diff --git a/.changeset/effect-4.0.6.md b/.changeset/effect-4.0.6.md new file mode 100644 index 0000000000..8c428dc04a --- /dev/null +++ b/.changeset/effect-4.0.6.md @@ -0,0 +1,9 @@ +--- +"effect-app": patch +"@effect-app/infra": patch +"@effect-app/vue": patch +"@effect-app/vue-components": patch +"@effect-app/cli": patch +--- + +Update Effect dependencies to 4.0.6. diff --git a/.changeset/rpc-parse-options.md b/.changeset/rpc-parse-options.md new file mode 100644 index 0000000000..3757094b04 --- /dev/null +++ b/.changeset/rpc-parse-options.md @@ -0,0 +1,10 @@ +--- +"effect-app": patch +"@effect-app/infra": patch +--- + +Adopt effect's `Rpc.ParseOptions` for RPC encode and decode concurrency, and drop the per-schema `parseOptions` concurrency annotations. + +Effect 4 ignores `parseOptions` annotations on schemas. Client and server RPC groups now carry `DefaultParseOptions` (unbounded unless `SCHEMA_CONCURRENCY` is set) via `annotateRpcParseOptions`, applied after prefix and middleware. `decodeEffectConcurrently` is unchanged for call sites that pass parse options themselves. + +Dependencies are on `effect@4.0.6`, which includes `Rpc.ParseOptions`. Peers require `effect@^4.0.6`. diff --git a/package.json b/package.json index bedc3c2e7e..718a908415 100644 --- a/package.json +++ b/package.json @@ -50,15 +50,15 @@ "@effect-app/infra": "workspace:*", "@effect/language-service": "0.87.5", "@effect/tsgo": "^0.56.0", - "@effect/platform-node": "4.0.5", - "@effect/vitest": "4.0.5", + "@effect/platform-node": "4.0.6", + "@effect/vitest": "4.0.6", "@tsconfig/strictest": "^2.0.8", "@types/node": "25.9.1", "@typescript-eslint/eslint-plugin": "8.60.0", "@typescript-eslint/parser": "8.60.0", "@typescript/native-preview": "7.0.0-dev.20260626.1", "dprint": "^0.54.0", - "effect": "4.0.5", + "effect": "4.0.6", "effect-app": "workspace:*", "eslint": "^10.4.1", "json5": "^2.2.3", diff --git a/packages/cli/package.json b/packages/cli/package.json index 1c03214afb..d37c7323c7 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -14,8 +14,8 @@ "effect-app-cli": "./bin.js" }, "dependencies": { - "@effect/platform-node": "4.0.5", - "effect": "4.0.5", + "@effect/platform-node": "4.0.6", + "effect": "4.0.6", "js-yaml": "4.2.0" }, "devDependencies": { diff --git a/packages/e2e/package.json b/packages/e2e/package.json index ec255856da..d01a8a4087 100644 --- a/packages/e2e/package.json +++ b/packages/e2e/package.json @@ -10,13 +10,13 @@ "effect-app": "workspace:*" }, "devDependencies": { - "@effect/atom-vue": "^4.0.5", - "@effect/platform-node": "4.0.5", - "@effect/vitest": "4.0.5", + "@effect/atom-vue": "^4.0.6", + "@effect/platform-node": "4.0.6", + "@effect/vitest": "4.0.6", "@tanstack/vue-query": "5.96.2", "@types/node": "25.9.1", "@vitejs/plugin-vue": "^6.0.7", - "effect": "^4.0.5", + "effect": "^4.0.6", "typescript": "~6.0.3", "vitest": "^5.0.5", "vue": "^3.5.35" diff --git a/packages/effect-app/package.json b/packages/effect-app/package.json index 6e4166e917..bce25ff254 100644 --- a/packages/effect-app/package.json +++ b/packages/effect-app/package.json @@ -25,7 +25,7 @@ "vitest": "^5.0.5" }, "peerDependencies": { - "effect": "^4.0.5" + "effect": "^4.0.6" }, "typesVersions": { "*": { diff --git a/packages/effect-app/src/Schema.ts b/packages/effect-app/src/Schema.ts index 01eecece89..336e1b2f25 100644 --- a/packages/effect-app/src/Schema.ts +++ b/packages/effect-app/src/Schema.ts @@ -6,7 +6,7 @@ import type { RequiredKeys } from "effect/Types" import type { Arbitrary as FastCheckArbitrary } from "fast-check" import type { NonEmptyReadonlyArray } from "./Array.ts" import { Email as EmailT, type Email as EmailType } from "./Schema/email.ts" -import { concurrencyUnbounded, withDefaultMake, withDefaultParseOptions } from "./Schema/ext.ts" +import { withDefaultMake, withDefaultParseOptions } from "./Schema/ext.ts" import type { FC } from "./Schema/FastCheck.ts" import { PhoneNumber as PhoneNumberT, type PhoneNumber as PhoneNumberType } from "./Schema/phoneNumber.ts" import { type AST } from "./Schema/schema.ts" @@ -156,7 +156,7 @@ export const Void: Void = S.Void export function Struct( fields: Fields ): Struct { - const result = S.Struct(fields).annotate(concurrencyUnbounded) + const result = S.Struct(fields) const allowVoidMake = (schema: any): any => { // Normalize omitted input to an empty object so optional/default-only structs can be constructed with make(). // Do not read schema.make first: Effect 4 caches it as a non-writable own property. @@ -194,7 +194,7 @@ export function Struct( } ;(result as any).mapFields = function(this: any, f: any, options?: any) { // eslint-disable-next-line @typescript-eslint/no-unsafe-call - const mapped = origMapFields.call(this, f, options).annotate(concurrencyUnbounded) + const mapped = origMapFields.call(this, f, options) return preserveCopyAndMethods(mapped) } ;(result as any).annotate = function(this: any, annotations?: any) { @@ -296,7 +296,7 @@ export type StructNestedEncoded = T extends { fields: infer Fields extends S. : StructNestedEncodedError export function NonEmptyArray(value: Value): S.NonEmptyArray { - return S.NonEmptyArray(value).annotate(concurrencyUnbounded) + return S.NonEmptyArray(value) } export function TaggedStruct( @@ -336,7 +336,7 @@ export function Record( key: Key, value: Value ): S.$Record { - return S.Record(key, value).annotate(concurrencyUnbounded) + return S.Record(key, value) } export declare namespace Record { export type Key = S.Record.Key diff --git a/packages/effect-app/src/Schema/Class.ts b/packages/effect-app/src/Schema/Class.ts index a13163e225..c52ef31c5c 100644 --- a/packages/effect-app/src/Schema/Class.ts +++ b/packages/effect-app/src/Schema/Class.ts @@ -8,7 +8,6 @@ import * as SchemaTransformation from "effect/SchemaTransformation" import type { Struct as EffectAppStruct } from "../Schema.ts" import * as SchemaAST from "../SchemaAST.ts" import { copyOrigin } from "../utils.ts" -import { concurrencyUnbounded } from "./ext.ts" import * as SchemaParser from "./SchemaParser.ts" type ClassAnnotations = S.Annotations.Declaration @@ -130,7 +129,7 @@ export const Class: = (identifier) => (fields, annotations, options) => { const relaxed = options?.strict === false // Build the original Schema.Class - const Base = (S.Class as any)(identifier)(fields, { ...concurrencyUnbounded, ...annotations }) + const Base = (S.Class as any)(identifier)(fields, annotations) // Get the original ast getter from the base class const originalAstDescriptor = Object.getOwnPropertyDescriptor(Base, "ast")! @@ -157,9 +156,6 @@ export const Class: = (identifier) => (tag, fields, annotations, options) => { const relaxed = options?.strict === false - const Base = (S.TaggedClass as any)(identifier)(tag, fields, { ...concurrencyUnbounded, ...annotations }) + const Base = (S.TaggedClass as any)(identifier)(tag, fields, annotations) const originalAstDescriptor = Object.getOwnPropertyDescriptor(Base, "ast")! const astCache = new WeakMap() const copyCache = new WeakMap>() @@ -219,9 +215,6 @@ export const TaggedClass: = (identifier) => (fields, annotations, options) => { const relaxed = options?.strict === false - const Base = (S.Error as any)(identifier)(fields, { ...concurrencyUnbounded, ...annotations }) + const Base = (S.Error as any)(identifier)(fields, annotations) const originalAstDescriptor = Object.getOwnPropertyDescriptor(Base, "ast")! const astCache = new WeakMap() const copyCache = new WeakMap>() @@ -264,9 +257,6 @@ export const ErrorClass: = (identifier) => (tag, fields, annotations, options) => { const relaxed = options?.strict === false - const Base = (S.TaggedError as any)(identifier)(tag, fields, { ...concurrencyUnbounded, ...annotations }) + const Base = (S.TaggedError as any)(identifier)(tag, fields, annotations) const originalAstDescriptor = Object.getOwnPropertyDescriptor(Base, "ast")! const astCache = new WeakMap() const copyCache = new WeakMap>() @@ -310,9 +300,6 @@ export const TaggedErrorClass: "unbounded" as const) } -/** - * Parse-options annotation used on schema constructors for decode paths where callers - * cannot currently pass parse options (notably some RPC / HttpApi integration paths). - * - * Keep this annotation in place so those framework-managed decodes still run with - * unbounded concurrency by default. - */ -export const concurrencyUnbounded = { parseOptions: DefaultParseOptions } as const - type DecodeLike = (schema: any) => (input: any, options?: SchemaAST.ParseOptions) => any export const withDefaultParseOptions = ( @@ -245,7 +236,7 @@ export const Literals = (value: ValueSchema) { return pipe( - S.Array(value).annotate(concurrencyUnbounded), + S.Array(value), (s) => Object.assign(s, { /** @@ -273,7 +264,7 @@ export function Array(value: ValueSchema) { export const ReadonlySetFromArray = (value: ValueSchema) => { const from = S .Array(value) - .annotate({ ...concurrencyUnbounded, expected: "an array of unique items that will be decoded as a ReadonlySet" }) + .annotate({ expected: "an array of unique items that will be decoded as a ReadonlySet" }) const to = S.instanceOf(Set) as S.instanceOf> const schema = from.pipe( S.decodeTo( @@ -297,7 +288,6 @@ export const ReadonlyMapFromArray = [keyof newM], `${ModuleName}.`> > - return rpcs + return annotateRpcParseOptions(rpcs) } // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/packages/effect-app/src/rpc.ts b/packages/effect-app/src/rpc.ts index e07797b678..ee58fad409 100644 --- a/packages/effect-app/src/rpc.ts +++ b/packages/effect-app/src/rpc.ts @@ -2,3 +2,4 @@ export * as Invalidation from "./rpc/Invalidation.ts" export * as MiddlewareMaker from "./rpc/MiddlewareMaker.ts" export * as RpcContextMap from "./rpc/RpcContextMap.ts" export * as RpcMiddleware from "./rpc/RpcMiddleware.ts" +export { annotateRpcParseOptions } from "./rpc/RpcParseOptions.ts" diff --git a/packages/effect-app/src/rpc/RpcParseOptions.ts b/packages/effect-app/src/rpc/RpcParseOptions.ts new file mode 100644 index 0000000000..06972139dc --- /dev/null +++ b/packages/effect-app/src/rpc/RpcParseOptions.ts @@ -0,0 +1,13 @@ +import { Rpc, type RpcGroup } from "effect/rpc" +import { DefaultParseOptions } from "../Schema/ext.ts" + +/** + * Apply {@link DefaultParseOptions} to every RPC in `group`. + * + * `RpcClient` and `RpcServer` read `Rpc.ParseOptions` when encoding and decoding + * payloads, successes, stream chunks, and exits. Schema `parseOptions` annotations + * are not consulted. Call this after `prefix` and `middleware`, which clone RPCs. + */ +export const annotateRpcParseOptions = ( + group: RpcGroup.RpcGroup +): RpcGroup.RpcGroup => group.annotateRpcs(Rpc.ParseOptions, DefaultParseOptions) diff --git a/packages/effect-app/test/rpc-parse-options.test.ts b/packages/effect-app/test/rpc-parse-options.test.ts new file mode 100644 index 0000000000..be704b03f3 --- /dev/null +++ b/packages/effect-app/test/rpc-parse-options.test.ts @@ -0,0 +1,193 @@ +import * as Context from "effect/Context" +import * as Effect from "effect/Effect" +import { HttpEffect } from "effect/http" +import * as Layer from "effect/Layer" +import * as EffectRequest from "effect/Request" +import * as RequestResolver from "effect/RequestResolver" +import { Rpc, RpcGroup, RpcMiddleware, RpcSerialization, RpcServer } from "effect/rpc" +import type * as SchemaAST from "effect/SchemaAST" +import * as SchemaTransformation from "effect/SchemaTransformation" +import { expect, test } from "vitest" +import { makeRpcClient, makeRpcGroupFromRequestsAndModuleName, NotLoggedInError } from "../src/client.js" +import { S } from "../src/index.js" +import { RpcContextMap } from "../src/rpc.js" +import { annotateRpcParseOptions } from "../src/rpc/RpcParseOptions.js" +import { DefaultParseOptions } from "../src/Schema/ext.js" + +const RequestContextMap = RpcContextMap.makeMap({ + allowAnonymous: RpcContextMap.makeInverted()(NotLoggedInError) +}) + +const stubMiddleware = { + requestContextMap: RequestContextMap.config, + requestContext: undefined +} + +const parseOptionsOf = (ast: SchemaAST.AST): unknown => { + const seen = new Set() + const visit = (node: SchemaAST.AST): unknown => { + if (seen.has(node)) return undefined + seen.add(node) + const parseOptions = node.annotations?.["parseOptions"] + if (parseOptions !== undefined) return parseOptions + if (node.encoding !== undefined) { + for (const link of node.encoding) { + const found = visit(link.to) + if (found !== undefined) return found + } + } + return undefined + } + return visit(ast) +} + +const expectNoParseOptions = (schema: { readonly ast: SchemaAST.AST }) => { + expect(parseOptionsOf(schema.ast)).toBeUndefined() +} + +const makeUser = (batches: Array>) => { + class GetUser extends EffectRequest.TaggedClass("GetUser")<{ readonly id: number }, string> {} + const resolver = RequestResolver.fromFunctionBatched((entries) => { + const ids = entries.map((entry) => entry.request.id) + batches.push(ids) + return ids.map((id) => `user-${id}`) + }) + return S.Number.pipe( + S.decodeTo( + S.String, + SchemaTransformation.transformEffect({ + decode: (id) => Effect.request(new GetUser({ id }), resolver), + encode: (user) => Effect.succeed(Number(user.slice("user-".length))) + }) + ) + ) +} + +const readExits = (body: string): Array => + body.split("\n").flatMap((line) => { + const trimmed = line.trim() + if (trimmed.length === 0) return [] + const parsed: unknown = JSON.parse(trimmed) + if (typeof parsed !== "object" || parsed === null || !("_tag" in parsed) || parsed._tag !== "Exit") { + return [] + } + return "exit" in parsed ? [parsed.exit] : [] + }) + +const postUsers = (annotate: boolean) => { + const batches: Array> = [] + const User = makeUser(batches) + const rpc = Rpc.make("getUsers", { + payload: S.Struct({ users: S.Array(User) }), + success: S.Array(S.String) + }) + const group = annotate ? annotateRpcParseOptions(RpcGroup.make(rpc)) : RpcGroup.make(rpc) + return Effect + .gen(function*() { + const httpEffect = yield* RpcServer.toHttpEffect(group).pipe( + Effect.provide(Layer.mergeAll( + group.toLayer({ + getUsers: ({ users }) => Effect.succeed(users) + }), + RpcSerialization.layerNdjson + )) + ) + const handler = HttpEffect.toWebHandler(httpEffect) + const response = yield* Effect.promise(() => + handler( + new Request("http://test/rpc", { + method: "POST", + body: `{"_tag":"Request","id":1,"tag":"getUsers","payload":{"users":[1,2,3]},"headers":[]}\n` + }) + ) + ) + const body = yield* Effect.promise(() => response.text()) + return { batches, body, exits: readExits(body) } + }) + .pipe(Effect.scoped) +} + +test("schema constructors do not carry parseOptions concurrency", () => { + expectNoParseOptions(S.Struct({ name: S.String })) + expectNoParseOptions(S.Struct({ name: S.String }).mapFields((fields) => ({ ...fields, extra: S.Finite }))) + expectNoParseOptions(S.Array(S.String)) + expectNoParseOptions(S.NonEmptyArray(S.String)) + expectNoParseOptions(S.Record(S.String, S.Finite)) + expectNoParseOptions(S.ReadonlySetFromArray(S.String)) + expectNoParseOptions(S.ReadonlyMapFromArray({ key: S.String, value: S.Finite })) + + class Person extends S.Class("Person")({ name: S.String }) {} + class Circle extends S.TaggedClass()("Circle", { radius: S.Finite }) {} + class Missing extends S.ErrorClass("Missing")({ reason: S.String }) {} + class Boom extends S.TaggedErrorClass()("Boom", { reason: S.String }) {} + + expectNoParseOptions(Person) + expectNoParseOptions(Person.mapFields((fields) => ({ ...fields, extra: S.Finite }))) + expectNoParseOptions(Circle) + expectNoParseOptions(Circle.mapFields((fields) => ({ ...fields }))) + expectNoParseOptions(Missing) + expectNoParseOptions(Boom) +}) + +test("schema array decode stays sequential unless the caller passes parse options", async () => { + const batches: Array> = [] + const User = makeUser(batches) + const schema = S.Array(User) + + expect(await Effect.runPromise(S.decodeUnknownEffect(schema)([1, 2, 3]))).toEqual(["user-1", "user-2", "user-3"]) + expect(batches).toEqual([[1], [2], [3]]) + + batches.length = 0 + expect(await Effect.runPromise(S.decodeUnknownEffectConcurrently(schema)([1, 2, 3]))).toEqual([ + "user-1", + "user-2", + "user-3" + ]) + expect(batches).toEqual([[1, 2, 3]]) +}) + +test("annotateRpcParseOptions sets DefaultParseOptions after prefix and middleware", () => { + class Marker extends RpcMiddleware.Service()("Marker") {} + const plain = RpcGroup.make(Rpc.make("ping", { success: S.String })) + const plainRpc = [...plain.requests.values()][0] + if (plainRpc === undefined) throw new Error("expected ping rpc") + expect(Context.getUnsafe(plainRpc.annotations, Rpc.ParseOptions).concurrency).toBeUndefined() + + const group = annotateRpcParseOptions(plain.prefix("Mod.").middleware(Marker)) + expect([...group.requests.keys()]).toEqual(["Mod.ping"]) + const rpc = group.requests.get("Mod.ping") + if (rpc === undefined) throw new Error("expected Mod.ping rpc") + expect(Context.getUnsafe(rpc.annotations, Rpc.ParseOptions)).toBe(DefaultParseOptions) + expect(rpc.middlewares.has(Marker)).toBe(true) +}) + +test("makeRpcGroupFromRequestsAndModuleName annotates every rpc", () => { + const { TaggedRequestFor } = makeRpcClient(stubMiddleware) + const Query = TaggedRequestFor("Users").Query + class GetStats extends Query()("GetStats", { ids: S.Array(S.String) }, { + success: S.Array(S.String) + }) {} + + const group = makeRpcGroupFromRequestsAndModuleName({ GetStats }, "Users") + const rpcs = [...group.requests.values()] + expect(rpcs.length).toBeGreaterThan(0) + for (const rpc of rpcs) { + expect(Context.getUnsafe(rpc.annotations, Rpc.ParseOptions)).toBe(DefaultParseOptions) + } +}) + +test("Rpc.ParseOptions makes server payload decode batch Effect.request calls", async () => { + const sequential = await Effect.runPromise(postUsers(false)) + expect(sequential.exits, sequential.body).toEqual([{ + _tag: "Success", + value: ["user-1", "user-2", "user-3"] + }]) + expect(sequential.batches).toEqual([[1], [2], [3]]) + + const concurrent = await Effect.runPromise(postUsers(true)) + expect(concurrent.exits, concurrent.body).toEqual([{ + _tag: "Success", + value: ["user-1", "user-2", "user-3"] + }]) + expect(concurrent.batches).toEqual([[1, 2, 3]]) +}) diff --git a/packages/infra/package.json b/packages/infra/package.json index 33005e8402..7d2c70add5 100644 --- a/packages/infra/package.json +++ b/packages/infra/package.json @@ -21,7 +21,7 @@ "devDependencies": { "@azure/cosmos": "^4.9.3", "@azure/service-bus": "^7.9.5", - "@effect/sql-sqlite-node": "4.0.5", + "@effect/sql-sqlite-node": "4.0.6", "@sentry/opentelemetry": "10.55.0", "@types/better-sqlite3": "^7.6.13", "@types/node": "25.9.1", @@ -39,12 +39,12 @@ "peerDependencies": { "@azure/cosmos": "^4.9.3", "@azure/service-bus": "^7.9.5", - "@effect/vitest": "^4.0.5", + "@effect/vitest": "^4.0.6", "@sendgrid/helpers": "^8.0.0", "@sendgrid/mail": "^8.1.6", "@sentry/node": "10.55.0", "@sentry/opentelemetry": "10.55.0", - "effect": "^4.0.5", + "effect": "^4.0.6", "jwt-decode": "^4.0.0", "redis": "^3.1.2", "redlock": "^4.2.0" diff --git a/packages/infra/src/routing.ts b/packages/infra/src/routing.ts index a3034e9459..7088abb64b 100644 --- a/packages/infra/src/routing.ts +++ b/packages/infra/src/routing.ts @@ -10,7 +10,7 @@ import * as DataDependencies from "effect-app/DataDependencies" import * as Effect from "effect-app/Effect" import { type HttpHeaders } from "effect-app/http" import * as Layer from "effect-app/Layer" -import { Invalidation } from "effect-app/rpc" +import { annotateRpcParseOptions, Invalidation } from "effect-app/rpc" import { type GetEffectContext, type GetEffectError, type RpcContextMap } from "effect-app/rpc/RpcContextMap" import * as S from "effect-app/Schema" import { type TypeTestId } from "effect-app/TypeTest" @@ -622,36 +622,38 @@ export const makeRouter = = Layer.Layer ] } - const rpcs = RpcGroup - .make( - ...(Object.values(mapped) as Array<(typeof mapped)[keyof typeof mapped]>).map(([resource]) => { - const isStream = resource.stream - const isCommand = resource.type === "command" - return (isCommand - ? isStream - ? Invalidation.makeStreamRpc(resource._tag, { + const rpcs = annotateRpcParseOptions( + RpcGroup + .make( + ...(Object.values(mapped) as Array<(typeof mapped)[keyof typeof mapped]>).map(([resource]) => { + const isStream = resource.stream + const isCommand = resource.type === "command" + return (isCommand + ? isStream + ? Invalidation.makeStreamRpc(resource._tag, { + payload: resource, + success: resource.success, + error: resource.error, + stream: true as const + }) + : Invalidation.makeCommandRpc(resource._tag, { + payload: resource, + success: resource.success, + error: resource.error + }) + : Rpc.make(resource._tag, { payload: resource, - success: resource.success, + success: isStream ? resource.success : Invalidation.QueryResponseWithMetaData(resource.success), error: resource.error, - stream: true as const - }) - : Invalidation.makeCommandRpc(resource._tag, { - payload: resource, - success: resource.success, - error: resource.error - }) - : Rpc.make(resource._tag, { - payload: resource, - success: isStream ? resource.success : Invalidation.QueryResponseWithMetaData(resource.success), - error: resource.error, - stream: isStream - })) - .annotate(mw.requestContext, resource.config ?? {}) - .annotate(RequestTypeAnnotation, resource.type) - }) - ) - .prefix(`${meta.moduleName}.`) - .middleware(mw) + stream: isStream + })) + .annotate(mw.requestContext, resource.config ?? {}) + .annotate(RequestTypeAnnotation, resource.type) + }) + ) + .prefix(`${meta.moduleName}.`) + .middleware(mw) + ) const rpc = rpcs .toLayer(Effect.gen(function*() { diff --git a/packages/vue-components/package.json b/packages/vue-components/package.json index 8edcaaabf0..2f98de8f9e 100644 --- a/packages/vue-components/package.json +++ b/packages/vue-components/package.json @@ -25,7 +25,7 @@ }, "peerDependencies": { "@mdi/js": "^7.4.47", - "effect": "^4.0.5", + "effect": "^4.0.6", "intl-messageformat": "^11.2.7", "mdi-js": "^1.0.1", "primeflex": "^4.0.0", diff --git a/packages/vue/package.json b/packages/vue/package.json index 506abf9752..a3abfbc29c 100644 --- a/packages/vue/package.json +++ b/packages/vue/package.json @@ -18,14 +18,14 @@ "query-string": "^9.4.0" }, "peerDependencies": { - "@effect/atom-vue": "^4.0.5", - "@effect/platform-browser": "^4.0.5", + "@effect/atom-vue": "^4.0.6", + "@effect/platform-browser": "^4.0.6", "@sentry/browser": "^10.55.0", - "effect": "^4.0.5", + "effect": "^4.0.6", "vue": "^3.5.35" }, "devDependencies": { - "@effect/vitest": "4.0.5", + "@effect/vitest": "4.0.6", "@formatjs/icu-messageformat-parser": "^3.5.10", "@types/node": "25.9.1", "@vitejs/plugin-vue": "^6.0.7", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 46da1f23d9..8b826ba7a1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -254,14 +254,14 @@ importers: specifier: 0.87.5 version: 0.87.5 "@effect/platform-node": - specifier: 4.0.5 - version: 4.0.5(effect@4.0.5)(redis@6.2.1(@opentelemetry/api@1.9.1)) + specifier: 4.0.6 + version: 4.0.6(effect@4.0.6)(redis@6.2.1(@opentelemetry/api@1.9.1)) "@effect/tsgo": specifier: ^0.56.0 version: 0.56.0 "@effect/vitest": - specifier: 4.0.5 - version: 4.0.5(effect@4.0.5)(vitest@5.0.5(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(jsdom@29.1.1)(vite@8.0.15(@types/node@25.9.1)(esbuild@0.27.2)(sass@1.100.0)(terser@5.45.0)(tsx@4.22.4)(yaml@2.9.0))) + specifier: 4.0.6 + version: 4.0.6(effect@4.0.6)(vitest@5.0.5(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(jsdom@29.1.1)(vite@8.0.15(@types/node@25.9.1)(esbuild@0.27.2)(sass@1.100.0)(terser@5.45.0)(tsx@4.22.4)(yaml@2.9.0))) "@tsconfig/strictest": specifier: ^2.0.8 version: 2.0.8 @@ -281,8 +281,8 @@ importers: specifier: ^0.54.0 version: 0.54.0 effect: - specifier: 4.0.5 - version: 4.0.5 + specifier: 4.0.6 + version: 4.0.6 effect-app: specifier: workspace:* version: link:packages/effect-app @@ -326,11 +326,11 @@ importers: packages/cli: dependencies: "@effect/platform-node": - specifier: 4.0.5 - version: 4.0.5(effect@4.0.5)(redis@6.2.1(@opentelemetry/api@1.9.1)) + specifier: 4.0.6 + version: 4.0.6(effect@4.0.6)(redis@6.2.1(@opentelemetry/api@1.9.1)) effect: - specifier: 4.0.5 - version: 4.0.5 + specifier: 4.0.6 + version: 4.0.6 js-yaml: specifier: 4.2.0 version: 4.2.0 @@ -363,14 +363,14 @@ importers: version: link:../effect-app devDependencies: "@effect/atom-vue": - specifier: ^4.0.5 - version: 4.0.5(effect@4.0.5)(vue@3.5.35(typescript@6.0.3(patch_hash=d2a2f8424c52b9d75deeb15b1f2367352c31126ecb19c4b469da7b943f946a54))) + specifier: ^4.0.6 + version: 4.0.6(effect@4.0.6)(vue@3.5.35(typescript@6.0.3(patch_hash=d2a2f8424c52b9d75deeb15b1f2367352c31126ecb19c4b469da7b943f946a54))) "@effect/platform-node": - specifier: 4.0.5 - version: 4.0.5(effect@4.0.5)(redis@6.2.1(@opentelemetry/api@1.9.1)) + specifier: 4.0.6 + version: 4.0.6(effect@4.0.6)(redis@6.2.1(@opentelemetry/api@1.9.1)) "@effect/vitest": - specifier: 4.0.5 - version: 4.0.5(effect@4.0.5)(vitest@5.0.5(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(jsdom@29.1.1)(vite@8.0.15(@types/node@25.9.1)(esbuild@0.28.0)(sass@1.100.0)(terser@5.45.0)(tsx@4.22.4)(yaml@2.9.0))) + specifier: 4.0.6 + version: 4.0.6(effect@4.0.6)(vitest@5.0.5(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(jsdom@29.1.1)(vite@8.0.15(@types/node@25.9.1)(esbuild@0.28.0)(sass@1.100.0)(terser@5.45.0)(tsx@4.22.4)(yaml@2.9.0))) "@tanstack/vue-query": specifier: 5.96.2 version: 5.96.2(vue@3.5.35(typescript@6.0.3(patch_hash=d2a2f8424c52b9d75deeb15b1f2367352c31126ecb19c4b469da7b943f946a54))) @@ -381,8 +381,8 @@ importers: specifier: ^6.0.7 version: 6.0.7(vite@8.0.15(@types/node@25.9.1)(esbuild@0.28.0)(sass@1.100.0)(terser@5.45.0)(tsx@4.22.4)(yaml@2.9.0))(vue@3.5.35(typescript@6.0.3(patch_hash=d2a2f8424c52b9d75deeb15b1f2367352c31126ecb19c4b469da7b943f946a54))) effect: - specifier: ^4.0.5 - version: 4.0.5 + specifier: ^4.0.6 + version: 4.0.6 typescript: specifier: ~6.0.3 version: 6.0.3(patch_hash=d2a2f8424c52b9d75deeb15b1f2367352c31126ecb19c4b469da7b943f946a54) @@ -399,8 +399,8 @@ importers: specifier: ^2.0.8 version: 2.0.8 effect: - specifier: ^4.0.5 - version: 4.0.5 + specifier: ^4.0.6 + version: 4.0.6 nanoid: specifier: ^5.1.11 version: 5.1.11 @@ -540,8 +540,8 @@ importers: packages/infra: dependencies: "@effect/vitest": - specifier: ^4.0.5 - version: 4.0.5(effect@4.0.5)(vitest@5.0.5(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(jsdom@29.1.1)(vite@8.0.15(@types/node@25.9.1)(esbuild@0.28.0)(sass@1.100.0)(terser@5.45.0)(tsx@4.22.4)(yaml@2.9.0))) + specifier: ^4.0.6 + version: 4.0.6(effect@4.0.6)(vitest@5.0.5(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(jsdom@29.1.1)(vite@8.0.15(@types/node@25.9.1)(esbuild@0.28.0)(sass@1.100.0)(terser@5.45.0)(tsx@4.22.4)(yaml@2.9.0))) "@faker-js/faker": specifier: ^8.4.1 version: 8.4.1 @@ -555,8 +555,8 @@ importers: specifier: 10.55.0 version: 10.55.0(supports-color@7.2.0) effect: - specifier: ^4.0.5 - version: 4.0.5 + specifier: ^4.0.6 + version: 4.0.6 effect-app: specifier: workspace:* version: link:../effect-app @@ -583,8 +583,8 @@ importers: specifier: ^7.9.5 version: 7.9.5(supports-color@7.2.0) "@effect/sql-sqlite-node": - specifier: 4.0.5 - version: 4.0.5(effect@4.0.5) + specifier: 4.0.6 + version: 4.0.6(effect@4.0.6) "@sentry/opentelemetry": specifier: 10.55.0 version: 10.55.0(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.6.1(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.6.1(@opentelemetry/api@1.9.1))(@opentelemetry/semantic-conventions@1.40.0) @@ -630,11 +630,11 @@ importers: packages/vue: dependencies: "@effect/atom-vue": - specifier: ^4.0.5 - version: 4.0.5(effect@4.0.5)(vue@3.5.35(typescript@6.0.3(patch_hash=d2a2f8424c52b9d75deeb15b1f2367352c31126ecb19c4b469da7b943f946a54))) + specifier: ^4.0.6 + version: 4.0.6(effect@4.0.6)(vue@3.5.35(typescript@6.0.3(patch_hash=d2a2f8424c52b9d75deeb15b1f2367352c31126ecb19c4b469da7b943f946a54))) "@effect/platform-browser": - specifier: ^4.0.5 - version: 4.0.5(effect@4.0.5) + specifier: ^4.0.6 + version: 4.0.6(effect@4.0.6) "@formatjs/intl": specifier: ^4.1.12 version: 4.1.12 @@ -651,8 +651,8 @@ importers: specifier: ^5.4.4 version: 5.4.4 effect: - specifier: ^4.0.5 - version: 4.0.5 + specifier: ^4.0.6 + version: 4.0.6 effect-app: specifier: workspace:* version: link:../effect-app @@ -664,8 +664,8 @@ importers: version: 3.5.35(typescript@6.0.3(patch_hash=d2a2f8424c52b9d75deeb15b1f2367352c31126ecb19c4b469da7b943f946a54)) devDependencies: "@effect/vitest": - specifier: 4.0.5 - version: 4.0.5(effect@4.0.5)(vitest@5.0.5(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(jsdom@29.1.1)(vite@8.0.15(@types/node@25.9.1)(esbuild@0.28.0)(sass@1.100.0)(terser@5.45.0)(tsx@4.22.4)(yaml@2.9.0))) + specifier: 4.0.6 + version: 4.0.6(effect@4.0.6)(vitest@5.0.5(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(jsdom@29.1.1)(vite@8.0.15(@types/node@25.9.1)(esbuild@0.28.0)(sass@1.100.0)(terser@5.45.0)(tsx@4.22.4)(yaml@2.9.0))) "@formatjs/icu-messageformat-parser": specifier: ^3.5.10 version: 3.5.10 @@ -705,8 +705,8 @@ importers: specifier: ^1.32.0 version: 1.32.0(vue@3.5.35(typescript@6.0.3(patch_hash=d2a2f8424c52b9d75deeb15b1f2367352c31126ecb19c4b469da7b943f946a54))) effect: - specifier: ^4.0.5 - version: 4.0.5 + specifier: ^4.0.6 + version: 4.0.6 effect-app: specifier: workspace:* version: link:../effect-app @@ -1393,12 +1393,12 @@ packages: cpu: [x64] os: [win32] - "@effect/atom-vue@4.0.5": + "@effect/atom-vue@4.0.6": resolution: { - integrity: sha512-GYnHSbdDE+2Q3AgQSNIxw0lsAUQngeP5RCW2xW9YH8KZ0eLq5DNUk3aHZyZy72M2KgM9dEU/z//vwMDBDB/c2g==, + integrity: sha512-JpbRYTBmVYyOBVVqnuzqgcoGbuBrw7r6skx6Al8YB2NFLUGskWqfJey+TY+fACr0qoUsGuDhlWYkzaiEm8/qmw==, } peerDependencies: - effect: ^4.0.5 + effect: ^4.0.6 vue: ^3.5.35 "@effect/language-service@0.87.5": @@ -1407,36 +1407,36 @@ packages: } hasBin: true - "@effect/platform-browser@4.0.5": + "@effect/platform-browser@4.0.6": resolution: { - integrity: sha512-o3H75xA91nkgIWOcp0XouIxxWxJEOFkTf7gG4qkQtZfy0tq46hJBI0U8jrUqsPruo2vCCNfM0yCoj5AG7F5BMQ==, + integrity: sha512-i4bf1+FlUnZypqK0pQ5cq46KhroTgqFGktFn2WorwIk+yc+RuZO6qsF+Sk6lY5uHwdHTOLGMlnlXSJ8u1UG42g==, } peerDependencies: - effect: ^4.0.5 + effect: ^4.0.6 - "@effect/platform-node-shared@4.0.5": + "@effect/platform-node-shared@4.0.6": resolution: { - integrity: sha512-a8rCdEkfZBGe9C6bsIjxVZuRp73svIyK5IHwHdDAVsvZXjrbkxusfI+rZ8WlX31RP1mKizwI/E72GQTNfdMWHw==, + integrity: sha512-0PhyC4uG30xGtsajnxjkcBWfTjNRzUh4ZKl1Kvo/MNyCTb+W2O+fBzf7wCRCoqt+nLudiq4V7vX40Q8XIBuMaQ==, } engines: { node: ">=18.0.0" } peerDependencies: - effect: ^4.0.5 + effect: ^4.0.6 - "@effect/platform-node@4.0.5": + "@effect/platform-node@4.0.6": resolution: { - integrity: sha512-v4KD5fWvEUa4QmWFpZFbBsRhmfnTP6ETCME2r9ifyXVDPjhfpxpTZl41prjgQFMqZBHdWWCl7YC5ImbXv5ui5g==, + integrity: sha512-gXvGHAy0/8PZbWvMMguloDUyeSP+rzYEo3FSEqUuGduxo9ORcGioDxPI+GOLCIFfam7VuNkmSUxBOiRMIrMrBw==, } engines: { node: ">=18.0.0" } peerDependencies: - effect: ^4.0.5 + effect: ^4.0.6 redis: ">=5.0.0 <7.0.0" - "@effect/sql-sqlite-node@4.0.5": + "@effect/sql-sqlite-node@4.0.6": resolution: { - integrity: sha512-SHza5qNwiI4dRtUk45wvUfvZq7vB950DW2wD6npzt2aH/tesFfPlZJSDBIvwhazoK+gvDknBOONfKyFZsIF+tg==, + integrity: sha512-mYWr1Ppoz3/XrY+ZBnYfdvZvdcAYJTO6VcWoNetBX5n2c0VKHqg9S8WO6EpgtV5WiR8/yHHmu3qkfAGHJmalcA==, } peerDependencies: - effect: ^4.0.5 + effect: ^4.0.6 "@effect/tsgo-darwin-arm64@0.56.0": resolution: { @@ -1493,12 +1493,12 @@ packages: } hasBin: true - "@effect/vitest@4.0.5": + "@effect/vitest@4.0.6": resolution: { - integrity: sha512-q64paZ/i4V2NiRhTyqCmT2KiDbFjlxc2EZrZca8GGt+Nf8844946HyJM5bPk/q+YeRkFSxIrlCs+J+LSfky/wA==, + integrity: sha512-oKNbtyUHK5+0o730CXwojULEu7f3+c235pYFD4U5T//o+hHrJbYbZU4bRkcv936tPr5IOnmujeyQiC/LuJJjHw==, } peerDependencies: - effect: ^4.0.5 + effect: ^4.0.6 vitest: ">=5.0.0 <6.0.0" "@emnapi/core@1.10.0": @@ -5829,9 +5829,9 @@ packages: engines: { node: ">=14" } hasBin: true - effect@4.0.5: + effect@4.0.6: resolution: { - integrity: sha512-BY4/28NlpOd99yzmV741xbVGmp+1U7tks4oB7F/Png64bTa5tm5eCA6W0RiXsnuLDDoJpjfVSo4Dnq4BZb/89Q==, + integrity: sha512-Tc3mkzuLR6t6P9dhLt0D2GrxYtmU5td4XEJEiXCmILz53IRIfJyHsKZWjTA3xEjiwpNTtXBgI4iqYPxMztBDrw==, } electron-to-chromium@1.5.267: @@ -9124,9 +9124,9 @@ packages: } engines: { node: ">=20.18.1" } - undici@8.11.2: + undici@8.12.3: resolution: { - integrity: sha512-u4UB2/IrKdU6lFxumHmmo1a3fCQO5tzQllRorfoRS63txhrB7xTpSn1PftwC4qEHkOaqP95fCWW4lJzwErwzhQ==, + integrity: sha512-+fdndUTwBP86cVjihTp21hZHo7d6oj1c/47XPA3lL8mM0XHs+C4EkcjmgRsFnFk2f/UONGDy9ItBItmXsLOQHw==, } engines: { node: ">=22.19.0" } @@ -10312,39 +10312,39 @@ snapshots: "@dprint/win32-x64@0.54.0": optional: true - "@effect/atom-vue@4.0.5(effect@4.0.5)(vue@3.5.35(typescript@6.0.3(patch_hash=d2a2f8424c52b9d75deeb15b1f2367352c31126ecb19c4b469da7b943f946a54)))": + "@effect/atom-vue@4.0.6(effect@4.0.6)(vue@3.5.35(typescript@6.0.3(patch_hash=d2a2f8424c52b9d75deeb15b1f2367352c31126ecb19c4b469da7b943f946a54)))": dependencies: - effect: 4.0.5 + effect: 4.0.6 vue: 3.5.35(typescript@6.0.3(patch_hash=d2a2f8424c52b9d75deeb15b1f2367352c31126ecb19c4b469da7b943f946a54)) "@effect/language-service@0.87.5": {} - "@effect/platform-browser@4.0.5(effect@4.0.5)": + "@effect/platform-browser@4.0.6(effect@4.0.6)": dependencies: - effect: 4.0.5 + effect: 4.0.6 - "@effect/platform-node-shared@4.0.5(effect@4.0.5)": + "@effect/platform-node-shared@4.0.6(effect@4.0.6)": dependencies: "@types/ws": 8.18.2 - effect: 4.0.5 + effect: 4.0.6 ws: 8.22.0 transitivePeerDependencies: - bufferutil - utf-8-validate - "@effect/platform-node@4.0.5(effect@4.0.5)(redis@6.2.1(@opentelemetry/api@1.9.1))": + "@effect/platform-node@4.0.6(effect@4.0.6)(redis@6.2.1(@opentelemetry/api@1.9.1))": dependencies: - "@effect/platform-node-shared": 4.0.5(effect@4.0.5) - effect: 4.0.5 + "@effect/platform-node-shared": 4.0.6(effect@4.0.6) + effect: 4.0.6 redis: 6.2.1(@opentelemetry/api@1.9.1) - undici: 8.11.2 + undici: 8.12.3 transitivePeerDependencies: - bufferutil - utf-8-validate - "@effect/sql-sqlite-node@4.0.5(effect@4.0.5)": + "@effect/sql-sqlite-node@4.0.6(effect@4.0.6)": dependencies: - effect: 4.0.5 + effect: 4.0.6 "@effect/tsgo-darwin-arm64@0.56.0": optional: true @@ -10377,14 +10377,14 @@ snapshots: "@effect/tsgo-win32-arm64": 0.56.0 "@effect/tsgo-win32-x64": 0.56.0 - "@effect/vitest@4.0.5(effect@4.0.5)(vitest@5.0.5(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(jsdom@29.1.1)(vite@8.0.15(@types/node@25.9.1)(esbuild@0.27.2)(sass@1.100.0)(terser@5.45.0)(tsx@4.22.4)(yaml@2.9.0)))": + "@effect/vitest@4.0.6(effect@4.0.6)(vitest@5.0.5(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(jsdom@29.1.1)(vite@8.0.15(@types/node@25.9.1)(esbuild@0.27.2)(sass@1.100.0)(terser@5.45.0)(tsx@4.22.4)(yaml@2.9.0)))": dependencies: - effect: 4.0.5 + effect: 4.0.6 vitest: 5.0.5(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(jsdom@29.1.1)(vite@8.0.15(@types/node@25.9.1)(esbuild@0.27.2)(sass@1.100.0)(terser@5.45.0)(tsx@4.22.4)(yaml@2.9.0)) - "@effect/vitest@4.0.5(effect@4.0.5)(vitest@5.0.5(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(jsdom@29.1.1)(vite@8.0.15(@types/node@25.9.1)(esbuild@0.28.0)(sass@1.100.0)(terser@5.45.0)(tsx@4.22.4)(yaml@2.9.0)))": + "@effect/vitest@4.0.6(effect@4.0.6)(vitest@5.0.5(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(jsdom@29.1.1)(vite@8.0.15(@types/node@25.9.1)(esbuild@0.28.0)(sass@1.100.0)(terser@5.45.0)(tsx@4.22.4)(yaml@2.9.0)))": dependencies: - effect: 4.0.5 + effect: 4.0.6 vitest: 5.0.5(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(jsdom@29.1.1)(vite@8.0.15(@types/node@25.9.1)(esbuild@0.28.0)(sass@1.100.0)(terser@5.45.0)(tsx@4.22.4)(yaml@2.9.0)) "@emnapi/core@1.10.0": @@ -12986,7 +12986,7 @@ snapshots: minimatch: 9.0.1 semver: 7.7.3 - effect@4.0.5: {} + effect@4.0.6: {} electron-to-chromium@1.5.267: optional: true @@ -15597,7 +15597,7 @@ snapshots: undici@7.25.0: {} - undici@8.11.2: {} + undici@8.12.3: {} unist-util-is@6.0.1: dependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index ddf9fed8d0..ce9067c297 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -17,13 +17,13 @@ minimumReleaseAgeExclude: - "@effect/tsgo-win32-x64@0.56.0" - "@effect/tsgo@0.56.0" - "@effect/language-service@0.87.5" - - "@effect/atom-vue@4.0.5" - - "@effect/platform-browser@4.0.5" - - "@effect/platform-node-shared@4.0.5" - - "@effect/platform-node@4.0.5" - - "@effect/sql-sqlite-node@4.0.5" - - "@effect/vitest@4.0.5" - - effect@4.0.5 + - "@effect/atom-vue@4.0.6" + - "@effect/platform-browser@4.0.6" + - "@effect/platform-node-shared@4.0.6" + - "@effect/platform-node@4.0.6" + - "@effect/sql-sqlite-node@4.0.6" + - "@effect/vitest@4.0.6" + - effect@4.0.6 overrides: date-fns: ^4.4.0 fast-check: ^4.9.0 diff --git a/repos/effect/.agents/skills/changesets/SKILL.md b/repos/effect/.agents/skills/changesets/SKILL.md index 5b6b4d5f29..36c078d791 100644 --- a/repos/effect/.agents/skills/changesets/SKILL.md +++ b/repos/effect/.agents/skills/changesets/SKILL.md @@ -18,8 +18,8 @@ focused validation are complete. 4. Record a reason no changeset is required, or update the PR's existing changeset. Create one only when the PR does not have one. When required, read [authoring.md](authoring.md). -5. Validate package names, frontmatter, bump policy, and a concise - consumer-facing description. +5. Validate package names, frontmatter, bump policy, and a one-sentence + consumer-facing description, longer only for breaks or required migration. Do not include unrelated worktree changes. @@ -71,5 +71,6 @@ output is mechanical evidence, not a semantic-version decision. The task is complete when every affected contract and published package is accounted for and either the no-changeset decision is explicit or the PR has one -consolidated, valid changeset by default. Its release note describes the consumer -impact and migration for every break without repeating the PR description. +consolidated, valid changeset by default. Its release note is one sentence of +consumer impact, plus migration steps for every break, with no implementation +detail. diff --git a/repos/effect/.agents/skills/changesets/authoring.md b/repos/effect/.agents/skills/changesets/authoring.md index 40d4ef9248..a1158ddf3e 100644 --- a/repos/effect/.agents/skills/changesets/authoring.md +++ b/repos/effect/.agents/skills/changesets/authoring.md @@ -9,17 +9,27 @@ Update the changeset already introduced by the PR. If there is none, create one "@effect/affected-package": patch --- -Describe the consumer-visible change and why it matters. +Fix `Effect.cachedInvalidateWithTTL` caching a stale result when invalidated while the effect is running. ``` -Default to one short paragraph of one to three sentences. State the changed -behavior or API and, for a break, the required migration. Leave motivation, -implementation details, benchmarks, test coverage, and the development history -in the PR description. +Write one short sentence, usually under 25 words, that names the changed API +and states what consumers will notice. That sentence is the whole entry unless +the change is breaking or consumers must migrate. -When several related consumer-visible effects cannot be stated clearly in that -paragraph, use compact bullets with one outcome per bullet. Keep them in this -file under the PR's shared purpose. +Leave out how the change works: internal modules, algorithms, data structures, +refactors, motivation, benchmarks, test coverage, and development history. They +belong in the PR description. Cut any sentence a consumer would not act on. + +- Bad: "Rework the `MemoMap` lookup so `Layer.provide` checks identity before + building, adding a cache keyed by layer reference and tests for shared + layers." +- Good: "Fix composed layers being rebuilt for every parent that references + them." + +Add detail only for a break or required migration: what existing code must +change and how. Use compact bullets, one break per bullet, when several breaks +need migration steps. Several non-breaking fixes or additions still share one +sentence. List every directly affected published package. Do not list packages merely because they share the fixed release group in `.changeset/config.json`. @@ -37,10 +47,10 @@ Choose the bump from current release policy: change requires, not the branch you want it to land on. - Ask when release intent is ambiguous. -Write for consumers. Use a `### Breaking changes` section when several breaks -need separate scanning. Include before/after examples only when they materially -clarify migration. +Use a `### Breaking changes` section when several breaks need separate +scanning. Include before/after examples only when they materially clarify +migration. Validate frontmatter against published package names and inspect nearby current -changesets for wording and release convention. Never run `changeset-version` or -`changeset-publish` as contributor validation. +changesets for release convention. Do not copy the length of longer entries. +Never run `changeset-version` or `changeset-publish` as contributor validation. diff --git a/repos/effect/.github/actions/setup/action.yaml b/repos/effect/.github/actions/setup/action.yaml index 0f97b61a17..7f3a2a2def 100644 --- a/repos/effect/.github/actions/setup/action.yaml +++ b/repos/effect/.github/actions/setup/action.yaml @@ -21,6 +21,8 @@ runs: with: node-version: 26.4.0 package-manager-cache: ${{ env.NSC_CONTAINER_REGISTRY == '' }} + cache: ${{ env.NSC_CONTAINER_REGISTRY == '' && 'pnpm' || '' }} + cache-dependency-path: pnpm-lock.yaml - name: Configure pnpm cache if: ${{ env.NSC_CONTAINER_REGISTRY != '' }} uses: namespacelabs/nscloud-cache-action@c5f8dab7560444c4bf8dbc64f1b203431873c547 # v1.6.1 diff --git a/repos/effect/.github/workflows/check.yml b/repos/effect/.github/workflows/check.yml index fbb21ced9b..6666cb00dd 100644 --- a/repos/effect/.github/workflows/check.yml +++ b/repos/effect/.github/workflows/check.yml @@ -50,13 +50,15 @@ jobs: - name: Install dependencies uses: ./.github/actions/setup with: - deno-version: v2.9.7 + deno-version: v2.9.6 - run: pnpm check env: # Leave memory headroom for the runner by collecting the Go heap earlier. GOMEMLIMIT: 4GiB - run: deno check . - run: pnpm test-types --target '>=5.9' + env: + TSTYCHE_FETCH_TIMEOUT: "120" build: name: Build @@ -116,6 +118,8 @@ jobs: runs-on: namespace-profile-linux-default env: EFFECT_INTEGRATION_TESTS: "1" + # MSSQL integration tests are too flaky for CI; run them locally instead. + EFFECT_MSSQL_INTEGRATION_TESTS: "0" # The ephemeral runner provides cleanup without a Ryuk sidecar. TESTCONTAINERS_RYUK_DISABLED: "true" permissions: @@ -130,7 +134,7 @@ jobs: bun: "" command: pnpm test --max-concurrency=10 - runtime: Deno - deno: v2.9.7 + deno: v2.9.6 bun: "" command: deno task test --max-concurrency=10 - runtime: Bun @@ -143,9 +147,8 @@ jobs: - name: Pre-pull test container images run: | docker pull ghcr.io/tursodatabase/libsql-server:main & - docker pull postgres:alpine & + docker pull postgres:14-alpine & docker pull mysql:lts & - docker pull mcr.microsoft.com/mssql/server:2022-latest & docker pull vitess/vttestserver:mysql80 & docker pull redis:alpine & docker pull coredns/coredns:1.14.7 & diff --git a/repos/effect/.github/workflows/release-queue.yml b/repos/effect/.github/workflows/release-queue.yml index c23e57485d..5f2bd71805 100644 --- a/repos/effect/.github/workflows/release-queue.yml +++ b/repos/effect/.github/workflows/release-queue.yml @@ -35,7 +35,7 @@ jobs: if: github.event.pull_request env: GITHUB_TOKEN: ${{ secrets.EFFECT_BOT_GH }} - - uses: Effect-TS/next-release-action@63daaa626282bc42fad44dcb89c7f7db9c53b786 + - uses: Effect-TS/next-release-action@9c7fe33d6e497924ae795d53c0c1cf75d48c96c2 with: github_token: ${{ secrets.EFFECT_BOT_GH }} base_branch: main diff --git a/repos/effect/ai-docs/package.json b/repos/effect/ai-docs/package.json index 13ef4319f8..31975a9fba 100644 --- a/repos/effect/ai-docs/package.json +++ b/repos/effect/ai-docs/package.json @@ -28,7 +28,7 @@ "@effect/vitest": "workspace:*", "effect": "workspace:*", "hono": "^4.13.13", - "nodemailer": "^10.1.0" + "nodemailer": "^10.1.2" }, "devDependencies": { "@types/nodemailer": "^8.0.2" diff --git a/repos/effect/ai-docs/src/06_schedule/10_schedules.ts b/repos/effect/ai-docs/src/06_schedule/10_schedules.ts index d42072add0..dfbe74e55d 100644 --- a/repos/effect/ai-docs/src/06_schedule/10_schedules.ts +++ b/repos/effect/ai-docs/src/06_schedule/10_schedules.ts @@ -5,14 +5,15 @@ */ import { Duration, Effect, Random, Schedule, Schema } from "effect" -// Production pattern: capped exponential backoff with jitter and max attempts. -// Delays start at 250ms, grow exponentially with jitter, and are capped at 10s. +// Production pattern: capped exponential backoff with jitter and at most 5 retries (6 attempts). +// Base delays start at 250ms and grow exponentially, capped at 10s before jitter. export const productionRetrySchedule = Schedule.min([ Schedule.exponential("250 millis"), // Cap the delay at 10 seconds to avoid excessively long waits. Schedule.spaced("10 seconds") ]).pipe( Schedule.jittered, + Schedule.upTo({ times: 5 }), Schedule.setInputType(), Schedule.while(({ input }) => input.retryable) ) @@ -44,8 +45,8 @@ export const fetchUserProfile = Effect.fn("fetchUserProfile")( // Use the schedule with `Effect.retry` to retry failures. export const loadUserWithRetry = fetchUserProfile("user-123").pipe( Effect.retry(productionRetrySchedule), - // If the effect still fails after exhausting the schedule, turn the error - // into a fatal one. + // Turn the final error into a defect, either after exhausting the retries + // or immediately for a non-retryable error. Effect.orDie ) diff --git a/repos/effect/ai-docs/tsconfig.json b/repos/effect/ai-docs/tsconfig.json index 8aad2d9366..d6ec9acb99 100644 --- a/repos/effect/ai-docs/tsconfig.json +++ b/repos/effect/ai-docs/tsconfig.json @@ -13,11 +13,13 @@ "plugins": [ { "name": "@effect/language-service", + "effectFn": ["untraced"], "namespaceImportPackages": [], "diagnosticSeverity": { "floatingEffect": "off", "preferSchemaOverJson": "off", - "unstableApiUsage": "off" + "unstableApiUsage": "off", + "effectFnOpportunity": "error" } } ] diff --git a/repos/effect/migration/annotations/effect__Cause.yaml b/repos/effect/migration/annotations/effect__Cause.yaml index cc4611e8a8..2b8219222e 100644 --- a/repos/effect/migration/annotations/effect__Cause.yaml +++ b/repos/effect/migration/annotations/effect__Cause.yaml @@ -5,8 +5,9 @@ effect/Cause#as: replacement: "Cause.map(self, () => error)" note: "Use Cause.map with a constant function; only Fail errors change and Die/Interrupt reasons remain." effect/Cause#Cause: + include: true replacement: "Cause.Cause" - note: "The name remains, but v4 Cause is a wrapper with readonly reasons: ReadonlyArray>, not the v3 Empty/Fail/Die/Interrupt/Sequential/Parallel tree." + note: "V4 Cause wraps readonly reasons: ReadonlyArray>, replacing the v3 cause tree. Finalizer and concurrent failure aggregation can convert typed errors to defects when a defect is present; Cause.combine itself does not perform this conversion." effect/Cause#Cause.Variance: replacement: "none" note: "The public variance helper was removed. Cause.Cause is directly branded by Cause.TypeId; application code should not reproduce the old variance member." diff --git a/repos/effect/migration/annotations/effect__Channel.yaml b/repos/effect/migration/annotations/effect__Channel.yaml index 06cbc912bf..cebd431275 100644 --- a/repos/effect/migration/annotations/effect__Channel.yaml +++ b/repos/effect/migration/annotations/effect__Channel.yaml @@ -12,7 +12,7 @@ note: "The inferred Channel.fromChunk match is not equivalent. Rebuild the buffered upstream-pull transform with Channel.fromTransform and Channel.toTransform." "effect/Channel#catchAll": replacement: "Channel.catch" - note: "Renamed to catch for typed-error recovery." + note: "Rename to Channel.catch. Causes containing defects skip typed recovery and convert typed errors to defects; interruptions alongside typed errors do not block recovery. Use Channel.catchCause for full-cause handling." "effect/Channel#catchAllCause": replacement: "Channel.catchCause" note: "Renamed to catchCause for full-cause recovery." @@ -146,11 +146,11 @@ replacement: "Channel.merge" note: "Use Channel.merge with haltStrategy left, right, both, or either for standard policies. Custom MergeDecision effects require a Pull-level redesign." "effect/Channel#orDieWith": - replacement: "Channel.catch" - note: "Use Channel.catch(self, (error) => Channel.die(f(error))); v4 Channel.orDie has no mapping callback." + replacement: "Channel.mapError + Channel.orDie" + note: "Use self.pipe(Channel.mapError(f), Channel.orDie). Mapping transforms every typed error while preserving existing defects and interruptions; Channel.catch skips the mapper for causes containing defects." "effect/Channel#orElse": replacement: "Channel.catch" - note: "Use Channel.catch(self, () => that()) and keep the fallback lazy." + note: "Use Channel.catch(self, () => that()) and keep the fallback lazy. Causes containing defects skip the fallback and convert typed errors to defects; interruptions alongside typed errors do not block recovery." "effect/Channel#provideLayer": replacement: "Channel.provide" note: "Both collapse into provide. V4 removes services supplied by the layer and retains remaining requirements; use options.local when a fresh layer instance is needed." @@ -217,3 +217,12 @@ "effect/Channel#zipRight": replacement: "Channel.concat" note: "Use concat for the sequential form; it preserves the right done value. Concurrent mode has no exact replacement." +effect/Channel#mapError: + include: true + replacement: "Channel.mapError" + note: "Maps every typed error in the cause, preserving defects, interruptions and reason order. Unlike typed recovery, mapping does not convert errors to defects." + +effect/Channel#orDie: + include: true + replacement: "Channel.orDie" + note: "Converts typed errors to defects. If a defect is already present, every typed error is converted to a defect, preserving reason order and interruptions." diff --git a/repos/effect/migration/annotations/effect__Effect.yaml b/repos/effect/migration/annotations/effect__Effect.yaml index 019969d243..c596249919 100644 --- a/repos/effect/migration/annotations/effect__Effect.yaml +++ b/repos/effect/migration/annotations/effect__Effect.yaml @@ -51,11 +51,12 @@ effect/Effect#cacheRequestResult: replacement: "none" note: "Direct request-cache mutation was removed; configure request resolution through `RequestResolver`. No direct public replacement exists in v4; rewrite the call site around the stated v4 primitive." effect/Effect#catch: + include: true replacement: "Effect.catch" - note: "Still exported in v4; update call sites for the revised signature, options, and channel inference." + note: "Typed recovery skips causes containing defects and converts their typed errors to defects, preserving reason order and interruptions. An interruption alongside a typed error does not block recovery; use Effect.catchCause for full-cause handling." effect/Effect#catchAll: replacement: "Effect.catch" - note: "Use the shortened v4 error-handler name. Adapt arguments and imports to the v4 API." + note: "Rename to Effect.catch. Causes containing defects skip typed recovery and convert typed errors to defects; interruptions alongside typed errors do not block recovery. Use Effect.catchCause for full-cause handling." effect/Effect#catchAllCause: replacement: "Effect.catchCause" note: "Use the shortened v4 cause-handler name. Adapt arguments and imports to the v4 API." @@ -64,7 +65,7 @@ effect/Effect#catchAllDefect: note: "Use the shortened v4 defect-handler name. Adapt arguments and imports to the v4 API." effect/Effect#catchSome: replacement: "Effect.catchFilter" - note: "Replace the Option-returning partial function with a `Filter` and handler. Adapt arguments and imports to the v4 API." + note: "Replace the Option-returning partial function with a Filter and handler. Causes containing defects skip typed recovery and convert typed errors to defects; interruptions alongside typed errors do not block recovery." effect/Effect#catchSomeCause: replacement: "Effect.catchCauseFilter" note: "Replace the Option-returning partial function with a cause `Filter` and handler. Adapt arguments and imports to the v4 API." @@ -151,7 +152,7 @@ effect/Effect#EffectUnifyIgnore: note: "The internal unification-ignore helper is no longer public; rely on v4 Effect inference. No direct public replacement exists in v4; rewrite the call site around the stated v4 primitive." effect/Effect#either: replacement: "Effect.result" - note: "V4 represents typed success/failure as `Result` instead of `Either`. Adapt arguments and imports to the v4 API." + note: "V4 represents typed success/failure as `Result` instead of `Either`. Adapt arguments and imports to the v4 API. Causes containing defects are not captured as Result failures and propagate with typed errors converted to defects; interruptions alongside typed errors do not block typed recovery." effect/Effect#ensureErrorType: replacement: "Effect.satisfiesErrorType" note: "Use the renamed compile-time channel constraint. Adapt arguments and imports to the v4 API." @@ -244,7 +245,7 @@ effect/Effect#if: note: "Select the branch lazily with a JavaScript conditional inside `Effect.suspend`. Adapt arguments and imports to the v4 API." effect/Effect#ignoreLogged: replacement: "Effect.ignore" - note: "Pass `{ log: true }` to the consolidated ignore combinator. Adapt arguments and imports to the v4 API." + note: "Use Effect.ignore(self, { log: true }). Causes containing defects are not ignored and propagate with typed errors converted to defects; interruptions alongside typed errors do not block typed recovery." effect/Effect#inheritFiberRefs: replacement: "none" note: "Bulk FiberRef inheritance was removed; propagate required context references explicitly. No direct public replacement exists in v4; rewrite the call site around the stated v4 primitive." @@ -337,7 +338,7 @@ effect/Effect#orDieWith: note: "Map the typed error to the desired defect, then convert failures to defects. Adapt arguments and imports to the v4 API." effect/Effect#orElse: replacement: "Effect.catch" - note: "Ignore the caught error and evaluate the fallback Effect. Adapt arguments and imports to the v4 API." + note: "Use Effect.catch(self, () => that()). Causes containing defects skip the fallback and convert typed errors to defects; interruptions alongside typed errors do not block recovery." effect/Effect#orElseFail: replacement: "Effect.mapError" note: "Replace every typed error with the lazily produced failure value. Adapt arguments and imports to the v4 API." @@ -662,8 +663,78 @@ effect/Effect#zipRight: replacement: "Effect.andThen" note: "Sequence the Effects and retain the second result. Adapt arguments and imports to the v4 API." "effect/Effect#repeatOrElse": + include: true replacement: "Effect.repeatOrElse" - note: "The fallback receives Option> instead of Option. Read metadata.output for the previous schedule output and metadata.attempt for the attempt count. The error also includes schedule failures." + note: "The fallback receives Option> instead of Option. Read metadata.output for the previous schedule output and metadata.attempt for the attempt count. The error also includes schedule failures. Causes containing defects from either the source or policy skip the fallback and convert typed errors to defects. Interruptions alongside typed errors do not block the fallback." "effect/Effect#tryPromise": replacement: "Effect.tryPromise" note: "Use the callback overload for Cause.UnknownError, or the object overload with try and catch to map failures to a custom error. The callback-only overload does not accept a custom error type parameter." +effect/Effect#catchTag: + include: true + replacement: "Effect.catchTag" + note: "Causes containing defects skip typed recovery and convert typed errors to defects, preserving reason order and interruptions. An interruption alongside a typed error does not block recovery." + +effect/Effect#catchTags: + include: true + replacement: "Effect.catchTags" + note: "Causes containing defects skip typed recovery and convert typed errors to defects, preserving reason order and interruptions. An interruption alongside a typed error does not block recovery." + +effect/Effect#catchIf: + include: true + replacement: "Effect.catchIf" + note: "Causes containing defects skip typed recovery and convert typed errors to defects, preserving reason order and interruptions. An interruption alongside a typed error does not block recovery." + +effect/Effect#match: + include: true + replacement: "Effect.match" + note: "Causes containing defects skip typed recovery and convert typed errors to defects, preserving reason order and interruptions. An interruption alongside a typed error does not block recovery." + +effect/Effect#matchEffect: + include: true + replacement: "Effect.matchEffect" + note: "Causes containing defects skip typed recovery and convert typed errors to defects, preserving reason order and interruptions. An interruption alongside a typed error does not block recovery." + +effect/Effect#orElseSucceed: + include: true + replacement: "Effect.orElseSucceed" + note: "Causes containing defects skip typed recovery and convert typed errors to defects, preserving reason order and interruptions. An interruption alongside a typed error does not block recovery." + +effect/Effect#option: + include: true + replacement: "Effect.option" + note: "Causes containing defects skip typed recovery and convert typed errors to defects, preserving reason order and interruptions. An interruption alongside a typed error does not block recovery." + +effect/Effect#ignore: + include: true + replacement: "Effect.ignore" + note: "Ignores recoverable typed errors only. Causes containing defects propagate with typed errors converted to defects; interruptions alongside typed errors do not prevent ignoring them." + +effect/Effect#mapError: + include: true + replacement: "Effect.mapError" + note: "Maps every typed error in the cause, preserving defects, interruptions and reason order. Unlike typed recovery, mapping does not convert errors to defects." + +effect/Effect#mapBoth: + include: true + replacement: "Effect.mapBoth" + note: "Maps every typed error in the cause, preserving defects, interruptions and reason order. Unlike typed recovery, mapping does not convert errors to defects." + +effect/Effect#orDie: + include: true + replacement: "Effect.orDie" + note: "Converts typed errors to defects. If a defect is already present, every typed error is converted to a defect, preserving reason order and interruptions." + +effect/Effect#retry: + include: true + replacement: "Effect.retry" + note: "Retries typed source errors, including those alongside interruptions. A source cause containing a defect is not retried and propagates with typed errors converted to defects." + +effect/Effect#retryOrElse: + include: true + replacement: "Effect.retryOrElse" + note: "A source cause containing a defect skips retries and the fallback, converting typed errors to defects. Interruptions alongside typed errors do not block retries. Policy failures propagate unchanged." + +effect/Effect#repeat: + include: true + replacement: "Effect.repeat" + note: "If the source or policy fails with a cause containing defects, typed errors are converted to defects. An interruption alongside a typed error does not block the typed failure path." diff --git a/repos/effect/migration/annotations/effect__Exit.yaml b/repos/effect/migration/annotations/effect__Exit.yaml index 5644d43681..4d352b33f1 100644 --- a/repos/effect/migration/annotations/effect__Exit.yaml +++ b/repos/effect/migration/annotations/effect__Exit.yaml @@ -70,3 +70,12 @@ effect/Exit#zipRight: effect/Exit#zipWith: replacement: "Exit.match" note: "No direct v4 equivalent. Nested-match both Exits: preserve a lone failure cause, call options.onFailure and Exit.failCause only when both fail, and call Exit.succeed(options.onSuccess(a, b)) when both succeed." +effect/Exit#mapError: + include: true + replacement: "Exit.mapError" + note: "Maps every typed error in the cause, preserving defects, interruptions and reason order. Unlike typed recovery, mapping does not convert errors to defects." + +effect/Exit#mapBoth: + include: true + replacement: "Exit.mapBoth" + note: "Maps every typed error in the cause, preserving defects, interruptions and reason order. Unlike typed recovery, mapping does not convert errors to defects." diff --git a/repos/effect/migration/annotations/effect__Layer.yaml b/repos/effect/migration/annotations/effect__Layer.yaml index 925c52e312..08314f0632 100644 --- a/repos/effect/migration/annotations/effect__Layer.yaml +++ b/repos/effect/migration/annotations/effect__Layer.yaml @@ -95,8 +95,8 @@ replacement: "Layer.fromBuild with Effect.matchCauseEffect over Layer.buildWithMemoMap" note: "Fold the source acquisition cause, then build the selected failure or success layer." "effect/Layer#memoize": - replacement: "automatic shared memoization under Effect.provide" - note: "Reuse the same Layer value; use { local: true } or Layer.fresh to opt out, or MemoMap APIs for manual control." + replacement: "enclosing Effect.provide or explicit MemoMap APIs" + note: "A layer already built by an enclosing provide or runtime is reused; sibling provide calls build their own copies. Use { local: true } or Layer.fresh for fresh builds, or MemoMap APIs for manual sharing." "effect/Layer#MemoMap": replacement: "Layer.MemoMap" note: "The interface remains and now supports parent-child ambient maps." diff --git a/repos/effect/migration/annotations/effect__Match.yaml b/repos/effect/migration/annotations/effect__Match.yaml index 2ed399401d..da050e1533 100644 --- a/repos/effect/migration/annotations/effect__Match.yaml +++ b/repos/effect/migration/annotations/effect__Match.yaml @@ -10,6 +10,10 @@ "effect/Match#Not": replacement: "Match.Not" note: "The case type is retained. Its evaluate method now receives any Match.fn selector arguments after the selected input; update custom case implementations that consume those arguments." +"effect/Match#not": + include: true + replacement: "Match.not" + note: "Object patterns now reject top-level keys absent from every variant of the remaining input type. Correct stale property names or narrow the input before matching; function, refinement, and array patterns are not subject to this object-key check." "effect/Match#SafeRefinementId": replacement: "none" note: "The public safe-refinement brand was internalized. Use Predicate.Refinement, Predicate.Predicate, or a built-in Match refinement instead of constructing the brand." @@ -21,7 +25,7 @@ note: "The public type-level namespace is retained with no call-site migration." "effect/Match#Types.ExtractAndNarrow": replacement: "Match.Types.ExtractAndNarrow" - note: "The type-only matching helper is retained unchanged." + note: "Array prefix patterns now narrow the required positions and preserve the unmatched tail of arrays and variadic tuples. Update explicit handler or helper result types that assumed a prefix pattern described the entire array; runtime prefix matching is unchanged." "effect/Match#Types.MaybeReplace": replacement: "Match.Types.MaybeReplace" note: "The type-only matching helper is retained unchanged." @@ -49,3 +53,15 @@ "effect/Match#When": replacement: "Match.When" note: "The case type is retained. Its evaluate method now receives any Match.fn selector arguments after the selected input; update custom case implementations that consume those arguments." +"effect/Match#when": + include: true + replacement: "Match.when" + note: "Object patterns now reject top-level keys absent from every variant of the remaining input type. Correct stale property names or narrow the input before matching. Array prefix patterns retain the unmatched tail in the handler type rather than treating the pattern as the entire array." +"effect/Match#whenAnd": + include: true + replacement: "Match.whenAnd" + note: "Each object pattern now rejects top-level keys absent from every variant of the remaining input type, including patterns spread from an array. Correct stale property names or narrow the input before matching." +"effect/Match#whenOr": + include: true + replacement: "Match.whenOr" + note: "Each object pattern now rejects top-level keys absent from every variant of the remaining input type, including patterns spread from an array. Correct stale property names or narrow the input before matching." diff --git a/repos/effect/migration/annotations/effect__Sink.yaml b/repos/effect/migration/annotations/effect__Sink.yaml index 636836d099..89b981b17b 100644 --- a/repos/effect/migration/annotations/effect__Sink.yaml +++ b/repos/effect/migration/annotations/effect__Sink.yaml @@ -171,16 +171,16 @@ effect/Sink#race: note: "Sink racing (race/raceBoth/raceWith) was removed from v4; broadcast the stream (Stream.broadcast) into two consumers and race the resulting run effects with Effect.race, or write a custom Channel." effect/Sink#raceBoth: replacement: "none" - note: "Removed with the race family; broadcast the stream and use Effect.raceBoth (or Effect.race) on the two Stream.run effects to learn which side won." + note: "Removed with the race family; broadcast the stream and tag each Stream.run effect's result before combining them with Effect.race to learn which side succeeded first." effect/Sink#raceWith: replacement: "none" - note: "Removed, along with the MergeDecision type it depended on; the closest is broadcasting the stream and combining the two run effects manually (Effect.raceWith on the run effects)." + note: "Removed, along with the MergeDecision type it depended on; broadcast the stream and combine the two run effects manually. Use Effect.race for the first success or Effect.raceFirst for the first completion; custom completion and interruption handling requires explicit fibers." effect/Sink#refineOrDie: replacement: "Sink.catch" - note: "Removed; rebuild with the typed-error handler: Sink.catch(self, (e) => Option.match(pf(e), { onSome: Effect.fail, onNone: () => Effect.die(e) })) — note Sink.catch replaces the result on recovery, so refined errors must be re-failed as shown." + note: "For recoverable typed errors, use Sink.catch(self, (e) => Option.match(pf(e), { onSome: Effect.fail, onNone: () => Effect.die(e) })). Causes containing defects skip pf and convert typed errors to defects; use Sink.catchCause for full-cause handling." effect/Sink#refineOrDieWith: replacement: "Sink.catch" - note: "Removed; same pattern as refineOrDie but die with the mapped defect: Sink.catch(self, (e) => Option.match(pf(e), { onSome: Effect.fail, onNone: () => Effect.die(f(e)) }))." + note: "Use the refineOrDie recipe with Effect.die(f(e)) for unrefined errors. Causes containing defects skip pf and f and convert typed errors to defects; use Sink.catchCause for full-cause handling." effect/Sink#splitWhere: replacement: "none" note: "Removed; it re-chunked input so the sink stopped before the first later element matching the predicate — closest v4 options are pre-splitting the stream (Stream.split / Stream.takeWhile) or a custom Sink.fromTransform that cuts pulled arrays at the predicate boundary and returns the rest as leftovers." @@ -202,3 +202,12 @@ effect/Sink#zipRight: effect/Sink#zipWith: replacement: "Sink.flatMap" note: "Removed; use self.pipe(Sink.flatMap((a) => Sink.map(that, (a2) => f(a, a2)))) — sequential only, the { concurrent: true } option has no v4 equivalent." +effect/Sink#mapError: + include: true + replacement: "Sink.mapError" + note: "Maps every typed error in the cause, preserving defects, interruptions and reason order. Unlike typed recovery, mapping does not convert errors to defects." + +effect/Sink#orElse: + include: true + replacement: "Sink.orElse" + note: "Causes containing defects skip the fallback and convert typed errors to defects; interruptions alongside typed errors do not block typed recovery." diff --git a/repos/effect/migration/annotations/effect__Stream.yaml b/repos/effect/migration/annotations/effect__Stream.yaml index e4396a5919..7ee4c80bec 100644 --- a/repos/effect/migration/annotations/effect__Stream.yaml +++ b/repos/effect/migration/annotations/effect__Stream.yaml @@ -9,11 +9,11 @@ effect/Stream#bindEffect: effect/Stream#catchTags: include: true replacement: "Stream.catchTags" - note: "Only keys present in the tagged error union are accepted; remove handlers for tags the stream cannot fail with. Untagged errors remain in the error channel." + note: "Only keys in the tagged error union are accepted; untagged errors remain in the error channel. Causes containing defects skip handlers and convert typed errors to defects; interruptions alongside typed errors do not block recovery." effect/Stream#mapBoth: include: true replacement: "Stream.mapBoth" - note: "Rename the options onSuccess to onElement and onFailure to onError. The callbacks still transform emitted elements and typed errors respectively." + note: "Rename the options onSuccess to onElement and onFailure to onError. The callbacks still transform emitted elements and typed errors respectively. Maps every typed error in the cause, preserving defects, interruptions and reason order. Unlike typed recovery, mapping does not convert errors to defects." effect/Stream#partition: include: true replacement: "Stream.partition" @@ -73,7 +73,7 @@ effect/Stream#async: note: "Stream.callback((queue) => Effect | void, { bufferSize?, strategy? }) replaces the Emit-based async; push with Queue.offer/offerAll, end with Queue.end, fail with Queue.fail." effect/Stream#asyncEffect: replacement: "Stream.callback" - note: "The register function of Stream.callback may return an Effect (run before the stream starts pulling), covering asyncEffect; signal end/failure through the provided Queue." + note: "The register function of Stream.callback may return an Effect, covering asyncEffect. Unlike v3, it is started before the stream pulls but not awaited: it runs synchronously until its first async boundary, so asynchronous setup may still be in progress when pulling begins. Signal end/failure through the provided Queue." effect/Stream#asyncPush: replacement: "Stream.callback" note: "Stream.callback's register effect can use Scope for acquire/release of the external subscription, replacing asyncPush; the Emit ops helpers become plain Queue operations." @@ -98,13 +98,13 @@ effect/Stream#bufferChunks: note: "Chunk->Array rename; buffers whole arrays (chunks) up to capacity with the same strategy options." effect/Stream#catchAll: replacement: "Stream.catch" - note: "Renamed to Stream.catch (exported keyword-style); same (error) => Stream handler for all typed failures." + note: "Rename to Stream.catch with an (error) => Stream handler. Causes containing defects skip typed recovery and convert typed errors to defects; interruptions alongside typed errors do not block recovery. Use Stream.catchCause for full-cause handling." effect/Stream#catchAllCause: replacement: "Stream.catchCause" note: "Renamed; handler receives the full Cause and returns a recovery stream, identical semantics." effect/Stream#catchSome: replacement: "Stream.catchFilter" - note: "Option-returning partial handler replaced by the Filter API: Stream.catchFilter(filter, f, orElse?) recovers matched errors, unmatched failures pass through (Stream.catchIf for refinement/predicate matching)." + note: "Use Stream.catchFilter(filter, f, orElse?) instead of an Option-returning partial handler. Causes containing defects skip typed recovery and convert typed errors to defects; interruptions alongside typed errors do not block recovery." effect/Stream#catchSomeCause: replacement: "Stream.catchCauseFilter" note: "Option-returning cause handler replaced by Stream.catchCauseFilter(filter, f, orElse?) using a Filter on the Cause (Stream.catchCauseIf for refinements)." @@ -149,7 +149,7 @@ effect/Stream#distributedWithDynamic: note: "Dynamic predicate-routed fan-out removed with distributedWith. Use Stream.broadcast/Stream.share for dynamic mirrors plus per-subscriber Stream.filter, or hand-roll routing into Queues via Stream.runForEach." effect/Stream#either: replacement: "Stream.result" - note: "Either is replaced by Result in v4: Stream.result yields Stream, never, R> (element -> Result.succeed, first error -> Result.fail and the stream ends, as before)." + note: "Either is replaced by Result in v4: Stream.result yields Stream, never, R> (element -> Result.succeed, first error -> Result.fail and the stream ends, as before). Causes containing defects are not emitted as Result failures and propagate with typed errors converted to defects; interruptions alongside typed errors do not block typed recovery." effect/Stream#ensuringWith: replacement: "Stream.onExit" note: "Renamed; Stream.onExit runs the finalizer with the Exit of the stream, identical shape." @@ -262,13 +262,13 @@ effect/Stream#orDieWith: note: "orDieWith removed; transform the error first, then convert failures to defects: `self.pipe(Stream.mapError(f), Stream.orDie)`." effect/Stream#orElse: replacement: "Stream.catch" - note: "v3 catchAll was renamed to Stream.catch in v4; orElse ignored the error, so write `Stream.catch(self, () => that())`." + note: "Use Stream.catch(self, () => that()). Causes containing defects skip the fallback and convert typed errors to defects; interruptions alongside typed errors do not block recovery." effect/Stream#orElseEither: replacement: "Stream.catch" - note: "Removed; Either is replaced by Result in v4. Emulate: `Stream.map(self, Result.succeed).pipe(Stream.catch(() => Stream.map(that(), Result.fail)))` (same encoding v4 Stream.mergeResult uses)." + note: "Replace Either with Result: Stream.map(self, Result.succeed).pipe(Stream.catch(() => Stream.map(that(), Result.fail))). Causes containing defects skip the fallback and convert typed errors to defects; interruptions alongside typed errors do not block recovery." effect/Stream#orElseFail: replacement: "Stream.mapError" - note: "Removed; it only replaced the failure value: `Stream.mapError(self, () => error())` or `Stream.catch(self, () => Stream.fail(error()))`." + note: "Use Stream.mapError(self, () => error()) to replace each typed error while preserving existing defects and interruptions. Stream.catch is not equivalent for causes containing defects because it skips typed recovery." effect/Stream#orElseIfEmptyChunk: replacement: "Stream.orElseIfEmpty" note: "Folded into Stream.orElseIfEmpty, which now takes a lazy fallback Stream: `Stream.orElseIfEmpty(self, () => Stream.fromArray(array))`; Chunk is replaced by plain arrays." @@ -301,10 +301,10 @@ effect/Stream#provideSomeLayer: note: "v4 Stream.provide accepts a Layer (or Context) and excludes only the provided services from R — same partial-provision semantics." effect/Stream#refineOrDie: replacement: "Stream.catch" - note: "Removed; emulate with `Stream.catch(self, (e) => { const r = pf(e); return Option.isSome(r) ? Stream.fail(r.value) : Stream.die(e) })` — refail refined errors, die on the rest." + note: "For recoverable typed errors, use Stream.catch(self, (e) => { const r = pf(e); return Option.isSome(r) ? Stream.fail(r.value) : Stream.die(e) }). Causes containing defects skip pf and convert typed errors to defects; use Stream.catchCause for full-cause handling." effect/Stream#refineOrDieWith: replacement: "Stream.catch" - note: "Removed; same as refineOrDie but die with the mapped defect: `Stream.die(f(e))` for unrefined errors." + note: "Use the refineOrDie recipe with Stream.die(f(e)) for unrefined errors. Causes containing defects skip pf and f and convert typed errors to defects; use Stream.catchCause for full-cause handling." effect/Stream#repeatEffect: replacement: "Stream.fromEffectRepeat" note: "Renamed; repeats the effect forever emitting each result." @@ -486,3 +486,27 @@ effect/Stream#zipAllWith: effect/Stream#zipWithChunks: replacement: "Stream.zipWithArray" note: "Renamed; the combiner now receives two NonEmptyReadonlyArrays and returns `[output: NonEmptyReadonlyArray, leftoverLeft: ReadonlyArray, leftoverRight: ReadonlyArray]` — the Either-wrapped leftover (ZipChunksResult) is replaced by the two explicit leftover arrays." +effect/Stream#mapError: + include: true + replacement: "Stream.mapError" + note: "Maps every typed error in the cause, preserving defects, interruptions and reason order. Unlike typed recovery, mapping does not convert errors to defects." + +effect/Stream#orDie: + include: true + replacement: "Stream.orDie" + note: "Converts typed errors to defects. If a defect is already present, every typed error is converted to a defect, preserving reason order and interruptions." + +effect/Stream#retry: + include: true + replacement: "Stream.retry" + note: "Retries typed source errors, including those alongside interruptions. A source cause containing a defect is not retried and propagates with typed errors converted to defects." + +effect/Stream#catchTag: + include: true + replacement: "Stream.catchTag" + note: "Causes containing defects skip typed recovery and convert typed errors to defects, preserving reason order and interruptions. An interruption alongside a typed error does not block recovery." + +effect/Stream#orElseSucceed: + include: true + replacement: "Stream.orElseSucceed" + note: "Causes containing defects skip typed recovery and convert typed errors to defects, preserving reason order and interruptions. An interruption alongside a typed error does not block recovery." diff --git a/repos/effect/migration/annotations/effect__ai-openai__Generated.yaml b/repos/effect/migration/annotations/effect__ai-openai__Generated.yaml index 5f96a15c45..6dc8673e8b 100644 --- a/repos/effect/migration/annotations/effect__ai-openai__Generated.yaml +++ b/repos/effect/migration/annotations/effect__ai-openai__Generated.yaml @@ -5,8 +5,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#AddUploadPartRequest": - replacement: "Generated.AddUploadPartRequest" - note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." + replacement: "Generated.AddUploadPartRequestMultipart" + note: "Renamed for multipart requests; file fields now accept File or Blob. Re-check the current Type/Encoded shape." "@effect/ai-openai/Generated#AdminApiKey": replacement: "Generated.AdminApiKey" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -29,17 +29,17 @@ replacement: "Generated.ApiKeyList" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#ApplyPatchCallOutputStatus": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ApplyPatchCallOutputStatus" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ApplyPatchCallOutputStatusParam": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ApplyPatchCallOutputStatusParam" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ApplyPatchCallStatus": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ApplyPatchCallStatus" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ApplyPatchCallStatusParam": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ApplyPatchCallStatusParam" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ApplyPatchCreateFileOperation": replacement: "Generated.ApplyPatchCreateFileOperation" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -65,8 +65,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#ApplyPatchOperationParam": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ApplyPatchOperationParam" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ApplyPatchToolCall": replacement: "Generated.ApplyPatchToolCall" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -110,8 +110,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#ApproximateLocation": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ApproximateLocation" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ApproximateLocationType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -179,14 +179,14 @@ replacement: "Generated.Attachment" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#AttachmentType": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.AttachmentType" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#AudioResponseFormat": replacement: "Generated.AudioResponseFormat" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#AudioTranscription": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.AudioTranscription" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#AudioTranscriptionModel": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -218,20 +218,20 @@ replacement: "Generated.AuditLogEventType" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#AutoChunkingStrategyRequestParam": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.AutoChunkingStrategyRequestParam" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#AutoChunkingStrategyRequestParamType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#AutomaticThreadTitlingParam": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.AutomaticThreadTitlingParam" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#Batch": replacement: "Generated.Batch" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#BatchError": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.BatchError" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#BatchFileExpirationAfter": replacement: "Generated.BatchFileExpirationAfter" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -242,8 +242,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#BatchRequestCounts": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.BatchRequestCounts" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#BatchStatus": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -428,32 +428,32 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#ChatkitConfigurationParam": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ChatkitConfigurationParam" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ChatkitWorkflow": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ChatkitWorkflow" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ChatkitWorkflowTracing": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ChatkitWorkflowTracing" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ChatModel": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#ChatSessionAutomaticThreadTitling": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ChatSessionAutomaticThreadTitling" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ChatSessionChatkitConfiguration": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ChatSessionChatkitConfiguration" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ChatSessionFileUpload": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ChatSessionFileUpload" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ChatSessionHistory": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ChatSessionHistory" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ChatSessionRateLimits": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ChatSessionRateLimits" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ChatSessionResource": replacement: "Generated.ChatSessionResource" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -461,8 +461,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#ChatSessionStatus": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ChatSessionStatus" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ChunkingStrategyRequestParam": replacement: "Generated.ChunkingStrategyRequestParam" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -470,8 +470,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#ClickButtonType": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ClickButtonType" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ClickParam": replacement: "Generated.ClickParam" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -494,8 +494,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#ClientToolCallStatus": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ClientToolCallStatus" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ClosedStatus": replacement: "Generated.ClosedStatus" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -569,8 +569,8 @@ replacement: "Generated.ComputerCallSafetyCheckParam" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#ComputerEnvironment": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ComputerEnvironment" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ComputerScreenshotContent": replacement: "Generated.ComputerScreenshotContent" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -623,8 +623,8 @@ replacement: "Generated.ContainerListResource" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#ContainerMemoryLimit": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ContainerMemoryLimit" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ContainerResource": replacement: "Generated.ContainerResource" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -662,8 +662,8 @@ replacement: "Generated.CreateAssistantRequest" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#CreateBatchRequest": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.CreateBatchRequest" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#CreateBatchRequestCompletionWindow": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -740,8 +740,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#CreateEvalCompletionsRunDataSource": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.CreateEvalCompletionsRunDataSource" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#CreateEvalCompletionsRunDataSourceInputMessagesEnumType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -749,8 +749,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#CreateEvalCustomDataSourceConfig": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.CreateEvalCustomDataSourceConfig" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#CreateEvalCustomDataSourceConfigType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -758,8 +758,8 @@ replacement: "Generated.CreateEvalItem" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#CreateEvalJsonlRunDataSource": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.CreateEvalJsonlRunDataSource" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#CreateEvalJsonlRunDataSourceType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -770,8 +770,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#CreateEvalLogsDataSourceConfig": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.CreateEvalLogsDataSourceConfig" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#CreateEvalLogsDataSourceConfigType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -779,8 +779,8 @@ replacement: "Generated.CreateEvalRequest" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#CreateEvalResponsesRunDataSource": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.CreateEvalResponsesRunDataSource" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#CreateEvalResponsesRunDataSourceInputMessagesEnumType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -791,14 +791,14 @@ replacement: "Generated.CreateEvalRunRequest" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#CreateEvalStoredCompletionsDataSourceConfig": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.CreateEvalStoredCompletionsDataSourceConfig" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#CreateEvalStoredCompletionsDataSourceConfigType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#CreateFileRequest": - replacement: "Generated.CreateFileRequest" - note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." + replacement: "Generated.CreateFileRequestMultipart" + note: "Renamed for multipart requests; file fields now accept File or Blob. Re-check the current Type/Encoded shape." "@effect/ai-openai/Generated#CreateFineTuningCheckpointPermissionRequest": replacement: "Generated.CreateFineTuningCheckpointPermissionRequest" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -824,8 +824,8 @@ replacement: "Generated.CreateGroupUserBody" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#CreateImageEditRequest": - replacement: "Generated.CreateImageEditRequest" - note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." + replacement: "Generated.CreateImageEditRequestMultipart" + note: "Renamed for multipart requests; file fields now accept File or Blob. Re-check the current Type/Encoded shape. JSON image edits instead use Generated.CreateImageEditRequestJson." "@effect/ai-openai/Generated#CreateImageEditRequestBackground": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -872,8 +872,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#CreateImageVariationRequest": - replacement: "Generated.CreateImageVariationRequest" - note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." + replacement: "Generated.CreateImageVariationRequestMultipart" + note: "Renamed for multipart requests; file fields now accept File or Blob. Re-check the current Type/Encoded shape." "@effect/ai-openai/Generated#CreateImageVariationRequestModelEnum": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -962,8 +962,8 @@ replacement: "Generated.CreateTranscription200" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#CreateTranscriptionRequest": - replacement: "Generated.CreateTranscriptionRequest" - note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." + replacement: "Generated.CreateTranscriptionRequestMultipart" + note: "Renamed for multipart requests; file fields now accept File or Blob. Re-check the current Type/Encoded shape." "@effect/ai-openai/Generated#CreateTranscriptionRequestModelEnum": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -983,8 +983,8 @@ replacement: "Generated.CreateTranslation200" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#CreateTranslationRequest": - replacement: "Generated.CreateTranslationRequest" - note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." + replacement: "Generated.CreateTranslationRequestMultipart" + note: "Renamed for multipart requests; file fields now accept File or Blob. Re-check the current Type/Encoded shape." "@effect/ai-openai/Generated#CreateTranslationRequestModelEnum": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -1136,8 +1136,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#DetailEnum": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.DetailEnum" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#DoubleClickAction": replacement: "Generated.DoubleClickAction" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -1145,8 +1145,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#DownloadFile200": - replacement: "Generated.DownloadFile200" - note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." + replacement: "none" + note: "The standalone response schema was removed. The generated client's downloadFile method returns Uint8Array, optionally paired with the HTTP response according to its config; downloadFileStream streams Uint8Array chunks." "@effect/ai-openai/Generated#Drag": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -1187,8 +1187,8 @@ replacement: "Generated.EvalApiError" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#EvalCustomDataSourceConfig": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.EvalCustomDataSourceConfig" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#EvalCustomDataSourceConfigType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -1259,8 +1259,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#EvalLogsDataSourceConfig": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.EvalLogsDataSourceConfig" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#EvalLogsDataSourceConfigType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -1301,8 +1301,8 @@ replacement: "Generated.EvalRunOutputItemResult" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#EvalStoredCompletionsDataSourceConfig": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.EvalStoredCompletionsDataSourceConfig" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#EvalStoredCompletionsDataSourceConfigType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -1313,8 +1313,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#ExpiresAfterParam": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ExpiresAfterParam" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ExpiresAfterParamAnchor": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -1322,8 +1322,8 @@ replacement: "Generated.FileAnnotation" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#FileAnnotationSource": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.FileAnnotationSource" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#FileAnnotationSourceType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -1373,11 +1373,11 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#FileUploadParam": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.FileUploadParam" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#Filters": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.Filters" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#FineTuneDPOHyperparameters": replacement: "Generated.FineTuneDPOHyperparameters" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -1496,8 +1496,8 @@ replacement: "Generated.FunctionAndCustomToolCallOutput" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#FunctionCallItemStatus": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.FunctionCallItemStatus" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#FunctionCallOutputItemParam": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -1511,11 +1511,11 @@ replacement: "Generated.FunctionParameters" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#FunctionShellAction": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.FunctionShellAction" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#FunctionShellActionParam": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.FunctionShellActionParam" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#FunctionShellCall": replacement: "Generated.FunctionShellCall" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -1526,8 +1526,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#FunctionShellCallItemStatus": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.FunctionShellCallItemStatus" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#FunctionShellCallOutput": replacement: "Generated.FunctionShellCallOutput" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -1556,8 +1556,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#FunctionShellCallOutputOutcomeParam": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.FunctionShellCallOutputOutcomeParam" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#FunctionShellCallOutputTimeoutOutcome": replacement: "Generated.FunctionShellCallOutputTimeoutOutcome" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -1658,8 +1658,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#GraderMulti": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.GraderMulti" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#GraderMultiType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -1694,8 +1694,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#GrammarSyntax1": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.GrammarSyntax1" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#Group": replacement: "Generated.Group" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -1739,17 +1739,17 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#HistoryParam": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.HistoryParam" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#HybridSearchOptions": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.HybridSearchOptions" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#Image": replacement: "Generated.Image" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#ImageDetail": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ImageDetail" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ImageGenInputUsageDetails": replacement: "Generated.ImageGenInputUsageDetails" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -1808,8 +1808,8 @@ replacement: "Generated.IncludeEnum" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#InferenceOptions": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.InferenceOptions" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#InputAudio": replacement: "Generated.InputAudio" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -2303,8 +2303,8 @@ replacement: "Generated.MCPToolCall" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#MCPToolCallStatus": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.MCPToolCallStatus" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#MCPToolCallType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -2390,8 +2390,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#MessageRole": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.MessageRole" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#MessageStatus": replacement: "OpenAiSchema.MessageStatus" note: "Use the focused v4 OpenAiSchema definition; the old generated export was removed when the OpenAI specification client was regenerated." @@ -2468,8 +2468,8 @@ replacement: "Generated.OrderEnum" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#OtherChunkingStrategyResponseParam": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.OtherChunkingStrategyResponseParam" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#OtherChunkingStrategyResponseParamType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -2675,14 +2675,14 @@ replacement: "Generated.PublicUpdateOrganizationRoleBody" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#RankerVersionType": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.RankerVersionType" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#RankingOptions": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.RankingOptions" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#RateLimitsParam": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.RateLimitsParam" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#RealtimeAudioFormats": replacement: "Generated.RealtimeAudioFormats" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -2975,8 +2975,8 @@ replacement: "Generated.RunStepCompletionUsage" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#RunStepDetailsMessageCreationObject": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.RunStepDetailsMessageCreationObject" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#RunStepDetailsMessageCreationObjectType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -2990,14 +2990,14 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#RunStepDetailsToolCallsCodeOutputImageObject": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.RunStepDetailsToolCallsCodeOutputImageObject" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#RunStepDetailsToolCallsCodeOutputImageObjectType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#RunStepDetailsToolCallsCodeOutputLogsObject": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.RunStepDetailsToolCallsCodeOutputLogsObject" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#RunStepDetailsToolCallsCodeOutputLogsObjectType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -3020,8 +3020,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#RunStepDetailsToolCallsObject": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.RunStepDetailsToolCallsObject" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#RunStepDetailsToolCallsObjectType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -3059,14 +3059,14 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#SearchContextSize": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.SearchContextSize" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#ServiceTier": replacement: "Generated.ServiceTier" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#ServiceTierEnum": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.ServiceTierEnum" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#SpecificApplyPatchParam": replacement: "Generated.SpecificApplyPatchParam" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -3083,14 +3083,14 @@ replacement: "Generated.StaticChunkingStrategy" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#StaticChunkingStrategyRequestParam": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.StaticChunkingStrategyRequestParam" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#StaticChunkingStrategyRequestParamType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#StaticChunkingStrategyResponseParam": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.StaticChunkingStrategyResponseParam" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#StaticChunkingStrategyResponseParamType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -3131,8 +3131,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#TaskType": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.TaskType" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#TextAnnotation": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -3269,8 +3269,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#TruncationEnum": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.TruncationEnum" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#Type": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -3335,8 +3335,8 @@ replacement: "Generated.UrlAnnotation" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#UrlAnnotationSource": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.UrlAnnotationSource" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#UrlAnnotationSourceType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -3614,8 +3614,8 @@ replacement: "Generated.VideoListResource" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." "@effect/ai-openai/Generated#VideoModel": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.VideoModel" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#VideoResource": replacement: "Generated.VideoResource" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -3623,14 +3623,14 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#VideoSeconds": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.VideoSeconds" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#VideoSize": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.VideoSize" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#VideoStatus": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.VideoStatus" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#VoiceIdsShared": replacement: "Generated.VoiceIdsShared" note: "Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed." @@ -3644,20 +3644,20 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#WebSearchActionFind": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.WebSearchActionFind" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#WebSearchActionFindType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#WebSearchActionOpenPage": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.WebSearchActionOpenPage" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#WebSearchActionOpenPageType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#WebSearchActionSearch": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.WebSearchActionSearch" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#WebSearchActionSearchType": replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." @@ -3707,8 +3707,8 @@ replacement: "none" note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." "@effect/ai-openai/Generated#WorkflowParam": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.WorkflowParam" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." "@effect/ai-openai/Generated#WorkflowTracingParam": - replacement: "none" - note: "Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper." + replacement: "Generated.WorkflowTracingParam" + note: "Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification." diff --git a/repos/effect/migration/annotations/effect__ai__AiError.yaml b/repos/effect/migration/annotations/effect__ai__AiError.yaml index b5c46028f5..07736140ec 100644 --- a/repos/effect/migration/annotations/effect__ai__AiError.yaml +++ b/repos/effect/migration/annotations/effect__ai__AiError.yaml @@ -5,14 +5,14 @@ replacement: "AiError.make + AiError.NetworkError" note: "Replace the old top-level request error with an AiError whose reason is NetworkError. NetworkError.fromRequestError converts a v4 HttpClientError.RequestError." "@effect/ai/AiError#HttpResponseError": - replacement: "AiError.make + AiError.reasonFromHttpStatus / AiError.InvalidOutputError" - note: "There is no single v4 response-error class. Wrap a semantic reason with AiError.make: use reasonFromHttpStatus for status failures and InvalidOutputError for decode or empty-body failures." + replacement: "AiError.make + AiError.reasonFromHttpStatus / AiError.InvalidResponseError / AiError.InvalidOutputError" + note: "There is no single v4 response-error class. Wrap a semantic reason with AiError.make: use reasonFromHttpStatus for status failures. Provider response-schema and SSE decoding failures use non-retryable InvalidResponseError. Lower-level HTTP client DecodeError and EmptyBodyError still map to retryable InvalidOutputError in provider adapters." "@effect/ai/AiError#MalformedInput": replacement: "AiError.make + AiError.InvalidUserInputError" note: "Replace the old top-level input error with an AiError whose reason is InvalidUserInputError. Use InvalidRequestError when the provider request parameters themselves are malformed." "@effect/ai/AiError#MalformedOutput": - replacement: "AiError.make + AiError.InvalidOutputError" - note: "Replace the old top-level output error with an AiError whose reason is InvalidOutputError. The old fromParseError helper becomes InvalidOutputError.fromSchemaError." + replacement: "AiError.make + AiError.InvalidOutputError / AiError.InvalidResponseError" + note: "Replace the old top-level output error with an AiError wrapper. Use retryable InvalidOutputError for invalid model-generated output and non-retryable InvalidResponseError for incompatible provider response formats, including response-schema and SSE decoding failures. Replace fromParseError with InvalidOutputError.fromSchemaError or InvalidResponseError.fromSchemaError according to what was being decoded; pass module and method to AiError.make." "@effect/ai/AiError#TypeId": replacement: "AiError.isAiError" note: "The AiError brand is private in v4. Use isAiError for runtime narrowing, or isAiErrorReason for a reason, instead of inspecting or constructing the type id." diff --git a/repos/effect/migration/annotations/effect__experimental__EventJournal.yaml b/repos/effect/migration/annotations/effect__experimental__EventJournal.yaml index a63c391dde..2a86c5ab48 100644 --- a/repos/effect/migration/annotations/effect__experimental__EventJournal.yaml +++ b/repos/effect/migration/annotations/effect__experimental__EventJournal.yaml @@ -5,8 +5,9 @@ replacement: none note: The v4 error marker is private; narrow with EventJournalError instead. "@effect/experimental/EventJournal#makeEntryId": + include: true replacement: effect/eventlog/EventJournal#makeEntryIdUnsafe - note: The unchecked EntryId constructor was renamed to makeEntryIdUnsafe. + note: The unchecked EntryId constructor was renamed to makeEntryIdUnsafe. Without explicit msecs, ids advance monotonically within the process using a logical millisecond and counter, so their timestamp can exceed wall-clock time when the clock stalls or moves backwards. Explicit msecs bypasses this sequence. "@effect/experimental/EventJournal#makeRemoteId": replacement: effect/eventlog/EventJournal#makeRemoteIdUnsafe note: The unchecked RemoteId constructor was renamed to makeRemoteIdUnsafe. @@ -16,3 +17,7 @@ "@effect/experimental/EventJournal#makeMemory": replacement: effect/eventlog/EventJournal#makeMemory note: The in-memory constructor moved into core Effect and now returns the Context.Service implementation through its Service projection. +"@effect/experimental/EventJournal#EventJournal": + include: true + replacement: effect/eventlog/EventJournal#EventJournal + note: Custom journal services must implement entriesById, writeMany, and writeRemoteAcks. writeFromRemote.effect now receives one array of entry/conflicts pairs per batch, not one pair per callback. Read conflicts before invoking it and commit the batch only after it succeeds. Replay entries in entry-id order; conflicts are stored entries with strictly greater ids, also in id order. Compaction replacements use the source entry as their ordering and conflict anchor. diff --git a/repos/effect/migration/annotations/effect__experimental__EventLog.yaml b/repos/effect/migration/annotations/effect__experimental__EventLog.yaml index 9d8229a1e3..dfb6f88910 100644 --- a/repos/effect/migration/annotations/effect__experimental__EventLog.yaml +++ b/repos/effect/migration/annotations/effect__experimental__EventLog.yaml @@ -2,8 +2,9 @@ replacement: effect/eventlog/EventLog#group note: Import group from the v4 EventLog module; it now requires the shared Registry service. "@effect/experimental/EventLog#Handlers": + include: true replacement: effect/eventlog/EventLog#Handlers - note: Import Handlers from the v4 EventLog module; handlers now also receive storeId. + note: Import Handlers from the v4 EventLog module; handlers now also receive storeId. Remote decode or handler failures leave the entire batch uncommitted and retry it after backoff instead of logging and committing past the failure. Replay handlers must tolerate repeated execution. Conflicts are stored entries with strictly greater ids, in id order; compacted replacements use their source entry as the conflict anchor. "@effect/experimental/EventLog#Handlers.Error": replacement: effect/eventlog/EventLog#Handlers.Error note: Import the retained Handlers.Error type from the v4 EventLog module. @@ -25,3 +26,11 @@ "@effect/experimental/EventLog#SchemaTypeId": replacement: effect/eventlog/EventLog#SchemaTypeId note: Import SchemaTypeId from the v4 EventLog module. +"@effect/experimental/EventLog#groupCompaction": + include: true + replacement: effect/eventlog/EventLog#groupCompaction + note: Use forward(entry) to retain an input entry unchanged rather than writing a replacement. Entries neither forwarded nor replaced are dropped; undecodable entries are preserved automatically. Replacements are anchored to the first decoded entry for the primary key and replayed in anchor-id order, then write order, with conflicts newer than that anchor. Collapsing entries across an intervening stored entry for the same event and primary key can break convergence. +"@effect/experimental/EventLog#makeClient": + include: true + replacement: effect/eventlog/EventLog#makeClient + note: The returned client now takes one { event, payload } object instead of separate event and payload arguments. The payload remains paired with its event tag, and each call delegates to EventLog.write; use EventLog.writeMany for an atomic batch of events. diff --git a/repos/effect/migration/annotations/effect__experimental__EventLogEncryption.yaml b/repos/effect/migration/annotations/effect__experimental__EventLogEncryption.yaml new file mode 100644 index 0000000000..a6ac8b0918 --- /dev/null +++ b/repos/effect/migration/annotations/effect__experimental__EventLogEncryption.yaml @@ -0,0 +1,12 @@ +"@effect/experimental/EventLogEncryption#EncryptedEntry": + include: true + replacement: effect/eventlog/EventLogEncryption#EncryptedBatch + note: Replace per-entry encrypted records with batches containing entryIds, iv, and encryptedBatch. Each batch encrypts multiple entries as one message and authenticates the store id and ordered entry ids; existing per-entry ciphertext is not compatible with this format. +"@effect/experimental/EventLogEncryption#EncryptedRemoteEntry": + include: true + replacement: effect/eventlog/EventLogEncryption#EncryptedRemoteBatch + note: Remote encrypted records are now batches with entryIds, iv, and encryptedBatch. sequence is the first entry sequence; subsequent entries have consecutive sequences. Existing per-entry ciphertext is not compatible with the batch format. +"@effect/experimental/EventLogEncryption#EventLogEncryption": + include: true + replacement: effect/eventlog/EventLogEncryption#EventLogEncryption + note: Use the Context.Service and its Service projection. encrypt and decrypt now take storeId after identity. encrypt returns EncryptedBatch records rather than an iv/encryptedEntries object; decrypt takes EncryptedRemoteBatch records and returns RemoteEntry records. Encryption authenticates the store id and ordered entry ids, so old per-entry ciphertext cannot be supplied as a batch. diff --git a/repos/effect/migration/annotations/effect__experimental__EventLogRemote.yaml b/repos/effect/migration/annotations/effect__experimental__EventLogRemote.yaml index 8e632aa219..c4989a140f 100644 --- a/repos/effect/migration/annotations/effect__experimental__EventLogRemote.yaml +++ b/repos/effect/migration/annotations/effect__experimental__EventLogRemote.yaml @@ -1,9 +1,11 @@ "@effect/experimental/EventLogRemote#Ack": - replacement: none - note: A write acknowledgement is now the void success of EventLogMessage.WriteSingleRpc or WriteChunkedRpc. + include: true + replacement: effect/eventlog/EventLogMessage#WriteAck + note: Write RPCs now return arrays of entryId/sequence acknowledgements in request order. WriteSingleRpc returns them directly; WriteChunkedRpc returns them on the completing chunk, with empty arrays for earlier chunks. "@effect/experimental/EventLogRemote#Changes": + include: true replacement: effect/eventlog/EventLogMessage#ChangesRpc - note: ChangesRpc replaces the separate request and response models with one streaming RPC. + note: ChangesRpc replaces the separate request and response models with one streaming RPC. Encrypted changes carry EncryptedRemoteBatch records. Built-in servers acknowledge writes from this connection instead of echoing them through changes. "@effect/experimental/EventLogRemote#decodeRequest": replacement: effect/eventlog/EventLogMessage#EventLogRemoteRpcs note: Generic RPC framing and RpcSerialization.layerSchemaBinary replace the module-specific request decoder. @@ -17,8 +19,9 @@ replacement: effect/eventlog/EventLogMessage#EventLogRemoteRpcs note: Generic RPC framing and RpcSerialization.layerSchemaBinary replace the module-specific response encoder. "@effect/experimental/EventLogRemote#EventLogRemote": + include: true replacement: effect/eventlog/EventLogRemote#EventLogRemote - note: Use the v4 Context.Service; methods now take storeId-aware options. + note: Use the v4 Context.Service; methods now take storeId-aware options. write may return entryId/sequence acknowledgement arrays; custom remotes can still return void when confirming uploads through changes instead. "@effect/experimental/EventLogRemote#fromSocket": replacement: effect/eventlog/EventLogRemote#makeEncrypted + effect/rpc/RpcClient#makeProtocolSocket note: Construct the encrypted remote separately from its generic RPC socket protocol. @@ -55,3 +58,7 @@ "@effect/experimental/EventLogRemote#StopChanges": replacement: none note: Interrupt the ChangesRpc stream instead of sending a StopChanges message. +"@effect/experimental/EventLogRemote#WriteEntries": + include: true + replacement: effect/eventlog/EventLogMessage#WriteEntries + note: Use the storeId-aware write payload with batches of EncryptedBatch instead of iv and encryptedEntries. Each batch contains entryIds, iv, and encryptedBatch; it is not compatible with the old per-entry ciphertext format. diff --git a/repos/effect/migration/annotations/effect__experimental__EventLogServer.yaml b/repos/effect/migration/annotations/effect__experimental__EventLogServer.yaml index 4968aa18bb..065a058b8f 100644 --- a/repos/effect/migration/annotations/effect__experimental__EventLogServer.yaml +++ b/repos/effect/migration/annotations/effect__experimental__EventLogServer.yaml @@ -14,4 +14,8 @@ "@effect/experimental/EventLogServer#Storage": include: true replacement: effect/eventlog/EventLogServerEncrypted#Storage - note: Use the encrypted server Storage service, which is storeId- and session-aware. + note: Use the storeId- and session-aware encrypted server Storage service. write and changes now operate on encrypted batches. write returns newly stored batches and skips a batch only when all its entry ids are already known. Custom storage must implement entrySequences to look up acknowledgements for skipped batches; each batch owns consecutive sequences starting at its sequence. +"@effect/experimental/EventLogServer#PersistedEntry": + include: true + replacement: effect/eventlog/EventLogEncryption#EncryptedBatch + note: The per-entry class was removed; encrypted storage now accepts batch records containing entryIds, iv, and encryptedBatch. The class encoding helpers and entryIdString getter are not retained. Old per-entry ciphertext cannot be reused as an encrypted batch. diff --git a/repos/effect/migration/annotations/effect__platform__FileSystem.yaml b/repos/effect/migration/annotations/effect__platform__FileSystem.yaml index a2b6995952..3646921417 100644 --- a/repos/effect/migration/annotations/effect__platform__FileSystem.yaml +++ b/repos/effect/migration/annotations/effect__platform__FileSystem.yaml @@ -7,6 +7,9 @@ "@effect/platform/FileSystem#File.Descriptor": replacement: "none" note: "Native file descriptors are no longer part of the portable File interface; use the File methods and keep any platform handle private in custom implementations." +"@effect/platform/FileSystem#File.Info": + replacement: "FileSystem.File.Info" + note: "The module moved to effect/FileSystem. dev is now bigint; ino, rdev, nlink, and blocks are Option. Update custom stat results, comparisons, arithmetic, and serialization rather than converting these values to number and losing precision. mode, uid, and gid remain numeric; size and blksize use ByteSize. Node and Bun preserve the 64-bit metadata exactly, while Deno may round large values before converting them to bigint." "@effect/platform/FileSystem#FileDescriptor": replacement: "none" note: "The descriptor branding constructor was removed with the public fd field; use File operations instead of exposing a native descriptor." diff --git a/repos/effect/migration/annotations/effect__platform__HttpMiddleware.yaml b/repos/effect/migration/annotations/effect__platform__HttpMiddleware.yaml index dd8f50df1c..bc9b63307a 100644 --- a/repos/effect/migration/annotations/effect__platform__HttpMiddleware.yaml +++ b/repos/effect/migration/annotations/effect__platform__HttpMiddleware.yaml @@ -25,3 +25,6 @@ "@effect/platform/HttpMiddleware#withTracerDisabledWhenEffect": replacement: "Effect.provideService(effect, HttpMiddleware.TracerDisabledWhen, predicate)" note: "Provide the TracerDisabledWhen reference locally to the effect." +"@effect/platform/HttpMiddleware#xForwardedHeaders": + replacement: "HttpMiddleware.xForwardedHeaders({ trustedProxies })" + note: "Now requires a trusted proxy policy; headers from untrusted peers are ignored and X-Forwarded-For is read from the right. HttpServerRequest.toURL no longer reads x-forwarded-proto directly." diff --git a/repos/effect/migration/annotations/effect__platform__Path.yaml b/repos/effect/migration/annotations/effect__platform__Path.yaml index c80325ae50..bbb536df2f 100644 --- a/repos/effect/migration/annotations/effect__platform__Path.yaml +++ b/repos/effect/migration/annotations/effect__platform__Path.yaml @@ -1,3 +1,6 @@ +"@effect/platform/Path#Path": + replacement: "Path.Path" + note: "The module moved to effect/Path. Custom service implementations must now provide resolveWithin as well. Prefer Path.make with the existing platform operations: it adds the type marker and derives resolveWithin, which returns an Option and checks lexical containment without following symlinks." "@effect/platform/Path#TypeId": replacement: "typeof Path.TypeId" note: "The module moved to effect/Path; the runtime marker remains exported but the separate type alias was removed." diff --git a/repos/effect/migration/annotations/effect__platform__UrlParams.yaml b/repos/effect/migration/annotations/effect__platform__UrlParams.yaml index 83fa4c75e9..8d5c9cb44b 100644 --- a/repos/effect/migration/annotations/effect__platform__UrlParams.yaml +++ b/repos/effect/migration/annotations/effect__platform__UrlParams.yaml @@ -51,4 +51,4 @@ note: "Retained and broadened to accept any UrlParams.Input." "@effect/platform/UrlParams#UrlParams": replacement: "UrlParams.UrlParams" - note: "Import UrlParams from effect/http. It is now a branded iterable object with a params field rather than a ReadonlyArray; construct it with UrlParams.make or UrlParams.fromInput." + note: "Import UrlParams from effect/http. It is now a branded iterable object with a params field rather than a ReadonlyArray; construct it with UrlParams.make or UrlParams.fromInput. Iteration and params values are string | Redacted.Redacted, not just string. getAll, getFirst, getLast and toRecord unwrap values; toString also serializes unwrapped values and can expose secrets." diff --git a/repos/effect/migration/annotations/effect__sql__SqlClient.yaml b/repos/effect/migration/annotations/effect__sql__SqlClient.yaml index e1daa30e39..505aeeff73 100644 --- a/repos/effect/migration/annotations/effect__sql__SqlClient.yaml +++ b/repos/effect/migration/annotations/effect__sql__SqlClient.yaml @@ -1,9 +1,20 @@ "@effect/sql/SqlClient#make": replacement: "effect/sql/SqlClient#make" - note: "Moved; custom clients rename MakeOptions.reactiveMailbox to reactiveQueue and may supply transactionService." + note: "Moved; custom clients must supply MakeOptions.driver, rename reactiveMailbox to reactiveQueue, and may supply transactionService. Driver names conventionally omit the @effect/sql- prefix, for example pg or pglite." "@effect/sql/SqlClient#SafeIntegers": replacement: "effect/sql/SqlClient#SafeIntegers" note: "Moved and changed from a Reference subclass to a Context.Reference value; provide it as a service." +"@effect/sql/SqlClient#SqlClient": + include: true + replacement: "effect/sql/SqlClient#SqlClient" + note: "Moved into core Effect. Custom service implementations must expose driver and afterCommit; SqlClient.make supplies both. driver identifies the driver independently of its SQL dialect. afterCommit registers effects to run after the outermost successful commit and connection release attempt, discarding rolled-back work. withTransaction now also defers contextual Reactivity.invalidate and mutation invalidations until that commit. Transactions bypassing makeWithTransaction, such as sqlite-do transactions, run callbacks and invalidations immediately." +"@effect/sql/SqlClient#makeWithTransaction": + include: true + replacement: "effect/sql/SqlClient#makeWithTransaction" + note: "Rename transactionTag to transactionService. acquireConnection is now a function receiving restore and returning Effect, rather than an effect returning a scope/connection pair. Register releases in the provided scope and wrap interruptible connection waits with restore; the wrapper closes the scope on transaction completion or failed acquisition." +"@effect/sql/SqlClient#SqlClient.MakeOptions": + replacement: "effect/sql/SqlClient#SqlClient.MakeOptions" + note: "Moved; driver is now required and conventionally names the driver package without @effect/sql-, for example pg or pglite. Rename reactiveMailbox to reactiveQueue; transactionService is optional." "@effect/sql/SqlClient#TransactionConnection": replacement: "effect/sql/SqlClient#TransactionConnection" note: "Now a factory keyed by client id, not a singleton tag. Prefer the client's transactionService; the payload type is TransactionConnection.Service." diff --git a/repos/effect/migration/annotations/effect__sql__SqlEventLogServer.yaml b/repos/effect/migration/annotations/effect__sql__SqlEventLogServer.yaml index bfe0a5c47c..c1d5c7c277 100644 --- a/repos/effect/migration/annotations/effect__sql__SqlEventLogServer.yaml +++ b/repos/effect/migration/annotations/effect__sql__SqlEventLogServer.yaml @@ -1,6 +1,12 @@ "@effect/sql/SqlEventLogServer#layerStorage": + include: true replacement: "effect/eventlog/SqlEventLogServerEncrypted#layerStorage" - note: "Moved to the encrypted server module with the same options and EventLogEncryption requirement." + note: "Moved to the encrypted server module; remove insertBatchSize. entryTablePrefix and remoteIdTable remain, as does the EventLogEncryption requirement. Storage uses encrypted batch tables and entry-id lookup tables, not the old per-entry ciphertext schema; existing encrypted tables are not automatically converted." "@effect/sql/SqlEventLogServer#makeStorage": + include: true replacement: "effect/eventlog/SqlEventLogServerEncrypted#makeStorage" - note: "Moved to the encrypted server module with the same SQL, encryption, and scope requirements." + note: "Moved to the encrypted server module; remove insertBatchSize. SQL, encryption, and scope requirements remain. Storage uses encrypted batch tables and entry-id lookup tables, not the old per-entry ciphertext schema; existing encrypted tables are not automatically converted." +"@effect/sql/SqlEventLogServer#layerStorageSubtle": + include: true + replacement: "effect/eventlog/SqlEventLogServerEncrypted#layerStorageSubtle" + note: "Moved to the encrypted server module; remove insertBatchSize. The layer still supplies Web Crypto encryption. Storage now uses encrypted batch tables and entry-id lookup tables; existing per-entry ciphertext tables are not automatically converted." diff --git a/repos/effect/migration/annotations/effect__vitest__index.yaml b/repos/effect/migration/annotations/effect__vitest__index.yaml index 515c9201aa..5db5f2cc09 100644 --- a/repos/effect/migration/annotations/effect__vitest__index.yaml +++ b/repos/effect/migration/annotations/effect__vitest__index.yaml @@ -269,8 +269,13 @@ replacement: "vitest#Matchers" note: "Augment vitest.Matchers for custom matchers. R is the matcher return type and T is the received value; @vitest/expect no longer shares Vitest's assertion state." "@effect/vitest/index#it": + include: true replacement: "@effect/vitest#it" - note: "Effect helpers retain their calling convention. Vitest 5 removes it.sequential; pass { concurrent: false } to the native test or as the Effect helper's third argument." + note: "Vitest 5 removes it.sequential; pass { concurrent: false } to the native test or as the Effect helper's third argument. Synchronous it.prop callbacks must return boolean or void: true and undefined pass, false falsifies, and other values (including Effects and Promises) throw TypeError. Use it.effect.prop for Effect callbacks." +"@effect/vitest/index#prop": + include: true + replacement: "@effect/vitest#prop" + note: "Synchronous property callbacks must return boolean or void: true and undefined pass, false falsifies, and other values (including Effects and Promises) throw TypeError. Move Effect callbacks to it.effect.prop instead of returning an Effect from prop." "@effect/vitest/index#test": replacement: "vitest#test" note: "Vitest 5 removes test.sequential and sequential options. Pass { concurrent: false } to opt out of inherited concurrency." diff --git a/repos/effect/migration/error-handling.md b/repos/effect/migration/error-handling.md index 93ab919eef..4638fc9a97 100644 --- a/repos/effect/migration/error-handling.md +++ b/repos/effect/migration/error-handling.md @@ -94,6 +94,73 @@ const program = Effect.fail(42).pipe( ) ``` +## Defects take precedence in mixed causes + +A failure cause can hold a typed error and a defect at the same time, for +example when a finalizer dies after the effect has already failed. In v4, +every operator that recovers from typed errors skips such causes. The handler +does not run, and the failure is propagated with each typed error converted to +a defect. + +In v3 only some operators behaved this way (`catchAll`, `matchEffect`, +`match`, `orElse`, `orElseSucceed`, `ignore`, `either`, `option` and +`Effect.retry`). `catchTag`, `catchTags`, `catchIf`, `catchSome` and the +`Stream` and `Channel` recovery operators ran the handler and dropped the +defect. In v4 the rule is the same for all of them: + +- `Effect`: `catch`, `catchTag`, `catchTags`, `catchIf`, `catchFilter`, + `catchReason`, `catchReasons`, `catchEager`, `matchEffect`, `match`, + `matchEager`, `orElseSucceed`, `ignore`, `option`, `result`, `orDie`, + `eventually`, `firstSuccessOf`, `retry`, `retryOrElse`, `repeat` and + `repeatOrElse` +- `Stream` and `Channel`: their typed recovery operators, including `catch`, + `catchTag`, `catchIf`, `catchFilter`, `catchReason`, `catchReasons`, + `ignore`, `orDie` and `retry` + +An interruption alongside a typed error does not prevent recovery. The first +typed error is handled; other typed errors and interrupts can be discarded. + +```ts +import { Effect } from "effect" + +const program = Effect.fail("error").pipe( + Effect.ensuring(Effect.die("finalizer defect")), + // The handler does not run: the cause contains a defect + Effect.catch(() => Effect.succeed("recovered")) +) +// fails with Die("error") + Die("finalizer defect") +``` + +Use `Effect.catchDefect` or `Effect.catchCause` to recover from causes that +contain defects. + +### Retry and repeat error paths + +`Effect.retry`, `retryOrElse`, `repeat` and `repeatOrElse` apply this rule to +source failures. `Fail(e) + Interrupt` now enters the typed error path instead +of propagating unchanged. `retry` retries it; when retries run out, or `repeat` +re-raises it, the final cause is `Fail(e)` without the interrupt. The `orElse` +handlers can now receive `e` from these causes. + +`repeat` and `repeatOrElse` also apply the rule to policy failures. A policy +failure containing `Fail(e) + Die(d)` propagates as `Die(e) + Die(d)` without +running `orElse`. In contrast, `retry` and `retryOrElse` do not recover policy +errors, so policy failures keep their original cause. + +## Error mapping preserves the whole cause + +Error mapping is not recovery. `Effect.mapError`, `mapErrorEager`, `mapBoth` +and `mapBothEager`, `Exit.mapError` and `mapBoth`, `Stream.mapError` and +`mapBoth`, and `Channel.mapError` now map every typed error in place while +preserving defects, interrupts and reason order. Eager and suspended inputs +follow the same rule. For example, `Fail(e) + Interrupt + Die(d)` becomes +`Fail(f(e)) + Interrupt + Die(d)`, not an elected defect cause. + +Previously these operators mapped only the first typed error and discarded +the remaining reasons. If a cause contains several typed errors, the mapper +now runs for each one. Unlike typed recovery, mapping neither skips errors +beside defects nor converts them to defects. + ## New in v4 - **`Effect.catchReason(errorTag, reasonTag, handler)`** — catches a specific diff --git a/repos/effect/migration/layer-memoization.md b/repos/effect/migration/layer-memoization.md index a67cc6e74d..7455b46aee 100644 --- a/repos/effect/migration/layer-memoization.md +++ b/repos/effect/migration/layer-memoization.md @@ -5,10 +5,12 @@ were memoized / deduplicated within a single `Effect.provide` call, but would **not** be shared across separate calls — so two `Effect.provide` calls with overlapping layers would silently build those layers twice. -In v4, the underlying `MemoMap` data structure which facilitates memoization of -`Layer`s is shared between `Effect.provide` calls (unless explicitly disabled -via the `{ local: true }` option). Thus, layers are automatically memoized / -deduplicated across `Effect.provide` calls. +In v4, a layer already built by an enclosing `Effect.provide` or runtime is +reused by nested `Effect.provide` calls. Each call forks the ambient `MemoMap`: it +can reuse parent entries, but newly built layers are not written back to the +parent. Sibling `Effect.provide` calls therefore build their own copies, just as +in v3. Use `{ local: true }` to build a fresh layer subtree without reusing the +enclosing memo map. ## Example @@ -30,7 +32,7 @@ const program = Effect.gen(function*() { return a.value }) -// Same layer provided twice in separate provide calls +// Same layer provided twice in nested provide calls const main = program.pipe( Effect.provide(MyServiceLayer), Effect.provide(MyServiceLayer) @@ -41,10 +43,23 @@ const main = program.pipe( Effect.runPromise(main) ``` -## Prefer Layer Composition Over Multipl Provides +The calls above are nested, not siblings. In contrast, the following builds +`MyServiceLayer` twice in v4 when no enclosing provide or runtime has already +built it: -Even though v4 memoizes across `provide` calls, **composing layers before -providing is still the recommended pattern**. Layer composition makes your +```ts +const siblings = Effect.gen(function*() { + yield* program.pipe(Effect.provide(MyServiceLayer)) + yield* program.pipe(Effect.provide(MyServiceLayer)) +}) + +Effect.runPromise(siblings) // "Building MyService" is logged TWICE +``` + +## Prefer Layer Composition Over Multiple Provides + +Even though nested v4 `provide` calls reuse already-built layers, **composing +layers before providing is still the recommended pattern**. Layer composition makes your dependency graph explicit and lets you see the full structure in one place: ```ts @@ -52,9 +67,8 @@ dependency graph explicit and lets you see the full structure in one place: const main = program.pipe(Effect.provide(MyServiceLayer)) ``` -The auto-memoization feature is a safety net to avoid the footguns associated -with multiple `Effect.provide` calls present in v3. It is **NOT** a substitute -for proper layer composition. +Reusing enclosing layers is **not** a substitute for proper layer composition +and does not deduplicate independent sibling builds. ## Opting Out of Shared Memoization @@ -80,15 +94,16 @@ const main = program.pipe( ### `Effect.provide` with `{ local: true }` New in v4. Builds the provided layer with a **local memo map** instead of the -fiber's shared one. The layer and all its sublayers are built from scratch and -are not shared with other `provide` calls. +fiber's ambient one. The layer and all its sublayers are built from scratch, +without reusing layers built by an enclosing `provide` or runtime. Nested +`provide` calls can still reuse layers built in this local map. ```ts import { Effect } from "effect" const main = program.pipe( - Effect.provide(MyServiceLayer), - Effect.provide(MyServiceLayer, { local: true }) + Effect.provide(MyServiceLayer, { local: true }), + Effect.provide(MyServiceLayer) ) // "Building MyService" is logged TWICE — local creates its own memo map ``` diff --git a/repos/effect/migration/schema.md b/repos/effect/migration/schema.md index 8f7aec0e64..f1164138fd 100644 --- a/repos/effect/migration/schema.md +++ b/repos/effect/migration/schema.md @@ -897,17 +897,16 @@ const NumberFromString = Schema.transformOrFail(Schema.String, Schema.Number, { v4 ```ts -import { Effect, Number, Schema, SchemaGetter, SchemaIssue } from "effect" +import { Effect, Number, Option, Schema, SchemaGetter, SchemaIssue } from "effect" const NumberFromString = Schema.String.pipe( Schema.decodeTo(Schema.Number, { - decode: SchemaGetter.transformEffect((s) => { - const n = Number.parse(s) - if (n === undefined) { - return Effect.fail(new SchemaIssue.InvalidValue()) - } - return Effect.succeed(n) - }), + decode: SchemaGetter.transformEffect((s) => + Option.match(Number.parse(s), { + onNone: () => Effect.fail(new SchemaIssue.InvalidValue()), + onSome: Effect.succeed + }) + ), encode: SchemaGetter.String() }) ) diff --git a/repos/effect/migration/v3-to-v4.md b/repos/effect/migration/v3-to-v4.md index 0adea8d81c..0993e3b03e 100644 --- a/repos/effect/migration/v3-to-v4.md +++ b/repos/effect/migration/v3-to-v4.md @@ -2,9 +2,9 @@ # v3 to v4 Migration Reference -Base: `e730f59d29db5f1e312be6476a95d07bcad45788` (`e730f59d29db5f1e312be6476a95d07bcad45788`) +Base: `origin/v3` (`cab7f2e3493e4a3dceda27cb1b6d28c4599cb939`) -Head: `origin/main` (`7547a99f81fa6f7e651d52f3d42081f399bfbca8`) +Head: `origin/main` (`4cddd3c41e71fc4990d37c42b1bcf2ce3780b41d`) This file is generated from the API diff and `migration/annotations/*.yaml`. @@ -1528,7 +1528,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.ActiveStatusType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.AddUploadPartRequest` -> `Generated.AddUploadPartRequest`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. +- `Generated.AddUploadPartRequest` -> `Generated.AddUploadPartRequestMultipart`: Renamed for multipart requests; file fields now accept File or Blob. Re-check the current Type/Encoded shape. - `Generated.AdminApiKey` -> `Generated.AdminApiKey`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. @@ -1544,13 +1544,13 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.ApiKeyList` -> `Generated.ApiKeyList`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.ApplyPatchCallOutputStatus` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ApplyPatchCallOutputStatus` -> `Generated.ApplyPatchCallOutputStatus`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. -- `Generated.ApplyPatchCallOutputStatusParam` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ApplyPatchCallOutputStatusParam` -> `Generated.ApplyPatchCallOutputStatusParam`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. -- `Generated.ApplyPatchCallStatus` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ApplyPatchCallStatus` -> `Generated.ApplyPatchCallStatus`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. -- `Generated.ApplyPatchCallStatusParam` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ApplyPatchCallStatusParam` -> `Generated.ApplyPatchCallStatusParam`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.ApplyPatchCreateFileOperation` -> `Generated.ApplyPatchCreateFileOperation`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. @@ -1568,7 +1568,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.ApplyPatchDeleteFileOperationType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.ApplyPatchOperationParam` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ApplyPatchOperationParam` -> `Generated.ApplyPatchOperationParam`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.ApplyPatchToolCall` -> `Generated.ApplyPatchToolCall`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. @@ -1598,7 +1598,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.ApplyPatchUpdateFileOperationType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.ApproximateLocation` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ApproximateLocation` -> `Generated.ApproximateLocation`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.ApproximateLocationType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -1644,11 +1644,11 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.Attachment` -> `Generated.Attachment`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.AttachmentType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.AttachmentType` -> `Generated.AttachmentType`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.AudioResponseFormat` -> `Generated.AudioResponseFormat`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.AudioTranscription` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.AudioTranscription` -> `Generated.AudioTranscription`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.AudioTranscriptionModel` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -1670,15 +1670,15 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.AuditLogEventType` -> `Generated.AuditLogEventType`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.AutoChunkingStrategyRequestParam` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.AutoChunkingStrategyRequestParam` -> `Generated.AutoChunkingStrategyRequestParam`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.AutoChunkingStrategyRequestParamType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.AutomaticThreadTitlingParam` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.AutomaticThreadTitlingParam` -> `Generated.AutomaticThreadTitlingParam`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.Batch` -> `Generated.Batch`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.BatchError` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.BatchError` -> `Generated.BatchError`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.BatchFileExpirationAfter` -> `Generated.BatchFileExpirationAfter`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. @@ -1686,7 +1686,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.BatchObject` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.BatchRequestCounts` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.BatchRequestCounts` -> `Generated.BatchRequestCounts`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.BatchStatus` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -1812,33 +1812,33 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.ChatModel` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.ChatSessionAutomaticThreadTitling` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ChatSessionAutomaticThreadTitling` -> `Generated.ChatSessionAutomaticThreadTitling`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. -- `Generated.ChatSessionChatkitConfiguration` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ChatSessionChatkitConfiguration` -> `Generated.ChatSessionChatkitConfiguration`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. -- `Generated.ChatSessionFileUpload` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ChatSessionFileUpload` -> `Generated.ChatSessionFileUpload`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. -- `Generated.ChatSessionHistory` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ChatSessionHistory` -> `Generated.ChatSessionHistory`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. -- `Generated.ChatSessionRateLimits` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ChatSessionRateLimits` -> `Generated.ChatSessionRateLimits`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.ChatSessionResource` -> `Generated.ChatSessionResource`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. - `Generated.ChatSessionResourceObject` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.ChatSessionStatus` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ChatSessionStatus` -> `Generated.ChatSessionStatus`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. -- `Generated.ChatkitConfigurationParam` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ChatkitConfigurationParam` -> `Generated.ChatkitConfigurationParam`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. -- `Generated.ChatkitWorkflow` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ChatkitWorkflow` -> `Generated.ChatkitWorkflow`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. -- `Generated.ChatkitWorkflowTracing` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ChatkitWorkflowTracing` -> `Generated.ChatkitWorkflowTracing`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.ChunkingStrategyRequestParam` -> `Generated.ChunkingStrategyRequestParam`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. - `Generated.ChunkingStrategyResponse` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.ClickButtonType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ClickButtonType` -> `Generated.ClickButtonType`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.ClickParam` -> `Generated.ClickParam`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. @@ -1854,7 +1854,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.ClientToolCallItemType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.ClientToolCallStatus` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ClientToolCallStatus` -> `Generated.ClientToolCallStatus`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.ClosedStatus` -> `Generated.ClosedStatus`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. @@ -1904,7 +1904,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.ComputerCallSafetyCheckParam` -> `Generated.ComputerCallSafetyCheckParam`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.ComputerEnvironment` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ComputerEnvironment` -> `Generated.ComputerEnvironment`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.ComputerScreenshotContent` -> `Generated.ComputerScreenshotContent`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. @@ -1940,7 +1940,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.ContainerListResource` -> `Generated.ContainerListResource`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.ContainerMemoryLimit` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ContainerMemoryLimit` -> `Generated.ContainerMemoryLimit`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.ContainerResource` -> `Generated.ContainerResource`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. @@ -1966,7 +1966,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.CreateAssistantRequest` -> `Generated.CreateAssistantRequest`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.CreateBatchRequest` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.CreateBatchRequest` -> `Generated.CreateBatchRequest`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.CreateBatchRequestCompletionWindow` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -2018,19 +2018,19 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.CreateEmbeddingResponseObject` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.CreateEvalCompletionsRunDataSource` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.CreateEvalCompletionsRunDataSource` -> `Generated.CreateEvalCompletionsRunDataSource`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.CreateEvalCompletionsRunDataSourceInputMessagesEnumType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. - `Generated.CreateEvalCompletionsRunDataSourceType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.CreateEvalCustomDataSourceConfig` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.CreateEvalCustomDataSourceConfig` -> `Generated.CreateEvalCustomDataSourceConfig`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.CreateEvalCustomDataSourceConfigType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. - `Generated.CreateEvalItem` -> `Generated.CreateEvalItem`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.CreateEvalJsonlRunDataSource` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.CreateEvalJsonlRunDataSource` -> `Generated.CreateEvalJsonlRunDataSource`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.CreateEvalJsonlRunDataSourceType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -2038,13 +2038,13 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.CreateEvalLabelModelGraderType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.CreateEvalLogsDataSourceConfig` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.CreateEvalLogsDataSourceConfig` -> `Generated.CreateEvalLogsDataSourceConfig`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.CreateEvalLogsDataSourceConfigType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. - `Generated.CreateEvalRequest` -> `Generated.CreateEvalRequest`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.CreateEvalResponsesRunDataSource` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.CreateEvalResponsesRunDataSource` -> `Generated.CreateEvalResponsesRunDataSource`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.CreateEvalResponsesRunDataSourceInputMessagesEnumType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -2052,11 +2052,11 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.CreateEvalRunRequest` -> `Generated.CreateEvalRunRequest`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.CreateEvalStoredCompletionsDataSourceConfig` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.CreateEvalStoredCompletionsDataSourceConfig` -> `Generated.CreateEvalStoredCompletionsDataSourceConfig`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.CreateEvalStoredCompletionsDataSourceConfigType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.CreateFileRequest` -> `Generated.CreateFileRequest`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. +- `Generated.CreateFileRequest` -> `Generated.CreateFileRequestMultipart`: Renamed for multipart requests; file fields now accept File or Blob. Re-check the current Type/Encoded shape. - `Generated.CreateFineTuningCheckpointPermissionRequest` -> `Generated.CreateFineTuningCheckpointPermissionRequest`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. @@ -2074,7 +2074,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.CreateGroupUserBody` -> `Generated.CreateGroupUserBody`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.CreateImageEditRequest` -> `Generated.CreateImageEditRequest`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. +- `Generated.CreateImageEditRequest` -> `Generated.CreateImageEditRequestMultipart`: Renamed for multipart requests; file fields now accept File or Blob. Re-check the current Type/Encoded shape. JSON image edits instead use Generated.CreateImageEditRequestJson. - `Generated.CreateImageEditRequestBackground` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -2106,7 +2106,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.CreateImageRequestStyle` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.CreateImageVariationRequest` -> `Generated.CreateImageVariationRequest`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. +- `Generated.CreateImageVariationRequest` -> `Generated.CreateImageVariationRequestMultipart`: Renamed for multipart requests; file fields now accept File or Blob. Re-check the current Type/Encoded shape. - `Generated.CreateImageVariationRequestModelEnum` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -2166,7 +2166,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.CreateTranscription200` -> `Generated.CreateTranscription200`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.CreateTranscriptionRequest` -> `Generated.CreateTranscriptionRequest`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. +- `Generated.CreateTranscriptionRequest` -> `Generated.CreateTranscriptionRequestMultipart`: Renamed for multipart requests; file fields now accept File or Blob. Re-check the current Type/Encoded shape. - `Generated.CreateTranscriptionRequestModelEnum` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -2180,7 +2180,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.CreateTranslation200` -> `Generated.CreateTranslation200`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.CreateTranslationRequest` -> `Generated.CreateTranslationRequest`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. +- `Generated.CreateTranslationRequest` -> `Generated.CreateTranslationRequestMultipart`: Renamed for multipart requests; file fields now accept File or Blob. Re-check the current Type/Encoded shape. - `Generated.CreateTranslationRequestModelEnum` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -2282,13 +2282,13 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.DeletedVideoResourceObject` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.DetailEnum` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.DetailEnum` -> `Generated.DetailEnum`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.DoubleClickAction` -> `Generated.DoubleClickAction`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. - `Generated.DoubleClickActionType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.DownloadFile200` -> `Generated.DownloadFile200`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. +- `Generated.DownloadFile200` -> `none`: The standalone response schema was removed. The generated client's downloadFile method returns Uint8Array, optionally paired with the HTTP response according to its config; downloadFileStream streams Uint8Array chunks. - `Generated.Drag` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -2316,7 +2316,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.EvalApiError` -> `Generated.EvalApiError`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.EvalCustomDataSourceConfig` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.EvalCustomDataSourceConfig` -> `Generated.EvalCustomDataSourceConfig`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.EvalCustomDataSourceConfigType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -2364,7 +2364,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.EvalListObject` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.EvalLogsDataSourceConfig` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.EvalLogsDataSourceConfig` -> `Generated.EvalLogsDataSourceConfig`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.EvalLogsDataSourceConfigType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -2392,7 +2392,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.EvalRunOutputItemResult` -> `Generated.EvalRunOutputItemResult`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.EvalStoredCompletionsDataSourceConfig` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.EvalStoredCompletionsDataSourceConfig` -> `Generated.EvalStoredCompletionsDataSourceConfig`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.EvalStoredCompletionsDataSourceConfigType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -2400,13 +2400,13 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.EvalStoredCompletionsSourceType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.ExpiresAfterParam` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ExpiresAfterParam` -> `Generated.ExpiresAfterParam`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.ExpiresAfterParamAnchor` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. - `Generated.FileAnnotation` -> `Generated.FileAnnotation`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.FileAnnotationSource` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.FileAnnotationSource` -> `Generated.FileAnnotationSource`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.FileAnnotationSourceType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -2440,9 +2440,9 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.FileSearchToolType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.FileUploadParam` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.FileUploadParam` -> `Generated.FileUploadParam`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. -- `Generated.Filters` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.Filters` -> `Generated.Filters`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.FineTuneDPOHyperparameters` -> `Generated.FineTuneDPOHyperparameters`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. @@ -2522,7 +2522,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.FunctionAndCustomToolCallOutput` -> `Generated.FunctionAndCustomToolCallOutput`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.FunctionCallItemStatus` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.FunctionCallItemStatus` -> `Generated.FunctionCallItemStatus`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.FunctionCallOutputItemParam` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -2532,9 +2532,9 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.FunctionParameters` -> `Generated.FunctionParameters`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.FunctionShellAction` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.FunctionShellAction` -> `Generated.FunctionShellAction`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. -- `Generated.FunctionShellActionParam` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.FunctionShellActionParam` -> `Generated.FunctionShellActionParam`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.FunctionShellCall` -> `Generated.FunctionShellCall`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. @@ -2542,7 +2542,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.FunctionShellCallItemParamType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.FunctionShellCallItemStatus` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.FunctionShellCallItemStatus` -> `Generated.FunctionShellCallItemStatus`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.FunctionShellCallOutput` -> `Generated.FunctionShellCallOutput`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. @@ -2562,7 +2562,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.FunctionShellCallOutputItemParamType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.FunctionShellCallOutputOutcomeParam` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.FunctionShellCallOutputOutcomeParam` -> `Generated.FunctionShellCallOutputOutcomeParam`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.FunctionShellCallOutputTimeoutOutcome` -> `Generated.FunctionShellCallOutputTimeoutOutcome`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. @@ -2630,7 +2630,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.GraderLabelModelType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.GraderMulti` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.GraderMulti` -> `Generated.GraderMulti`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.GraderMultiType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -2654,7 +2654,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.GraderTextSimilarityType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.GrammarSyntax1` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.GrammarSyntax1` -> `Generated.GrammarSyntax1`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.Group` -> `Generated.Group`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. @@ -2684,13 +2684,13 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.GroupUserDeletedResourceObject` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.HistoryParam` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.HistoryParam` -> `Generated.HistoryParam`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. -- `Generated.HybridSearchOptions` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.HybridSearchOptions` -> `Generated.HybridSearchOptions`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.Image` -> `Generated.Image`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.ImageDetail` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ImageDetail` -> `Generated.ImageDetail`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.ImageGenInputUsageDetails` -> `Generated.ImageGenInputUsageDetails`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. @@ -2730,7 +2730,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.IncludeEnum` -> `Generated.IncludeEnum`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.InferenceOptions` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.InferenceOptions` -> `Generated.InferenceOptions`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.InputAudio` -> `Generated.InputAudio`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. @@ -3058,7 +3058,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.MCPToolCall` -> `Generated.MCPToolCall`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.MCPToolCallStatus` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.MCPToolCallStatus` -> `Generated.MCPToolCallStatus`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.MCPToolCallType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -3116,7 +3116,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.MessageRequestContentTextObjectType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.MessageRole` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.MessageRole` -> `Generated.MessageRole`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.MessageStatus` -> `OpenAiSchema.MessageStatus`: Use the focused v4 OpenAiSchema definition; the old generated export was removed when the OpenAI specification client was regenerated. @@ -3168,7 +3168,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.OrderEnum` -> `Generated.OrderEnum`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.OtherChunkingStrategyResponseParam` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.OtherChunkingStrategyResponseParam` -> `Generated.OtherChunkingStrategyResponseParam`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.OtherChunkingStrategyResponseParamType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -3306,11 +3306,11 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.PublicUpdateOrganizationRoleBody` -> `Generated.PublicUpdateOrganizationRoleBody`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.RankerVersionType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.RankerVersionType` -> `Generated.RankerVersionType`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. -- `Generated.RankingOptions` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.RankingOptions` -> `Generated.RankingOptions`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. -- `Generated.RateLimitsParam` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.RateLimitsParam` -> `Generated.RateLimitsParam`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.RealtimeAudioFormats` -> `Generated.RealtimeAudioFormats`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. @@ -3506,7 +3506,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.RunStepCompletionUsage` -> `Generated.RunStepCompletionUsage`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.RunStepDetailsMessageCreationObject` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.RunStepDetailsMessageCreationObject` -> `Generated.RunStepDetailsMessageCreationObject`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.RunStepDetailsMessageCreationObjectType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -3516,11 +3516,11 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.RunStepDetailsToolCallsCodeObjectType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.RunStepDetailsToolCallsCodeOutputImageObject` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.RunStepDetailsToolCallsCodeOutputImageObject` -> `Generated.RunStepDetailsToolCallsCodeOutputImageObject`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.RunStepDetailsToolCallsCodeOutputImageObjectType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.RunStepDetailsToolCallsCodeOutputLogsObject` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.RunStepDetailsToolCallsCodeOutputLogsObject` -> `Generated.RunStepDetailsToolCallsCodeOutputLogsObject`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.RunStepDetailsToolCallsCodeOutputLogsObjectType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -3536,7 +3536,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.RunStepDetailsToolCallsFunctionObjectType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.RunStepDetailsToolCallsObject` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.RunStepDetailsToolCallsObject` -> `Generated.RunStepDetailsToolCallsObject`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.RunStepDetailsToolCallsObjectType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -3562,11 +3562,11 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.ScrollType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.SearchContextSize` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.SearchContextSize` -> `Generated.SearchContextSize`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.ServiceTier` -> `Generated.ServiceTier`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.ServiceTierEnum` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.ServiceTierEnum` -> `Generated.ServiceTierEnum`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.SpecificApplyPatchParam` -> `Generated.SpecificApplyPatchParam`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. @@ -3578,11 +3578,11 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.StaticChunkingStrategy` -> `Generated.StaticChunkingStrategy`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.StaticChunkingStrategyRequestParam` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.StaticChunkingStrategyRequestParam` -> `Generated.StaticChunkingStrategyRequestParam`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.StaticChunkingStrategyRequestParamType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.StaticChunkingStrategyResponseParam` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.StaticChunkingStrategyResponseParam` -> `Generated.StaticChunkingStrategyResponseParam`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.StaticChunkingStrategyResponseParamType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -3610,7 +3610,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.TaskItemType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.TaskType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.TaskType` -> `Generated.TaskType`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.TextAnnotation` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -3702,7 +3702,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.TranscriptionWord` -> `Generated.TranscriptionWord`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.TruncationEnum` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.TruncationEnum` -> `Generated.TruncationEnum`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.Type` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -3746,7 +3746,7 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.UrlAnnotation` -> `Generated.UrlAnnotation`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.UrlAnnotationSource` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.UrlAnnotationSource` -> `Generated.UrlAnnotationSource`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.UrlAnnotationSourceType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -3932,17 +3932,17 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.VideoListResource` -> `Generated.VideoListResource`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. -- `Generated.VideoModel` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.VideoModel` -> `Generated.VideoModel`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.VideoResource` -> `Generated.VideoResource`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. - `Generated.VideoResourceObject` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.VideoSeconds` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.VideoSeconds` -> `Generated.VideoSeconds`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. -- `Generated.VideoSize` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.VideoSize` -> `Generated.VideoSize`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. -- `Generated.VideoStatus` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.VideoStatus` -> `Generated.VideoStatus`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.VoiceIdsShared` -> `Generated.VoiceIdsShared`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. @@ -3952,15 +3952,15 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.WaitType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.WebSearchActionFind` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.WebSearchActionFind` -> `Generated.WebSearchActionFind`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.WebSearchActionFindType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.WebSearchActionOpenPage` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.WebSearchActionOpenPage` -> `Generated.WebSearchActionOpenPage`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.WebSearchActionOpenPageType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.WebSearchActionSearch` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.WebSearchActionSearch` -> `Generated.WebSearchActionSearch`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.WebSearchActionSearchType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. @@ -3994,9 +3994,9 @@ effect/rpc/Utils (barrel: effect/rpc) - `Generated.WidgetMessageItemType` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. -- `Generated.WorkflowParam` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.WorkflowParam` -> `Generated.WorkflowParam`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. -- `Generated.WorkflowTracingParam` -> `none`: Removed when the OpenAI specification client was regenerated; use the current Generated or OpenAiSchema request/response schema for the relevant endpoint instead of this old snapshot helper. +- `Generated.WorkflowTracingParam` -> `Generated.WorkflowTracingParam`: Available again in the regenerated v4 OpenAI client; re-check the schema's Type/Encoded shape against the current specification. - `Generated.make` -> `Generated.make`: Still generated in v4 from the current OpenAI specification; re-check the schema's Type/Encoded shape because the generated definition changed. @@ -4914,11 +4914,11 @@ effect/rpc/Utils (barrel: effect/rpc) - `AiError.HttpRequestError` -> `AiError.make + AiError.NetworkError`: Replace the old top-level request error with an AiError whose reason is NetworkError. NetworkError.fromRequestError converts a v4 HttpClientError.RequestError. -- `AiError.HttpResponseError` -> `AiError.make + AiError.reasonFromHttpStatus / AiError.InvalidOutputError`: There is no single v4 response-error class. Wrap a semantic reason with AiError.make: use reasonFromHttpStatus for status failures and InvalidOutputError for decode or empty-body failures. +- `AiError.HttpResponseError` -> `AiError.make + AiError.reasonFromHttpStatus / AiError.InvalidResponseError / AiError.InvalidOutputError`: There is no single v4 response-error class. Wrap a semantic reason with AiError.make: use reasonFromHttpStatus for status failures. Provider response-schema and SSE decoding failures use non-retryable InvalidResponseError. Lower-level HTTP client DecodeError and EmptyBodyError still map to retryable InvalidOutputError in provider adapters. - `AiError.MalformedInput` -> `AiError.make + AiError.InvalidUserInputError`: Replace the old top-level input error with an AiError whose reason is InvalidUserInputError. Use InvalidRequestError when the provider request parameters themselves are malformed. -- `AiError.MalformedOutput` -> `AiError.make + AiError.InvalidOutputError`: Replace the old top-level output error with an AiError whose reason is InvalidOutputError. The old fromParseError helper becomes InvalidOutputError.fromSchemaError. +- `AiError.MalformedOutput` -> `AiError.make + AiError.InvalidOutputError / AiError.InvalidResponseError`: Replace the old top-level output error with an AiError wrapper. Use retryable InvalidOutputError for invalid model-generated output and non-retryable InvalidResponseError for incompatible provider response formats, including response-schema and SSE decoding failures. Replace fromParseError with InvalidOutputError.fromSchemaError or InvalidResponseError.fromSchemaError according to what was being decoded; pass module and method to AiError.make. - `AiError.TypeId` -> `AiError.isAiError`: The AiError brand is private in v4. Use isAiError for runtime narrowing, or isAiErrorReason for a reason, instead of inspecting or constructing the type id. @@ -5706,6 +5706,8 @@ effect/rpc/Utils (barrel: effect/rpc) - `ClusterSchema.ClientTracingEnabled` -> `effect/cluster/ClusterSchema#ClientTracingEnabled`: Moved into core Effect and changed from a Context.Tag subclass to a Context.Reference value. +- `ClusterSchema.InterruptOnTermination`: TODO: needs guidance + - `ClusterSchema.Persisted` -> `effect/cluster/ClusterSchema#Persisted`: Moved into core Effect and changed from a Context.Tag subclass to a Context.Reference value with the same false default. - `ClusterSchema.ShardGroup` -> `effect/cluster/ClusterSchema#ShardGroup`: Moved into core Effect and changed from a Context.Tag subclass to a Context.Reference value. @@ -5948,9 +5950,11 @@ effect/rpc/Utils (barrel: effect/rpc) - `EventJournal.ErrorTypeId` -> `none`: The v4 error marker is private; narrow with EventJournalError instead. +- `EventJournal.EventJournal` -> `effect/eventlog/EventJournal#EventJournal`: Custom journal services must implement entriesById, writeMany, and writeRemoteAcks. writeFromRemote.effect now receives one array of entry/conflicts pairs per batch, not one pair per callback. Read conflicts before invoking it and commit the batch only after it succeeds. Replay entries in entry-id order; conflicts are stored entries with strictly greater ids, also in id order. Compaction replacements use the source entry as their ordering and conflict anchor. + - `EventJournal.RemoteIdTypeId` -> `effect/eventlog/EventJournal#RemoteIdTypeId`: Import RemoteIdTypeId from the v4 EventJournal module; it is now a string brand. -- `EventJournal.makeEntryId` -> `effect/eventlog/EventJournal#makeEntryIdUnsafe`: The unchecked EntryId constructor was renamed to makeEntryIdUnsafe. +- `EventJournal.makeEntryId` -> `effect/eventlog/EventJournal#makeEntryIdUnsafe`: The unchecked EntryId constructor was renamed to makeEntryIdUnsafe. Without explicit msecs, ids advance monotonically within the process using a logical millisecond and counter, so their timestamp can exceed wall-clock time when the clock stalls or moves backwards. Explicit msecs bypasses this sequence. - `EventJournal.makeMemory` -> `effect/eventlog/EventJournal#makeMemory`: The in-memory constructor moved into core Effect and now returns the Context.Service implementation through its Service projection. @@ -5958,7 +5962,7 @@ effect/rpc/Utils (barrel: effect/rpc) ### `@effect/experimental/EventLog` -- `EventLog.Handlers` -> `effect/eventlog/EventLog#Handlers`: Import Handlers from the v4 EventLog module; handlers now also receive storeId. +- `EventLog.Handlers` -> `effect/eventlog/EventLog#Handlers`: Import Handlers from the v4 EventLog module; handlers now also receive storeId. Remote decode or handler failures leave the entire batch uncommitted and retry it after backoff instead of logging and committing past the failure. Replay handlers must tolerate repeated execution. Conflicts are stored entries with strictly greater ids, in id order; compacted replacements use their source entry as the conflict anchor. - `EventLog.HandlersTypeId` -> `effect/eventlog/EventLog#HandlersTypeId`: Import HandlersTypeId from the v4 EventLog module. @@ -5966,17 +5970,29 @@ effect/rpc/Utils (barrel: effect/rpc) - `EventLog.group` -> `effect/eventlog/EventLog#group`: Import group from the v4 EventLog module; it now requires the shared Registry service. +- `EventLog.groupCompaction` -> `effect/eventlog/EventLog#groupCompaction`: Use forward(entry) to retain an input entry unchanged rather than writing a replacement. Entries neither forwarded nor replaced are dropped; undecodable entries are preserved automatically. Replacements are anchored to the first decoded entry for the primary key and replayed in anchor-id order, then write order, with conflicts newer than that anchor. Collapsing entries across an intervening stored entry for the same event and primary key can break convergence. + - `EventLog.layer` -> `effect/eventlog/EventLog#layer`: The v4 layer takes both the schema and handler layer; use layerEventLog for runtime only. - `EventLog.layerIdentityKvs` -> `none`: Compose KeyValueStore.toSchemaStore, EventLog.IdentitySchema, EventLog.makeIdentity, and Layer.effect manually. +- `EventLog.makeClient` -> `effect/eventlog/EventLog#makeClient`: The returned client now takes one { event, payload } object instead of separate event and payload arguments. The payload remains paired with its event tag, and each call delegates to EventLog.write; use EventLog.writeMany for an atomic batch of events. + +### `@effect/experimental/EventLogEncryption` + +- `EventLogEncryption.EncryptedEntry` -> `effect/eventlog/EventLogEncryption#EncryptedBatch`: Replace per-entry encrypted records with batches containing entryIds, iv, and encryptedBatch. Each batch encrypts multiple entries as one message and authenticates the store id and ordered entry ids; existing per-entry ciphertext is not compatible with this format. + +- `EventLogEncryption.EncryptedRemoteEntry` -> `effect/eventlog/EventLogEncryption#EncryptedRemoteBatch`: Remote encrypted records are now batches with entryIds, iv, and encryptedBatch. sequence is the first entry sequence; subsequent entries have consecutive sequences. Existing per-entry ciphertext is not compatible with the batch format. + +- `EventLogEncryption.EventLogEncryption` -> `effect/eventlog/EventLogEncryption#EventLogEncryption`: Use the Context.Service and its Service projection. encrypt and decrypt now take storeId after identity. encrypt returns EncryptedBatch records rather than an iv/encryptedEntries object; decrypt takes EncryptedRemoteBatch records and returns RemoteEntry records. Encryption authenticates the store id and ordered entry ids, so old per-entry ciphertext cannot be supplied as a batch. + ### `@effect/experimental/EventLogRemote` -- `EventLogRemote.Ack` -> `none`: A write acknowledgement is now the void success of EventLogMessage.WriteSingleRpc or WriteChunkedRpc. +- `EventLogRemote.Ack` -> `effect/eventlog/EventLogMessage#WriteAck`: Write RPCs now return arrays of entryId/sequence acknowledgements in request order. WriteSingleRpc returns them directly; WriteChunkedRpc returns them on the completing chunk, with empty arrays for earlier chunks. -- `EventLogRemote.Changes` -> `effect/eventlog/EventLogMessage#ChangesRpc`: ChangesRpc replaces the separate request and response models with one streaming RPC. +- `EventLogRemote.Changes` -> `effect/eventlog/EventLogMessage#ChangesRpc`: ChangesRpc replaces the separate request and response models with one streaming RPC. Encrypted changes carry EncryptedRemoteBatch records. Built-in servers acknowledge writes from this connection instead of echoing them through changes. -- `EventLogRemote.EventLogRemote` -> `effect/eventlog/EventLogRemote#EventLogRemote`: Use the v4 Context.Service; methods now take storeId-aware options. +- `EventLogRemote.EventLogRemote` -> `effect/eventlog/EventLogRemote#EventLogRemote`: Use the v4 Context.Service; methods now take storeId-aware options. write may return entryId/sequence acknowledgement arrays; custom remotes can still return void when confirming uploads through changes instead. - `EventLogRemote.Hello` -> `effect/eventlog/EventLogMessage#HelloResponse`: HelloResponse replaces Hello and includes the v4 authentication challenge; HelloRpc defines the endpoint. @@ -5996,6 +6012,8 @@ effect/rpc/Utils (barrel: effect/rpc) - `EventLogRemote.StopChanges` -> `none`: Interrupt the ChangesRpc stream instead of sending a StopChanges message. +- `EventLogRemote.WriteEntries` -> `effect/eventlog/EventLogMessage#WriteEntries`: Use the storeId-aware write payload with batches of EncryptedBatch instead of iv and encryptedEntries. Each batch contains entryIds, iv, and encryptedBatch; it is not compatible with the old per-entry ciphertext format. + - `EventLogRemote.decodeRequest` -> `effect/eventlog/EventLogMessage#EventLogRemoteRpcs`: Generic RPC framing and RpcSerialization.layerSchemaBinary replace the module-specific request decoder. - `EventLogRemote.decodeResponse` -> `effect/eventlog/EventLogMessage#EventLogRemoteRpcs`: Generic RPC framing and RpcSerialization.layerSchemaBinary replace the module-specific response decoder. @@ -6012,7 +6030,9 @@ effect/rpc/Utils (barrel: effect/rpc) ### `@effect/experimental/EventLogServer` -- `EventLogServer.Storage` -> `effect/eventlog/EventLogServerEncrypted#Storage`: Use the encrypted server Storage service, which is storeId- and session-aware. +- `EventLogServer.PersistedEntry` -> `effect/eventlog/EventLogEncryption#EncryptedBatch`: The per-entry class was removed; encrypted storage now accepts batch records containing entryIds, iv, and encryptedBatch. The class encoding helpers and entryIdString getter are not retained. Old per-entry ciphertext cannot be reused as an encrypted batch. + +- `EventLogServer.Storage` -> `effect/eventlog/EventLogServerEncrypted#Storage`: Use the storeId- and session-aware encrypted server Storage service. write and changes now operate on encrypted batches. write returns newly stored batches and skips a batch only when all its entry ids are already known. Custom storage must implement entrySequences to look up acknowledgements for skipped batches; each batch owns consecutive sequences starting at its sequence. - `EventLogServer.layerStorageMemory` -> `effect/eventlog/EventLogServerEncrypted#layerStorageMemory`: Use the encrypted server memory storage layer; storage is now storeId- and session-aware. @@ -6508,6 +6528,8 @@ effect/rpc/Utils (barrel: effect/rpc) - `Undici.errors` -> `undici.errors`: Import the upstream errors object directly; individual classes follow the Undici 8 API. +- `Undici.errors.ResponseError`: TODO: needs guidance + - `Undici.errors.ResponseStatusCodeError` -> `undici.errors.ResponseError`: Undici 8 replaced ResponseStatusCodeError with ResponseError; construct it with message, statusCode, and the headers/body object. - `Undici.interceptors` -> `undici.interceptors`: Import the upstream interceptors object and apply returned interceptors with dispatcher.compose(...). @@ -6518,6 +6540,8 @@ effect/rpc/Utils (barrel: effect/rpc) - `Undici.interceptors.RedirectInterceptorOpts` -> `undici.interceptors.RedirectInterceptorOpts`: Import the same namespace type; Undici 8 adds throwOnMaxRedirect and redirect header-stripping options. +- `Undici.interceptors.ResponseErrorInterceptorOpts`: TODO: needs guidance + ### `@effect/platform/ChannelSchema` - `ChannelSchema.decode` -> `ChannelSchema.decode`: The API moved to effect/ChannelSchema; update the import and adapt the schema to the v4 Schema.Constraint model. @@ -6626,6 +6650,8 @@ effect/rpc/Utils (barrel: effect/rpc) - `FileSystem.File.Descriptor` -> `none`: Native file descriptors are no longer part of the portable File interface; use the File methods and keep any platform handle private in custom implementations. +- `FileSystem.File.Info` -> `FileSystem.File.Info`: The module moved to effect/FileSystem. dev is now bigint; ino, rdev, nlink, and blocks are Option\. Update custom stat results, comparisons, arithmetic, and serialization rather than converting these values to number and losing precision. mode, uid, and gid remain numeric; size and blksize use ByteSize. Node and Bun preserve the 64-bit metadata exactly, while Deno may round large values before converting them to bigint. + - `FileSystem.FileDescriptor` -> `none`: The descriptor branding constructor was removed with the public fd field; use File operations instead of exposing a native descriptor. - `FileSystem.FileTypeId` -> `typeof FileSystem.FileTypeId`: The runtime marker remains exported, but the separate type alias was removed. @@ -7070,6 +7096,8 @@ effect/rpc/Utils (barrel: effect/rpc) - `HttpClient.makeWith` -> `HttpClient.makeWith`: Retained with the preprocess and postprocess constructor pattern. +- `HttpClient.mapRequest`: TODO: needs guidance + - `HttpClient.retry` -> `HttpClient.retry`: Retained; the Schedule error channel is included in the resulting client error type. - `HttpClient.tap` -> `effect/http/HttpClient#tap`: Moved to the v4 HTTP module with the same response-effect callback and client error/service widening. @@ -7206,6 +7234,8 @@ effect/rpc/Utils (barrel: effect/rpc) - `HttpMiddleware.withTracerDisabledWhenEffect` -> `Effect.provideService(effect, HttpMiddleware.TracerDisabledWhen, predicate)`: Provide the TracerDisabledWhen reference locally to the effect. +- `HttpMiddleware.xForwardedHeaders` -> `HttpMiddleware.xForwardedHeaders({ trustedProxies })`: Now requires a trusted proxy policy; headers from untrusted peers are ignored and X-Forwarded-For is read from the right. HttpServerRequest.toURL no longer reads x-forwarded-proto directly. + ### `@effect/platform/HttpPlatform` - `HttpPlatform.HttpPlatform` -> `HttpPlatform.HttpPlatform`: The service is now a Context.Service class; use its Service member for the implementation type. Path-backed offset and bytesToRead accept ByteSize.Input, while chunkSize and all Web File range options use number. @@ -7526,6 +7556,8 @@ effect/rpc/Utils (barrel: effect/rpc) ### `@effect/platform/Path` +- `Path.Path` -> `Path.Path`: The module moved to effect/Path. Custom service implementations must now provide resolveWithin as well. Prefer Path.make with the existing platform operations: it adds the type marker and derives resolveWithin, which returns an Option and checks lexical containment without following symlinks. + - `Path.TypeId` -> `typeof Path.TypeId`: The module moved to effect/Path; the runtime marker remains exported but the separate type alias was removed. ### `@effect/platform/PlatformConfigProvider` @@ -7568,6 +7600,8 @@ effect/rpc/Utils (barrel: effect/rpc) - `Socket.layerWebSocket` -> `Socket.layerWebSocket`: The constructor remains in effect/socket/Socket; its URL may now also be an Effect. +- `Socket.makeWebSocket`: TODO: needs guidance + - `Socket.toChannelMap` -> `none`: The v4 Socket read side is an Effect that never completes via Cause.Done; map frames by acquiring Socket.readerBytes or Socket.readerString, or Effect.map the reader from socket.reader, and use Socket.toChannel or Socket.toChannelString for duplex channels. ### `@effect/platform/SocketServer` @@ -7610,11 +7644,13 @@ effect/rpc/Utils (barrel: effect/rpc) ### `@effect/platform/UrlParams` +- `UrlParams.Coercible` -> `Template.PrimitiveValue`: TODO: needs guidance + - `UrlParams.CoercibleRecord` -> `UrlParams.CoercibleRecord`: The recursive interface became a generic mapped type preserving the input shape. - `UrlParams.Input` -> `UrlParams.Input`: Retained and broadened to include UrlParams itself. -- `UrlParams.UrlParams` -> `UrlParams.UrlParams`: Import UrlParams from effect/http. It is now a branded iterable object with a params field rather than a ReadonlyArray; construct it with UrlParams.make or UrlParams.fromInput. +- `UrlParams.UrlParams` -> `UrlParams.UrlParams`: Import UrlParams from effect/http. It is now a branded iterable object with a params field rather than a ReadonlyArray; construct it with UrlParams.make or UrlParams.fromInput. Iteration and params values are string | Redacted.Redacted, not just string. getAll, getFirst, getLast and toRecord unwrap values; toString also serializes unwrapped values and can expose secrets. - `UrlParams.makeUrl` -> `Url.make`: Moved to Url, returns Result, and takes string | undefined for the hash. @@ -8060,13 +8096,17 @@ effect/rpc/Utils (barrel: effect/rpc) - `SqlClient.SafeIntegers` -> `effect/sql/SqlClient#SafeIntegers`: Moved and changed from a Reference subclass to a Context.Reference value; provide it as a service. -- `SqlClient.SqlClient.MakeOptions`: TODO: needs guidance +- `SqlClient.SqlClient` -> `effect/sql/SqlClient#SqlClient`: Moved into core Effect. Custom service implementations must expose driver and afterCommit; SqlClient.make supplies both. driver identifies the driver independently of its SQL dialect. afterCommit registers effects to run after the outermost successful commit and connection release attempt, discarding rolled-back work. withTransaction now also defers contextual Reactivity.invalidate and mutation invalidations until that commit. Transactions bypassing makeWithTransaction, such as sqlite-do transactions, run callbacks and invalidations immediately. + +- `SqlClient.SqlClient.MakeOptions` -> `effect/sql/SqlClient#SqlClient.MakeOptions`: Moved; driver is now required and conventionally names the driver package without @effect/sql-, for example pg or pglite. Rename reactiveMailbox to reactiveQueue; transactionService is optional. - `SqlClient.TransactionConnection` -> `effect/sql/SqlClient#TransactionConnection`: Now a factory keyed by client id, not a singleton tag. Prefer the client's transactionService; the payload type is TransactionConnection.Service. - `SqlClient.TypeId` -> `none`: The brand is private in v4; do not inspect or attach it, and obtain clients through SqlClient or SqlClient.make. -- `SqlClient.make` -> `effect/sql/SqlClient#make`: Moved; custom clients rename MakeOptions.reactiveMailbox to reactiveQueue and may supply transactionService. +- `SqlClient.make` -> `effect/sql/SqlClient#make`: Moved; custom clients must supply MakeOptions.driver, rename reactiveMailbox to reactiveQueue, and may supply transactionService. Driver names conventionally omit the @effect/sql- prefix, for example pg or pglite. + +- `SqlClient.makeWithTransaction` -> `effect/sql/SqlClient#makeWithTransaction`: Rename transactionTag to transactionService. acquireConnection is now a function receiving restore and returning Effect\, rather than an effect returning a scope/connection pair. Register releases in the provided scope and wrap interruptible connection waits with restore; the wrapper closes the scope on transaction completion or failed acquisition. ### `@effect/sql/SqlConnection` @@ -8084,9 +8124,11 @@ effect/rpc/Utils (barrel: effect/rpc) ### `@effect/sql/SqlEventLogServer` -- `SqlEventLogServer.layerStorage` -> `effect/eventlog/SqlEventLogServerEncrypted#layerStorage`: Moved to the encrypted server module with the same options and EventLogEncryption requirement. +- `SqlEventLogServer.layerStorage` -> `effect/eventlog/SqlEventLogServerEncrypted#layerStorage`: Moved to the encrypted server module; remove insertBatchSize. entryTablePrefix and remoteIdTable remain, as does the EventLogEncryption requirement. Storage uses encrypted batch tables and entry-id lookup tables, not the old per-entry ciphertext schema; existing encrypted tables are not automatically converted. + +- `SqlEventLogServer.layerStorageSubtle` -> `effect/eventlog/SqlEventLogServerEncrypted#layerStorageSubtle`: Moved to the encrypted server module; remove insertBatchSize. The layer still supplies Web Crypto encryption. Storage now uses encrypted batch tables and entry-id lookup tables; existing per-entry ciphertext tables are not automatically converted. -- `SqlEventLogServer.makeStorage` -> `effect/eventlog/SqlEventLogServerEncrypted#makeStorage`: Moved to the encrypted server module with the same SQL, encryption, and scope requirements. +- `SqlEventLogServer.makeStorage` -> `effect/eventlog/SqlEventLogServerEncrypted#makeStorage`: Moved to the encrypted server module; remove insertBatchSize. SQL, encryption, and scope requirements remain. Storage uses encrypted batch tables and entry-id lookup tables, not the old per-entry ciphertext schema; existing encrypted tables are not automatically converted. ### `@effect/sql/SqlPersistedQueue` @@ -8098,6 +8140,10 @@ effect/rpc/Utils (barrel: effect/rpc) - `SqlResolver.SqlResolver` -> `RequestResolver.RequestResolver>`: The wrapper interface was removed; constructors return RequestResolvers. Execute them with effect/sql/SqlResolver#request. +- `SqlResolver.findById`: TODO: needs guidance + +- `SqlResolver.ordered`: TODO: needs guidance + - `SqlResolver.void` -> `effect/sql/SqlResolver#void`: Moved, but remove the leading tag and withContext arguments; it now returns a RequestResolver synchronously and runs through SqlResolver.request. ### `@effect/sql/SqlSchema` @@ -8108,6 +8154,8 @@ effect/rpc/Utils (barrel: effect/rpc) ### `@effect/sql/Statement` +- `Statement.Fragment`: TODO: needs guidance + - `Statement.FragmentId` -> `none`: The v4 fragment brand is private; use Fragment, fragment, and isFragment instead of direct type-id access. - `Statement.Statement` -> `effect/sql/Statement#Statement`: Moved; the nested Transformer type is now top-level and its callback receives Fiber.Fiber rather than FiberRefs.FiberRefs. @@ -8680,7 +8728,9 @@ effect/rpc/Utils (barrel: effect/rpc) - `index.expect` -> `vitest#expect`: This was a Vitest re-export, not Effect API. Import it directly from vitest; @effect/vitest/index is not a valid v4 route. -- `index.it` -> `@effect/vitest#it`: Effect helpers retain their calling convention. Vitest 5 removes it.sequential; pass { concurrent: false } to the native test or as the Effect helper's third argument. +- `index.it` -> `@effect/vitest#it`: Vitest 5 removes it.sequential; pass { concurrent: false } to the native test or as the Effect helper's third argument. Synchronous it.prop callbacks must return boolean or void: true and undefined pass, false falsifies, and other values (including Effects and Promises) throw TypeError. Use it.effect.prop for Effect callbacks. + +- `index.prop` -> `@effect/vitest#prop`: Synchronous property callbacks must return boolean or void: true and undefined pass, false falsifies, and other values (including Effects and Promises) throw TypeError. Move Effect callbacks to it.effect.prop instead of returning an Effect from prop. - `index.scoped` -> `@effect/vitest#effect`: V4 effect tests are scoped and provide the test environment. Replace scoped(...) with effect(...), and it.scoped(...) with it.effect(...). @@ -8980,7 +9030,7 @@ Arbitrary.schema(schema) ### `effect/Cause` -- `Cause.Cause` -> `Cause.Cause`: The name remains, but v4 Cause\ is a wrapper with readonly reasons: ReadonlyArray\\>, not the v3 Empty/Fail/Die/Interrupt/Sequential/Parallel tree. +- `Cause.Cause` -> `Cause.Cause`: V4 Cause\ wraps readonly reasons: ReadonlyArray\\>, replacing the v3 cause tree. Finalizer and concurrent failure aggregation can convert typed errors to defects when a defect is present; Cause.combine itself does not perform this conversion. - `Cause.Cause.Variance` -> `none`: The public variance helper was removed. Cause.Cause is directly branded by Cause.TypeId; application code should not reproduce the old variance member. @@ -9144,7 +9194,7 @@ Arbitrary.schema(schema) - `Channel.bufferChunk` -> `none`: The inferred Channel.fromChunk match is not equivalent. Rebuild the buffered upstream-pull transform with Channel.fromTransform and Channel.toTransform. -- `Channel.catchAll` -> `Channel.catch`: Renamed to catch for typed-error recovery. +- `Channel.catchAll` -> `Channel.catch`: Rename to Channel.catch. Causes containing defects skip typed recovery and convert typed errors to defects; interruptions alongside typed errors do not block recovery. Use Channel.catchCause for full-cause handling. - `Channel.catchAllCause` -> `Channel.catchCause`: Renamed to catchCause for full-cause recovery. @@ -9190,6 +9240,8 @@ Arbitrary.schema(schema) - `Channel.isChannelException` -> `none`: Removed implementation artifact. Channel.pipeToOrFail now handles upstream failures without exposing the v3 exception wrapper. +- `Channel.mapError` -> `Channel.mapError`: Maps every typed error in the cause, preserving defects, interruptions and reason order. Unlike typed recovery, mapping does not convert errors to defects. + - `Channel.mapErrorCause` -> `Channel.catchCause`: Use Channel.catchCause(self, (cause) =\> Channel.failCause(f(cause))). - `Channel.mapInputContext` -> `Channel.updateContext`: Renamed to updateContext for transforming the channel requirement Context. @@ -9222,9 +9274,11 @@ Arbitrary.schema(schema) - `Channel.mergeWith` -> `Channel.merge`: Use Channel.merge with haltStrategy left, right, both, or either for standard policies. Custom MergeDecision effects require a Pull-level redesign. -- `Channel.orDieWith` -> `Channel.catch`: Use Channel.catch(self, (error) =\> Channel.die(f(error))); v4 Channel.orDie has no mapping callback. +- `Channel.orDie` -> `Channel.orDie`: Converts typed errors to defects. If a defect is already present, every typed error is converted to a defect, preserving reason order and interruptions. + +- `Channel.orDieWith` -> `Channel.mapError + Channel.orDie`: Use self.pipe(Channel.mapError(f), Channel.orDie). Mapping transforms every typed error while preserving existing defects and interruptions; Channel.catch skips the mapper for causes containing defects. -- `Channel.orElse` -> `Channel.catch`: Use Channel.catch(self, () =\> that()) and keep the fallback lazy. +- `Channel.orElse` -> `Channel.catch`: Use Channel.catch(self, () =\> that()) and keep the fallback lazy. Causes containing defects skip the fallback and convert typed errors to defects; interruptions alongside typed errors do not block recovery. - `Channel.provideLayer` -> `Channel.provide`: Both collapse into provide. V4 removes services supplied by the layer and retains remaining requirements; use options.local when a fresh layer instance is needed. @@ -9904,20 +9958,26 @@ Arbitrary.schema(schema) - `Effect.cachedFunction` -> `none`: The function memoizer was removed; use `Cache` for keyed effectful caching. No direct public replacement exists in v4; rewrite the call site around the stated v4 primitive. -- `Effect.catch` -> `Effect.catch`: Still exported in v4; update call sites for the revised signature, options, and channel inference. +- `Effect.catch` -> `Effect.catch`: Typed recovery skips causes containing defects and converts their typed errors to defects, preserving reason order and interruptions. An interruption alongside a typed error does not block recovery; use Effect.catchCause for full-cause handling. -- `Effect.catchAll` -> `Effect.catch`: Use the shortened v4 error-handler name. Adapt arguments and imports to the v4 API. +- `Effect.catchAll` -> `Effect.catch`: Rename to Effect.catch. Causes containing defects skip typed recovery and convert typed errors to defects; interruptions alongside typed errors do not block recovery. Use Effect.catchCause for full-cause handling. - `Effect.catchAllCause` -> `Effect.catchCause`: Use the shortened v4 cause-handler name. Adapt arguments and imports to the v4 API. - `Effect.catchAllDefect` -> `Effect.catchDefect`: Use the shortened v4 defect-handler name. Adapt arguments and imports to the v4 API. -- `Effect.catchSome` -> `Effect.catchFilter`: Replace the Option-returning partial function with a `Filter` and handler. Adapt arguments and imports to the v4 API. +- `Effect.catchIf` -> `Effect.catchIf`: Causes containing defects skip typed recovery and convert typed errors to defects, preserving reason order and interruptions. An interruption alongside a typed error does not block recovery. + +- `Effect.catchSome` -> `Effect.catchFilter`: Replace the Option-returning partial function with a Filter and handler. Causes containing defects skip typed recovery and convert typed errors to defects; interruptions alongside typed errors do not block recovery. - `Effect.catchSomeCause` -> `Effect.catchCauseFilter`: Replace the Option-returning partial function with a cause `Filter` and handler. Adapt arguments and imports to the v4 API. - `Effect.catchSomeDefect` -> `none`: Use `Effect.catchDefect` and branch explicitly, re-dying for unmatched defects. No direct public replacement exists in v4; rewrite the call site around the stated v4 primitive. +- `Effect.catchTag` -> `Effect.catchTag`: Causes containing defects skip typed recovery and convert typed errors to defects, preserving reason order and interruptions. An interruption alongside a typed error does not block recovery. + +- `Effect.catchTags` -> `Effect.catchTags`: Causes containing defects skip typed recovery and convert typed errors to defects, preserving reason order and interruptions. An interruption alongside a typed error does not block recovery. + - `Effect.cause` -> `Effect.exit`: Inspect `Exit.Failure.cause`; v4 no longer exposes an Effect-only cause extractor. Adapt arguments and imports to the v4 API. - `Effect.checkInterruptible` -> `none`: Interruptibility introspection was removed; structure the region explicitly with `Effect.interruptible` or `Effect.uninterruptible`. No direct public replacement exists in v4; rewrite the call site around the stated v4 primitive. @@ -9954,7 +10014,7 @@ Arbitrary.schema(schema) - `Effect.dropWhile` -> `none`: Use an explicit `Effect.gen` loop, or `Array.dropWhile` when the predicate is pure. No direct public replacement exists in v4; rewrite the call site around the stated v4 primitive. -- `Effect.either` -> `Effect.result`: V4 represents typed success/failure as `Result` instead of `Either`. Adapt arguments and imports to the v4 API. +- `Effect.either` -> `Effect.result`: V4 represents typed success/failure as `Result` instead of `Either`. Adapt arguments and imports to the v4 API. Causes containing defects are not captured as Result failures and propagate with typed errors converted to defects; interruptions alongside typed errors do not block typed recovery. - `Effect.ensureErrorType` -> `Effect.satisfiesErrorType`: Use the renamed compile-time channel constraint. Adapt arguments and imports to the v4 API. @@ -10014,7 +10074,9 @@ Arbitrary.schema(schema) - `Effect.if` -> `Effect.suspend`: Select the branch lazily with a JavaScript conditional inside `Effect.suspend`. Adapt arguments and imports to the v4 API. -- `Effect.ignoreLogged` -> `Effect.ignore`: Pass `{ log: true }` to the consolidated ignore combinator. Adapt arguments and imports to the v4 API. +- `Effect.ignore` -> `Effect.ignore`: Ignores recoverable typed errors only. Causes containing defects propagate with typed errors converted to defects; interruptions alongside typed errors do not prevent ignoring them. + +- `Effect.ignoreLogged` -> `Effect.ignore`: Use Effect.ignore(self, { log: true }). Causes containing defects are not ignored and propagate with typed errors converted to defects; interruptions alongside typed errors do not block typed recovery. - `Effect.inheritFiberRefs` -> `none`: Bulk FiberRef inheritance was removed; propagate required context references explicitly. No direct public replacement exists in v4; rewrite the call site around the stated v4 primitive. @@ -10052,10 +10114,18 @@ Arbitrary.schema(schema) - `Effect.mapAccum` -> `Effect.reduce`: Carry `[state, output]` through an effectful reduction. Adapt arguments and imports to the v4 API. +- `Effect.mapBoth` -> `Effect.mapBoth`: Maps every typed error in the cause, preserving defects, interruptions and reason order. Unlike typed recovery, mapping does not convert errors to defects. + +- `Effect.mapError` -> `Effect.mapError`: Maps every typed error in the cause, preserving defects, interruptions and reason order. Unlike typed recovery, mapping does not convert errors to defects. + - `Effect.mapErrorCause` -> `Effect.catchCause + Effect.failCause`: Transform the Cause in a cause handler and fail with the mapped Cause. Adapt arguments and imports to the v4 API. - `Effect.mapInputContext` -> `Effect.contextWith + Effect.provide`: Build the required context from the incoming context and provide it explicitly. Adapt arguments and imports to the v4 API. +- `Effect.match` -> `Effect.match`: Causes containing defects skip typed recovery and convert typed errors to defects, preserving reason order and interruptions. An interruption alongside a typed error does not block recovery. + +- `Effect.matchEffect` -> `Effect.matchEffect`: Causes containing defects skip typed recovery and convert typed errors to defects, preserving reason order and interruptions. An interruption alongside a typed error does not block recovery. + - `Effect.merge` -> `Effect.catch`: Recover each typed error with `Effect.succeed` so both channels become success values. Adapt arguments and imports to the v4 API. - `Effect.mergeAll` -> `Effect.reduce`: Reduce the input effects with an effectful accumulator. Adapt arguments and imports to the v4 API. @@ -10068,14 +10138,20 @@ Arbitrary.schema(schema) - `Effect.once` -> `Effect.cached`: Create the cached Effect once, then execute the returned Effect repeatedly. Adapt arguments and imports to the v4 API. +- `Effect.option` -> `Effect.option`: Causes containing defects skip typed recovery and convert typed errors to defects, preserving reason order and interruptions. An interruption alongside a typed error does not block recovery. + - `Effect.optionFromOptional` -> `Effect.catchTag`: Map success to `Option.some` and recover `NoSuchElementError` with `Option.none`. Adapt arguments and imports to the v4 API. +- `Effect.orDie` -> `Effect.orDie`: Converts typed errors to defects. If a defect is already present, every typed error is converted to a defect, preserving reason order and interruptions. + - `Effect.orDieWith` -> `Effect.mapError + Effect.orDie`: Map the typed error to the desired defect, then convert failures to defects. Adapt arguments and imports to the v4 API. -- `Effect.orElse` -> `Effect.catch`: Ignore the caught error and evaluate the fallback Effect. Adapt arguments and imports to the v4 API. +- `Effect.orElse` -> `Effect.catch`: Use Effect.catch(self, () =\> that()). Causes containing defects skip the fallback and convert typed errors to defects; interruptions alongside typed errors do not block recovery. - `Effect.orElseFail` -> `Effect.mapError`: Replace every typed error with the lazily produced failure value. Adapt arguments and imports to the v4 API. +- `Effect.orElseSucceed` -> `Effect.orElseSucceed`: Causes containing defects skip typed recovery and convert typed errors to defects, preserving reason order and interruptions. An interruption alongside a typed error does not block recovery. + - `Effect.parallelErrors` -> `Effect.all`: Use `{ mode: "result", concurrency: "unbounded" }` and collect failures explicitly. Adapt arguments and imports to the v4 API. - `Effect.parallelFinalizers` -> `none`: Parallel finalizer strategy mutation was removed; fork independent cleanup explicitly when ordering is irrelevant. No direct public replacement exists in v4; rewrite the call site around the stated v4 primitive. @@ -10098,8 +10174,16 @@ Arbitrary.schema(schema) - `Effect.reduceWhile` -> `none`: Use an explicit `Effect.gen` loop that checks the accumulator before each step. No direct public replacement exists in v4; rewrite the call site around the stated v4 primitive. +- `Effect.repeat` -> `Effect.repeat`: If the source or policy fails with a cause containing defects, typed errors are converted to defects. An interruption alongside a typed error does not block the typed failure path. + - `Effect.repeatN` -> `Effect.repeat`: Pass `{ times: n }` to the consolidated repeat combinator. Adapt arguments and imports to the v4 API. +- `Effect.repeatOrElse` -> `Effect.repeatOrElse`: The fallback receives Option\\> instead of Option\. Read metadata.output for the previous schedule output and metadata.attempt for the attempt count. The error also includes schedule failures. Causes containing defects from either the source or policy skip the fallback and convert typed errors to defects. Interruptions alongside typed errors do not block the fallback. + +- `Effect.retry` -> `Effect.retry`: Retries typed source errors, including those alongside interruptions. A source cause containing a defect is not retried and propagates with typed errors converted to defects. + +- `Effect.retryOrElse` -> `Effect.retryOrElse`: A source cause containing a defect skips retries and the fallback, converting typed errors to defects. Interruptions alongside typed errors do not block retries. Policy failures propagate unchanged. + - `Effect.runRequestBlock` -> `none`: The request-runtime block runner is internal; submit requests with `Effect.request`. No direct public replacement exists in v4; rewrite the call site around the stated v4 primitive. - `Effect.runtime` -> `Effect.context + Effect.runForkWith`: Capture services as a Context and use the corresponding `run*With` function. Adapt arguments and imports to the v4 API. @@ -10476,6 +10560,10 @@ Arbitrary.schema(schema) - `Exit.isInterrupted` -> `Exit.hasInterrupts`: Direct semantic rename; true for a Failure whose Cause contains at least one Interrupt reason, false for Success. +- `Exit.mapBoth` -> `Exit.mapBoth`: Maps every typed error in the cause, preserving defects, interruptions and reason order. Unlike typed recovery, mapping does not convert errors to defects. + +- `Exit.mapError` -> `Exit.mapError`: Maps every typed error in the cause, preserving defects, interruptions and reason order. Unlike typed recovery, mapping does not convert errors to defects. + - `Exit.mapErrorCause` -> `Exit.match + Exit.failCause / Exit.succeed`: No direct v4 combinator. Use Exit.match(self, { onFailure: cause =\> Exit.failCause(f(cause)), onSuccess: Exit.succeed }); f now receives the flattened v4 Cause representation. Cause.map is only equivalent when f merely maps typed errors. - `Exit.matchEffect` -> `Effect.matchCauseEffectEager`: Direct cause-aware migration because Exit is an Effect in v4. Use the same onFailure/onSuccess handlers; the Eager variant preserves v3's immediate branch selection for resolved Exit values. @@ -10990,6 +11078,8 @@ Depend on fast-check and import it directly. Pass the former character arbitrary FastCheck.string({ ...constraints, unit: arbitrary }) ``` +- `FastCheck.stringify`: TODO: needs guidance + - `FastCheck.subarray`: TODO: needs guidance - `FastCheck.toStringMethod`: TODO: needs guidance @@ -11681,7 +11771,7 @@ JsonSchema.toDocumentDraft07(Schema.toJsonSchemaDocument(schema)) - `Layer.matchCause` -> `Layer.fromBuild with Effect.matchCauseEffect over Layer.buildWithMemoMap`: Fold the source acquisition cause, then build the selected failure or success layer. -- `Layer.memoize` -> `automatic shared memoization under Effect.provide`: Reuse the same Layer value; use { local: true } or Layer.fresh to opt out, or MemoMap APIs for manual control. +- `Layer.memoize` -> `enclosing Effect.provide or explicit MemoMap APIs`: A layer already built by an enclosing provide or runtime is reused; sibling provide calls build their own copies. Use { local: true } or Layer.fresh for fresh builds, or MemoMap APIs for manual sharing. - `Layer.orElse` -> `Layer.catch(self, () => fallback())`: Expand the removed lazy fallback alias with Layer.catch. @@ -12005,7 +12095,7 @@ JsonSchema.toDocumentDraft07(Schema.toJsonSchemaDocument(schema)) - `Match.Types` -> `Match.Types`: The public type-level namespace is retained with no call-site migration. -- `Match.Types.ExtractAndNarrow` -> `Match.Types.ExtractAndNarrow`: The type-only matching helper is retained unchanged. +- `Match.Types.ExtractAndNarrow` -> `Match.Types.ExtractAndNarrow`: Array prefix patterns now narrow the required positions and preserve the unmatched tail of arrays and variadic tuples. Update explicit handler or helper result types that assumed a prefix pattern described the entire array; runtime prefix matching is unchanged. - `Match.Types.MaybeReplace` -> `Match.Types.MaybeReplace`: The type-only matching helper is retained unchanged. @@ -12027,6 +12117,14 @@ JsonSchema.toDocumentDraft07(Schema.toJsonSchemaDocument(schema)) - `Match.either` -> `Match.result`: Renamed finalizer with a container change: matched Right and unmatched Left become Result.Success and Result.Failure. +- `Match.not` -> `Match.not`: Object patterns now reject top-level keys absent from every variant of the remaining input type. Correct stale property names or narrow the input before matching; function, refinement, and array patterns are not subject to this object-key check. + +- `Match.when` -> `Match.when`: Object patterns now reject top-level keys absent from every variant of the remaining input type. Correct stale property names or narrow the input before matching. Array prefix patterns retain the unmatched tail in the handler type rather than treating the pattern as the entire array. + +- `Match.whenAnd` -> `Match.whenAnd`: Each object pattern now rejects top-level keys absent from every variant of the remaining input type, including patterns spread from an array. Correct stale property names or narrow the input before matching. + +- `Match.whenOr` -> `Match.whenOr`: Each object pattern now rejects top-level keys absent from every variant of the remaining input type, including patterns spread from an array. Correct stale property names or narrow the input before matching. + ### `effect/MergeDecision` - `MergeDecision.Await` -> `Channel.merge`: Replace Channel.mergeWith decision callbacks with Channel.merge and haltStrategy. V4 has no MergeDecision values; custom effectful exit folding must be restructured. @@ -15537,21 +15635,25 @@ Schema.toFormatter(schema) - `Sink.leftover` -> `Sink.succeed`: Removed as a standalone constructor; Sink.succeed now takes optional leftovers: Sink.succeed(void 0, leftovers) where leftovers is a NonEmptyReadonlyArray\ instead of Chunk\. +- `Sink.mapError` -> `Sink.mapError`: Maps every typed error in the cause, preserving defects, interruptions and reason order. Unlike typed recovery, mapping does not convert errors to defects. + - `Sink.mapInputChunks` -> `Sink.mapInputArray`: Renamed; f maps NonEmptyReadonlyArray\ =\> NonEmptyReadonlyArray\ (must stay non-empty) instead of Chunk =\> Chunk. - `Sink.mapInputChunksEffect` -> `Sink.mapInputArrayEffect`: Renamed; f maps NonEmptyReadonlyArray\ =\> Effect\\> (must stay non-empty) instead of Chunk =\> Effect\. - `Sink.mkString` -> `Sink.reduceArray`: Removed as a built-in; equivalent one-liner: Sink.reduceArray(() =\> "", (s, arr) =\> s + arr.join("")). +- `Sink.orElse` -> `Sink.orElse`: Causes containing defects skip the fallback and convert typed errors to defects; interruptions alongside typed errors do not block typed recovery. + - `Sink.race` -> `none`: Sink racing (race/raceBoth/raceWith) was removed from v4; broadcast the stream (Stream.broadcast) into two consumers and race the resulting run effects with Effect.race, or write a custom Channel. -- `Sink.raceBoth` -> `none`: Removed with the race family; broadcast the stream and use Effect.raceBoth (or Effect.race) on the two Stream.run effects to learn which side won. +- `Sink.raceBoth` -> `none`: Removed with the race family; broadcast the stream and tag each Stream.run effect's result before combining them with Effect.race to learn which side succeeded first. -- `Sink.raceWith` -> `none`: Removed, along with the MergeDecision type it depended on; the closest is broadcasting the stream and combining the two run effects manually (Effect.raceWith on the run effects). +- `Sink.raceWith` -> `none`: Removed, along with the MergeDecision type it depended on; broadcast the stream and combine the two run effects manually. Use Effect.race for the first success or Effect.raceFirst for the first completion; custom completion and interruption handling requires explicit fibers. -- `Sink.refineOrDie` -> `Sink.catch`: Removed; rebuild with the typed-error handler: Sink.catch(self, (e) =\> Option.match(pf(e), { onSome: Effect.fail, onNone: () =\> Effect.die(e) })) — note Sink.catch replaces the result on recovery, so refined errors must be re-failed as shown. +- `Sink.refineOrDie` -> `Sink.catch`: For recoverable typed errors, use Sink.catch(self, (e) =\> Option.match(pf(e), { onSome: Effect.fail, onNone: () =\> Effect.die(e) })). Causes containing defects skip pf and convert typed errors to defects; use Sink.catchCause for full-cause handling. -- `Sink.refineOrDieWith` -> `Sink.catch`: Removed; same pattern as refineOrDie but die with the mapped defect: Sink.catch(self, (e) =\> Option.match(pf(e), { onSome: Effect.fail, onNone: () =\> Effect.die(f(e)) })). +- `Sink.refineOrDieWith` -> `Sink.catch`: Use the refineOrDie recipe with Effect.die(f(e)) for unrefined errors. Causes containing defects skip pf and f and convert typed errors to defects; use Sink.catchCause for full-cause handling. - `Sink.splitWhere` -> `none`: Removed; it re-chunked input so the sink stopped before the first later element matching the predicate — closest v4 options are pre-splitting the stream (Stream.split / Stream.takeWhile) or a custom Sink.fromTransform that cuts pulled arrays at the predicate boundary and returns the rest as leftovers. @@ -15685,7 +15787,7 @@ Schema.toFormatter(schema) - `Stream.async` -> `Stream.callback`: Stream.callback((queue) =\> Effect | void, { bufferSize?, strategy? }) replaces the Emit-based async; push with Queue.offer/offerAll, end with Queue.end, fail with Queue.fail. -- `Stream.asyncEffect` -> `Stream.callback`: The register function of Stream.callback may return an Effect (run before the stream starts pulling), covering asyncEffect; signal end/failure through the provided Queue. +- `Stream.asyncEffect` -> `Stream.callback`: The register function of Stream.callback may return an Effect, covering asyncEffect. Unlike v3, it is started before the stream pulls but not awaited: it runs synchronously until its first async boundary, so asynchronous setup may still be in progress when pulling begins. Signal end/failure through the provided Queue. - `Stream.asyncPush` -> `Stream.callback`: Stream.callback's register effect can use Scope for acquire/release of the external subscription, replacing asyncPush; the Emit ops helpers become plain Queue operations. @@ -15715,15 +15817,17 @@ Stream.unwrap(Effect.map(Stream.peel(self, Sink.take(n)), ([head, rest]) => f(he - `Stream.bufferChunks` -> `Stream.bufferArray`: Chunk-\>Array rename; buffers whole arrays (chunks) up to capacity with the same strategy options. -- `Stream.catchAll` -> `Stream.catch`: Renamed to Stream.catch (exported keyword-style); same (error) =\> Stream handler for all typed failures. +- `Stream.catchAll` -> `Stream.catch`: Rename to Stream.catch with an (error) =\> Stream handler. Causes containing defects skip typed recovery and convert typed errors to defects; interruptions alongside typed errors do not block recovery. Use Stream.catchCause for full-cause handling. - `Stream.catchAllCause` -> `Stream.catchCause`: Renamed; handler receives the full Cause\ and returns a recovery stream, identical semantics. -- `Stream.catchSome` -> `Stream.catchFilter`: Option-returning partial handler replaced by the Filter API: Stream.catchFilter(filter, f, orElse?) recovers matched errors, unmatched failures pass through (Stream.catchIf for refinement/predicate matching). +- `Stream.catchSome` -> `Stream.catchFilter`: Use Stream.catchFilter(filter, f, orElse?) instead of an Option-returning partial handler. Causes containing defects skip typed recovery and convert typed errors to defects; interruptions alongside typed errors do not block recovery. - `Stream.catchSomeCause` -> `Stream.catchCauseFilter`: Option-returning cause handler replaced by Stream.catchCauseFilter(filter, f, orElse?) using a Filter on the Cause (Stream.catchCauseIf for refinements). -- `Stream.catchTags` -> `Stream.catchTags`: Only keys present in the tagged error union are accepted; remove handlers for tags the stream cannot fail with. Untagged errors remain in the error channel. +- `Stream.catchTag` -> `Stream.catchTag`: Causes containing defects skip typed recovery and convert typed errors to defects, preserving reason order and interruptions. An interruption alongside a typed error does not block recovery. + +- `Stream.catchTags` -> `Stream.catchTags`: Only keys in the tagged error union are accepted; untagged errors remain in the error channel. Causes containing defects skip handlers and convert typed errors to defects; interruptions alongside typed errors do not block recovery. - `Stream.chunksWith` -> `Stream.flattenArray(f(Stream.chunks(self)))`: No dedicated combinator; expose chunk structure with Stream.chunks (Stream\\>), transform, then re-flatten with Stream.flattenArray. @@ -15751,7 +15855,7 @@ Stream.unwrap(Effect.map(Stream.peel(self, Sink.take(n)), ([head, rest]) => f(he - `Stream.distributedWithDynamic` -> `none`: Dynamic predicate-routed fan-out removed with distributedWith. Use Stream.broadcast/Stream.share for dynamic mirrors plus per-subscriber Stream.filter, or hand-roll routing into Queues via Stream.runForEach. -- `Stream.either` -> `Stream.result`: Either is replaced by Result in v4: Stream.result yields Stream\, never, R\> (element -\> Result.succeed, first error -\> Result.fail and the stream ends, as before). +- `Stream.either` -> `Stream.result`: Either is replaced by Result in v4: Stream.result yields Stream\, never, R\> (element -\> Result.succeed, first error -\> Result.fail and the stream ends, as before). Causes containing defects are not emitted as Result failures and propagate with typed errors converted to defects; interruptions alongside typed errors do not block typed recovery. - `Stream.ensuringWith` -> `Stream.onExit`: Renamed; Stream.onExit runs the finalizer with the Exit\ of the stream, identical shape. @@ -15801,7 +15905,7 @@ Stream.unwrap(Effect.map(Stream.peel(self, Sink.take(n)), ([head, rest]) => f(he - `Stream.let` -> `Stream.let`: Retained in v4 and aligned with Effect.let: the field name may re-bind an existing key (the computed value replaces it) and the record type is Simplify\ & Record\\>. -- `Stream.mapBoth` -> `Stream.mapBoth`: Rename the options onSuccess to onElement and onFailure to onError. The callbacks still transform emitted elements and typed errors respectively. +- `Stream.mapBoth` -> `Stream.mapBoth`: Rename the options onSuccess to onElement and onFailure to onError. The callbacks still transform emitted elements and typed errors respectively. Maps every typed error in the cause, preserving defects, interruptions and reason order. Unlike typed recovery, mapping does not convert errors to defects. - `Stream.mapChunks` -> `Stream.mapArray`: Chunk-\>Array rename; transforms each emitted chunk as a NonEmptyReadonlyArray. @@ -15815,6 +15919,8 @@ Stream.unwrap(Effect.map(Stream.peel(self, Sink.take(n)), ([head, rest]) => f(he - `Stream.mapConcatEffect` -> `Stream.flattenIterable(Stream.mapEffect(self, f))`: Removed; effectfully map each element to an Iterable and flatten with Stream.flattenIterable. +- `Stream.mapError` -> `Stream.mapError`: Maps every typed error in the cause, preserving defects, interruptions and reason order. Unlike typed recovery, mapping does not convert errors to defects. + - `Stream.mapErrorCause` -> `Stream.catchCause((cause) => Stream.failCause(f(cause)))`: Removed; transform the full Cause by catching it and re-failing with the mapped cause. - `Stream.mapInputContext` -> `Stream.updateContext`: Renamed; same contravariant (Context\) =\> Context\ mapping of the required services. @@ -15837,18 +15943,22 @@ Stream.mergeAll(Object.entries(streams).map(([_tag, s]) => Stream.map(s, (value) - `Stream.onDone` -> `Stream.onEnd`: Renamed; v4 onEnd takes an Effect value (not a () =\> Effect thunk) run when the stream ends successfully, and its error type may add to the stream's. +- `Stream.orDie` -> `Stream.orDie`: Converts typed errors to defects. If a defect is already present, every typed error is converted to a defect, preserving reason order and interruptions. + - `Stream.orDieWith` -> `Stream.orDie`: orDieWith removed; transform the error first, then convert failures to defects: `self.pipe(Stream.mapError(f), Stream.orDie)`. -- `Stream.orElse` -> `Stream.catch`: v3 catchAll was renamed to Stream.catch in v4; orElse ignored the error, so write `Stream.catch(self, () => that())`. +- `Stream.orElse` -> `Stream.catch`: Use Stream.catch(self, () =\> that()). Causes containing defects skip the fallback and convert typed errors to defects; interruptions alongside typed errors do not block recovery. -- `Stream.orElseEither` -> `Stream.catch`: Removed; Either is replaced by Result in v4. Emulate: `Stream.map(self, Result.succeed).pipe(Stream.catch(() => Stream.map(that(), Result.fail)))` (same encoding v4 Stream.mergeResult uses). +- `Stream.orElseEither` -> `Stream.catch`: Replace Either with Result: Stream.map(self, Result.succeed).pipe(Stream.catch(() =\> Stream.map(that(), Result.fail))). Causes containing defects skip the fallback and convert typed errors to defects; interruptions alongside typed errors do not block recovery. -- `Stream.orElseFail` -> `Stream.mapError`: Removed; it only replaced the failure value: `Stream.mapError(self, () => error())` or `Stream.catch(self, () => Stream.fail(error()))`. +- `Stream.orElseFail` -> `Stream.mapError`: Use Stream.mapError(self, () =\> error()) to replace each typed error while preserving existing defects and interruptions. Stream.catch is not equivalent for causes containing defects because it skips typed recovery. - `Stream.orElseIfEmptyChunk` -> `Stream.orElseIfEmpty`: Folded into Stream.orElseIfEmpty, which now takes a lazy fallback Stream: `Stream.orElseIfEmpty(self, () => Stream.fromArray(array))`; Chunk is replaced by plain arrays. - `Stream.orElseIfEmptyStream` -> `Stream.orElseIfEmpty`: Direct rename: v4 Stream.orElseIfEmpty takes a LazyArg\ fallback, identical semantics. +- `Stream.orElseSucceed` -> `Stream.orElseSucceed`: Causes containing defects skip typed recovery and convert typed errors to defects, preserving reason order and interruptions. An interruption alongside a typed error does not block recovery. + - `Stream.paginateChunk` -> `Stream.paginate`: v4 Stream.paginate is effectful and array-based: `paginate(s, (s) => Effect, Option]>)`; wrap the pure step in Effect.succeed and use an array instead of a Chunk. - `Stream.paginateChunkEffect` -> `Stream.paginate`: v4 Stream.paginate has exactly this shape; only Chunk becomes ReadonlyArray. @@ -15867,9 +15977,9 @@ Stream.mergeAll(Object.entries(streams).map(([_tag, s]) => Stream.map(s, (value) - `Stream.provideSomeLayer` -> `Stream.provide`: v4 Stream.provide accepts a Layer (or Context) and excludes only the provided services from R — same partial-provision semantics. -- `Stream.refineOrDie` -> `Stream.catch`: Removed; emulate with `Stream.catch(self, (e) => { const r = pf(e); return Option.isSome(r) ? Stream.fail(r.value) : Stream.die(e) })` — refail refined errors, die on the rest. +- `Stream.refineOrDie` -> `Stream.catch`: For recoverable typed errors, use Stream.catch(self, (e) =\> { const r = pf(e); return Option.isSome(r) ? Stream.fail(r.value) : Stream.die(e) }). Causes containing defects skip pf and convert typed errors to defects; use Stream.catchCause for full-cause handling. -- `Stream.refineOrDieWith` -> `Stream.catch`: Removed; same as refineOrDie but die with the mapped defect: `Stream.die(f(e))` for unrefined errors. +- `Stream.refineOrDieWith` -> `Stream.catch`: Use the refineOrDie recipe with Stream.die(f(e)) for unrefined errors. Causes containing defects skip pf and f and convert typed errors to defects; use Stream.catchCause for full-cause handling. - `Stream.repeatEffect` -> `Stream.fromEffectRepeat`: Renamed; repeats the effect forever emitting each result. @@ -15889,6 +15999,8 @@ Stream.mergeAll(Object.entries(streams).map(([_tag, s]) => Stream.map(s, (value) - `Stream.repeatWith` -> `none`: Removed; v4 Stream.repeat(schedule) covers the repetition but drops the schedule outputs and the onElement/onSchedule unification. Hand-roll if schedule outputs must appear in the stream. +- `Stream.retry` -> `Stream.retry`: Retries typed source errors, including those alongside interruptions. A source cause containing a defect is not retried and propagates with typed errors converted to defects. + - `Stream.runFoldScoped` -> `Stream.runFold`: Scoped run variants are gone; v4 run functions manage the stream's scope internally and the initial value is now a LazyArg: `Stream.runFold(self, () => s, f)`. For enclosing-scope control, pull manually via `Stream.toPull` (Effect\). - `Stream.runFoldScopedEffect` -> `Stream.runFoldEffect`: Scoped run variants are gone; use `Stream.runFoldEffect(self, () => s, f)` — scope is managed internally, initial value is a LazyArg. Use Stream.toPull for manual scoped consumption. diff --git a/repos/effect/package.json b/repos/effect/package.json index 3be5cbf06f..30ef6be55c 100644 --- a/repos/effect/package.json +++ b/repos/effect/package.json @@ -46,7 +46,7 @@ "@effect/doctest": "workspace:^", "@effect/jsdocs": "workspace:^", "@effect/oxc": "workspace:^", - "@effect/tsgo": "^0.51.1", + "@effect/tsgo": "^0.53.4", "@effect/utils": "workspace:^", "@effect/vitest": "workspace:^", "@faker-js/faker": "^10.6.0", @@ -55,21 +55,21 @@ "@rollup/plugin-terser": "^1.0.0", "@types/jscodeshift": "^17.3.0", "@types/node": "^26.6.5", - "@vitest/coverage-v8": "^5.0.3", - "@vitest/web-worker": "^5.0.3", + "@vitest/coverage-v8": "^5.0.5", + "@vitest/web-worker": "^5.0.5", "ast-types": "^0.16.3", "babel-plugin-annotate-pure-calls": "^0.5.0", - "dprint": "^0.61.1", + "dprint": "^0.63.0", "fast-check": "^4.10.2", "glob": "^13.0.6", - "happy-dom": "^20.14.6", + "happy-dom": "^21.0.3", "jscodeshift": "^17.4.0", "lalph": "^0.3.139", "madge": "^8.0.0", - "oxlint": "^1.87.0", + "oxlint": "^1.88.0", "pkg-pr-new": "0.0.91", - "playwright": "^1.64.0", - "rollup": "^4.64.5", + "playwright": "^1.64.1", + "rollup": "^4.64.9", "rollup-plugin-bundle-stats": "^4.22.3", "rollup-plugin-esbuild": "^6.2.1", "rollup-plugin-visualizer": "^7.1.1", @@ -77,8 +77,8 @@ "tstyche": "^7.2.5", "typescript": "^7.0.2", "vite": "^8.3.4", - "vitest": "^5.0.3", + "vitest": "^5.0.5", "vitest-websocket-mock": "^0.8.0", - "zod": "^4.6.5" + "zod": "^4.6.7" } } diff --git a/repos/effect/packages/ai/anthropic/CHANGELOG.md b/repos/effect/packages/ai/anthropic/CHANGELOG.md index d1d417677d..f2f1016771 100644 --- a/repos/effect/packages/ai/anthropic/CHANGELOG.md +++ b/repos/effect/packages/ai/anthropic/CHANGELOG.md @@ -1,5 +1,25 @@ # @effect/ai-anthropic +## 4.0.6 + +### Patch Changes + +- [#9500](https://github.com/Effect-TS/effect/pull/9500) [`8e9616a`](https://github.com/Effect-TS/effect/commit/8e9616a2b6da0849686ffc11988026411022a408) Thanks @typedrat! - Fix assistant cache breakpoints by sending `cache_control` on the last eligible content block, preserving part-level precedence and skipping thinking blocks. + +- [#9709](https://github.com/Effect-TS/effect/pull/9709) [`85e2aee`](https://github.com/Effect-TS/effect/commit/85e2aeee686963c26ca40d204f982deebb9d6117) Thanks @tim-smart! - Add typed HTTP date and `Retry-After` header values to `Headers`, and parse them consistently across the HTTP client, OTLP exporter, GraphQL client, AI clients and static server. + + - `Headers.parseHttpDate` / `Headers.formatHttpDate` read all three RFC 9110 date formats as UTC, whatever the process time zone, and emit IMF-fixdate. + - `Headers.parseRetryAfter`, `Headers.formatRetryAfter` and `Headers.retryAfterDelay` accept whole non-negative seconds of any size, read exactly, or an HTTP date, and the delay is never negative. HTTP dates accept the `23:59:60` leap second, read as the following second. + - `Headers.RetryAfter` holds either a `Duration` (`RetryAfterDuration`) or a `DateTime.Utc` (`RetryAfterDateTime`). `Headers.retryAfterDuration` builds the first from any `Duration.Input`: infinite durations throw, finite negative ones become zero, and fractional seconds are rounded up when formatted. + - Parse failures are reported as `Headers.HeaderParseError`. + - `Schema.DateTimeUtcFromHttpDate` and `Schema.RetryAfterFromString` expose the same parsing as string codecs for HTTP API headers. + - Invalid `Retry-After` values such as `-5`, `1.5`, `1e3` or `Infinity` are now ignored everywhere. `HttpClient.retryTransient` and `HttpClient.withRateLimiter` no longer honour fractional seconds, and the AI clients no longer produce negative or infinite `retryAfter` durations. + - `HttpStaticServer` reads `If-Modified-Since` and `Last-Modified` as UTC; asctime dates were previously shifted by the process time zone. + +- [#9430](https://github.com/Effect-TS/effect/pull/9430) [`48102ac`](https://github.com/Effect-TS/effect/commit/48102ac592ef89fed9d5d10afd2daece0a0ee310) Thanks @effect-bot! - Add non-retryable `AiError.InvalidResponseError` for provider response schema and SSE decoding failures, with optional HTTP context; model-generated output validation remains retryable. Update exhaustive matches on `AiErrorReason` to handle the new tag, and update serialized-error consumers to accept it. +- Updated dependencies [[`ce13a48`](https://github.com/Effect-TS/effect/commit/ce13a4872e516187851c5d9336d4df309cc8f0a4), [`f3bbbb4`](https://github.com/Effect-TS/effect/commit/f3bbbb448d431a9d1205ff09ba47cb81af0c7199), [`5547315`](https://github.com/Effect-TS/effect/commit/554731500e43eef47ff68b569116b74006e91d87), [`6093cfb`](https://github.com/Effect-TS/effect/commit/6093cfbbb673d36ecd18edea955763db003c8c26), [`30319d5`](https://github.com/Effect-TS/effect/commit/30319d506c9177d1cc1e0b10372e1ddb776aaf0a), [`74a7596`](https://github.com/Effect-TS/effect/commit/74a759699a8582dbe660b21dcda21cc9db08f613), [`bfb32a3`](https://github.com/Effect-TS/effect/commit/bfb32a3cf5f8d0f02f0dfdb63dcf5237fde755b0), [`f8bf7f4`](https://github.com/Effect-TS/effect/commit/f8bf7f4bcc3ad543cd94cbeb010cb164e5dedcb0), [`7f3d3df`](https://github.com/Effect-TS/effect/commit/7f3d3dfbb7b9e6e041c4ad9654c4532f72d2f16e), [`db1046d`](https://github.com/Effect-TS/effect/commit/db1046d1fa0e4b5818c51d28017a89579b6261d6), [`3c4f88a`](https://github.com/Effect-TS/effect/commit/3c4f88aa4d1d4d958b42fe8d3f1195f0ca990f8a), [`916e709`](https://github.com/Effect-TS/effect/commit/916e70968acc4e443885c67012d855b7cc6f9bca), [`916f843`](https://github.com/Effect-TS/effect/commit/916f8436225153a81a049bdc1feaedcafe4d1ded), [`66ec757`](https://github.com/Effect-TS/effect/commit/66ec7574d23b7943584059b41136bc21ee9d4c53), [`daac06f`](https://github.com/Effect-TS/effect/commit/daac06f79a6ab8c7d3e1b66b427c06f7d64dcc8f), [`c2552ec`](https://github.com/Effect-TS/effect/commit/c2552ec2e031b409b2bef67309312b0168480a39), [`b02c1cd`](https://github.com/Effect-TS/effect/commit/b02c1cde36c890941a235224af8ecb035197582a), [`1855f1d`](https://github.com/Effect-TS/effect/commit/1855f1d359535339bc7640b67e6bb7b12752ba19), [`683ed2c`](https://github.com/Effect-TS/effect/commit/683ed2cb491f9529379c1d5c63933689ff57fd2b), [`ac68940`](https://github.com/Effect-TS/effect/commit/ac68940f7e4246b1beb43f9d451a67f41241ab82), [`22a8401`](https://github.com/Effect-TS/effect/commit/22a84017e7222f332611c1bace05f9fa60c7cf67), [`c53c194`](https://github.com/Effect-TS/effect/commit/c53c19451171a00776b400a8548394ac514b628a), [`ca6d1cc`](https://github.com/Effect-TS/effect/commit/ca6d1cc5f803cd501e85431642b79b552322a76a), [`47940c5`](https://github.com/Effect-TS/effect/commit/47940c55c9e0eb855615094a2aca6e0812307bb2), [`e8b24ef`](https://github.com/Effect-TS/effect/commit/e8b24ef9516706971db948efb2d3eab84ad85951), [`2e8aae0`](https://github.com/Effect-TS/effect/commit/2e8aae0080a8d3e6d00c5ee0bc4f49f0b074acc2), [`ad7497b`](https://github.com/Effect-TS/effect/commit/ad7497bbc1bc70935f8f1ae395ff501532e2e770), [`7f504a3`](https://github.com/Effect-TS/effect/commit/7f504a35edcf44952c55ecbb0916726bac4d3e52), [`99e365d`](https://github.com/Effect-TS/effect/commit/99e365d61a1fdaabeb31dc58e5752cade8a09e1f), [`954067d`](https://github.com/Effect-TS/effect/commit/954067da7fbd20a20c10d5d0de6ffe4690e6bdfd), [`3e1a035`](https://github.com/Effect-TS/effect/commit/3e1a035d9c470ced858b7757f6d102b3aa9be008), [`41a1fe7`](https://github.com/Effect-TS/effect/commit/41a1fe74e6338e8d13d6d4f3daa03f71a826c2c4), [`bcb3a86`](https://github.com/Effect-TS/effect/commit/bcb3a86335a8ed18da39300984c0fbae187fdc6b), [`739d3d1`](https://github.com/Effect-TS/effect/commit/739d3d18728cce0b6d907f0a3f24e04ab0b53560), [`ed4da35`](https://github.com/Effect-TS/effect/commit/ed4da351fb9d29e4dfd6c60bd89ba4c007d264dc), [`3d84e19`](https://github.com/Effect-TS/effect/commit/3d84e19f8f14bccf0aa57b11825590c3bae162f4), [`cc00906`](https://github.com/Effect-TS/effect/commit/cc0090657eeb89debdd54af5fcb99d544f46c1bf), [`09ba1f5`](https://github.com/Effect-TS/effect/commit/09ba1f59952c46cd22229d41236742c3e1f53a34), [`9ff1a73`](https://github.com/Effect-TS/effect/commit/9ff1a73d995702f4c4c335888fe6a186f3f710c7), [`06bc532`](https://github.com/Effect-TS/effect/commit/06bc532b71cd3684f010956774356864dd359372), [`97bfd84`](https://github.com/Effect-TS/effect/commit/97bfd84bb147e10ed562e4c1ffa589cdaf83a596), [`25efaef`](https://github.com/Effect-TS/effect/commit/25efaef0d81d25b2651ef4de474e929d3180810a), [`c10335b`](https://github.com/Effect-TS/effect/commit/c10335be125c7846ae7b914eeb138244b660275b), [`a808972`](https://github.com/Effect-TS/effect/commit/a80897262648e4222ce48943e538f39a6b3f69d6), [`20e844c`](https://github.com/Effect-TS/effect/commit/20e844c7ef27b0e4b4f7911d603c45b1de2b0bc3), [`b9f73f3`](https://github.com/Effect-TS/effect/commit/b9f73f37c4ac538e6a327ed80ac5a0708858c540), [`5739878`](https://github.com/Effect-TS/effect/commit/5739878ded09d83d71d657e3b8d4264f4e4936d9), [`28acdf3`](https://github.com/Effect-TS/effect/commit/28acdf3820b66056037bb7a24cd0e8d3de3fb06d), [`59b6396`](https://github.com/Effect-TS/effect/commit/59b63967c1817ae5acb1811037f5aaa964219b7e), [`0bccc9c`](https://github.com/Effect-TS/effect/commit/0bccc9c3dd5349b20f8d2e509ac5482d0c26a616), [`fed70de`](https://github.com/Effect-TS/effect/commit/fed70de0dff8af24c5f814bd0d985ab70c70420b), [`82648a2`](https://github.com/Effect-TS/effect/commit/82648a21bf228145a14d2d1390006d67f74d89c7), [`afb5f32`](https://github.com/Effect-TS/effect/commit/afb5f326cc14c9849d8d7ddefde1039ae036e017), [`28813f5`](https://github.com/Effect-TS/effect/commit/28813f53a5026984fdc13d3971147ea5dc20104c), [`0fae934`](https://github.com/Effect-TS/effect/commit/0fae934a13aac12f9036b0733edd5dde5ce029f3), [`f260830`](https://github.com/Effect-TS/effect/commit/f2608305861269fc2fbedcbbdcd71e4fc5f98ffc), [`19b63eb`](https://github.com/Effect-TS/effect/commit/19b63eb81b2bccf033aadb565f52a6b0375d26a1), [`14db221`](https://github.com/Effect-TS/effect/commit/14db221f8f584100ff50fd1173b1a99a1caa1d06), [`679f956`](https://github.com/Effect-TS/effect/commit/679f956c4a7527854ff78db071dd7aa417211a48), [`b72707c`](https://github.com/Effect-TS/effect/commit/b72707c2937328fd3bc70f29a6792d84c4256484), [`44c8c1d`](https://github.com/Effect-TS/effect/commit/44c8c1d1c6f01ba9887a051b365e9fa89fcfc685), [`1e7909b`](https://github.com/Effect-TS/effect/commit/1e7909beb05bc740dbd330509e60bb185d7eeed1), [`0611a65`](https://github.com/Effect-TS/effect/commit/0611a657f76836f9ca275f64764b82270b4eda10), [`2fadc65`](https://github.com/Effect-TS/effect/commit/2fadc654aa655fbd1912ce51eb1b764a757ac362), [`feb6ce5`](https://github.com/Effect-TS/effect/commit/feb6ce59a38ab15fc6f89545bd5f0992dd58a06a), [`5a2ed53`](https://github.com/Effect-TS/effect/commit/5a2ed5323884a435bfcfb5e3a6f2ae29547458ad), [`c8d26a4`](https://github.com/Effect-TS/effect/commit/c8d26a46731cb26a9b9612d9fde823d8813cfc04), [`0c8b446`](https://github.com/Effect-TS/effect/commit/0c8b446709c737c10d0e11af4313bb29fdff2588), [`05c3409`](https://github.com/Effect-TS/effect/commit/05c340955944374e3ae47a1de0f7cb9b41a3b0e6), [`b04191b`](https://github.com/Effect-TS/effect/commit/b04191b2941534754282c51ef2df2f965ac43a5c), [`05c3409`](https://github.com/Effect-TS/effect/commit/05c340955944374e3ae47a1de0f7cb9b41a3b0e6), [`4067e64`](https://github.com/Effect-TS/effect/commit/4067e6428776ae573c8cc72f7eaebd078a4ff8c7), [`e04563d`](https://github.com/Effect-TS/effect/commit/e04563dcd6ddeea74c088a12194c5a73e597ba2f), [`02d7fc5`](https://github.com/Effect-TS/effect/commit/02d7fc583053c47dea3b5e1d9d6be164aeb321b6), [`561d2a1`](https://github.com/Effect-TS/effect/commit/561d2a1ddfa3c4340510f23da7adc50c89d97863), [`8fe2dcc`](https://github.com/Effect-TS/effect/commit/8fe2dcc2137788a71177d42a9f67fb7003224868), [`a8086fa`](https://github.com/Effect-TS/effect/commit/a8086fa26a2c9a9f5f7f1d7590bbc494a0feb02a), [`88a33b4`](https://github.com/Effect-TS/effect/commit/88a33b4122f7ffbbc1423add793a79abffd99c75), [`e77372a`](https://github.com/Effect-TS/effect/commit/e77372a8a2e7caec11999636e04c6e31c52e315f), [`3dd4388`](https://github.com/Effect-TS/effect/commit/3dd438810df34ed3b3d2e3e35ccf84ccb00feb85), [`ba2b00c`](https://github.com/Effect-TS/effect/commit/ba2b00c0658b65775809e0052ec9612432600c31), [`a049d4a`](https://github.com/Effect-TS/effect/commit/a049d4ad25f17cf4985284a8d15b07aea2a33242), [`84aeede`](https://github.com/Effect-TS/effect/commit/84aeede040460a2b24bcf0db7a5d0a9b0e3c42d4), [`43e5e19`](https://github.com/Effect-TS/effect/commit/43e5e198d16df038b60764f5dd0218cdcf828f16), [`77bbf16`](https://github.com/Effect-TS/effect/commit/77bbf16e50d6b0ea090288cc5d3f11b20706a027), [`6af3a47`](https://github.com/Effect-TS/effect/commit/6af3a47d5b5d698c3ba896c37684266cd465bef1), [`aac8d21`](https://github.com/Effect-TS/effect/commit/aac8d21911333dedb649f4f9d302c28d14d46369), [`84c8dc4`](https://github.com/Effect-TS/effect/commit/84c8dc47b09067362749620f75068d6f28044660), [`78d1988`](https://github.com/Effect-TS/effect/commit/78d198818d4fd547ce2e9431302f66cb00dc7ad3), [`5676a2a`](https://github.com/Effect-TS/effect/commit/5676a2ac5a918f8f91fc7838d7adf1b6f5e86a6d), [`4b437bf`](https://github.com/Effect-TS/effect/commit/4b437bf4645fabc8aaeaa3b5a88e8c9048b9e4f5), [`1af11fd`](https://github.com/Effect-TS/effect/commit/1af11fda1bccf823f74121b5125aa3befa084b18), [`716f8df`](https://github.com/Effect-TS/effect/commit/716f8dfa178f849e5b288464373bd132860c2e19), [`ee27718`](https://github.com/Effect-TS/effect/commit/ee27718a02125328a7478812b1b7d7c1d8532578), [`4d86d1b`](https://github.com/Effect-TS/effect/commit/4d86d1b8887d48fa24acf6ad54f33ef3d55a7cd8), [`5462ed7`](https://github.com/Effect-TS/effect/commit/5462ed7b3a4c37206c47e401d6be293dc540350f), [`658837c`](https://github.com/Effect-TS/effect/commit/658837c3b249d982df56f2990247499746e36672), [`00261cc`](https://github.com/Effect-TS/effect/commit/00261cc12c31ecc7878f80670439731c905dc2bd), [`7cccb93`](https://github.com/Effect-TS/effect/commit/7cccb9385729bc0aed6630edea95d7db15439ac4), [`7c98228`](https://github.com/Effect-TS/effect/commit/7c982281cf73958833054175fc86db385e043cf6), [`5a8e248`](https://github.com/Effect-TS/effect/commit/5a8e2481546352e2ad6233dad49420b14c59f624), [`8037a65`](https://github.com/Effect-TS/effect/commit/8037a65f40cae9163ab9e0e687e289f717ae570b), [`25912db`](https://github.com/Effect-TS/effect/commit/25912db9d493227a2d6fa45cebcfc7507e638d8a), [`7f32606`](https://github.com/Effect-TS/effect/commit/7f32606baac43a67d171ae93b77c7b29791c9692), [`0386f88`](https://github.com/Effect-TS/effect/commit/0386f88295ac70cbc1f90ab8cceb0981d848c628), [`403298a`](https://github.com/Effect-TS/effect/commit/403298ac457fb634eca2c013fed26d37fc80445f), [`5a3ebc0`](https://github.com/Effect-TS/effect/commit/5a3ebc082124f9e10106b2347736e66fc74ebc36), [`fac2191`](https://github.com/Effect-TS/effect/commit/fac21911964b742a38e1d4e73bd4f95aec4911dc), [`fdb12dc`](https://github.com/Effect-TS/effect/commit/fdb12dcd61b6505eed02e6f7a3c88a85d4c48e8d), [`97909b8`](https://github.com/Effect-TS/effect/commit/97909b89b167da54a56e6f82f1c4db8bfedae04f), [`c3983f4`](https://github.com/Effect-TS/effect/commit/c3983f4db2b3052a58e8045cce270fd7ccc4e2b2), [`3a5e65e`](https://github.com/Effect-TS/effect/commit/3a5e65e409c9569e23bb7af61df401347e9ec420), [`8b40fbc`](https://github.com/Effect-TS/effect/commit/8b40fbc93caf52f3a4c69020849f0051f100be65), [`3f3e119`](https://github.com/Effect-TS/effect/commit/3f3e119ad61c2167bc6c7170c3fb68cd1eb3421e), [`7c7b8ed`](https://github.com/Effect-TS/effect/commit/7c7b8eda54e44f9cddc1188f994f7fdddf901055), [`d4f71bc`](https://github.com/Effect-TS/effect/commit/d4f71bc42507b638d149fb8806d26ea626d17956), [`c66241e`](https://github.com/Effect-TS/effect/commit/c66241e791c4b51d8a65051142d01682590f9bf8), [`f0e941e`](https://github.com/Effect-TS/effect/commit/f0e941ee2f90f0a891d1228a90da91386f0258cb), [`5f8a6e5`](https://github.com/Effect-TS/effect/commit/5f8a6e5108aa6c6cd4043a3c9f4dc93d43369623), [`2800a09`](https://github.com/Effect-TS/effect/commit/2800a0910f7f4ad7cfa21dfc840edf8fe6e67ca7), [`e9749b0`](https://github.com/Effect-TS/effect/commit/e9749b04c10badabcdf71e6958de5ac56a5316e9), [`eaef09c`](https://github.com/Effect-TS/effect/commit/eaef09c11b6ed2029c513eea0aa3256dcc6f517f), [`a3c56e5`](https://github.com/Effect-TS/effect/commit/a3c56e5b5e756e879f1cfb90230e8eae620e33df), [`bb61152`](https://github.com/Effect-TS/effect/commit/bb611522fcfb4d317b33acdf5ee24a480d042435), [`e9c842b`](https://github.com/Effect-TS/effect/commit/e9c842bc294abf658d698bb17732ae13465b3817), [`6d8c8e2`](https://github.com/Effect-TS/effect/commit/6d8c8e2075b15ae84ae96e93a0d295cb9221b496), [`1f28187`](https://github.com/Effect-TS/effect/commit/1f281878ef559ef356af2d0e11b381aba857cae1), [`0b9842b`](https://github.com/Effect-TS/effect/commit/0b9842b76bbc1b016c8abc46432052e1f60b2fa3), [`c29ddd8`](https://github.com/Effect-TS/effect/commit/c29ddd899c53c5d8194c4a0dfef2134304dda799), [`84c3405`](https://github.com/Effect-TS/effect/commit/84c34057ceab9fe6782b73f4a04a31304e0d14de), [`3934c0c`](https://github.com/Effect-TS/effect/commit/3934c0c61857c03468c08c2477f49a6b977d28ff), [`048e7c8`](https://github.com/Effect-TS/effect/commit/048e7c87125c4ace5a61895feac3d049a317bbfc), [`1121334`](https://github.com/Effect-TS/effect/commit/1121334ce68c379051011a2ffea37b0a30ea9a47), [`9a680d6`](https://github.com/Effect-TS/effect/commit/9a680d6985c6a27d7ebb29cd9eb674f3fe524505), [`85e2aee`](https://github.com/Effect-TS/effect/commit/85e2aeee686963c26ca40d204f982deebb9d6117), [`34ec440`](https://github.com/Effect-TS/effect/commit/34ec4407a14ce045ca14cc76c77582884a7e1ebe), [`343a149`](https://github.com/Effect-TS/effect/commit/343a149fa7d5a9a0a45841b1b6800b0ecfa16a32), [`915117b`](https://github.com/Effect-TS/effect/commit/915117b523cb6e5cbc98b5ff1c119a8d0d1f6b9a), [`7f823c1`](https://github.com/Effect-TS/effect/commit/7f823c10d20ef53a1081e0d0e2008a4116140799), [`6e614e6`](https://github.com/Effect-TS/effect/commit/6e614e6feb4281a9e81f5ec3be48e7905ff4292c), [`9990b39`](https://github.com/Effect-TS/effect/commit/9990b3977a544247cd4123edde8353a55bf77192), [`22bf781`](https://github.com/Effect-TS/effect/commit/22bf7812aa8a20c676efade943c342cde4ef1cbd), [`41f08b7`](https://github.com/Effect-TS/effect/commit/41f08b75aba40e95cd6c743ec6fcf1f5022f7b5b), [`e8482ca`](https://github.com/Effect-TS/effect/commit/e8482cad9651fa8fe8036fe6292a286994897d0d), [`5ae85fa`](https://github.com/Effect-TS/effect/commit/5ae85fa8ac1fc5e12d5fe8fe7388ac504788ae3b), [`c477a0a`](https://github.com/Effect-TS/effect/commit/c477a0a8010bd0a988221bc00c446848314e1c3c), [`8fbb39e`](https://github.com/Effect-TS/effect/commit/8fbb39eb92577915831f19a83b51be376e844613), [`1c70828`](https://github.com/Effect-TS/effect/commit/1c7082847453e9c2be0cc267a39ca554c52f4013), [`8207d1e`](https://github.com/Effect-TS/effect/commit/8207d1e91ead8aac8f01d9e13b49624b64237c8b), [`7886b84`](https://github.com/Effect-TS/effect/commit/7886b847658664f64392a14efda86f44b3d1e170), [`4839c60`](https://github.com/Effect-TS/effect/commit/4839c601bdda845d9a54d77f578f64372475ab04), [`2595d83`](https://github.com/Effect-TS/effect/commit/2595d831f702d4a748f97e33e71fd5250f58af56), [`f5ae278`](https://github.com/Effect-TS/effect/commit/f5ae27827a82db3f8973b70745fc6f14e7d8fa80), [`7a8ce00`](https://github.com/Effect-TS/effect/commit/7a8ce0098a5f22cbb6c110b16e850fb3fcbcadf2), [`688d302`](https://github.com/Effect-TS/effect/commit/688d30275ff114e50b4f0d343c6227ee33d67348), [`c7072e4`](https://github.com/Effect-TS/effect/commit/c7072e46356a19615625ae0fa569b1c3a79d50be), [`425cedb`](https://github.com/Effect-TS/effect/commit/425cedb5801c9527c116a64d280d9e7c2820f922), [`88bc680`](https://github.com/Effect-TS/effect/commit/88bc68001c6c70984b03a5500232e9b5aab14085), [`268e09c`](https://github.com/Effect-TS/effect/commit/268e09c94c9e8f46142b5c9f29ce0a22bfbbc3aa), [`7e94505`](https://github.com/Effect-TS/effect/commit/7e94505edfc9d81ea08cb96a7d935806495bf61e), [`afa5da3`](https://github.com/Effect-TS/effect/commit/afa5da3c8a9848215e49535dd3d39179bfd67715), [`c29ddd8`](https://github.com/Effect-TS/effect/commit/c29ddd899c53c5d8194c4a0dfef2134304dda799), [`48102ac`](https://github.com/Effect-TS/effect/commit/48102ac592ef89fed9d5d10afd2daece0a0ee310), [`463eca3`](https://github.com/Effect-TS/effect/commit/463eca3609a6a9681247abbcc71a6f10b261a1d9), [`e7162ea`](https://github.com/Effect-TS/effect/commit/e7162eae7f04af1d583b1f31fdd767f91ec1c82e), [`95752db`](https://github.com/Effect-TS/effect/commit/95752db5eb3b6e59fb8749e9792704f8b167f36c), [`4577a28`](https://github.com/Effect-TS/effect/commit/4577a28bcdb50b0d184775f8c1b7ad890d86e255), [`973d599`](https://github.com/Effect-TS/effect/commit/973d599a3e8611626764a0e3e8f79edbcf30dc7c), [`2f066a5`](https://github.com/Effect-TS/effect/commit/2f066a58e90316dd88a219fcea94200beda54219), [`f0ccbb6`](https://github.com/Effect-TS/effect/commit/f0ccbb6917be5c08364b0b8adbd0c4c19e7f2de1), [`2dda6ce`](https://github.com/Effect-TS/effect/commit/2dda6cefc641dbea2ea3279a9b96c0332b277a40), [`3da7c0a`](https://github.com/Effect-TS/effect/commit/3da7c0aabab07c8fc682aea7559536bf31705853), [`5a67e0b`](https://github.com/Effect-TS/effect/commit/5a67e0b3a8118dd7be709a56d6d7af4f10e9c4f3), [`73b3a6e`](https://github.com/Effect-TS/effect/commit/73b3a6e3c38795400251d9c5d73f05c88acd90cd), [`c6f9e4c`](https://github.com/Effect-TS/effect/commit/c6f9e4cbe6f9c8f0a03fefde18df4d35e98a24df), [`be02676`](https://github.com/Effect-TS/effect/commit/be026768ee29a1b1a860c1fd17f2a0896c01f61e), [`26de861`](https://github.com/Effect-TS/effect/commit/26de86100cda8617e077ce66f0a6bb8dfe738fb6), [`9313799`](https://github.com/Effect-TS/effect/commit/931379911a74d5e2207859c295b1acc6a5e2917e), [`d5c1e5b`](https://github.com/Effect-TS/effect/commit/d5c1e5bd425dfe983dff1f8c853c0b7eaa772f87), [`11bb31f`](https://github.com/Effect-TS/effect/commit/11bb31fec0bf2fdfc025009a04de4c69bffa433a), [`8c23e46`](https://github.com/Effect-TS/effect/commit/8c23e46a8280e3f2c831d1c0ca3f106c92f8e071), [`9c96011`](https://github.com/Effect-TS/effect/commit/9c960111de202c0f1dd6989e5b8836150e0ca694), [`0d50570`](https://github.com/Effect-TS/effect/commit/0d50570bbf078804092fcd4a47ccd2d0019f989c), [`9ed2bf1`](https://github.com/Effect-TS/effect/commit/9ed2bf18da0e1ff4fecc31b7cbac2976ec999e00), [`76ec912`](https://github.com/Effect-TS/effect/commit/76ec9120f43f4fdb08b9c5170ff9e3fd82f22457), [`af2756d`](https://github.com/Effect-TS/effect/commit/af2756dd1d196aa3f199ce21c6d2dc3ada3b67b5), [`3ec8100`](https://github.com/Effect-TS/effect/commit/3ec8100c793b8a22381ad115106b93824d840f64), [`93abf02`](https://github.com/Effect-TS/effect/commit/93abf020876af0a6b6b8b25621c80760b7996762), [`1e94993`](https://github.com/Effect-TS/effect/commit/1e949935a1065e5140245a514ce3a799061143e3), [`99a4616`](https://github.com/Effect-TS/effect/commit/99a4616264e31de65fa8d3dc19576309b1a3f84a), [`0a7978e`](https://github.com/Effect-TS/effect/commit/0a7978ea8e70e51700345b12f8c19952dde05255), [`b4a060b`](https://github.com/Effect-TS/effect/commit/b4a060b95a6473d54e277dc005172b54aab9bddf), [`274f4a4`](https://github.com/Effect-TS/effect/commit/274f4a4dac5ee2d9dbb6358cc3dd3afe345d4705), [`25f29be`](https://github.com/Effect-TS/effect/commit/25f29be77bdcd5c2a7be8e2710e88f5a10c4cb8d), [`9036440`](https://github.com/Effect-TS/effect/commit/9036440edca01c24371d0c2c1044be5e5d3cb65e), [`415f005`](https://github.com/Effect-TS/effect/commit/415f0053eb848d562febcb5b011e15f31a99bead), [`191308b`](https://github.com/Effect-TS/effect/commit/191308bd0f5ee04557570f2f960c6e06c429bcfd), [`4b96570`](https://github.com/Effect-TS/effect/commit/4b96570ed28d93b14e958742933d227dc5e74b15), [`d84ec3f`](https://github.com/Effect-TS/effect/commit/d84ec3fab0346c01bc59f6624d1c046ca4140087), [`392aa1c`](https://github.com/Effect-TS/effect/commit/392aa1cb275ac9fdb9b4b1831d8c06196bba54b9), [`99e9bd9`](https://github.com/Effect-TS/effect/commit/99e9bd98a354d83202e2522f3ade9ee4bf4a862c), [`6f6f45f`](https://github.com/Effect-TS/effect/commit/6f6f45f480a3f191aed6a82ca5c301f60ef2cf84), [`13fc07e`](https://github.com/Effect-TS/effect/commit/13fc07edeb939ca6229b858c9d9a8c5243ac21b5), [`a4b38dd`](https://github.com/Effect-TS/effect/commit/a4b38dde7f3fdc1b54a49bffdaca912b4c47c76e), [`8b18580`](https://github.com/Effect-TS/effect/commit/8b185807bb99f6de3d9564447e2f4e08d2f8863d), [`0c477fe`](https://github.com/Effect-TS/effect/commit/0c477fe417d3ecab86bb8d9afd0c392e9cb8a3da), [`9a5331d`](https://github.com/Effect-TS/effect/commit/9a5331db94bfdca4f48317d062fbeb96054a9049), [`a706d73`](https://github.com/Effect-TS/effect/commit/a706d73db8a8d460877c5d7e8a793a52d45be307), [`c9a1e72`](https://github.com/Effect-TS/effect/commit/c9a1e72c5965aa9fe01a669ebfd523f803f5a189), [`89eaaf8`](https://github.com/Effect-TS/effect/commit/89eaaf8581908e248c0b1e47baa7906e0c541635), [`8215811`](https://github.com/Effect-TS/effect/commit/8215811d51d28d37b4732a6ab03de4047ada62ea), [`ad69018`](https://github.com/Effect-TS/effect/commit/ad6901857e05e7b436a0149b5ece7f3a36d9ff33), [`a733eba`](https://github.com/Effect-TS/effect/commit/a733eba9f699737a80a9e7276521980bd929075a), [`7282d5f`](https://github.com/Effect-TS/effect/commit/7282d5f1bcc06d7065429ef81a3d17c64043fde4), [`da33572`](https://github.com/Effect-TS/effect/commit/da33572776763e45802ce8c2fe14ec101f9a5c34), [`d9e2d12`](https://github.com/Effect-TS/effect/commit/d9e2d1221768e9a3e5e00165cb95b86e0c126cc0), [`dc923b8`](https://github.com/Effect-TS/effect/commit/dc923b847ac7d9673b4f0aa7c136557287b2e5e3), [`1b74086`](https://github.com/Effect-TS/effect/commit/1b740862e3d25ec5d4b6f86b9fe792b6dabbf84c), [`8a5507b`](https://github.com/Effect-TS/effect/commit/8a5507ba00f641edf71227d79b9f90efdb634fe2), [`23460fa`](https://github.com/Effect-TS/effect/commit/23460fad704c730926bffe3cc6874268ef6680ec), [`0dbbea6`](https://github.com/Effect-TS/effect/commit/0dbbea65a93c59147a4bc407cfcda38ea26341b2), [`70aad3f`](https://github.com/Effect-TS/effect/commit/70aad3ffe617984f5b599aeed1bc7f04899aca55), [`0909236`](https://github.com/Effect-TS/effect/commit/090923611b209a19f3281a7b1d0c15a9b653a0c7), [`a9e66d8`](https://github.com/Effect-TS/effect/commit/a9e66d889017b302355ba5b8bc2b2050ca1ea5aa), [`ee5c0d5`](https://github.com/Effect-TS/effect/commit/ee5c0d5f43348747b04b5f7bf1b39355614c2f2c), [`631c5d6`](https://github.com/Effect-TS/effect/commit/631c5d6d9a81e6389964b878764fb9aea1538247), [`7e60c1a`](https://github.com/Effect-TS/effect/commit/7e60c1ae62a96c768f9c67d706f494213947f22f), [`4cddd3c`](https://github.com/Effect-TS/effect/commit/4cddd3c41e71fc4990d37c42b1bcf2ce3780b41d), [`8727e5d`](https://github.com/Effect-TS/effect/commit/8727e5d2bec8e2d729b7aeafb725a4ab21d533d2), [`2bc97fa`](https://github.com/Effect-TS/effect/commit/2bc97faa0e06b3973780dae26029e2c59f5dba38), [`f9a7f05`](https://github.com/Effect-TS/effect/commit/f9a7f056ba31aef435933acf082ec8c01ac7decc), [`d978cca`](https://github.com/Effect-TS/effect/commit/d978cca02914bec0ddf66bb2d68d90b220096fb6), [`b334ccf`](https://github.com/Effect-TS/effect/commit/b334ccf361a0f11362e7aabdbf70b35f53a39d61), [`0b50cfc`](https://github.com/Effect-TS/effect/commit/0b50cfc644313afe4c318073893231ec1946eb5f), [`5b7d2b5`](https://github.com/Effect-TS/effect/commit/5b7d2b5c90bc77eb9f8be1b91166c53f89aa1d0c), [`6ee26b1`](https://github.com/Effect-TS/effect/commit/6ee26b16b51b380c41dbfa84d612aa525b420fce), [`eab7e6a`](https://github.com/Effect-TS/effect/commit/eab7e6acfc7e422857565b98b9eddd3a98de1b92), [`e618f10`](https://github.com/Effect-TS/effect/commit/e618f1050f41424439eb82478fcf1c5da5a3244d), [`da14b1f`](https://github.com/Effect-TS/effect/commit/da14b1f18d71a18fc431b56d82cffe09aeb92708), [`3314fc3`](https://github.com/Effect-TS/effect/commit/3314fc30c22d5c9d737f3df7201f4a41424f701b), [`66db624`](https://github.com/Effect-TS/effect/commit/66db624670032a02144d42b6e64b83b324c813cd), [`4b8a50d`](https://github.com/Effect-TS/effect/commit/4b8a50db9649603f382f33640698020ca79547d0), [`8d52668`](https://github.com/Effect-TS/effect/commit/8d52668533e9d4a6d6b3be9fe58bd5e69fa80b65), [`1de1055`](https://github.com/Effect-TS/effect/commit/1de10559e7263747751bfb032fd4b0fd8aa28724), [`db130e6`](https://github.com/Effect-TS/effect/commit/db130e64557fafe370819d19f78e3b19a21275d6), [`5fa5fa5`](https://github.com/Effect-TS/effect/commit/5fa5fa5b9bff77352fb9ff2d424bf46bd6309986)]: + - effect@4.0.6 + ## 4.0.5 ### Patch Changes diff --git a/repos/effect/packages/ai/anthropic/package.json b/repos/effect/packages/ai/anthropic/package.json index e38e713202..73e99363ee 100644 --- a/repos/effect/packages/ai/anthropic/package.json +++ b/repos/effect/packages/ai/anthropic/package.json @@ -1,6 +1,6 @@ { "name": "@effect/ai-anthropic", - "version": "4.0.5", + "version": "4.0.6", "type": "module", "license": "MIT", "description": "An Anthropic provider integration for Effect AI SDK", diff --git a/repos/effect/packages/ai/anthropic/src/AnthropicClient.ts b/repos/effect/packages/ai/anthropic/src/AnthropicClient.ts index 077d5b3ce4..e29e04f53d 100644 --- a/repos/effect/packages/ai/anthropic/src/AnthropicClient.ts +++ b/repos/effect/packages/ai/anthropic/src/AnthropicClient.ts @@ -318,7 +318,7 @@ export const make = Effect.fnUntraced( > => client.betaMessagesPost({ ...options, config: { includeResponse: true } }).pipe( Effect.catchTags({ - BetaMessagesPost4XX: (error) => Effect.fail(Errors.mapClientError(error, "createMessage")), + BetaMessagesPost4XX: (error) => Errors.mapClientError(error, "createMessage"), HttpClientError: (error) => Errors.mapHttpClientError(error, "createMessage"), SchemaError: (error) => Effect.fail(Errors.mapSchemaError(error, "createMessage")) }), diff --git a/repos/effect/packages/ai/anthropic/src/AnthropicLanguageModel.ts b/repos/effect/packages/ai/anthropic/src/AnthropicLanguageModel.ts index 8f3e30f944..5d172ff560 100644 --- a/repos/effect/packages/ai/anthropic/src/AnthropicLanguageModel.ts +++ b/repos/effect/packages/ai/anthropic/src/AnthropicLanguageModel.ts @@ -1068,6 +1068,7 @@ const prepareMessages = Effect.fnUntraced( for (let j = 0; j < group.messages.length; j++) { const message = group.messages[j] const isLastMessage = j === group.messages.length - 1 + const messageStart = content.length for (let k = 0; k < message.content.length; k++) { const part = message.content[k] @@ -1077,17 +1078,7 @@ const prepareMessages = Effect.fnUntraced( } const isLastPart = k === message.content.length - 1 - - // Attempt to get the cache control from the part first. If - // the part does not have cache control defined and we are - // evaluating the last part for this message, also check the - // message for cache control. - const cacheControl = getCacheControl(part) ?? ( - isLastPart ? getCacheControl(message) : undefined - ) - - // TODO: use cache_control in content blocks - void cacheControl + const partStart = content.length switch (part.type) { case "text": { @@ -1290,7 +1281,11 @@ const prepareMessages = Effect.fnUntraced( break } } + + setLastCacheControl(content, partStart, getCacheControl(part)) } + + setLastCacheControl(content, messageStart, getCacheControl(message)) } messages.push({ role: "assistant", content }) @@ -2995,6 +2990,28 @@ const getCacheControl = ( | Prompt.ToolMessagePart ): typeof Generated.CacheControlEphemeral.Encoded | null => part.options.anthropic?.cacheControl ?? null +// Skip thinking blocks without crossing the part/message boundary. +// Preserve existing part-level breakpoints over message-level ones. +const setLastCacheControl = ( + content: Array, + start: number, + cacheControl: typeof Generated.CacheControlEphemeral.Encoded | null +): void => { + if (cacheControl === null) { + return + } + for (let i = content.length - 1; i >= start; i--) { + const block = content[i] + if (block.type === "thinking" || block.type === "redacted_thinking") { + continue + } + if (Predicate.isNullish(block.cache_control)) { + content[i] = { ...block, cache_control: cacheControl } + } + return + } +} + const getDocumentMetadata = (part: Prompt.FilePart): { readonly title: string | null readonly context: string | null diff --git a/repos/effect/packages/ai/anthropic/src/internal/errors.ts b/repos/effect/packages/ai/anthropic/src/internal/errors.ts index 8576ad8feb..fedae9e223 100644 --- a/repos/effect/packages/ai/anthropic/src/internal/errors.ts +++ b/repos/effect/packages/ai/anthropic/src/internal/errors.ts @@ -1,9 +1,11 @@ import * as AiError from "effect/ai/AiError" import type * as Response from "effect/ai/Response" -import * as Duration from "effect/Duration" +import * as Clock from "effect/Clock" +import type * as Duration from "effect/Duration" import * as Effect from "effect/Effect" import type * as Sse from "effect/encoding/Sse" import { dual } from "effect/Function" +import * as Headers from "effect/http/Headers" import type * as HttpClientError from "effect/http/HttpClientError" import type * as HttpClientRequest from "effect/http/HttpClientRequest" import type * as HttpClientResponse from "effect/http/HttpClientResponse" @@ -11,6 +13,7 @@ import * as Number from "effect/Number" import * as Option from "effect/Option" import * as Predicate from "effect/Predicate" import * as Redactable from "effect/Redactable" +import * as Result from "effect/Result" import * as Schema from "effect/Schema" import type { AnthropicErrorMetadata } from "../AnthropicError.ts" import type * as Generated from "../Generated.ts" @@ -50,7 +53,7 @@ export const mapSchemaError = dual< AiError.make({ module: "AnthropicClient", method, - reason: AiError.InvalidOutputError.fromSchemaError(error) + reason: AiError.InvalidResponseError.fromSchemaError(error) })) /** @internal */ @@ -61,31 +64,38 @@ export const mapSseError = dual< AiError.make({ module: "AnthropicClient", method, - reason: new AiError.InvalidOutputError({ description: error.message }) + reason: new AiError.InvalidResponseError({ description: error.message }) })) +type ClientError = Generated.AnthropicClientError + /** @internal */ export const mapClientError = dual< - (method: string) => (error: Generated.AnthropicClientError) => AiError.AiError, - (error: Generated.AnthropicClientError, method: string) => AiError.AiError ->(2, (error, method) => { - const { request, response, cause } = error - const status = response.status - const headers = response.headers as Record - const metadata: AnthropicErrorMetadata = { - errorType: cause.error.type, - requestId: cause.request_id - } - const http = buildHttpContext({ request, response, body: JSON.stringify(cause) }) - const reason = mapStatusCodeToReason({ - status, - headers, - message: cause.error.message, - metadata, - http + (method: string) => (error: ClientError) => Effect.Effect, + (error: ClientError, method: string) => Effect.Effect +>( + 2, + Effect.fnUntraced(function*(error: ClientError, method: string) { + const now = yield* Clock.currentTimeMillis + const { request, response, cause } = error + const status = response.status + const headers = response.headers as Record + const metadata: AnthropicErrorMetadata = { + errorType: cause.error.type, + requestId: cause.request_id + } + const http = buildHttpContext({ request, response, body: JSON.stringify(cause) }) + const reason = mapStatusCodeToReason({ + now, + status, + headers, + message: cause.error.message, + metadata, + http + }) + return yield* AiError.make({ module: "AnthropicClient", method, reason }) }) - return AiError.make({ module: "AnthropicClient", method, reason }) -}) +) /** @internal */ export const mapHttpClientError = dual< @@ -178,7 +188,9 @@ const mapStatusCodeError = Effect.fnUntraced(function*( } const decoded = Schema.decodeUnknownOption(AnthropicErrorBody)(json) + const now = yield* Clock.currentTimeMillis const reason = mapStatusCodeToReason({ + now, status, headers, message: Option.isSome(decoded) ? decoded.value.error.message : undefined, @@ -196,16 +208,16 @@ const mapStatusCodeError = Effect.fnUntraced(function*( // Rate Limits // ============================================================================= +const retryAfterFromHeader = (raw: string, now: number): Duration.Duration | undefined => { + const retryAfter = Headers.parseRetryAfter(raw) + return Result.isSuccess(retryAfter) ? Headers.retryAfterDelay(retryAfter.success, now) : undefined +} + /** @internal */ -export const parseRateLimitHeaders = (headers: Record) => { +export const parseRateLimitHeaders = (headers: Record, now: number) => { const retryAfterRaw = headers["retry-after"] - let retryAfter: Duration.Duration | undefined - if (Predicate.isNotUndefined(retryAfterRaw)) { - const parsed = Number.parse(retryAfterRaw) - if (Option.isSome(parsed)) { - retryAfter = Duration.seconds(parsed.value) - } - } + // an invalid Retry-After is treated as absent + const retryAfter = retryAfterRaw === undefined ? undefined : retryAfterFromHeader(retryAfterRaw, now) const requestsLimitRaw = headers["anthropic-ratelimit-requests-limit"] const requestsRemainingRaw = headers["anthropic-ratelimit-requests-remaining"] const tokensLimitRaw = headers["anthropic-ratelimit-tokens-limit"] @@ -261,8 +273,10 @@ export const buildHttpContext = (params: { // ============================================================================= /** @internal */ -export const mapStatusCodeToReason = ({ status, headers, message, metadata, http }: { +export const mapStatusCodeToReason = ({ status, headers, message, metadata, http, now }: { readonly status: number + /** Current time in epoch milliseconds, used to measure a Retry-After date. */ + readonly now: number readonly headers: Record readonly message: string | undefined readonly metadata: AnthropicErrorMetadata @@ -312,7 +326,7 @@ export const mapStatusCodeToReason = ({ status, headers, message, metadata, http http }) case 429: { - const { retryAfter, ...rateLimitMetadata } = parseRateLimitHeaders(headers) + const { retryAfter, ...rateLimitMetadata } = parseRateLimitHeaders(headers, now) return new AiError.RateLimitError({ retryAfter, metadata: { diff --git a/repos/effect/packages/ai/anthropic/test/AnthropicClient.test.ts b/repos/effect/packages/ai/anthropic/test/AnthropicClient.test.ts index e77c4d2f06..a63fe80bef 100644 --- a/repos/effect/packages/ai/anthropic/test/AnthropicClient.test.ts +++ b/repos/effect/packages/ai/anthropic/test/AnthropicClient.test.ts @@ -49,6 +49,34 @@ describe("AnthropicClient", () => { body: { error: { message: "gateway rate limit" } } })))) + it.effect.each([ + { name: "ignores a negative Retry-After", header: "-5", expected: undefined }, + { name: "ignores a non-finite Retry-After", header: "Infinity", expected: undefined }, + { name: "honors an HTTP-date Retry-After", header: "Thu, 01 Jan 1970 00:00:30 GMT", expected: 30_000 } + ])("$name", ({ expected, header }) => + Effect.gen(function*() { + const client = yield* AnthropicClient.AnthropicClient + const error = yield* client.createMessage({ + payload: { + model: "claude-sonnet-4-20250514", + max_tokens: 1, + messages: [{ role: "user", content: "hello" }] + } + }).pipe(Effect.flip) + + assert.strictEqual(error.reason._tag, "RateLimitError") + if (error.reason._tag !== "RateLimitError") return + assert.strictEqual( + error.reason.retryAfter === undefined ? undefined : Duration.toMillis(error.reason.retryAfter), + expected + ) + }).pipe(Effect.provide(makeTestLayer({ + _tag: "Json", + status: 429, + headers: { "retry-after": header }, + body: { error: { message: "gateway rate limit" } } + })))) + it.effect("redacts the API key in AI error context", () => Effect.gen(function*() { const client = yield* AnthropicClient.AnthropicClient @@ -127,6 +155,7 @@ describe("AnthropicClient", () => { it("preserves and truncates a fallback HTTP response", () => { const body = `${"a".repeat(200)}b` const reason = Errors.mapStatusCodeToReason({ + now: 0, status: 400, headers: {}, message: undefined, diff --git a/repos/effect/packages/ai/anthropic/test/AnthropicLanguageModel.test.ts b/repos/effect/packages/ai/anthropic/test/AnthropicLanguageModel.test.ts index 2d89752262..a45b0087b7 100644 --- a/repos/effect/packages/ai/anthropic/test/AnthropicLanguageModel.test.ts +++ b/repos/effect/packages/ai/anthropic/test/AnthropicLanguageModel.test.ts @@ -1365,56 +1365,55 @@ describe("AnthropicLanguageModel", () => { }) describe("generateObject", () => { - const getRequest = ( + const getRequest = Effect.fnUntraced(function*( model: string, config?: { readonly structuredOutputs?: boolean | undefined } - ) => - Effect.gen(function*() { - let capturedRequest: HttpClientRequest.HttpClientRequest | undefined = undefined - const layer = AnthropicClient.layer({ apiKey: Redacted.make("sk-test-key") }).pipe( - Layer.provide(Layer.succeed( - HttpClient.HttpClient, - makeHttpClient((request) => { - capturedRequest = request - return Effect.succeed(jsonResponse(request, { - id: "msg_test_1", - type: "message", - role: "assistant", - model, - content: [{ type: "text", text: JSON.stringify({ name: "John", age: 30 }) }], - stop_reason: "end_turn", - stop_sequence: null, - usage: { - cache_creation: null, - cache_creation_input_tokens: null, - cache_read_input_tokens: null, - inference_geo: null, - input_tokens: 10, - output_tokens: 5, - service_tier: null - } - })) - }) - )) - ) - - // Assert the request shape; the response outcome is irrelevant here. - yield* LanguageModel.generateObject({ - prompt: "Give me a person", - schema: Schema.Struct({ name: Schema.String, age: Schema.Number }) - }).pipe( - Effect.provide(AnthropicLanguageModel.model(model, config)), - Effect.provide(layer), - Effect.ignore - ) - - assert.isDefined(capturedRequest) - if (capturedRequest === undefined) { - return yield* Effect.die(new Error("Expected a captured request")) - } + ) { + let capturedRequest: HttpClientRequest.HttpClientRequest | undefined = undefined + const layer = AnthropicClient.layer({ apiKey: Redacted.make("sk-test-key") }).pipe( + Layer.provide(Layer.succeed( + HttpClient.HttpClient, + makeHttpClient((request) => { + capturedRequest = request + return Effect.succeed(jsonResponse(request, { + id: "msg_test_1", + type: "message", + role: "assistant", + model, + content: [{ type: "text", text: JSON.stringify({ name: "John", age: 30 }) }], + stop_reason: "end_turn", + stop_sequence: null, + usage: { + cache_creation: null, + cache_creation_input_tokens: null, + cache_read_input_tokens: null, + inference_geo: null, + input_tokens: 10, + output_tokens: 5, + service_tier: null + } + })) + }) + )) + ) + + // Assert the request shape; the response outcome is irrelevant here. + yield* LanguageModel.generateObject({ + prompt: "Give me a person", + schema: Schema.Struct({ name: Schema.String, age: Schema.Number }) + }).pipe( + Effect.provide(AnthropicLanguageModel.model(model, config)), + Effect.provide(layer), + Effect.ignore + ) + + assert.isDefined(capturedRequest) + if (capturedRequest === undefined) { + return yield* Effect.die(new Error("Expected a captured request")) + } - return yield* getRequestBody(capturedRequest) - }) + return yield* getRequestBody(capturedRequest) + }) it.effect("uses native structured output and 128K for Claude 4.6", () => Effect.gen(function*() { @@ -1548,50 +1547,49 @@ describe("AnthropicLanguageModel", () => { }) describe("system messages", () => { - const getRequest = ( + const getRequest = Effect.fnUntraced(function*( model: string, prompt: Prompt.RawInput, config?: { readonly midConversationSystemMessages?: boolean | undefined } - ) => - Effect.gen(function*() { - let capturedRequest: HttpClientRequest.HttpClientRequest | undefined = undefined - const layer = AnthropicClient.layer({ apiKey: Redacted.make("sk-test-key") }).pipe( - Layer.provide(Layer.succeed( - HttpClient.HttpClient, - makeHttpClient((request) => { - capturedRequest = request - return Effect.succeed(jsonResponse(request, { - id: "msg_test_1", - type: "message", - role: "assistant", - model, - content: [{ type: "text", text: "Done" }], - stop_reason: "end_turn", - stop_sequence: null, - usage: { - cache_creation: null, - cache_creation_input_tokens: null, - cache_read_input_tokens: null, - inference_geo: null, - input_tokens: 1, - output_tokens: 1, - service_tier: null - } - })) - }) - )) - ) - - yield* LanguageModel.generateText({ prompt, disableToolCallResolution: true }).pipe( - Effect.provide(AnthropicLanguageModel.model(model, config)), - Effect.provide(layer) - ) - - if (capturedRequest === undefined) { - return yield* Effect.die(new Error("Expected a captured request")) - } - return yield* getRequestBody(capturedRequest) - }) + ) { + let capturedRequest: HttpClientRequest.HttpClientRequest | undefined = undefined + const layer = AnthropicClient.layer({ apiKey: Redacted.make("sk-test-key") }).pipe( + Layer.provide(Layer.succeed( + HttpClient.HttpClient, + makeHttpClient((request) => { + capturedRequest = request + return Effect.succeed(jsonResponse(request, { + id: "msg_test_1", + type: "message", + role: "assistant", + model, + content: [{ type: "text", text: "Done" }], + stop_reason: "end_turn", + stop_sequence: null, + usage: { + cache_creation: null, + cache_creation_input_tokens: null, + cache_read_input_tokens: null, + inference_geo: null, + input_tokens: 1, + output_tokens: 1, + service_tier: null + } + })) + }) + )) + ) + + yield* LanguageModel.generateText({ prompt, disableToolCallResolution: true }).pipe( + Effect.provide(AnthropicLanguageModel.model(model, config)), + Effect.provide(layer) + ) + + if (capturedRequest === undefined) { + return yield* Effect.die(new Error("Expected a captured request")) + } + return yield* getRequestBody(capturedRequest) + }) const conversation = Prompt.make([ { role: "system", content: "A" }, @@ -1661,6 +1659,154 @@ describe("AnthropicLanguageModel", () => { )) }) + describe("assistant cache control", () => { + const ephemeral = { type: "ephemeral" } as const + const cached = { anthropic: { cacheControl: ephemeral } } + + const getAssistantContent = Effect.fnUntraced(function*(prompt: Prompt.RawInput) { + let capturedRequest: HttpClientRequest.HttpClientRequest | undefined = undefined + const layer = AnthropicClient.layer({ apiKey: Redacted.make("sk-test-key") }).pipe( + Layer.provide(Layer.succeed( + HttpClient.HttpClient, + makeHttpClient((request) => { + capturedRequest = request + return Effect.succeed(jsonResponse(request, { + id: "msg_test_1", + type: "message", + role: "assistant", + model: "claude-sonnet-4-20250514", + content: [{ type: "text", text: "Done" }], + stop_reason: "end_turn", + stop_sequence: null, + usage: { + cache_creation: null, + cache_creation_input_tokens: null, + cache_read_input_tokens: null, + inference_geo: null, + input_tokens: 1, + output_tokens: 1, + service_tier: null + } + })) + }) + )) + ) + + yield* LanguageModel.generateText({ prompt, disableToolCallResolution: true }).pipe( + Effect.provide(AnthropicLanguageModel.model("claude-sonnet-4-20250514")), + Effect.provide(layer) + ) + + if (capturedRequest === undefined) { + return yield* Effect.die(new Error("Expected a captured request")) + } + const body = yield* getRequestBody(capturedRequest) + return body.messages.find((message: any) => message.role === "assistant").content as Array + }) + + const thinking = { + type: "reasoning", + text: "Thinking", + options: { anthropic: { info: { type: "thinking", signature: "sig" } } } + } as const + + const redactedThinking = { + type: "reasoning", + text: "", + options: { anthropic: { info: { type: "redacted_thinking", redactedData: "hidden" } } } + } as const + + it.effect("places part and message breakpoints without overriding a part's TTL", () => + Effect.gen(function*() { + const oneHour = { type: "ephemeral", ttl: "1h" } as const + const content = yield* getAssistantContent([ + { role: "user", content: "Question" }, + { + role: "assistant", + content: [ + { type: "text", text: "Answer", options: { anthropic: { cacheControl: oneHour } } }, + { type: "tool-call", id: "toolu_1", name: "lookup", params: {} }, + { + type: "tool-call", + id: "toolu_2", + name: "lookup", + params: {}, + options: { anthropic: { cacheControl: oneHour } } + } + ], + options: cached + }, + { + role: "assistant", + content: [{ type: "tool-call", id: "toolu_3", name: "lookup", params: {} }], + options: cached + }, + { role: "user", content: "Thanks" } + ]) + + assert.deepStrictEqual(content, [ + { type: "text", text: "Answer", cache_control: oneHour }, + { type: "tool_use", id: "toolu_1", name: "lookup", input: {} }, + { type: "tool_use", id: "toolu_2", name: "lookup", input: {}, cache_control: oneHour }, + { type: "tool_use", id: "toolu_3", name: "lookup", input: {}, cache_control: ephemeral } + ]) + })) + + it.effect("places a message breakpoint before trailing thinking and skipped parts", () => + Effect.gen(function*() { + const content = yield* getAssistantContent([ + { role: "user", content: "Question" }, + { + role: "assistant", + content: [ + { type: "text", text: "Answer" }, + thinking, + redactedThinking, + { type: "file", mediaType: "image/png", data: "aGVsbG8=" } + ], + options: cached + }, + { role: "user", content: "Thanks" } + ]) + + assert.deepStrictEqual(content, [ + { type: "text", text: "Answer", cache_control: ephemeral }, + { type: "thinking", thinking: "Thinking", signature: "sig" }, + { type: "redacted_thinking", data: "hidden" } + ]) + })) + + it.effect("drops thinking-only breakpoints without marking other parts or messages", () => + Effect.gen(function*() { + const content = yield* getAssistantContent([ + { role: "user", content: "Question" }, + { role: "assistant", content: "Unmarked" }, + { + role: "assistant", + content: [ + { type: "text", text: "Answer" }, + { ...thinking, options: { anthropic: { ...thinking.options.anthropic, ...cached.anthropic } } }, + { + ...redactedThinking, + options: { anthropic: { ...redactedThinking.options.anthropic, ...cached.anthropic } } + } + ] + }, + { role: "assistant", content: [thinking, redactedThinking], options: cached }, + { role: "user", content: "Thanks" } + ]) + + assert.deepStrictEqual(content, [ + { type: "text", text: "Unmarked" }, + { type: "text", text: "Answer" }, + { type: "thinking", thinking: "Thinking", signature: "sig" }, + { type: "redacted_thinking", data: "hidden" }, + { type: "thinking", thinking: "Thinking", signature: "sig" }, + { type: "redacted_thinking", data: "hidden" } + ]) + })) + }) + // The packaged `Memory_20250818` tool ships `customName: "AnthropicMemory"` / // `providerName: "memory"`, and is a client-executed provider tool. These // tests cover the round-trip that was broken on beta.98 (see #2615): @@ -1837,14 +1983,13 @@ const jsonResponse = ( }) ) -const getRequestBody = (request: HttpClientRequest.HttpClientRequest) => - Effect.gen(function*() { - const body = request.body - if (body._tag !== "Uint8Array") { - return yield* Effect.die(new Error("Expected Uint8Array body")) - } - return JSON.parse(new TextDecoder().decode(body.body)) - }) +const getRequestBody = Effect.fnUntraced(function*(request: HttpClientRequest.HttpClientRequest) { + const body = request.body + if (body._tag !== "Uint8Array") { + return yield* Effect.die(new Error("Expected Uint8Array body")) + } + return JSON.parse(new TextDecoder().decode(body.body)) +}) const toSseBody = (events: ReadonlyArray): string => events.map((event) => `event: message_stream\ndata: ${JSON.stringify(event)}\n\n`).join("") diff --git a/repos/effect/packages/ai/cloudflare/CHANGELOG.md b/repos/effect/packages/ai/cloudflare/CHANGELOG.md index 8428dc82ff..43ca6ca50c 100644 --- a/repos/effect/packages/ai/cloudflare/CHANGELOG.md +++ b/repos/effect/packages/ai/cloudflare/CHANGELOG.md @@ -1,5 +1,25 @@ # @effect/ai-cloudflare +## 4.0.6 + +### Patch Changes + +- [#9528](https://github.com/Effect-TS/effect/pull/9528) [`0163546`](https://github.com/Effect-TS/effect/commit/016354639aa7d21d48366f3fd47041992a51d815) Thanks @third774! - Add optional AI Gateway routing via `gatewayId` or `CLOUDFLARE_AI_GATEWAY_ID`, sent as the `cf-aig-gateway-id` header. + +- [#9709](https://github.com/Effect-TS/effect/pull/9709) [`85e2aee`](https://github.com/Effect-TS/effect/commit/85e2aeee686963c26ca40d204f982deebb9d6117) Thanks @tim-smart! - Add typed HTTP date and `Retry-After` header values to `Headers`, and parse them consistently across the HTTP client, OTLP exporter, GraphQL client, AI clients and static server. + + - `Headers.parseHttpDate` / `Headers.formatHttpDate` read all three RFC 9110 date formats as UTC, whatever the process time zone, and emit IMF-fixdate. + - `Headers.parseRetryAfter`, `Headers.formatRetryAfter` and `Headers.retryAfterDelay` accept whole non-negative seconds of any size, read exactly, or an HTTP date, and the delay is never negative. HTTP dates accept the `23:59:60` leap second, read as the following second. + - `Headers.RetryAfter` holds either a `Duration` (`RetryAfterDuration`) or a `DateTime.Utc` (`RetryAfterDateTime`). `Headers.retryAfterDuration` builds the first from any `Duration.Input`: infinite durations throw, finite negative ones become zero, and fractional seconds are rounded up when formatted. + - Parse failures are reported as `Headers.HeaderParseError`. + - `Schema.DateTimeUtcFromHttpDate` and `Schema.RetryAfterFromString` expose the same parsing as string codecs for HTTP API headers. + - Invalid `Retry-After` values such as `-5`, `1.5`, `1e3` or `Infinity` are now ignored everywhere. `HttpClient.retryTransient` and `HttpClient.withRateLimiter` no longer honour fractional seconds, and the AI clients no longer produce negative or infinite `retryAfter` durations. + - `HttpStaticServer` reads `If-Modified-Since` and `Last-Modified` as UTC; asctime dates were previously shifted by the process time zone. + +- [#9430](https://github.com/Effect-TS/effect/pull/9430) [`48102ac`](https://github.com/Effect-TS/effect/commit/48102ac592ef89fed9d5d10afd2daece0a0ee310) Thanks @effect-bot! - Add non-retryable `AiError.InvalidResponseError` for provider response schema and SSE decoding failures, with optional HTTP context; model-generated output validation remains retryable. Update exhaustive matches on `AiErrorReason` to handle the new tag, and update serialized-error consumers to accept it. +- Updated dependencies [[`ce13a48`](https://github.com/Effect-TS/effect/commit/ce13a4872e516187851c5d9336d4df309cc8f0a4), [`f3bbbb4`](https://github.com/Effect-TS/effect/commit/f3bbbb448d431a9d1205ff09ba47cb81af0c7199), [`5547315`](https://github.com/Effect-TS/effect/commit/554731500e43eef47ff68b569116b74006e91d87), [`6093cfb`](https://github.com/Effect-TS/effect/commit/6093cfbbb673d36ecd18edea955763db003c8c26), [`30319d5`](https://github.com/Effect-TS/effect/commit/30319d506c9177d1cc1e0b10372e1ddb776aaf0a), [`74a7596`](https://github.com/Effect-TS/effect/commit/74a759699a8582dbe660b21dcda21cc9db08f613), [`bfb32a3`](https://github.com/Effect-TS/effect/commit/bfb32a3cf5f8d0f02f0dfdb63dcf5237fde755b0), [`f8bf7f4`](https://github.com/Effect-TS/effect/commit/f8bf7f4bcc3ad543cd94cbeb010cb164e5dedcb0), [`7f3d3df`](https://github.com/Effect-TS/effect/commit/7f3d3dfbb7b9e6e041c4ad9654c4532f72d2f16e), [`db1046d`](https://github.com/Effect-TS/effect/commit/db1046d1fa0e4b5818c51d28017a89579b6261d6), [`3c4f88a`](https://github.com/Effect-TS/effect/commit/3c4f88aa4d1d4d958b42fe8d3f1195f0ca990f8a), [`916e709`](https://github.com/Effect-TS/effect/commit/916e70968acc4e443885c67012d855b7cc6f9bca), [`916f843`](https://github.com/Effect-TS/effect/commit/916f8436225153a81a049bdc1feaedcafe4d1ded), [`66ec757`](https://github.com/Effect-TS/effect/commit/66ec7574d23b7943584059b41136bc21ee9d4c53), [`daac06f`](https://github.com/Effect-TS/effect/commit/daac06f79a6ab8c7d3e1b66b427c06f7d64dcc8f), [`c2552ec`](https://github.com/Effect-TS/effect/commit/c2552ec2e031b409b2bef67309312b0168480a39), [`b02c1cd`](https://github.com/Effect-TS/effect/commit/b02c1cde36c890941a235224af8ecb035197582a), [`1855f1d`](https://github.com/Effect-TS/effect/commit/1855f1d359535339bc7640b67e6bb7b12752ba19), [`683ed2c`](https://github.com/Effect-TS/effect/commit/683ed2cb491f9529379c1d5c63933689ff57fd2b), [`ac68940`](https://github.com/Effect-TS/effect/commit/ac68940f7e4246b1beb43f9d451a67f41241ab82), [`22a8401`](https://github.com/Effect-TS/effect/commit/22a84017e7222f332611c1bace05f9fa60c7cf67), [`c53c194`](https://github.com/Effect-TS/effect/commit/c53c19451171a00776b400a8548394ac514b628a), [`ca6d1cc`](https://github.com/Effect-TS/effect/commit/ca6d1cc5f803cd501e85431642b79b552322a76a), [`47940c5`](https://github.com/Effect-TS/effect/commit/47940c55c9e0eb855615094a2aca6e0812307bb2), [`e8b24ef`](https://github.com/Effect-TS/effect/commit/e8b24ef9516706971db948efb2d3eab84ad85951), [`2e8aae0`](https://github.com/Effect-TS/effect/commit/2e8aae0080a8d3e6d00c5ee0bc4f49f0b074acc2), [`ad7497b`](https://github.com/Effect-TS/effect/commit/ad7497bbc1bc70935f8f1ae395ff501532e2e770), [`7f504a3`](https://github.com/Effect-TS/effect/commit/7f504a35edcf44952c55ecbb0916726bac4d3e52), [`99e365d`](https://github.com/Effect-TS/effect/commit/99e365d61a1fdaabeb31dc58e5752cade8a09e1f), [`954067d`](https://github.com/Effect-TS/effect/commit/954067da7fbd20a20c10d5d0de6ffe4690e6bdfd), [`3e1a035`](https://github.com/Effect-TS/effect/commit/3e1a035d9c470ced858b7757f6d102b3aa9be008), [`41a1fe7`](https://github.com/Effect-TS/effect/commit/41a1fe74e6338e8d13d6d4f3daa03f71a826c2c4), [`bcb3a86`](https://github.com/Effect-TS/effect/commit/bcb3a86335a8ed18da39300984c0fbae187fdc6b), [`739d3d1`](https://github.com/Effect-TS/effect/commit/739d3d18728cce0b6d907f0a3f24e04ab0b53560), [`ed4da35`](https://github.com/Effect-TS/effect/commit/ed4da351fb9d29e4dfd6c60bd89ba4c007d264dc), [`3d84e19`](https://github.com/Effect-TS/effect/commit/3d84e19f8f14bccf0aa57b11825590c3bae162f4), [`cc00906`](https://github.com/Effect-TS/effect/commit/cc0090657eeb89debdd54af5fcb99d544f46c1bf), [`09ba1f5`](https://github.com/Effect-TS/effect/commit/09ba1f59952c46cd22229d41236742c3e1f53a34), [`9ff1a73`](https://github.com/Effect-TS/effect/commit/9ff1a73d995702f4c4c335888fe6a186f3f710c7), [`06bc532`](https://github.com/Effect-TS/effect/commit/06bc532b71cd3684f010956774356864dd359372), [`97bfd84`](https://github.com/Effect-TS/effect/commit/97bfd84bb147e10ed562e4c1ffa589cdaf83a596), [`25efaef`](https://github.com/Effect-TS/effect/commit/25efaef0d81d25b2651ef4de474e929d3180810a), [`c10335b`](https://github.com/Effect-TS/effect/commit/c10335be125c7846ae7b914eeb138244b660275b), [`a808972`](https://github.com/Effect-TS/effect/commit/a80897262648e4222ce48943e538f39a6b3f69d6), [`20e844c`](https://github.com/Effect-TS/effect/commit/20e844c7ef27b0e4b4f7911d603c45b1de2b0bc3), [`b9f73f3`](https://github.com/Effect-TS/effect/commit/b9f73f37c4ac538e6a327ed80ac5a0708858c540), [`5739878`](https://github.com/Effect-TS/effect/commit/5739878ded09d83d71d657e3b8d4264f4e4936d9), [`28acdf3`](https://github.com/Effect-TS/effect/commit/28acdf3820b66056037bb7a24cd0e8d3de3fb06d), [`59b6396`](https://github.com/Effect-TS/effect/commit/59b63967c1817ae5acb1811037f5aaa964219b7e), [`0bccc9c`](https://github.com/Effect-TS/effect/commit/0bccc9c3dd5349b20f8d2e509ac5482d0c26a616), [`fed70de`](https://github.com/Effect-TS/effect/commit/fed70de0dff8af24c5f814bd0d985ab70c70420b), [`82648a2`](https://github.com/Effect-TS/effect/commit/82648a21bf228145a14d2d1390006d67f74d89c7), [`afb5f32`](https://github.com/Effect-TS/effect/commit/afb5f326cc14c9849d8d7ddefde1039ae036e017), [`28813f5`](https://github.com/Effect-TS/effect/commit/28813f53a5026984fdc13d3971147ea5dc20104c), [`0fae934`](https://github.com/Effect-TS/effect/commit/0fae934a13aac12f9036b0733edd5dde5ce029f3), [`f260830`](https://github.com/Effect-TS/effect/commit/f2608305861269fc2fbedcbbdcd71e4fc5f98ffc), [`19b63eb`](https://github.com/Effect-TS/effect/commit/19b63eb81b2bccf033aadb565f52a6b0375d26a1), [`14db221`](https://github.com/Effect-TS/effect/commit/14db221f8f584100ff50fd1173b1a99a1caa1d06), [`679f956`](https://github.com/Effect-TS/effect/commit/679f956c4a7527854ff78db071dd7aa417211a48), [`b72707c`](https://github.com/Effect-TS/effect/commit/b72707c2937328fd3bc70f29a6792d84c4256484), [`44c8c1d`](https://github.com/Effect-TS/effect/commit/44c8c1d1c6f01ba9887a051b365e9fa89fcfc685), [`1e7909b`](https://github.com/Effect-TS/effect/commit/1e7909beb05bc740dbd330509e60bb185d7eeed1), [`0611a65`](https://github.com/Effect-TS/effect/commit/0611a657f76836f9ca275f64764b82270b4eda10), [`2fadc65`](https://github.com/Effect-TS/effect/commit/2fadc654aa655fbd1912ce51eb1b764a757ac362), [`feb6ce5`](https://github.com/Effect-TS/effect/commit/feb6ce59a38ab15fc6f89545bd5f0992dd58a06a), [`5a2ed53`](https://github.com/Effect-TS/effect/commit/5a2ed5323884a435bfcfb5e3a6f2ae29547458ad), [`c8d26a4`](https://github.com/Effect-TS/effect/commit/c8d26a46731cb26a9b9612d9fde823d8813cfc04), [`0c8b446`](https://github.com/Effect-TS/effect/commit/0c8b446709c737c10d0e11af4313bb29fdff2588), [`05c3409`](https://github.com/Effect-TS/effect/commit/05c340955944374e3ae47a1de0f7cb9b41a3b0e6), [`b04191b`](https://github.com/Effect-TS/effect/commit/b04191b2941534754282c51ef2df2f965ac43a5c), [`05c3409`](https://github.com/Effect-TS/effect/commit/05c340955944374e3ae47a1de0f7cb9b41a3b0e6), [`4067e64`](https://github.com/Effect-TS/effect/commit/4067e6428776ae573c8cc72f7eaebd078a4ff8c7), [`e04563d`](https://github.com/Effect-TS/effect/commit/e04563dcd6ddeea74c088a12194c5a73e597ba2f), [`02d7fc5`](https://github.com/Effect-TS/effect/commit/02d7fc583053c47dea3b5e1d9d6be164aeb321b6), [`561d2a1`](https://github.com/Effect-TS/effect/commit/561d2a1ddfa3c4340510f23da7adc50c89d97863), [`8fe2dcc`](https://github.com/Effect-TS/effect/commit/8fe2dcc2137788a71177d42a9f67fb7003224868), [`a8086fa`](https://github.com/Effect-TS/effect/commit/a8086fa26a2c9a9f5f7f1d7590bbc494a0feb02a), [`88a33b4`](https://github.com/Effect-TS/effect/commit/88a33b4122f7ffbbc1423add793a79abffd99c75), [`e77372a`](https://github.com/Effect-TS/effect/commit/e77372a8a2e7caec11999636e04c6e31c52e315f), [`3dd4388`](https://github.com/Effect-TS/effect/commit/3dd438810df34ed3b3d2e3e35ccf84ccb00feb85), [`ba2b00c`](https://github.com/Effect-TS/effect/commit/ba2b00c0658b65775809e0052ec9612432600c31), [`a049d4a`](https://github.com/Effect-TS/effect/commit/a049d4ad25f17cf4985284a8d15b07aea2a33242), [`84aeede`](https://github.com/Effect-TS/effect/commit/84aeede040460a2b24bcf0db7a5d0a9b0e3c42d4), [`43e5e19`](https://github.com/Effect-TS/effect/commit/43e5e198d16df038b60764f5dd0218cdcf828f16), [`77bbf16`](https://github.com/Effect-TS/effect/commit/77bbf16e50d6b0ea090288cc5d3f11b20706a027), [`6af3a47`](https://github.com/Effect-TS/effect/commit/6af3a47d5b5d698c3ba896c37684266cd465bef1), [`aac8d21`](https://github.com/Effect-TS/effect/commit/aac8d21911333dedb649f4f9d302c28d14d46369), [`84c8dc4`](https://github.com/Effect-TS/effect/commit/84c8dc47b09067362749620f75068d6f28044660), [`78d1988`](https://github.com/Effect-TS/effect/commit/78d198818d4fd547ce2e9431302f66cb00dc7ad3), [`5676a2a`](https://github.com/Effect-TS/effect/commit/5676a2ac5a918f8f91fc7838d7adf1b6f5e86a6d), [`4b437bf`](https://github.com/Effect-TS/effect/commit/4b437bf4645fabc8aaeaa3b5a88e8c9048b9e4f5), [`1af11fd`](https://github.com/Effect-TS/effect/commit/1af11fda1bccf823f74121b5125aa3befa084b18), [`716f8df`](https://github.com/Effect-TS/effect/commit/716f8dfa178f849e5b288464373bd132860c2e19), [`ee27718`](https://github.com/Effect-TS/effect/commit/ee27718a02125328a7478812b1b7d7c1d8532578), [`4d86d1b`](https://github.com/Effect-TS/effect/commit/4d86d1b8887d48fa24acf6ad54f33ef3d55a7cd8), [`5462ed7`](https://github.com/Effect-TS/effect/commit/5462ed7b3a4c37206c47e401d6be293dc540350f), [`658837c`](https://github.com/Effect-TS/effect/commit/658837c3b249d982df56f2990247499746e36672), [`00261cc`](https://github.com/Effect-TS/effect/commit/00261cc12c31ecc7878f80670439731c905dc2bd), [`7cccb93`](https://github.com/Effect-TS/effect/commit/7cccb9385729bc0aed6630edea95d7db15439ac4), [`7c98228`](https://github.com/Effect-TS/effect/commit/7c982281cf73958833054175fc86db385e043cf6), [`5a8e248`](https://github.com/Effect-TS/effect/commit/5a8e2481546352e2ad6233dad49420b14c59f624), [`8037a65`](https://github.com/Effect-TS/effect/commit/8037a65f40cae9163ab9e0e687e289f717ae570b), [`25912db`](https://github.com/Effect-TS/effect/commit/25912db9d493227a2d6fa45cebcfc7507e638d8a), [`7f32606`](https://github.com/Effect-TS/effect/commit/7f32606baac43a67d171ae93b77c7b29791c9692), [`0386f88`](https://github.com/Effect-TS/effect/commit/0386f88295ac70cbc1f90ab8cceb0981d848c628), [`403298a`](https://github.com/Effect-TS/effect/commit/403298ac457fb634eca2c013fed26d37fc80445f), [`5a3ebc0`](https://github.com/Effect-TS/effect/commit/5a3ebc082124f9e10106b2347736e66fc74ebc36), [`fac2191`](https://github.com/Effect-TS/effect/commit/fac21911964b742a38e1d4e73bd4f95aec4911dc), [`fdb12dc`](https://github.com/Effect-TS/effect/commit/fdb12dcd61b6505eed02e6f7a3c88a85d4c48e8d), [`97909b8`](https://github.com/Effect-TS/effect/commit/97909b89b167da54a56e6f82f1c4db8bfedae04f), [`c3983f4`](https://github.com/Effect-TS/effect/commit/c3983f4db2b3052a58e8045cce270fd7ccc4e2b2), [`3a5e65e`](https://github.com/Effect-TS/effect/commit/3a5e65e409c9569e23bb7af61df401347e9ec420), [`8b40fbc`](https://github.com/Effect-TS/effect/commit/8b40fbc93caf52f3a4c69020849f0051f100be65), [`3f3e119`](https://github.com/Effect-TS/effect/commit/3f3e119ad61c2167bc6c7170c3fb68cd1eb3421e), [`7c7b8ed`](https://github.com/Effect-TS/effect/commit/7c7b8eda54e44f9cddc1188f994f7fdddf901055), [`d4f71bc`](https://github.com/Effect-TS/effect/commit/d4f71bc42507b638d149fb8806d26ea626d17956), [`c66241e`](https://github.com/Effect-TS/effect/commit/c66241e791c4b51d8a65051142d01682590f9bf8), [`f0e941e`](https://github.com/Effect-TS/effect/commit/f0e941ee2f90f0a891d1228a90da91386f0258cb), [`5f8a6e5`](https://github.com/Effect-TS/effect/commit/5f8a6e5108aa6c6cd4043a3c9f4dc93d43369623), [`2800a09`](https://github.com/Effect-TS/effect/commit/2800a0910f7f4ad7cfa21dfc840edf8fe6e67ca7), [`e9749b0`](https://github.com/Effect-TS/effect/commit/e9749b04c10badabcdf71e6958de5ac56a5316e9), [`eaef09c`](https://github.com/Effect-TS/effect/commit/eaef09c11b6ed2029c513eea0aa3256dcc6f517f), [`a3c56e5`](https://github.com/Effect-TS/effect/commit/a3c56e5b5e756e879f1cfb90230e8eae620e33df), [`bb61152`](https://github.com/Effect-TS/effect/commit/bb611522fcfb4d317b33acdf5ee24a480d042435), [`e9c842b`](https://github.com/Effect-TS/effect/commit/e9c842bc294abf658d698bb17732ae13465b3817), [`6d8c8e2`](https://github.com/Effect-TS/effect/commit/6d8c8e2075b15ae84ae96e93a0d295cb9221b496), [`1f28187`](https://github.com/Effect-TS/effect/commit/1f281878ef559ef356af2d0e11b381aba857cae1), [`0b9842b`](https://github.com/Effect-TS/effect/commit/0b9842b76bbc1b016c8abc46432052e1f60b2fa3), [`c29ddd8`](https://github.com/Effect-TS/effect/commit/c29ddd899c53c5d8194c4a0dfef2134304dda799), [`84c3405`](https://github.com/Effect-TS/effect/commit/84c34057ceab9fe6782b73f4a04a31304e0d14de), [`3934c0c`](https://github.com/Effect-TS/effect/commit/3934c0c61857c03468c08c2477f49a6b977d28ff), [`048e7c8`](https://github.com/Effect-TS/effect/commit/048e7c87125c4ace5a61895feac3d049a317bbfc), [`1121334`](https://github.com/Effect-TS/effect/commit/1121334ce68c379051011a2ffea37b0a30ea9a47), [`9a680d6`](https://github.com/Effect-TS/effect/commit/9a680d6985c6a27d7ebb29cd9eb674f3fe524505), [`85e2aee`](https://github.com/Effect-TS/effect/commit/85e2aeee686963c26ca40d204f982deebb9d6117), [`34ec440`](https://github.com/Effect-TS/effect/commit/34ec4407a14ce045ca14cc76c77582884a7e1ebe), [`343a149`](https://github.com/Effect-TS/effect/commit/343a149fa7d5a9a0a45841b1b6800b0ecfa16a32), [`915117b`](https://github.com/Effect-TS/effect/commit/915117b523cb6e5cbc98b5ff1c119a8d0d1f6b9a), [`7f823c1`](https://github.com/Effect-TS/effect/commit/7f823c10d20ef53a1081e0d0e2008a4116140799), [`6e614e6`](https://github.com/Effect-TS/effect/commit/6e614e6feb4281a9e81f5ec3be48e7905ff4292c), [`9990b39`](https://github.com/Effect-TS/effect/commit/9990b3977a544247cd4123edde8353a55bf77192), [`22bf781`](https://github.com/Effect-TS/effect/commit/22bf7812aa8a20c676efade943c342cde4ef1cbd), [`41f08b7`](https://github.com/Effect-TS/effect/commit/41f08b75aba40e95cd6c743ec6fcf1f5022f7b5b), [`e8482ca`](https://github.com/Effect-TS/effect/commit/e8482cad9651fa8fe8036fe6292a286994897d0d), [`5ae85fa`](https://github.com/Effect-TS/effect/commit/5ae85fa8ac1fc5e12d5fe8fe7388ac504788ae3b), [`c477a0a`](https://github.com/Effect-TS/effect/commit/c477a0a8010bd0a988221bc00c446848314e1c3c), [`8fbb39e`](https://github.com/Effect-TS/effect/commit/8fbb39eb92577915831f19a83b51be376e844613), [`1c70828`](https://github.com/Effect-TS/effect/commit/1c7082847453e9c2be0cc267a39ca554c52f4013), [`8207d1e`](https://github.com/Effect-TS/effect/commit/8207d1e91ead8aac8f01d9e13b49624b64237c8b), [`7886b84`](https://github.com/Effect-TS/effect/commit/7886b847658664f64392a14efda86f44b3d1e170), [`4839c60`](https://github.com/Effect-TS/effect/commit/4839c601bdda845d9a54d77f578f64372475ab04), [`2595d83`](https://github.com/Effect-TS/effect/commit/2595d831f702d4a748f97e33e71fd5250f58af56), [`f5ae278`](https://github.com/Effect-TS/effect/commit/f5ae27827a82db3f8973b70745fc6f14e7d8fa80), [`7a8ce00`](https://github.com/Effect-TS/effect/commit/7a8ce0098a5f22cbb6c110b16e850fb3fcbcadf2), [`688d302`](https://github.com/Effect-TS/effect/commit/688d30275ff114e50b4f0d343c6227ee33d67348), [`c7072e4`](https://github.com/Effect-TS/effect/commit/c7072e46356a19615625ae0fa569b1c3a79d50be), [`425cedb`](https://github.com/Effect-TS/effect/commit/425cedb5801c9527c116a64d280d9e7c2820f922), [`88bc680`](https://github.com/Effect-TS/effect/commit/88bc68001c6c70984b03a5500232e9b5aab14085), [`268e09c`](https://github.com/Effect-TS/effect/commit/268e09c94c9e8f46142b5c9f29ce0a22bfbbc3aa), [`7e94505`](https://github.com/Effect-TS/effect/commit/7e94505edfc9d81ea08cb96a7d935806495bf61e), [`afa5da3`](https://github.com/Effect-TS/effect/commit/afa5da3c8a9848215e49535dd3d39179bfd67715), [`c29ddd8`](https://github.com/Effect-TS/effect/commit/c29ddd899c53c5d8194c4a0dfef2134304dda799), [`48102ac`](https://github.com/Effect-TS/effect/commit/48102ac592ef89fed9d5d10afd2daece0a0ee310), [`463eca3`](https://github.com/Effect-TS/effect/commit/463eca3609a6a9681247abbcc71a6f10b261a1d9), [`e7162ea`](https://github.com/Effect-TS/effect/commit/e7162eae7f04af1d583b1f31fdd767f91ec1c82e), [`95752db`](https://github.com/Effect-TS/effect/commit/95752db5eb3b6e59fb8749e9792704f8b167f36c), [`4577a28`](https://github.com/Effect-TS/effect/commit/4577a28bcdb50b0d184775f8c1b7ad890d86e255), [`973d599`](https://github.com/Effect-TS/effect/commit/973d599a3e8611626764a0e3e8f79edbcf30dc7c), [`2f066a5`](https://github.com/Effect-TS/effect/commit/2f066a58e90316dd88a219fcea94200beda54219), [`f0ccbb6`](https://github.com/Effect-TS/effect/commit/f0ccbb6917be5c08364b0b8adbd0c4c19e7f2de1), [`2dda6ce`](https://github.com/Effect-TS/effect/commit/2dda6cefc641dbea2ea3279a9b96c0332b277a40), [`3da7c0a`](https://github.com/Effect-TS/effect/commit/3da7c0aabab07c8fc682aea7559536bf31705853), [`5a67e0b`](https://github.com/Effect-TS/effect/commit/5a67e0b3a8118dd7be709a56d6d7af4f10e9c4f3), [`73b3a6e`](https://github.com/Effect-TS/effect/commit/73b3a6e3c38795400251d9c5d73f05c88acd90cd), [`c6f9e4c`](https://github.com/Effect-TS/effect/commit/c6f9e4cbe6f9c8f0a03fefde18df4d35e98a24df), [`be02676`](https://github.com/Effect-TS/effect/commit/be026768ee29a1b1a860c1fd17f2a0896c01f61e), [`26de861`](https://github.com/Effect-TS/effect/commit/26de86100cda8617e077ce66f0a6bb8dfe738fb6), [`9313799`](https://github.com/Effect-TS/effect/commit/931379911a74d5e2207859c295b1acc6a5e2917e), [`d5c1e5b`](https://github.com/Effect-TS/effect/commit/d5c1e5bd425dfe983dff1f8c853c0b7eaa772f87), [`11bb31f`](https://github.com/Effect-TS/effect/commit/11bb31fec0bf2fdfc025009a04de4c69bffa433a), [`8c23e46`](https://github.com/Effect-TS/effect/commit/8c23e46a8280e3f2c831d1c0ca3f106c92f8e071), [`9c96011`](https://github.com/Effect-TS/effect/commit/9c960111de202c0f1dd6989e5b8836150e0ca694), [`0d50570`](https://github.com/Effect-TS/effect/commit/0d50570bbf078804092fcd4a47ccd2d0019f989c), [`9ed2bf1`](https://github.com/Effect-TS/effect/commit/9ed2bf18da0e1ff4fecc31b7cbac2976ec999e00), [`76ec912`](https://github.com/Effect-TS/effect/commit/76ec9120f43f4fdb08b9c5170ff9e3fd82f22457), [`af2756d`](https://github.com/Effect-TS/effect/commit/af2756dd1d196aa3f199ce21c6d2dc3ada3b67b5), [`3ec8100`](https://github.com/Effect-TS/effect/commit/3ec8100c793b8a22381ad115106b93824d840f64), [`93abf02`](https://github.com/Effect-TS/effect/commit/93abf020876af0a6b6b8b25621c80760b7996762), [`1e94993`](https://github.com/Effect-TS/effect/commit/1e949935a1065e5140245a514ce3a799061143e3), [`99a4616`](https://github.com/Effect-TS/effect/commit/99a4616264e31de65fa8d3dc19576309b1a3f84a), [`0a7978e`](https://github.com/Effect-TS/effect/commit/0a7978ea8e70e51700345b12f8c19952dde05255), [`b4a060b`](https://github.com/Effect-TS/effect/commit/b4a060b95a6473d54e277dc005172b54aab9bddf), [`274f4a4`](https://github.com/Effect-TS/effect/commit/274f4a4dac5ee2d9dbb6358cc3dd3afe345d4705), [`25f29be`](https://github.com/Effect-TS/effect/commit/25f29be77bdcd5c2a7be8e2710e88f5a10c4cb8d), [`9036440`](https://github.com/Effect-TS/effect/commit/9036440edca01c24371d0c2c1044be5e5d3cb65e), [`415f005`](https://github.com/Effect-TS/effect/commit/415f0053eb848d562febcb5b011e15f31a99bead), [`191308b`](https://github.com/Effect-TS/effect/commit/191308bd0f5ee04557570f2f960c6e06c429bcfd), [`4b96570`](https://github.com/Effect-TS/effect/commit/4b96570ed28d93b14e958742933d227dc5e74b15), [`d84ec3f`](https://github.com/Effect-TS/effect/commit/d84ec3fab0346c01bc59f6624d1c046ca4140087), [`392aa1c`](https://github.com/Effect-TS/effect/commit/392aa1cb275ac9fdb9b4b1831d8c06196bba54b9), [`99e9bd9`](https://github.com/Effect-TS/effect/commit/99e9bd98a354d83202e2522f3ade9ee4bf4a862c), [`6f6f45f`](https://github.com/Effect-TS/effect/commit/6f6f45f480a3f191aed6a82ca5c301f60ef2cf84), [`13fc07e`](https://github.com/Effect-TS/effect/commit/13fc07edeb939ca6229b858c9d9a8c5243ac21b5), [`a4b38dd`](https://github.com/Effect-TS/effect/commit/a4b38dde7f3fdc1b54a49bffdaca912b4c47c76e), [`8b18580`](https://github.com/Effect-TS/effect/commit/8b185807bb99f6de3d9564447e2f4e08d2f8863d), [`0c477fe`](https://github.com/Effect-TS/effect/commit/0c477fe417d3ecab86bb8d9afd0c392e9cb8a3da), [`9a5331d`](https://github.com/Effect-TS/effect/commit/9a5331db94bfdca4f48317d062fbeb96054a9049), [`a706d73`](https://github.com/Effect-TS/effect/commit/a706d73db8a8d460877c5d7e8a793a52d45be307), [`c9a1e72`](https://github.com/Effect-TS/effect/commit/c9a1e72c5965aa9fe01a669ebfd523f803f5a189), [`89eaaf8`](https://github.com/Effect-TS/effect/commit/89eaaf8581908e248c0b1e47baa7906e0c541635), [`8215811`](https://github.com/Effect-TS/effect/commit/8215811d51d28d37b4732a6ab03de4047ada62ea), [`ad69018`](https://github.com/Effect-TS/effect/commit/ad6901857e05e7b436a0149b5ece7f3a36d9ff33), [`a733eba`](https://github.com/Effect-TS/effect/commit/a733eba9f699737a80a9e7276521980bd929075a), [`7282d5f`](https://github.com/Effect-TS/effect/commit/7282d5f1bcc06d7065429ef81a3d17c64043fde4), [`da33572`](https://github.com/Effect-TS/effect/commit/da33572776763e45802ce8c2fe14ec101f9a5c34), [`d9e2d12`](https://github.com/Effect-TS/effect/commit/d9e2d1221768e9a3e5e00165cb95b86e0c126cc0), [`dc923b8`](https://github.com/Effect-TS/effect/commit/dc923b847ac7d9673b4f0aa7c136557287b2e5e3), [`1b74086`](https://github.com/Effect-TS/effect/commit/1b740862e3d25ec5d4b6f86b9fe792b6dabbf84c), [`8a5507b`](https://github.com/Effect-TS/effect/commit/8a5507ba00f641edf71227d79b9f90efdb634fe2), [`23460fa`](https://github.com/Effect-TS/effect/commit/23460fad704c730926bffe3cc6874268ef6680ec), [`0dbbea6`](https://github.com/Effect-TS/effect/commit/0dbbea65a93c59147a4bc407cfcda38ea26341b2), [`70aad3f`](https://github.com/Effect-TS/effect/commit/70aad3ffe617984f5b599aeed1bc7f04899aca55), [`0909236`](https://github.com/Effect-TS/effect/commit/090923611b209a19f3281a7b1d0c15a9b653a0c7), [`a9e66d8`](https://github.com/Effect-TS/effect/commit/a9e66d889017b302355ba5b8bc2b2050ca1ea5aa), [`ee5c0d5`](https://github.com/Effect-TS/effect/commit/ee5c0d5f43348747b04b5f7bf1b39355614c2f2c), [`631c5d6`](https://github.com/Effect-TS/effect/commit/631c5d6d9a81e6389964b878764fb9aea1538247), [`7e60c1a`](https://github.com/Effect-TS/effect/commit/7e60c1ae62a96c768f9c67d706f494213947f22f), [`4cddd3c`](https://github.com/Effect-TS/effect/commit/4cddd3c41e71fc4990d37c42b1bcf2ce3780b41d), [`8727e5d`](https://github.com/Effect-TS/effect/commit/8727e5d2bec8e2d729b7aeafb725a4ab21d533d2), [`2bc97fa`](https://github.com/Effect-TS/effect/commit/2bc97faa0e06b3973780dae26029e2c59f5dba38), [`f9a7f05`](https://github.com/Effect-TS/effect/commit/f9a7f056ba31aef435933acf082ec8c01ac7decc), [`d978cca`](https://github.com/Effect-TS/effect/commit/d978cca02914bec0ddf66bb2d68d90b220096fb6), [`b334ccf`](https://github.com/Effect-TS/effect/commit/b334ccf361a0f11362e7aabdbf70b35f53a39d61), [`0b50cfc`](https://github.com/Effect-TS/effect/commit/0b50cfc644313afe4c318073893231ec1946eb5f), [`5b7d2b5`](https://github.com/Effect-TS/effect/commit/5b7d2b5c90bc77eb9f8be1b91166c53f89aa1d0c), [`6ee26b1`](https://github.com/Effect-TS/effect/commit/6ee26b16b51b380c41dbfa84d612aa525b420fce), [`eab7e6a`](https://github.com/Effect-TS/effect/commit/eab7e6acfc7e422857565b98b9eddd3a98de1b92), [`e618f10`](https://github.com/Effect-TS/effect/commit/e618f1050f41424439eb82478fcf1c5da5a3244d), [`da14b1f`](https://github.com/Effect-TS/effect/commit/da14b1f18d71a18fc431b56d82cffe09aeb92708), [`3314fc3`](https://github.com/Effect-TS/effect/commit/3314fc30c22d5c9d737f3df7201f4a41424f701b), [`66db624`](https://github.com/Effect-TS/effect/commit/66db624670032a02144d42b6e64b83b324c813cd), [`4b8a50d`](https://github.com/Effect-TS/effect/commit/4b8a50db9649603f382f33640698020ca79547d0), [`8d52668`](https://github.com/Effect-TS/effect/commit/8d52668533e9d4a6d6b3be9fe58bd5e69fa80b65), [`1de1055`](https://github.com/Effect-TS/effect/commit/1de10559e7263747751bfb032fd4b0fd8aa28724), [`db130e6`](https://github.com/Effect-TS/effect/commit/db130e64557fafe370819d19f78e3b19a21275d6), [`5fa5fa5`](https://github.com/Effect-TS/effect/commit/5fa5fa5b9bff77352fb9ff2d424bf46bd6309986)]: + - effect@4.0.6 + ## 4.0.5 ### Patch Changes diff --git a/repos/effect/packages/ai/cloudflare/README.md b/repos/effect/packages/ai/cloudflare/README.md index 35a9d0f27b..041eeb3b1c 100644 --- a/repos/effect/packages/ai/cloudflare/README.md +++ b/repos/effect/packages/ai/cloudflare/README.md @@ -57,3 +57,7 @@ Instructions and descriptions can be strings, JSON objects, or JSON arrays. A ra The client unwraps Cloudflare's REST envelope and maps failures to `AiError`. It does not retry automatically. Rate-limit errors include `Retry-After` when provided. The decision model validates answers and rescales small probability drift from four-decimal rounding. Cloudflare limits each request to 64 questions, classification to 2–255 choices, and ratings to 2–10 levels. See the [Clef API documentation](https://developers.cloudflare.com/workers-ai/models/clef/) for request limits. + +## AI Gateway + +Set `CLOUDFLARE_AI_GATEWAY_ID` when using `CloudflareClient.layerConfig()` to route requests through a Cloudflare AI Gateway. You can also pass `gatewayId` to `CloudflareClient.layer`. The client adds the `cf-aig-gateway-id` header. Use `transformClient` to add other `cf-aig-*` headers. diff --git a/repos/effect/packages/ai/cloudflare/package.json b/repos/effect/packages/ai/cloudflare/package.json index d8c9a520aa..3d8ff105b9 100644 --- a/repos/effect/packages/ai/cloudflare/package.json +++ b/repos/effect/packages/ai/cloudflare/package.json @@ -1,6 +1,6 @@ { "name": "@effect/ai-cloudflare", - "version": "4.0.5", + "version": "4.0.6", "type": "module", "license": "MIT", "description": "A Cloudflare Workers AI decision provider for Effect", diff --git a/repos/effect/packages/ai/cloudflare/src/CloudflareClient.ts b/repos/effect/packages/ai/cloudflare/src/CloudflareClient.ts index 73eba727b6..2e58de0585 100644 --- a/repos/effect/packages/ai/cloudflare/src/CloudflareClient.ts +++ b/repos/effect/packages/ai/cloudflare/src/CloudflareClient.ts @@ -44,7 +44,7 @@ export class CloudflareClient {} /** - * Account, authentication, API base URL, and HTTP customization options. + * Account, authentication, AI Gateway, API base URL, and HTTP customization options. * * @stability unstable * @category options @@ -53,6 +53,7 @@ export class CloudflareClient export interface Options { readonly accountId: string readonly apiKey: Redacted.Redacted + readonly gatewayId?: string | undefined readonly apiUrl?: string | undefined readonly transformClient?: ((client: HttpClient.HttpClient) => HttpClient.HttpClient) | undefined } @@ -76,7 +77,10 @@ export const make = Effect.fnUntraced( HttpClient.mapRequest(flow( HttpClientRequest.prependUrl(options.apiUrl ?? "https://api.cloudflare.com/client/v4"), HttpClientRequest.bearerToken(Redacted.value(options.apiKey)), - HttpClientRequest.acceptJson + HttpClientRequest.acceptJson, + options.gatewayId === undefined + ? identity + : HttpClientRequest.setHeader("cf-aig-gateway-id", options.gatewayId) )), HttpClient.filterStatusOk, options.transformClient ?? identity @@ -105,7 +109,7 @@ export const make = Effect.fnUntraced( Effect.fail(AiError.make({ module: "CloudflareClient", method: "createDecisions", - reason: AiError.InvalidOutputError.fromSchemaError(error) + reason: AiError.InvalidResponseError.fromSchemaError(error) })) }) ) @@ -124,7 +128,8 @@ export const layer = (options: Options): Layer.Layer | undefined readonly apiKey?: Config.Config> | undefined + readonly gatewayId?: Config.Config | undefined readonly apiUrl?: Config.Config | undefined readonly transformClient?: ((client: HttpClient.HttpClient) => HttpClient.HttpClient) | undefined }): Layer.Layer => @@ -142,6 +148,8 @@ export const layerConfig = (options?: { return yield* make({ accountId: yield* (options?.accountId ?? Config.String("CLOUDFLARE_ACCOUNT_ID")), apiKey: yield* (options?.apiKey ?? Config.Redacted("CLOUDFLARE_API_TOKEN")), + gatewayId: yield* (options?.gatewayId ?? + Config.String("CLOUDFLARE_AI_GATEWAY_ID").pipe(Config.withDefault(undefined))), apiUrl: options?.apiUrl ? yield* options.apiUrl : undefined, transformClient: options?.transformClient }) diff --git a/repos/effect/packages/ai/cloudflare/src/internal/errors.ts b/repos/effect/packages/ai/cloudflare/src/internal/errors.ts index b2f5caad43..cd2fb92fb3 100644 --- a/repos/effect/packages/ai/cloudflare/src/internal/errors.ts +++ b/repos/effect/packages/ai/cloudflare/src/internal/errors.ts @@ -1,12 +1,14 @@ import * as AiError from "effect/ai/AiError" -import * as Duration from "effect/Duration" +import * as Clock from "effect/Clock" +import type * as Duration from "effect/Duration" import * as Effect from "effect/Effect" +import * as Headers from "effect/http/Headers" import type * as HttpClientError from "effect/http/HttpClientError" import type * as HttpClientRequest from "effect/http/HttpClientRequest" import type * as HttpClientResponse from "effect/http/HttpClientResponse" -import * as Num from "effect/Number" import * as Option from "effect/Option" import * as Redactable from "effect/Redactable" +import * as Result from "effect/Result" import * as Schema from "effect/Schema" const requestDetails = (request: HttpClientRequest.HttpClientRequest) => ({ @@ -23,12 +25,11 @@ const ErrorBody = Schema.Struct({ const decodeErrorBody = Schema.decodeUnknownOption(ErrorBody) -const retryAfter = (raw: string | undefined): Duration.Duration | undefined => { - if (raw === undefined || raw.trim() === "") return undefined - const seconds = Option.getOrUndefined(Num.parse(raw)) - if (seconds !== undefined) return Number.isFinite(seconds) && seconds >= 0 ? Duration.seconds(seconds) : undefined - const date = Date.parse(raw) - return Number.isNaN(date) ? undefined : Duration.millis(Math.max(0, date - Date.now())) +// an invalid Retry-After is treated as absent +const retryAfter = (raw: string | undefined, now: number): Duration.Duration | undefined => { + if (raw === undefined) return undefined + const retryAfter = Headers.parseRetryAfter(raw) + return Result.isSuccess(retryAfter) ? Headers.retryAfterDelay(retryAfter.success, now) : undefined } const fail = (reason: AiError.AiErrorReason) => @@ -57,7 +58,7 @@ export const mapResponseError = Effect.fnUntraced( if (status === 429) { return yield* fail( new AiError.RateLimitError({ - retryAfter: retryAfter(response.headers["retry-after"]), + retryAfter: retryAfter(response.headers["retry-after"], yield* Clock.currentTimeMillis), http, metadata: { cloudflare: { rayId: response.headers["cf-ray"] ?? null, errors: details?.errors ?? [] } } }) diff --git a/repos/effect/packages/ai/cloudflare/test/CloudflareClient.test.ts b/repos/effect/packages/ai/cloudflare/test/CloudflareClient.test.ts index daef1a985f..b1fe28db19 100644 --- a/repos/effect/packages/ai/cloudflare/test/CloudflareClient.test.ts +++ b/repos/effect/packages/ai/cloudflare/test/CloudflareClient.test.ts @@ -31,8 +31,11 @@ const httpLayer = ( ) ) -const clientLayer = (response: (request: HttpClientRequest.HttpClientRequest) => Response) => - CloudflareClient.layer({ accountId: "test-account", apiKey: Redacted.make("test-token") }).pipe( +const clientLayer = ( + response: (request: HttpClientRequest.HttpClientRequest) => Response, + options?: Partial +) => + CloudflareClient.layer({ accountId: "test-account", apiKey: Redacted.make("test-token"), ...options }).pipe( Layer.provide(httpLayer((request) => Effect.succeed(HttpClientResponse.fromWeb(request, response(request))))) ) @@ -53,6 +56,7 @@ describe("CloudflareClient", () => { "https://api.cloudflare.com/client/v4/accounts/test-account/ai/run/@cf/cloudflare/clef-flash" ) assert.strictEqual(request.headers.authorization, "Bearer test-token") + assert.strictEqual(request.headers["cf-aig-gateway-id"], undefined) assert.strictEqual(request.body._tag, "Uint8Array") if (request.body._tag === "Uint8Array") { assert.deepStrictEqual(JSON.parse(new TextDecoder().decode(request.body.body)), payload) @@ -60,7 +64,15 @@ describe("CloudflareClient", () => { return Response.json({ success: true, result, errors: [], messages: [] }) })))) - it.effect("layerConfig reads Cloudflare credentials from the environment", () => + it.effect("routes requests through the configured AI Gateway", () => + createDecisions.pipe( + Effect.provide(clientLayer((request) => { + assert.strictEqual(request.headers["cf-aig-gateway-id"], "test-gateway") + return Response.json({ success: true, result }) + }, { gatewayId: "test-gateway" })) + )) + + it.effect("layerConfig reads Cloudflare credentials and AI Gateway configuration from the environment", () => createDecisions.pipe( Effect.provide(CloudflareClient.layerConfig({ apiUrl: Config.String("TEST_API_URL"), @@ -70,6 +82,7 @@ describe("CloudflareClient", () => { env: { CLOUDFLARE_ACCOUNT_ID: "configured-account", CLOUDFLARE_API_TOKEN: "configured-token", + CLOUDFLARE_AI_GATEWAY_ID: "configured-gateway", TEST_API_URL: "https://example.com/api" } }))), @@ -79,6 +92,7 @@ describe("CloudflareClient", () => { "https://example.com/api/accounts/configured-account/ai/run/@cf/cloudflare/clef-flash" ) assert.strictEqual(request.headers.authorization, "Bearer configured-token") + assert.strictEqual(request.headers["cf-aig-gateway-id"], "configured-gateway") assert.strictEqual(request.headers["x-test"], "configured") return Effect.succeed(HttpClientResponse.fromWeb(request, Response.json({ success: true, result }))) })) @@ -148,10 +162,10 @@ describe("CloudflareClient", () => { } }).pipe(Effect.provide(clientLayer(() => new Response("Bad gateway", { status: 502 }))))) - it.effect("maps a success body without a result envelope to InvalidOutputError", () => + it.effect("maps a success body without a result envelope to a non-retryable InvalidResponseError", () => Effect.gen(function*() { const error = yield* Effect.flip(createDecisions) - assert.strictEqual(error.reason._tag, "InvalidOutputError") + assert.strictEqual(error.reason._tag, "InvalidResponseError") }).pipe(Effect.provide(clientLayer(() => Response.json(result))))) it.effect("maps TransportError to NetworkError", () => diff --git a/repos/effect/packages/ai/openai-compat/CHANGELOG.md b/repos/effect/packages/ai/openai-compat/CHANGELOG.md index 915ba4c40a..6bac3e05fa 100644 --- a/repos/effect/packages/ai/openai-compat/CHANGELOG.md +++ b/repos/effect/packages/ai/openai-compat/CHANGELOG.md @@ -1,5 +1,27 @@ # @effect/ai-openai-compat +## 4.0.6 + +### Patch Changes + +- [#9552](https://github.com/Effect-TS/effect/pull/9552) [`44c8c1d`](https://github.com/Effect-TS/effect/commit/44c8c1d1c6f01ba9887a051b365e9fa89fcfc685) Thanks @wmaurer! - Add an optional `batchKey` to `EmbeddingModel.make` to group concurrent `embed` calls by provider-relevant context. OpenAI and OpenAI-compatible providers now separate batches with different configuration overrides or client transforms. Undeclared context, such as per-call `FetchHttpClient.RequestInit`, still uses the first caller's value. + +- [#9709](https://github.com/Effect-TS/effect/pull/9709) [`85e2aee`](https://github.com/Effect-TS/effect/commit/85e2aeee686963c26ca40d204f982deebb9d6117) Thanks @tim-smart! - Add typed HTTP date and `Retry-After` header values to `Headers`, and parse them consistently across the HTTP client, OTLP exporter, GraphQL client, AI clients and static server. + + - `Headers.parseHttpDate` / `Headers.formatHttpDate` read all three RFC 9110 date formats as UTC, whatever the process time zone, and emit IMF-fixdate. + - `Headers.parseRetryAfter`, `Headers.formatRetryAfter` and `Headers.retryAfterDelay` accept whole non-negative seconds of any size, read exactly, or an HTTP date, and the delay is never negative. HTTP dates accept the `23:59:60` leap second, read as the following second. + - `Headers.RetryAfter` holds either a `Duration` (`RetryAfterDuration`) or a `DateTime.Utc` (`RetryAfterDateTime`). `Headers.retryAfterDuration` builds the first from any `Duration.Input`: infinite durations throw, finite negative ones become zero, and fractional seconds are rounded up when formatted. + - Parse failures are reported as `Headers.HeaderParseError`. + - `Schema.DateTimeUtcFromHttpDate` and `Schema.RetryAfterFromString` expose the same parsing as string codecs for HTTP API headers. + - Invalid `Retry-After` values such as `-5`, `1.5`, `1e3` or `Infinity` are now ignored everywhere. `HttpClient.retryTransient` and `HttpClient.withRateLimiter` no longer honour fractional seconds, and the AI clients no longer produce negative or infinite `retryAfter` durations. + - `HttpStaticServer` reads `If-Modified-Since` and `Last-Modified` as UTC; asctime dates were previously shifted by the process time zone. + +- [#9430](https://github.com/Effect-TS/effect/pull/9430) [`48102ac`](https://github.com/Effect-TS/effect/commit/48102ac592ef89fed9d5d10afd2daece0a0ee310) Thanks @effect-bot! - Add non-retryable `AiError.InvalidResponseError` for provider response schema and SSE decoding failures, with optional HTTP context; model-generated output validation remains retryable. Update exhaustive matches on `AiErrorReason` to handle the new tag, and update serialized-error consumers to accept it. + +- [#9242](https://github.com/Effect-TS/effect/pull/9242) [`12b2726`](https://github.com/Effect-TS/effect/commit/12b2726651a6d6eeb2fd66539d34c1401f9afd94) Thanks @effect-bot! - Consistently nest authentication, internal provider, and unknown error metadata under the `openai` or `openai-compat` provider key, including error codes, error types, and request IDs. +- Updated dependencies [[`ce13a48`](https://github.com/Effect-TS/effect/commit/ce13a4872e516187851c5d9336d4df309cc8f0a4), [`f3bbbb4`](https://github.com/Effect-TS/effect/commit/f3bbbb448d431a9d1205ff09ba47cb81af0c7199), [`5547315`](https://github.com/Effect-TS/effect/commit/554731500e43eef47ff68b569116b74006e91d87), [`6093cfb`](https://github.com/Effect-TS/effect/commit/6093cfbbb673d36ecd18edea955763db003c8c26), [`30319d5`](https://github.com/Effect-TS/effect/commit/30319d506c9177d1cc1e0b10372e1ddb776aaf0a), [`74a7596`](https://github.com/Effect-TS/effect/commit/74a759699a8582dbe660b21dcda21cc9db08f613), [`bfb32a3`](https://github.com/Effect-TS/effect/commit/bfb32a3cf5f8d0f02f0dfdb63dcf5237fde755b0), [`f8bf7f4`](https://github.com/Effect-TS/effect/commit/f8bf7f4bcc3ad543cd94cbeb010cb164e5dedcb0), [`7f3d3df`](https://github.com/Effect-TS/effect/commit/7f3d3dfbb7b9e6e041c4ad9654c4532f72d2f16e), [`db1046d`](https://github.com/Effect-TS/effect/commit/db1046d1fa0e4b5818c51d28017a89579b6261d6), [`3c4f88a`](https://github.com/Effect-TS/effect/commit/3c4f88aa4d1d4d958b42fe8d3f1195f0ca990f8a), [`916e709`](https://github.com/Effect-TS/effect/commit/916e70968acc4e443885c67012d855b7cc6f9bca), [`916f843`](https://github.com/Effect-TS/effect/commit/916f8436225153a81a049bdc1feaedcafe4d1ded), [`66ec757`](https://github.com/Effect-TS/effect/commit/66ec7574d23b7943584059b41136bc21ee9d4c53), [`daac06f`](https://github.com/Effect-TS/effect/commit/daac06f79a6ab8c7d3e1b66b427c06f7d64dcc8f), [`c2552ec`](https://github.com/Effect-TS/effect/commit/c2552ec2e031b409b2bef67309312b0168480a39), [`b02c1cd`](https://github.com/Effect-TS/effect/commit/b02c1cde36c890941a235224af8ecb035197582a), [`1855f1d`](https://github.com/Effect-TS/effect/commit/1855f1d359535339bc7640b67e6bb7b12752ba19), [`683ed2c`](https://github.com/Effect-TS/effect/commit/683ed2cb491f9529379c1d5c63933689ff57fd2b), [`ac68940`](https://github.com/Effect-TS/effect/commit/ac68940f7e4246b1beb43f9d451a67f41241ab82), [`22a8401`](https://github.com/Effect-TS/effect/commit/22a84017e7222f332611c1bace05f9fa60c7cf67), [`c53c194`](https://github.com/Effect-TS/effect/commit/c53c19451171a00776b400a8548394ac514b628a), [`ca6d1cc`](https://github.com/Effect-TS/effect/commit/ca6d1cc5f803cd501e85431642b79b552322a76a), [`47940c5`](https://github.com/Effect-TS/effect/commit/47940c55c9e0eb855615094a2aca6e0812307bb2), [`e8b24ef`](https://github.com/Effect-TS/effect/commit/e8b24ef9516706971db948efb2d3eab84ad85951), [`2e8aae0`](https://github.com/Effect-TS/effect/commit/2e8aae0080a8d3e6d00c5ee0bc4f49f0b074acc2), [`ad7497b`](https://github.com/Effect-TS/effect/commit/ad7497bbc1bc70935f8f1ae395ff501532e2e770), [`7f504a3`](https://github.com/Effect-TS/effect/commit/7f504a35edcf44952c55ecbb0916726bac4d3e52), [`99e365d`](https://github.com/Effect-TS/effect/commit/99e365d61a1fdaabeb31dc58e5752cade8a09e1f), [`954067d`](https://github.com/Effect-TS/effect/commit/954067da7fbd20a20c10d5d0de6ffe4690e6bdfd), [`3e1a035`](https://github.com/Effect-TS/effect/commit/3e1a035d9c470ced858b7757f6d102b3aa9be008), [`41a1fe7`](https://github.com/Effect-TS/effect/commit/41a1fe74e6338e8d13d6d4f3daa03f71a826c2c4), [`bcb3a86`](https://github.com/Effect-TS/effect/commit/bcb3a86335a8ed18da39300984c0fbae187fdc6b), [`739d3d1`](https://github.com/Effect-TS/effect/commit/739d3d18728cce0b6d907f0a3f24e04ab0b53560), [`ed4da35`](https://github.com/Effect-TS/effect/commit/ed4da351fb9d29e4dfd6c60bd89ba4c007d264dc), [`3d84e19`](https://github.com/Effect-TS/effect/commit/3d84e19f8f14bccf0aa57b11825590c3bae162f4), [`cc00906`](https://github.com/Effect-TS/effect/commit/cc0090657eeb89debdd54af5fcb99d544f46c1bf), [`09ba1f5`](https://github.com/Effect-TS/effect/commit/09ba1f59952c46cd22229d41236742c3e1f53a34), [`9ff1a73`](https://github.com/Effect-TS/effect/commit/9ff1a73d995702f4c4c335888fe6a186f3f710c7), [`06bc532`](https://github.com/Effect-TS/effect/commit/06bc532b71cd3684f010956774356864dd359372), [`97bfd84`](https://github.com/Effect-TS/effect/commit/97bfd84bb147e10ed562e4c1ffa589cdaf83a596), [`25efaef`](https://github.com/Effect-TS/effect/commit/25efaef0d81d25b2651ef4de474e929d3180810a), [`c10335b`](https://github.com/Effect-TS/effect/commit/c10335be125c7846ae7b914eeb138244b660275b), [`a808972`](https://github.com/Effect-TS/effect/commit/a80897262648e4222ce48943e538f39a6b3f69d6), [`20e844c`](https://github.com/Effect-TS/effect/commit/20e844c7ef27b0e4b4f7911d603c45b1de2b0bc3), [`b9f73f3`](https://github.com/Effect-TS/effect/commit/b9f73f37c4ac538e6a327ed80ac5a0708858c540), [`5739878`](https://github.com/Effect-TS/effect/commit/5739878ded09d83d71d657e3b8d4264f4e4936d9), [`28acdf3`](https://github.com/Effect-TS/effect/commit/28acdf3820b66056037bb7a24cd0e8d3de3fb06d), [`59b6396`](https://github.com/Effect-TS/effect/commit/59b63967c1817ae5acb1811037f5aaa964219b7e), [`0bccc9c`](https://github.com/Effect-TS/effect/commit/0bccc9c3dd5349b20f8d2e509ac5482d0c26a616), [`fed70de`](https://github.com/Effect-TS/effect/commit/fed70de0dff8af24c5f814bd0d985ab70c70420b), [`82648a2`](https://github.com/Effect-TS/effect/commit/82648a21bf228145a14d2d1390006d67f74d89c7), [`afb5f32`](https://github.com/Effect-TS/effect/commit/afb5f326cc14c9849d8d7ddefde1039ae036e017), [`28813f5`](https://github.com/Effect-TS/effect/commit/28813f53a5026984fdc13d3971147ea5dc20104c), [`0fae934`](https://github.com/Effect-TS/effect/commit/0fae934a13aac12f9036b0733edd5dde5ce029f3), [`f260830`](https://github.com/Effect-TS/effect/commit/f2608305861269fc2fbedcbbdcd71e4fc5f98ffc), [`19b63eb`](https://github.com/Effect-TS/effect/commit/19b63eb81b2bccf033aadb565f52a6b0375d26a1), [`14db221`](https://github.com/Effect-TS/effect/commit/14db221f8f584100ff50fd1173b1a99a1caa1d06), [`679f956`](https://github.com/Effect-TS/effect/commit/679f956c4a7527854ff78db071dd7aa417211a48), [`b72707c`](https://github.com/Effect-TS/effect/commit/b72707c2937328fd3bc70f29a6792d84c4256484), [`44c8c1d`](https://github.com/Effect-TS/effect/commit/44c8c1d1c6f01ba9887a051b365e9fa89fcfc685), [`1e7909b`](https://github.com/Effect-TS/effect/commit/1e7909beb05bc740dbd330509e60bb185d7eeed1), [`0611a65`](https://github.com/Effect-TS/effect/commit/0611a657f76836f9ca275f64764b82270b4eda10), [`2fadc65`](https://github.com/Effect-TS/effect/commit/2fadc654aa655fbd1912ce51eb1b764a757ac362), [`feb6ce5`](https://github.com/Effect-TS/effect/commit/feb6ce59a38ab15fc6f89545bd5f0992dd58a06a), [`5a2ed53`](https://github.com/Effect-TS/effect/commit/5a2ed5323884a435bfcfb5e3a6f2ae29547458ad), [`c8d26a4`](https://github.com/Effect-TS/effect/commit/c8d26a46731cb26a9b9612d9fde823d8813cfc04), [`0c8b446`](https://github.com/Effect-TS/effect/commit/0c8b446709c737c10d0e11af4313bb29fdff2588), [`05c3409`](https://github.com/Effect-TS/effect/commit/05c340955944374e3ae47a1de0f7cb9b41a3b0e6), [`b04191b`](https://github.com/Effect-TS/effect/commit/b04191b2941534754282c51ef2df2f965ac43a5c), [`05c3409`](https://github.com/Effect-TS/effect/commit/05c340955944374e3ae47a1de0f7cb9b41a3b0e6), [`4067e64`](https://github.com/Effect-TS/effect/commit/4067e6428776ae573c8cc72f7eaebd078a4ff8c7), [`e04563d`](https://github.com/Effect-TS/effect/commit/e04563dcd6ddeea74c088a12194c5a73e597ba2f), [`02d7fc5`](https://github.com/Effect-TS/effect/commit/02d7fc583053c47dea3b5e1d9d6be164aeb321b6), [`561d2a1`](https://github.com/Effect-TS/effect/commit/561d2a1ddfa3c4340510f23da7adc50c89d97863), [`8fe2dcc`](https://github.com/Effect-TS/effect/commit/8fe2dcc2137788a71177d42a9f67fb7003224868), [`a8086fa`](https://github.com/Effect-TS/effect/commit/a8086fa26a2c9a9f5f7f1d7590bbc494a0feb02a), [`88a33b4`](https://github.com/Effect-TS/effect/commit/88a33b4122f7ffbbc1423add793a79abffd99c75), [`e77372a`](https://github.com/Effect-TS/effect/commit/e77372a8a2e7caec11999636e04c6e31c52e315f), [`3dd4388`](https://github.com/Effect-TS/effect/commit/3dd438810df34ed3b3d2e3e35ccf84ccb00feb85), [`ba2b00c`](https://github.com/Effect-TS/effect/commit/ba2b00c0658b65775809e0052ec9612432600c31), [`a049d4a`](https://github.com/Effect-TS/effect/commit/a049d4ad25f17cf4985284a8d15b07aea2a33242), [`84aeede`](https://github.com/Effect-TS/effect/commit/84aeede040460a2b24bcf0db7a5d0a9b0e3c42d4), [`43e5e19`](https://github.com/Effect-TS/effect/commit/43e5e198d16df038b60764f5dd0218cdcf828f16), [`77bbf16`](https://github.com/Effect-TS/effect/commit/77bbf16e50d6b0ea090288cc5d3f11b20706a027), [`6af3a47`](https://github.com/Effect-TS/effect/commit/6af3a47d5b5d698c3ba896c37684266cd465bef1), [`aac8d21`](https://github.com/Effect-TS/effect/commit/aac8d21911333dedb649f4f9d302c28d14d46369), [`84c8dc4`](https://github.com/Effect-TS/effect/commit/84c8dc47b09067362749620f75068d6f28044660), [`78d1988`](https://github.com/Effect-TS/effect/commit/78d198818d4fd547ce2e9431302f66cb00dc7ad3), [`5676a2a`](https://github.com/Effect-TS/effect/commit/5676a2ac5a918f8f91fc7838d7adf1b6f5e86a6d), [`4b437bf`](https://github.com/Effect-TS/effect/commit/4b437bf4645fabc8aaeaa3b5a88e8c9048b9e4f5), [`1af11fd`](https://github.com/Effect-TS/effect/commit/1af11fda1bccf823f74121b5125aa3befa084b18), [`716f8df`](https://github.com/Effect-TS/effect/commit/716f8dfa178f849e5b288464373bd132860c2e19), [`ee27718`](https://github.com/Effect-TS/effect/commit/ee27718a02125328a7478812b1b7d7c1d8532578), [`4d86d1b`](https://github.com/Effect-TS/effect/commit/4d86d1b8887d48fa24acf6ad54f33ef3d55a7cd8), [`5462ed7`](https://github.com/Effect-TS/effect/commit/5462ed7b3a4c37206c47e401d6be293dc540350f), [`658837c`](https://github.com/Effect-TS/effect/commit/658837c3b249d982df56f2990247499746e36672), [`00261cc`](https://github.com/Effect-TS/effect/commit/00261cc12c31ecc7878f80670439731c905dc2bd), [`7cccb93`](https://github.com/Effect-TS/effect/commit/7cccb9385729bc0aed6630edea95d7db15439ac4), [`7c98228`](https://github.com/Effect-TS/effect/commit/7c982281cf73958833054175fc86db385e043cf6), [`5a8e248`](https://github.com/Effect-TS/effect/commit/5a8e2481546352e2ad6233dad49420b14c59f624), [`8037a65`](https://github.com/Effect-TS/effect/commit/8037a65f40cae9163ab9e0e687e289f717ae570b), [`25912db`](https://github.com/Effect-TS/effect/commit/25912db9d493227a2d6fa45cebcfc7507e638d8a), [`7f32606`](https://github.com/Effect-TS/effect/commit/7f32606baac43a67d171ae93b77c7b29791c9692), [`0386f88`](https://github.com/Effect-TS/effect/commit/0386f88295ac70cbc1f90ab8cceb0981d848c628), [`403298a`](https://github.com/Effect-TS/effect/commit/403298ac457fb634eca2c013fed26d37fc80445f), [`5a3ebc0`](https://github.com/Effect-TS/effect/commit/5a3ebc082124f9e10106b2347736e66fc74ebc36), [`fac2191`](https://github.com/Effect-TS/effect/commit/fac21911964b742a38e1d4e73bd4f95aec4911dc), [`fdb12dc`](https://github.com/Effect-TS/effect/commit/fdb12dcd61b6505eed02e6f7a3c88a85d4c48e8d), [`97909b8`](https://github.com/Effect-TS/effect/commit/97909b89b167da54a56e6f82f1c4db8bfedae04f), [`c3983f4`](https://github.com/Effect-TS/effect/commit/c3983f4db2b3052a58e8045cce270fd7ccc4e2b2), [`3a5e65e`](https://github.com/Effect-TS/effect/commit/3a5e65e409c9569e23bb7af61df401347e9ec420), [`8b40fbc`](https://github.com/Effect-TS/effect/commit/8b40fbc93caf52f3a4c69020849f0051f100be65), [`3f3e119`](https://github.com/Effect-TS/effect/commit/3f3e119ad61c2167bc6c7170c3fb68cd1eb3421e), [`7c7b8ed`](https://github.com/Effect-TS/effect/commit/7c7b8eda54e44f9cddc1188f994f7fdddf901055), [`d4f71bc`](https://github.com/Effect-TS/effect/commit/d4f71bc42507b638d149fb8806d26ea626d17956), [`c66241e`](https://github.com/Effect-TS/effect/commit/c66241e791c4b51d8a65051142d01682590f9bf8), [`f0e941e`](https://github.com/Effect-TS/effect/commit/f0e941ee2f90f0a891d1228a90da91386f0258cb), [`5f8a6e5`](https://github.com/Effect-TS/effect/commit/5f8a6e5108aa6c6cd4043a3c9f4dc93d43369623), [`2800a09`](https://github.com/Effect-TS/effect/commit/2800a0910f7f4ad7cfa21dfc840edf8fe6e67ca7), [`e9749b0`](https://github.com/Effect-TS/effect/commit/e9749b04c10badabcdf71e6958de5ac56a5316e9), [`eaef09c`](https://github.com/Effect-TS/effect/commit/eaef09c11b6ed2029c513eea0aa3256dcc6f517f), [`a3c56e5`](https://github.com/Effect-TS/effect/commit/a3c56e5b5e756e879f1cfb90230e8eae620e33df), [`bb61152`](https://github.com/Effect-TS/effect/commit/bb611522fcfb4d317b33acdf5ee24a480d042435), [`e9c842b`](https://github.com/Effect-TS/effect/commit/e9c842bc294abf658d698bb17732ae13465b3817), [`6d8c8e2`](https://github.com/Effect-TS/effect/commit/6d8c8e2075b15ae84ae96e93a0d295cb9221b496), [`1f28187`](https://github.com/Effect-TS/effect/commit/1f281878ef559ef356af2d0e11b381aba857cae1), [`0b9842b`](https://github.com/Effect-TS/effect/commit/0b9842b76bbc1b016c8abc46432052e1f60b2fa3), [`c29ddd8`](https://github.com/Effect-TS/effect/commit/c29ddd899c53c5d8194c4a0dfef2134304dda799), [`84c3405`](https://github.com/Effect-TS/effect/commit/84c34057ceab9fe6782b73f4a04a31304e0d14de), [`3934c0c`](https://github.com/Effect-TS/effect/commit/3934c0c61857c03468c08c2477f49a6b977d28ff), [`048e7c8`](https://github.com/Effect-TS/effect/commit/048e7c87125c4ace5a61895feac3d049a317bbfc), [`1121334`](https://github.com/Effect-TS/effect/commit/1121334ce68c379051011a2ffea37b0a30ea9a47), [`9a680d6`](https://github.com/Effect-TS/effect/commit/9a680d6985c6a27d7ebb29cd9eb674f3fe524505), [`85e2aee`](https://github.com/Effect-TS/effect/commit/85e2aeee686963c26ca40d204f982deebb9d6117), [`34ec440`](https://github.com/Effect-TS/effect/commit/34ec4407a14ce045ca14cc76c77582884a7e1ebe), [`343a149`](https://github.com/Effect-TS/effect/commit/343a149fa7d5a9a0a45841b1b6800b0ecfa16a32), [`915117b`](https://github.com/Effect-TS/effect/commit/915117b523cb6e5cbc98b5ff1c119a8d0d1f6b9a), [`7f823c1`](https://github.com/Effect-TS/effect/commit/7f823c10d20ef53a1081e0d0e2008a4116140799), [`6e614e6`](https://github.com/Effect-TS/effect/commit/6e614e6feb4281a9e81f5ec3be48e7905ff4292c), [`9990b39`](https://github.com/Effect-TS/effect/commit/9990b3977a544247cd4123edde8353a55bf77192), [`22bf781`](https://github.com/Effect-TS/effect/commit/22bf7812aa8a20c676efade943c342cde4ef1cbd), [`41f08b7`](https://github.com/Effect-TS/effect/commit/41f08b75aba40e95cd6c743ec6fcf1f5022f7b5b), [`e8482ca`](https://github.com/Effect-TS/effect/commit/e8482cad9651fa8fe8036fe6292a286994897d0d), [`5ae85fa`](https://github.com/Effect-TS/effect/commit/5ae85fa8ac1fc5e12d5fe8fe7388ac504788ae3b), [`c477a0a`](https://github.com/Effect-TS/effect/commit/c477a0a8010bd0a988221bc00c446848314e1c3c), [`8fbb39e`](https://github.com/Effect-TS/effect/commit/8fbb39eb92577915831f19a83b51be376e844613), [`1c70828`](https://github.com/Effect-TS/effect/commit/1c7082847453e9c2be0cc267a39ca554c52f4013), [`8207d1e`](https://github.com/Effect-TS/effect/commit/8207d1e91ead8aac8f01d9e13b49624b64237c8b), [`7886b84`](https://github.com/Effect-TS/effect/commit/7886b847658664f64392a14efda86f44b3d1e170), [`4839c60`](https://github.com/Effect-TS/effect/commit/4839c601bdda845d9a54d77f578f64372475ab04), [`2595d83`](https://github.com/Effect-TS/effect/commit/2595d831f702d4a748f97e33e71fd5250f58af56), [`f5ae278`](https://github.com/Effect-TS/effect/commit/f5ae27827a82db3f8973b70745fc6f14e7d8fa80), [`7a8ce00`](https://github.com/Effect-TS/effect/commit/7a8ce0098a5f22cbb6c110b16e850fb3fcbcadf2), [`688d302`](https://github.com/Effect-TS/effect/commit/688d30275ff114e50b4f0d343c6227ee33d67348), [`c7072e4`](https://github.com/Effect-TS/effect/commit/c7072e46356a19615625ae0fa569b1c3a79d50be), [`425cedb`](https://github.com/Effect-TS/effect/commit/425cedb5801c9527c116a64d280d9e7c2820f922), [`88bc680`](https://github.com/Effect-TS/effect/commit/88bc68001c6c70984b03a5500232e9b5aab14085), [`268e09c`](https://github.com/Effect-TS/effect/commit/268e09c94c9e8f46142b5c9f29ce0a22bfbbc3aa), [`7e94505`](https://github.com/Effect-TS/effect/commit/7e94505edfc9d81ea08cb96a7d935806495bf61e), [`afa5da3`](https://github.com/Effect-TS/effect/commit/afa5da3c8a9848215e49535dd3d39179bfd67715), [`c29ddd8`](https://github.com/Effect-TS/effect/commit/c29ddd899c53c5d8194c4a0dfef2134304dda799), [`48102ac`](https://github.com/Effect-TS/effect/commit/48102ac592ef89fed9d5d10afd2daece0a0ee310), [`463eca3`](https://github.com/Effect-TS/effect/commit/463eca3609a6a9681247abbcc71a6f10b261a1d9), [`e7162ea`](https://github.com/Effect-TS/effect/commit/e7162eae7f04af1d583b1f31fdd767f91ec1c82e), [`95752db`](https://github.com/Effect-TS/effect/commit/95752db5eb3b6e59fb8749e9792704f8b167f36c), [`4577a28`](https://github.com/Effect-TS/effect/commit/4577a28bcdb50b0d184775f8c1b7ad890d86e255), [`973d599`](https://github.com/Effect-TS/effect/commit/973d599a3e8611626764a0e3e8f79edbcf30dc7c), [`2f066a5`](https://github.com/Effect-TS/effect/commit/2f066a58e90316dd88a219fcea94200beda54219), [`f0ccbb6`](https://github.com/Effect-TS/effect/commit/f0ccbb6917be5c08364b0b8adbd0c4c19e7f2de1), [`2dda6ce`](https://github.com/Effect-TS/effect/commit/2dda6cefc641dbea2ea3279a9b96c0332b277a40), [`3da7c0a`](https://github.com/Effect-TS/effect/commit/3da7c0aabab07c8fc682aea7559536bf31705853), [`5a67e0b`](https://github.com/Effect-TS/effect/commit/5a67e0b3a8118dd7be709a56d6d7af4f10e9c4f3), [`73b3a6e`](https://github.com/Effect-TS/effect/commit/73b3a6e3c38795400251d9c5d73f05c88acd90cd), [`c6f9e4c`](https://github.com/Effect-TS/effect/commit/c6f9e4cbe6f9c8f0a03fefde18df4d35e98a24df), [`be02676`](https://github.com/Effect-TS/effect/commit/be026768ee29a1b1a860c1fd17f2a0896c01f61e), [`26de861`](https://github.com/Effect-TS/effect/commit/26de86100cda8617e077ce66f0a6bb8dfe738fb6), [`9313799`](https://github.com/Effect-TS/effect/commit/931379911a74d5e2207859c295b1acc6a5e2917e), [`d5c1e5b`](https://github.com/Effect-TS/effect/commit/d5c1e5bd425dfe983dff1f8c853c0b7eaa772f87), [`11bb31f`](https://github.com/Effect-TS/effect/commit/11bb31fec0bf2fdfc025009a04de4c69bffa433a), [`8c23e46`](https://github.com/Effect-TS/effect/commit/8c23e46a8280e3f2c831d1c0ca3f106c92f8e071), [`9c96011`](https://github.com/Effect-TS/effect/commit/9c960111de202c0f1dd6989e5b8836150e0ca694), [`0d50570`](https://github.com/Effect-TS/effect/commit/0d50570bbf078804092fcd4a47ccd2d0019f989c), [`9ed2bf1`](https://github.com/Effect-TS/effect/commit/9ed2bf18da0e1ff4fecc31b7cbac2976ec999e00), [`76ec912`](https://github.com/Effect-TS/effect/commit/76ec9120f43f4fdb08b9c5170ff9e3fd82f22457), [`af2756d`](https://github.com/Effect-TS/effect/commit/af2756dd1d196aa3f199ce21c6d2dc3ada3b67b5), [`3ec8100`](https://github.com/Effect-TS/effect/commit/3ec8100c793b8a22381ad115106b93824d840f64), [`93abf02`](https://github.com/Effect-TS/effect/commit/93abf020876af0a6b6b8b25621c80760b7996762), [`1e94993`](https://github.com/Effect-TS/effect/commit/1e949935a1065e5140245a514ce3a799061143e3), [`99a4616`](https://github.com/Effect-TS/effect/commit/99a4616264e31de65fa8d3dc19576309b1a3f84a), [`0a7978e`](https://github.com/Effect-TS/effect/commit/0a7978ea8e70e51700345b12f8c19952dde05255), [`b4a060b`](https://github.com/Effect-TS/effect/commit/b4a060b95a6473d54e277dc005172b54aab9bddf), [`274f4a4`](https://github.com/Effect-TS/effect/commit/274f4a4dac5ee2d9dbb6358cc3dd3afe345d4705), [`25f29be`](https://github.com/Effect-TS/effect/commit/25f29be77bdcd5c2a7be8e2710e88f5a10c4cb8d), [`9036440`](https://github.com/Effect-TS/effect/commit/9036440edca01c24371d0c2c1044be5e5d3cb65e), [`415f005`](https://github.com/Effect-TS/effect/commit/415f0053eb848d562febcb5b011e15f31a99bead), [`191308b`](https://github.com/Effect-TS/effect/commit/191308bd0f5ee04557570f2f960c6e06c429bcfd), [`4b96570`](https://github.com/Effect-TS/effect/commit/4b96570ed28d93b14e958742933d227dc5e74b15), [`d84ec3f`](https://github.com/Effect-TS/effect/commit/d84ec3fab0346c01bc59f6624d1c046ca4140087), [`392aa1c`](https://github.com/Effect-TS/effect/commit/392aa1cb275ac9fdb9b4b1831d8c06196bba54b9), [`99e9bd9`](https://github.com/Effect-TS/effect/commit/99e9bd98a354d83202e2522f3ade9ee4bf4a862c), [`6f6f45f`](https://github.com/Effect-TS/effect/commit/6f6f45f480a3f191aed6a82ca5c301f60ef2cf84), [`13fc07e`](https://github.com/Effect-TS/effect/commit/13fc07edeb939ca6229b858c9d9a8c5243ac21b5), [`a4b38dd`](https://github.com/Effect-TS/effect/commit/a4b38dde7f3fdc1b54a49bffdaca912b4c47c76e), [`8b18580`](https://github.com/Effect-TS/effect/commit/8b185807bb99f6de3d9564447e2f4e08d2f8863d), [`0c477fe`](https://github.com/Effect-TS/effect/commit/0c477fe417d3ecab86bb8d9afd0c392e9cb8a3da), [`9a5331d`](https://github.com/Effect-TS/effect/commit/9a5331db94bfdca4f48317d062fbeb96054a9049), [`a706d73`](https://github.com/Effect-TS/effect/commit/a706d73db8a8d460877c5d7e8a793a52d45be307), [`c9a1e72`](https://github.com/Effect-TS/effect/commit/c9a1e72c5965aa9fe01a669ebfd523f803f5a189), [`89eaaf8`](https://github.com/Effect-TS/effect/commit/89eaaf8581908e248c0b1e47baa7906e0c541635), [`8215811`](https://github.com/Effect-TS/effect/commit/8215811d51d28d37b4732a6ab03de4047ada62ea), [`ad69018`](https://github.com/Effect-TS/effect/commit/ad6901857e05e7b436a0149b5ece7f3a36d9ff33), [`a733eba`](https://github.com/Effect-TS/effect/commit/a733eba9f699737a80a9e7276521980bd929075a), [`7282d5f`](https://github.com/Effect-TS/effect/commit/7282d5f1bcc06d7065429ef81a3d17c64043fde4), [`da33572`](https://github.com/Effect-TS/effect/commit/da33572776763e45802ce8c2fe14ec101f9a5c34), [`d9e2d12`](https://github.com/Effect-TS/effect/commit/d9e2d1221768e9a3e5e00165cb95b86e0c126cc0), [`dc923b8`](https://github.com/Effect-TS/effect/commit/dc923b847ac7d9673b4f0aa7c136557287b2e5e3), [`1b74086`](https://github.com/Effect-TS/effect/commit/1b740862e3d25ec5d4b6f86b9fe792b6dabbf84c), [`8a5507b`](https://github.com/Effect-TS/effect/commit/8a5507ba00f641edf71227d79b9f90efdb634fe2), [`23460fa`](https://github.com/Effect-TS/effect/commit/23460fad704c730926bffe3cc6874268ef6680ec), [`0dbbea6`](https://github.com/Effect-TS/effect/commit/0dbbea65a93c59147a4bc407cfcda38ea26341b2), [`70aad3f`](https://github.com/Effect-TS/effect/commit/70aad3ffe617984f5b599aeed1bc7f04899aca55), [`0909236`](https://github.com/Effect-TS/effect/commit/090923611b209a19f3281a7b1d0c15a9b653a0c7), [`a9e66d8`](https://github.com/Effect-TS/effect/commit/a9e66d889017b302355ba5b8bc2b2050ca1ea5aa), [`ee5c0d5`](https://github.com/Effect-TS/effect/commit/ee5c0d5f43348747b04b5f7bf1b39355614c2f2c), [`631c5d6`](https://github.com/Effect-TS/effect/commit/631c5d6d9a81e6389964b878764fb9aea1538247), [`7e60c1a`](https://github.com/Effect-TS/effect/commit/7e60c1ae62a96c768f9c67d706f494213947f22f), [`4cddd3c`](https://github.com/Effect-TS/effect/commit/4cddd3c41e71fc4990d37c42b1bcf2ce3780b41d), [`8727e5d`](https://github.com/Effect-TS/effect/commit/8727e5d2bec8e2d729b7aeafb725a4ab21d533d2), [`2bc97fa`](https://github.com/Effect-TS/effect/commit/2bc97faa0e06b3973780dae26029e2c59f5dba38), [`f9a7f05`](https://github.com/Effect-TS/effect/commit/f9a7f056ba31aef435933acf082ec8c01ac7decc), [`d978cca`](https://github.com/Effect-TS/effect/commit/d978cca02914bec0ddf66bb2d68d90b220096fb6), [`b334ccf`](https://github.com/Effect-TS/effect/commit/b334ccf361a0f11362e7aabdbf70b35f53a39d61), [`0b50cfc`](https://github.com/Effect-TS/effect/commit/0b50cfc644313afe4c318073893231ec1946eb5f), [`5b7d2b5`](https://github.com/Effect-TS/effect/commit/5b7d2b5c90bc77eb9f8be1b91166c53f89aa1d0c), [`6ee26b1`](https://github.com/Effect-TS/effect/commit/6ee26b16b51b380c41dbfa84d612aa525b420fce), [`eab7e6a`](https://github.com/Effect-TS/effect/commit/eab7e6acfc7e422857565b98b9eddd3a98de1b92), [`e618f10`](https://github.com/Effect-TS/effect/commit/e618f1050f41424439eb82478fcf1c5da5a3244d), [`da14b1f`](https://github.com/Effect-TS/effect/commit/da14b1f18d71a18fc431b56d82cffe09aeb92708), [`3314fc3`](https://github.com/Effect-TS/effect/commit/3314fc30c22d5c9d737f3df7201f4a41424f701b), [`66db624`](https://github.com/Effect-TS/effect/commit/66db624670032a02144d42b6e64b83b324c813cd), [`4b8a50d`](https://github.com/Effect-TS/effect/commit/4b8a50db9649603f382f33640698020ca79547d0), [`8d52668`](https://github.com/Effect-TS/effect/commit/8d52668533e9d4a6d6b3be9fe58bd5e69fa80b65), [`1de1055`](https://github.com/Effect-TS/effect/commit/1de10559e7263747751bfb032fd4b0fd8aa28724), [`db130e6`](https://github.com/Effect-TS/effect/commit/db130e64557fafe370819d19f78e3b19a21275d6), [`5fa5fa5`](https://github.com/Effect-TS/effect/commit/5fa5fa5b9bff77352fb9ff2d424bf46bd6309986)]: + - effect@4.0.6 + ## 4.0.5 ### Patch Changes diff --git a/repos/effect/packages/ai/openai-compat/package.json b/repos/effect/packages/ai/openai-compat/package.json index c908e4b93d..209dc06547 100644 --- a/repos/effect/packages/ai/openai-compat/package.json +++ b/repos/effect/packages/ai/openai-compat/package.json @@ -1,6 +1,6 @@ { "name": "@effect/ai-openai-compat", - "version": "4.0.5", + "version": "4.0.6", "type": "module", "license": "MIT", "description": "An OpenAI compat integration for Effect", diff --git a/repos/effect/packages/ai/openai-compat/src/OpenAiEmbeddingModel.ts b/repos/effect/packages/ai/openai-compat/src/OpenAiEmbeddingModel.ts index d7cc366043..e7e24be114 100644 --- a/repos/effect/packages/ai/openai-compat/src/OpenAiEmbeddingModel.ts +++ b/repos/effect/packages/ai/openai-compat/src/OpenAiEmbeddingModel.ts @@ -18,6 +18,7 @@ import * as Layer from "effect/Layer" import type { Simplify } from "effect/Types" import type { CreateEmbedding200, CreateEmbeddingRequestJson } from "./OpenAiClient.ts" import { OpenAiClient } from "./OpenAiClient.ts" +import { OpenAiConfig } from "./OpenAiConfig.ts" /** * A model identifier accepted by an OpenAI-compatible embeddings endpoint. @@ -108,6 +109,10 @@ export const model = ( * * **Gotchas** * + * Concurrent `embed` calls share a request when their `Config` and + * `OpenAiConfig` values are equal. Other per-call context, such as + * `FetchHttpClient.RequestInit`, takes the first caller's value for the batch. + * * Provider responses must contain one numeric vector for every requested input * with unique, in-range `index` values; otherwise embedding operations fail with * `AiError.InvalidOutputError`. @@ -131,6 +136,7 @@ export const make = Effect.fnUntraced(function*({ model, config: providerConfig ) return yield* EmbeddingModel.make({ + batchKey: (context) => [Context.getOrUndefined(context, Config), Context.getOrUndefined(context, OpenAiConfig)], embedMany: Effect.fnUntraced(function*({ inputs }) { const config = yield* makeConfig const response = yield* client.createEmbedding({ ...config, input: inputs }) diff --git a/repos/effect/packages/ai/openai-compat/src/internal/errors.ts b/repos/effect/packages/ai/openai-compat/src/internal/errors.ts index b80a6b413b..11f73613fc 100644 --- a/repos/effect/packages/ai/openai-compat/src/internal/errors.ts +++ b/repos/effect/packages/ai/openai-compat/src/internal/errors.ts @@ -1,16 +1,19 @@ import * as AiError from "effect/ai/AiError" import type * as Response from "effect/ai/Response" import * as Arr from "effect/Array" -import * as Duration from "effect/Duration" +import * as Clock from "effect/Clock" +import type * as Duration from "effect/Duration" import * as Effect from "effect/Effect" import type * as Sse from "effect/encoding/Sse" import { dual } from "effect/Function" +import * as Headers from "effect/http/Headers" import type * as HttpClientError from "effect/http/HttpClientError" import type * as HttpClientRequest from "effect/http/HttpClientRequest" import type * as HttpClientResponse from "effect/http/HttpClientResponse" import * as Number from "effect/Number" import * as Option from "effect/Option" import * as Redactable from "effect/Redactable" +import * as Result from "effect/Result" import * as Schema from "effect/Schema" import * as SchemaTransformation from "effect/SchemaTransformation" import * as String from "effect/String" @@ -47,7 +50,7 @@ export const mapSchemaError = dual< AiError.make({ module: "OpenAiClient", method, - reason: AiError.InvalidOutputError.fromSchemaError(error) + reason: AiError.InvalidResponseError.fromSchemaError(error) })) /** @internal */ @@ -58,7 +61,7 @@ export const mapSseError = dual< AiError.make({ module: "OpenAiClient", method, - reason: new AiError.InvalidOutputError({ description: error.message }) + reason: new AiError.InvalidResponseError({ description: error.message }) })) /** @internal */ @@ -140,7 +143,9 @@ const mapStatusCodeError = Effect.fnUntraced(function*( ) const decoded = OpenAiErrorBodyJson(body) + const now = yield* Clock.currentTimeMillis const reason = mapStatusCodeToReason({ + now, status, headers, message: Option.isSome(decoded) ? decoded.value.error.message : undefined, @@ -159,16 +164,16 @@ const mapStatusCodeError = Effect.fnUntraced(function*( return yield* AiError.make({ module: "OpenAiClient", method, reason }) }) +const retryAfterFromHeader = (raw: string, now: number): Duration.Duration | undefined => { + const retryAfter = Headers.parseRetryAfter(raw) + return Result.isSuccess(retryAfter) ? Headers.retryAfterDelay(retryAfter.success, now) : undefined +} + /** @internal */ -export const parseRateLimitHeaders = (headers: Record) => { +export const parseRateLimitHeaders = (headers: Record, now: number) => { const retryAfterRaw = headers["retry-after"] - let retryAfter: Duration.Duration | undefined - if (retryAfterRaw !== undefined) { - const parsed = Number.parse(retryAfterRaw) - if (Option.isSome(parsed)) { - retryAfter = Duration.seconds(parsed.value) - } - } + // an invalid Retry-After is treated as absent + const retryAfter = retryAfterRaw === undefined ? undefined : retryAfterFromHeader(retryAfterRaw, now) const remainingRaw = headers["x-ratelimit-remaining-requests"] const remaining = remainingRaw !== undefined ? Option.getOrNull(Number.parse(remainingRaw)) : null return { @@ -208,8 +213,10 @@ export const buildHttpContext = (params: { }) /** @internal */ -export const mapStatusCodeToReason = ({ status, headers, message, metadata, http }: { +export const mapStatusCodeToReason = ({ status, headers, message, metadata, http, now }: { readonly status: number + /** Current time in epoch milliseconds, used to measure a Retry-After date. */ + readonly now: number readonly headers: Record readonly message: string | undefined readonly metadata: OpenAiErrorMetadata @@ -237,14 +244,14 @@ export const mapStatusCodeToReason = ({ status, headers, message, metadata, http return new AiError.AuthenticationError({ kind: "InvalidKey", description: errorDescription, - metadata, + metadata: { "openai-compat": metadata }, http }) case 403: return new AiError.AuthenticationError({ kind: "InsufficientPermissions", description: errorDescription, - metadata, + metadata: { "openai-compat": metadata }, http }) case 404: @@ -272,7 +279,7 @@ export const mapStatusCodeToReason = ({ status, headers, message, metadata, http http }) } - const { retryAfter, ...rateLimitMetadata } = parseRateLimitHeaders(headers) + const { retryAfter, ...rateLimitMetadata } = parseRateLimitHeaders(headers, now) return new AiError.RateLimitError({ retryAfter, metadata: { @@ -288,13 +295,13 @@ export const mapStatusCodeToReason = ({ status, headers, message, metadata, http if (status >= 500) { return new AiError.InternalProviderError({ description: message ?? "Server error", - metadata, + metadata: { "openai-compat": metadata }, http }) } return new AiError.UnknownError({ description: message, - metadata, + metadata: { "openai-compat": metadata }, http }) } diff --git a/repos/effect/packages/ai/openai-compat/test/OpenAiClient.test.ts b/repos/effect/packages/ai/openai-compat/test/OpenAiClient.test.ts index 8cbd49581c..89845ee4fb 100644 --- a/repos/effect/packages/ai/openai-compat/test/OpenAiClient.test.ts +++ b/repos/effect/packages/ai/openai-compat/test/OpenAiClient.test.ts @@ -407,6 +407,7 @@ describe("OpenAiClient", () => { it("preserves and truncates a fallback HTTP response", () => { const body = `${"a".repeat(200)}b` const reason = Errors.mapStatusCodeToReason({ + now: 0, status: 400, headers: {}, message: undefined, @@ -596,15 +597,14 @@ const makeResponse = ( ) } -const getRequestBody = (request: HttpClientRequest.HttpClientRequest) => - Effect.gen(function*() { - const body = request.body - if (body._tag === "Uint8Array") { - const text = new TextDecoder().decode(body.body) - return JSON.parse(text) - } - return yield* Effect.die(new Error("Expected Uint8Array body")) - }) +const getRequestBody = Effect.fnUntraced(function*(request: HttpClientRequest.HttpClientRequest) { + const body = request.body + if (body._tag === "Uint8Array") { + const text = new TextDecoder().decode(body.body) + return JSON.parse(text) + } + return yield* Effect.die(new Error("Expected Uint8Array body")) +}) const toSseBody = (events: ReadonlyArray): string => events.map((event) => { diff --git a/repos/effect/packages/ai/openai-compat/test/OpenAiEmbeddingModel.test.ts b/repos/effect/packages/ai/openai-compat/test/OpenAiEmbeddingModel.test.ts index b7f6132c2d..f414e6c965 100644 --- a/repos/effect/packages/ai/openai-compat/test/OpenAiEmbeddingModel.test.ts +++ b/repos/effect/packages/ai/openai-compat/test/OpenAiEmbeddingModel.test.ts @@ -198,15 +198,14 @@ const jsonResponse = ( }) ) -const getRequestBody = (request: HttpClientRequest.HttpClientRequest) => - Effect.gen(function*() { - const body = request.body - if (body._tag === "Uint8Array") { - const text = new TextDecoder().decode(body.body) - return JSON.parse(text) - } - return yield* Effect.die(new Error("Expected Uint8Array body")) - }) +const getRequestBody = Effect.fnUntraced(function*(request: HttpClientRequest.HttpClientRequest) { + const body = request.body + if (body._tag === "Uint8Array") { + const text = new TextDecoder().decode(body.body) + return JSON.parse(text) + } + return yield* Effect.die(new Error("Expected Uint8Array body")) +}) const noopOpenAiClient: OpenAiClient.Service = { client: undefined as unknown as OpenAiClient.Service["client"], diff --git a/repos/effect/packages/ai/openai-compat/test/OpenAiLanguageModel.test.ts b/repos/effect/packages/ai/openai-compat/test/OpenAiLanguageModel.test.ts index c1ae663ba7..b231ba6516 100644 --- a/repos/effect/packages/ai/openai-compat/test/OpenAiLanguageModel.test.ts +++ b/repos/effect/packages/ai/openai-compat/test/OpenAiLanguageModel.test.ts @@ -2466,15 +2466,14 @@ const sseResponse = ( }) ) -const getRequestBody = (request: HttpClientRequest.HttpClientRequest) => - Effect.gen(function*() { - const body = request.body - if (body._tag === "Uint8Array") { - const text = new TextDecoder().decode(body.body) - return JSON.parse(text) - } - return yield* Effect.die(new Error("Expected Uint8Array body")) - }) +const getRequestBody = Effect.fnUntraced(function*(request: HttpClientRequest.HttpClientRequest) { + const body = request.body + if (body._tag === "Uint8Array") { + const text = new TextDecoder().decode(body.body) + return JSON.parse(text) + } + return yield* Effect.die(new Error("Expected Uint8Array body")) +}) const decodeToolParamsFromStream = ( parts: ReadonlyArray, diff --git a/repos/effect/packages/ai/openai/CHANGELOG.md b/repos/effect/packages/ai/openai/CHANGELOG.md index 1ed5bd3f26..ee9a9f4791 100644 --- a/repos/effect/packages/ai/openai/CHANGELOG.md +++ b/repos/effect/packages/ai/openai/CHANGELOG.md @@ -1,5 +1,37 @@ # @effect/ai-openai +## 4.0.6 + +### Patch Changes + +- [#9437](https://github.com/Effect-TS/effect/pull/9437) [`fed70de`](https://github.com/Effect-TS/effect/commit/fed70de0dff8af24c5f814bd0d985ab70c70420b) Thanks @effect-bot! - Add `DecisionModel.decideSettled` to retain valid answers alongside per-decision `Result` failures (`Missing`, `Refused`, `NoDistribution`, or `Invalid`). Provider failures still fail the effect; `decide` remains all-or-nothing. + + OpenRouter now passes missing distributions to core validation. OpenAI reports refusals per decision and wrong-kind score answers as invalid rather than missing. + + Update hand-built services to implement `decideSettled`, handle optional provider `probabilities` and refusal answers, and use settled outcomes instead of the removed `OpenAiContentPolicyMetadata` and `metadata.openai.refusedDecisions`. + +- [#9552](https://github.com/Effect-TS/effect/pull/9552) [`44c8c1d`](https://github.com/Effect-TS/effect/commit/44c8c1d1c6f01ba9887a051b365e9fa89fcfc685) Thanks @wmaurer! - Add an optional `batchKey` to `EmbeddingModel.make` to group concurrent `embed` calls by provider-relevant context. OpenAI and OpenAI-compatible providers now separate batches with different configuration overrides or client transforms. Undeclared context, such as per-call `FetchHttpClient.RequestInit`, still uses the first caller's value. + +- [#9709](https://github.com/Effect-TS/effect/pull/9709) [`85e2aee`](https://github.com/Effect-TS/effect/commit/85e2aeee686963c26ca40d204f982deebb9d6117) Thanks @tim-smart! - Add typed HTTP date and `Retry-After` header values to `Headers`, and parse them consistently across the HTTP client, OTLP exporter, GraphQL client, AI clients and static server. + + - `Headers.parseHttpDate` / `Headers.formatHttpDate` read all three RFC 9110 date formats as UTC, whatever the process time zone, and emit IMF-fixdate. + - `Headers.parseRetryAfter`, `Headers.formatRetryAfter` and `Headers.retryAfterDelay` accept whole non-negative seconds of any size, read exactly, or an HTTP date, and the delay is never negative. HTTP dates accept the `23:59:60` leap second, read as the following second. + - `Headers.RetryAfter` holds either a `Duration` (`RetryAfterDuration`) or a `DateTime.Utc` (`RetryAfterDateTime`). `Headers.retryAfterDuration` builds the first from any `Duration.Input`: infinite durations throw, finite negative ones become zero, and fractional seconds are rounded up when formatted. + - Parse failures are reported as `Headers.HeaderParseError`. + - `Schema.DateTimeUtcFromHttpDate` and `Schema.RetryAfterFromString` expose the same parsing as string codecs for HTTP API headers. + - Invalid `Retry-After` values such as `-5`, `1.5`, `1e3` or `Infinity` are now ignored everywhere. `HttpClient.retryTransient` and `HttpClient.withRateLimiter` no longer honour fractional seconds, and the AI clients no longer produce negative or infinite `retryAfter` durations. + - `HttpStaticServer` reads `If-Modified-Since` and `Last-Modified` as UTC; asctime dates were previously shifted by the process time zone. + +- [#9430](https://github.com/Effect-TS/effect/pull/9430) [`48102ac`](https://github.com/Effect-TS/effect/commit/48102ac592ef89fed9d5d10afd2daece0a0ee310) Thanks @effect-bot! - Add non-retryable `AiError.InvalidResponseError` for provider response schema and SSE decoding failures, with optional HTTP context; model-generated output validation remains retryable. Update exhaustive matches on `AiErrorReason` to handle the new tag, and update serialized-error consumers to accept it. + +- [#9242](https://github.com/Effect-TS/effect/pull/9242) [`12b2726`](https://github.com/Effect-TS/effect/commit/12b2726651a6d6eeb2fd66539d34c1401f9afd94) Thanks @effect-bot! - Consistently nest authentication, internal provider, and unknown error metadata under the `openai` or `openai-compat` provider key, including error codes, error types, and request IDs. + +- [#9667](https://github.com/Effect-TS/effect/pull/9667) [`225f2c6`](https://github.com/Effect-TS/effect/commit/225f2c6b8f05a849abaea063043b4a09c92c703b) Thanks @effect-bot! - Support `promptCacheBreakpoint` on OpenAI image and file prompt parts, sending `prompt_cache_breakpoint` on `input_image` and `input_file` content blocks. + +- [#9670](https://github.com/Effect-TS/effect/pull/9670) [`ab5c1a8`](https://github.com/Effect-TS/effect/commit/ab5c1a8a3581ed3c2b05130b9e58fb7bae41dfe5) Thanks @effect-bot! - Support the `safety_identifier` and `prompt_cache_retention` Responses request options in `OpenAiSchema.CreateResponse` and `OpenAiLanguageModel.Config`. +- Updated dependencies [[`ce13a48`](https://github.com/Effect-TS/effect/commit/ce13a4872e516187851c5d9336d4df309cc8f0a4), [`f3bbbb4`](https://github.com/Effect-TS/effect/commit/f3bbbb448d431a9d1205ff09ba47cb81af0c7199), [`5547315`](https://github.com/Effect-TS/effect/commit/554731500e43eef47ff68b569116b74006e91d87), [`6093cfb`](https://github.com/Effect-TS/effect/commit/6093cfbbb673d36ecd18edea955763db003c8c26), [`30319d5`](https://github.com/Effect-TS/effect/commit/30319d506c9177d1cc1e0b10372e1ddb776aaf0a), [`74a7596`](https://github.com/Effect-TS/effect/commit/74a759699a8582dbe660b21dcda21cc9db08f613), [`bfb32a3`](https://github.com/Effect-TS/effect/commit/bfb32a3cf5f8d0f02f0dfdb63dcf5237fde755b0), [`f8bf7f4`](https://github.com/Effect-TS/effect/commit/f8bf7f4bcc3ad543cd94cbeb010cb164e5dedcb0), [`7f3d3df`](https://github.com/Effect-TS/effect/commit/7f3d3dfbb7b9e6e041c4ad9654c4532f72d2f16e), [`db1046d`](https://github.com/Effect-TS/effect/commit/db1046d1fa0e4b5818c51d28017a89579b6261d6), [`3c4f88a`](https://github.com/Effect-TS/effect/commit/3c4f88aa4d1d4d958b42fe8d3f1195f0ca990f8a), [`916e709`](https://github.com/Effect-TS/effect/commit/916e70968acc4e443885c67012d855b7cc6f9bca), [`916f843`](https://github.com/Effect-TS/effect/commit/916f8436225153a81a049bdc1feaedcafe4d1ded), [`66ec757`](https://github.com/Effect-TS/effect/commit/66ec7574d23b7943584059b41136bc21ee9d4c53), [`daac06f`](https://github.com/Effect-TS/effect/commit/daac06f79a6ab8c7d3e1b66b427c06f7d64dcc8f), [`c2552ec`](https://github.com/Effect-TS/effect/commit/c2552ec2e031b409b2bef67309312b0168480a39), [`b02c1cd`](https://github.com/Effect-TS/effect/commit/b02c1cde36c890941a235224af8ecb035197582a), [`1855f1d`](https://github.com/Effect-TS/effect/commit/1855f1d359535339bc7640b67e6bb7b12752ba19), [`683ed2c`](https://github.com/Effect-TS/effect/commit/683ed2cb491f9529379c1d5c63933689ff57fd2b), [`ac68940`](https://github.com/Effect-TS/effect/commit/ac68940f7e4246b1beb43f9d451a67f41241ab82), [`22a8401`](https://github.com/Effect-TS/effect/commit/22a84017e7222f332611c1bace05f9fa60c7cf67), [`c53c194`](https://github.com/Effect-TS/effect/commit/c53c19451171a00776b400a8548394ac514b628a), [`ca6d1cc`](https://github.com/Effect-TS/effect/commit/ca6d1cc5f803cd501e85431642b79b552322a76a), [`47940c5`](https://github.com/Effect-TS/effect/commit/47940c55c9e0eb855615094a2aca6e0812307bb2), [`e8b24ef`](https://github.com/Effect-TS/effect/commit/e8b24ef9516706971db948efb2d3eab84ad85951), [`2e8aae0`](https://github.com/Effect-TS/effect/commit/2e8aae0080a8d3e6d00c5ee0bc4f49f0b074acc2), [`ad7497b`](https://github.com/Effect-TS/effect/commit/ad7497bbc1bc70935f8f1ae395ff501532e2e770), [`7f504a3`](https://github.com/Effect-TS/effect/commit/7f504a35edcf44952c55ecbb0916726bac4d3e52), [`99e365d`](https://github.com/Effect-TS/effect/commit/99e365d61a1fdaabeb31dc58e5752cade8a09e1f), [`954067d`](https://github.com/Effect-TS/effect/commit/954067da7fbd20a20c10d5d0de6ffe4690e6bdfd), [`3e1a035`](https://github.com/Effect-TS/effect/commit/3e1a035d9c470ced858b7757f6d102b3aa9be008), [`41a1fe7`](https://github.com/Effect-TS/effect/commit/41a1fe74e6338e8d13d6d4f3daa03f71a826c2c4), [`bcb3a86`](https://github.com/Effect-TS/effect/commit/bcb3a86335a8ed18da39300984c0fbae187fdc6b), [`739d3d1`](https://github.com/Effect-TS/effect/commit/739d3d18728cce0b6d907f0a3f24e04ab0b53560), [`ed4da35`](https://github.com/Effect-TS/effect/commit/ed4da351fb9d29e4dfd6c60bd89ba4c007d264dc), [`3d84e19`](https://github.com/Effect-TS/effect/commit/3d84e19f8f14bccf0aa57b11825590c3bae162f4), [`cc00906`](https://github.com/Effect-TS/effect/commit/cc0090657eeb89debdd54af5fcb99d544f46c1bf), [`09ba1f5`](https://github.com/Effect-TS/effect/commit/09ba1f59952c46cd22229d41236742c3e1f53a34), [`9ff1a73`](https://github.com/Effect-TS/effect/commit/9ff1a73d995702f4c4c335888fe6a186f3f710c7), [`06bc532`](https://github.com/Effect-TS/effect/commit/06bc532b71cd3684f010956774356864dd359372), [`97bfd84`](https://github.com/Effect-TS/effect/commit/97bfd84bb147e10ed562e4c1ffa589cdaf83a596), [`25efaef`](https://github.com/Effect-TS/effect/commit/25efaef0d81d25b2651ef4de474e929d3180810a), [`c10335b`](https://github.com/Effect-TS/effect/commit/c10335be125c7846ae7b914eeb138244b660275b), [`a808972`](https://github.com/Effect-TS/effect/commit/a80897262648e4222ce48943e538f39a6b3f69d6), [`20e844c`](https://github.com/Effect-TS/effect/commit/20e844c7ef27b0e4b4f7911d603c45b1de2b0bc3), [`b9f73f3`](https://github.com/Effect-TS/effect/commit/b9f73f37c4ac538e6a327ed80ac5a0708858c540), [`5739878`](https://github.com/Effect-TS/effect/commit/5739878ded09d83d71d657e3b8d4264f4e4936d9), [`28acdf3`](https://github.com/Effect-TS/effect/commit/28acdf3820b66056037bb7a24cd0e8d3de3fb06d), [`59b6396`](https://github.com/Effect-TS/effect/commit/59b63967c1817ae5acb1811037f5aaa964219b7e), [`0bccc9c`](https://github.com/Effect-TS/effect/commit/0bccc9c3dd5349b20f8d2e509ac5482d0c26a616), [`fed70de`](https://github.com/Effect-TS/effect/commit/fed70de0dff8af24c5f814bd0d985ab70c70420b), [`82648a2`](https://github.com/Effect-TS/effect/commit/82648a21bf228145a14d2d1390006d67f74d89c7), [`afb5f32`](https://github.com/Effect-TS/effect/commit/afb5f326cc14c9849d8d7ddefde1039ae036e017), [`28813f5`](https://github.com/Effect-TS/effect/commit/28813f53a5026984fdc13d3971147ea5dc20104c), [`0fae934`](https://github.com/Effect-TS/effect/commit/0fae934a13aac12f9036b0733edd5dde5ce029f3), [`f260830`](https://github.com/Effect-TS/effect/commit/f2608305861269fc2fbedcbbdcd71e4fc5f98ffc), [`19b63eb`](https://github.com/Effect-TS/effect/commit/19b63eb81b2bccf033aadb565f52a6b0375d26a1), [`14db221`](https://github.com/Effect-TS/effect/commit/14db221f8f584100ff50fd1173b1a99a1caa1d06), [`679f956`](https://github.com/Effect-TS/effect/commit/679f956c4a7527854ff78db071dd7aa417211a48), [`b72707c`](https://github.com/Effect-TS/effect/commit/b72707c2937328fd3bc70f29a6792d84c4256484), [`44c8c1d`](https://github.com/Effect-TS/effect/commit/44c8c1d1c6f01ba9887a051b365e9fa89fcfc685), [`1e7909b`](https://github.com/Effect-TS/effect/commit/1e7909beb05bc740dbd330509e60bb185d7eeed1), [`0611a65`](https://github.com/Effect-TS/effect/commit/0611a657f76836f9ca275f64764b82270b4eda10), [`2fadc65`](https://github.com/Effect-TS/effect/commit/2fadc654aa655fbd1912ce51eb1b764a757ac362), [`feb6ce5`](https://github.com/Effect-TS/effect/commit/feb6ce59a38ab15fc6f89545bd5f0992dd58a06a), [`5a2ed53`](https://github.com/Effect-TS/effect/commit/5a2ed5323884a435bfcfb5e3a6f2ae29547458ad), [`c8d26a4`](https://github.com/Effect-TS/effect/commit/c8d26a46731cb26a9b9612d9fde823d8813cfc04), [`0c8b446`](https://github.com/Effect-TS/effect/commit/0c8b446709c737c10d0e11af4313bb29fdff2588), [`05c3409`](https://github.com/Effect-TS/effect/commit/05c340955944374e3ae47a1de0f7cb9b41a3b0e6), [`b04191b`](https://github.com/Effect-TS/effect/commit/b04191b2941534754282c51ef2df2f965ac43a5c), [`05c3409`](https://github.com/Effect-TS/effect/commit/05c340955944374e3ae47a1de0f7cb9b41a3b0e6), [`4067e64`](https://github.com/Effect-TS/effect/commit/4067e6428776ae573c8cc72f7eaebd078a4ff8c7), [`e04563d`](https://github.com/Effect-TS/effect/commit/e04563dcd6ddeea74c088a12194c5a73e597ba2f), [`02d7fc5`](https://github.com/Effect-TS/effect/commit/02d7fc583053c47dea3b5e1d9d6be164aeb321b6), [`561d2a1`](https://github.com/Effect-TS/effect/commit/561d2a1ddfa3c4340510f23da7adc50c89d97863), [`8fe2dcc`](https://github.com/Effect-TS/effect/commit/8fe2dcc2137788a71177d42a9f67fb7003224868), [`a8086fa`](https://github.com/Effect-TS/effect/commit/a8086fa26a2c9a9f5f7f1d7590bbc494a0feb02a), [`88a33b4`](https://github.com/Effect-TS/effect/commit/88a33b4122f7ffbbc1423add793a79abffd99c75), [`e77372a`](https://github.com/Effect-TS/effect/commit/e77372a8a2e7caec11999636e04c6e31c52e315f), [`3dd4388`](https://github.com/Effect-TS/effect/commit/3dd438810df34ed3b3d2e3e35ccf84ccb00feb85), [`ba2b00c`](https://github.com/Effect-TS/effect/commit/ba2b00c0658b65775809e0052ec9612432600c31), [`a049d4a`](https://github.com/Effect-TS/effect/commit/a049d4ad25f17cf4985284a8d15b07aea2a33242), [`84aeede`](https://github.com/Effect-TS/effect/commit/84aeede040460a2b24bcf0db7a5d0a9b0e3c42d4), [`43e5e19`](https://github.com/Effect-TS/effect/commit/43e5e198d16df038b60764f5dd0218cdcf828f16), [`77bbf16`](https://github.com/Effect-TS/effect/commit/77bbf16e50d6b0ea090288cc5d3f11b20706a027), [`6af3a47`](https://github.com/Effect-TS/effect/commit/6af3a47d5b5d698c3ba896c37684266cd465bef1), [`aac8d21`](https://github.com/Effect-TS/effect/commit/aac8d21911333dedb649f4f9d302c28d14d46369), [`84c8dc4`](https://github.com/Effect-TS/effect/commit/84c8dc47b09067362749620f75068d6f28044660), [`78d1988`](https://github.com/Effect-TS/effect/commit/78d198818d4fd547ce2e9431302f66cb00dc7ad3), [`5676a2a`](https://github.com/Effect-TS/effect/commit/5676a2ac5a918f8f91fc7838d7adf1b6f5e86a6d), [`4b437bf`](https://github.com/Effect-TS/effect/commit/4b437bf4645fabc8aaeaa3b5a88e8c9048b9e4f5), [`1af11fd`](https://github.com/Effect-TS/effect/commit/1af11fda1bccf823f74121b5125aa3befa084b18), [`716f8df`](https://github.com/Effect-TS/effect/commit/716f8dfa178f849e5b288464373bd132860c2e19), [`ee27718`](https://github.com/Effect-TS/effect/commit/ee27718a02125328a7478812b1b7d7c1d8532578), [`4d86d1b`](https://github.com/Effect-TS/effect/commit/4d86d1b8887d48fa24acf6ad54f33ef3d55a7cd8), [`5462ed7`](https://github.com/Effect-TS/effect/commit/5462ed7b3a4c37206c47e401d6be293dc540350f), [`658837c`](https://github.com/Effect-TS/effect/commit/658837c3b249d982df56f2990247499746e36672), [`00261cc`](https://github.com/Effect-TS/effect/commit/00261cc12c31ecc7878f80670439731c905dc2bd), [`7cccb93`](https://github.com/Effect-TS/effect/commit/7cccb9385729bc0aed6630edea95d7db15439ac4), [`7c98228`](https://github.com/Effect-TS/effect/commit/7c982281cf73958833054175fc86db385e043cf6), [`5a8e248`](https://github.com/Effect-TS/effect/commit/5a8e2481546352e2ad6233dad49420b14c59f624), [`8037a65`](https://github.com/Effect-TS/effect/commit/8037a65f40cae9163ab9e0e687e289f717ae570b), [`25912db`](https://github.com/Effect-TS/effect/commit/25912db9d493227a2d6fa45cebcfc7507e638d8a), [`7f32606`](https://github.com/Effect-TS/effect/commit/7f32606baac43a67d171ae93b77c7b29791c9692), [`0386f88`](https://github.com/Effect-TS/effect/commit/0386f88295ac70cbc1f90ab8cceb0981d848c628), [`403298a`](https://github.com/Effect-TS/effect/commit/403298ac457fb634eca2c013fed26d37fc80445f), [`5a3ebc0`](https://github.com/Effect-TS/effect/commit/5a3ebc082124f9e10106b2347736e66fc74ebc36), [`fac2191`](https://github.com/Effect-TS/effect/commit/fac21911964b742a38e1d4e73bd4f95aec4911dc), [`fdb12dc`](https://github.com/Effect-TS/effect/commit/fdb12dcd61b6505eed02e6f7a3c88a85d4c48e8d), [`97909b8`](https://github.com/Effect-TS/effect/commit/97909b89b167da54a56e6f82f1c4db8bfedae04f), [`c3983f4`](https://github.com/Effect-TS/effect/commit/c3983f4db2b3052a58e8045cce270fd7ccc4e2b2), [`3a5e65e`](https://github.com/Effect-TS/effect/commit/3a5e65e409c9569e23bb7af61df401347e9ec420), [`8b40fbc`](https://github.com/Effect-TS/effect/commit/8b40fbc93caf52f3a4c69020849f0051f100be65), [`3f3e119`](https://github.com/Effect-TS/effect/commit/3f3e119ad61c2167bc6c7170c3fb68cd1eb3421e), [`7c7b8ed`](https://github.com/Effect-TS/effect/commit/7c7b8eda54e44f9cddc1188f994f7fdddf901055), [`d4f71bc`](https://github.com/Effect-TS/effect/commit/d4f71bc42507b638d149fb8806d26ea626d17956), [`c66241e`](https://github.com/Effect-TS/effect/commit/c66241e791c4b51d8a65051142d01682590f9bf8), [`f0e941e`](https://github.com/Effect-TS/effect/commit/f0e941ee2f90f0a891d1228a90da91386f0258cb), [`5f8a6e5`](https://github.com/Effect-TS/effect/commit/5f8a6e5108aa6c6cd4043a3c9f4dc93d43369623), [`2800a09`](https://github.com/Effect-TS/effect/commit/2800a0910f7f4ad7cfa21dfc840edf8fe6e67ca7), [`e9749b0`](https://github.com/Effect-TS/effect/commit/e9749b04c10badabcdf71e6958de5ac56a5316e9), [`eaef09c`](https://github.com/Effect-TS/effect/commit/eaef09c11b6ed2029c513eea0aa3256dcc6f517f), [`a3c56e5`](https://github.com/Effect-TS/effect/commit/a3c56e5b5e756e879f1cfb90230e8eae620e33df), [`bb61152`](https://github.com/Effect-TS/effect/commit/bb611522fcfb4d317b33acdf5ee24a480d042435), [`e9c842b`](https://github.com/Effect-TS/effect/commit/e9c842bc294abf658d698bb17732ae13465b3817), [`6d8c8e2`](https://github.com/Effect-TS/effect/commit/6d8c8e2075b15ae84ae96e93a0d295cb9221b496), [`1f28187`](https://github.com/Effect-TS/effect/commit/1f281878ef559ef356af2d0e11b381aba857cae1), [`0b9842b`](https://github.com/Effect-TS/effect/commit/0b9842b76bbc1b016c8abc46432052e1f60b2fa3), [`c29ddd8`](https://github.com/Effect-TS/effect/commit/c29ddd899c53c5d8194c4a0dfef2134304dda799), [`84c3405`](https://github.com/Effect-TS/effect/commit/84c34057ceab9fe6782b73f4a04a31304e0d14de), [`3934c0c`](https://github.com/Effect-TS/effect/commit/3934c0c61857c03468c08c2477f49a6b977d28ff), [`048e7c8`](https://github.com/Effect-TS/effect/commit/048e7c87125c4ace5a61895feac3d049a317bbfc), [`1121334`](https://github.com/Effect-TS/effect/commit/1121334ce68c379051011a2ffea37b0a30ea9a47), [`9a680d6`](https://github.com/Effect-TS/effect/commit/9a680d6985c6a27d7ebb29cd9eb674f3fe524505), [`85e2aee`](https://github.com/Effect-TS/effect/commit/85e2aeee686963c26ca40d204f982deebb9d6117), [`34ec440`](https://github.com/Effect-TS/effect/commit/34ec4407a14ce045ca14cc76c77582884a7e1ebe), [`343a149`](https://github.com/Effect-TS/effect/commit/343a149fa7d5a9a0a45841b1b6800b0ecfa16a32), [`915117b`](https://github.com/Effect-TS/effect/commit/915117b523cb6e5cbc98b5ff1c119a8d0d1f6b9a), [`7f823c1`](https://github.com/Effect-TS/effect/commit/7f823c10d20ef53a1081e0d0e2008a4116140799), [`6e614e6`](https://github.com/Effect-TS/effect/commit/6e614e6feb4281a9e81f5ec3be48e7905ff4292c), [`9990b39`](https://github.com/Effect-TS/effect/commit/9990b3977a544247cd4123edde8353a55bf77192), [`22bf781`](https://github.com/Effect-TS/effect/commit/22bf7812aa8a20c676efade943c342cde4ef1cbd), [`41f08b7`](https://github.com/Effect-TS/effect/commit/41f08b75aba40e95cd6c743ec6fcf1f5022f7b5b), [`e8482ca`](https://github.com/Effect-TS/effect/commit/e8482cad9651fa8fe8036fe6292a286994897d0d), [`5ae85fa`](https://github.com/Effect-TS/effect/commit/5ae85fa8ac1fc5e12d5fe8fe7388ac504788ae3b), [`c477a0a`](https://github.com/Effect-TS/effect/commit/c477a0a8010bd0a988221bc00c446848314e1c3c), [`8fbb39e`](https://github.com/Effect-TS/effect/commit/8fbb39eb92577915831f19a83b51be376e844613), [`1c70828`](https://github.com/Effect-TS/effect/commit/1c7082847453e9c2be0cc267a39ca554c52f4013), [`8207d1e`](https://github.com/Effect-TS/effect/commit/8207d1e91ead8aac8f01d9e13b49624b64237c8b), [`7886b84`](https://github.com/Effect-TS/effect/commit/7886b847658664f64392a14efda86f44b3d1e170), [`4839c60`](https://github.com/Effect-TS/effect/commit/4839c601bdda845d9a54d77f578f64372475ab04), [`2595d83`](https://github.com/Effect-TS/effect/commit/2595d831f702d4a748f97e33e71fd5250f58af56), [`f5ae278`](https://github.com/Effect-TS/effect/commit/f5ae27827a82db3f8973b70745fc6f14e7d8fa80), [`7a8ce00`](https://github.com/Effect-TS/effect/commit/7a8ce0098a5f22cbb6c110b16e850fb3fcbcadf2), [`688d302`](https://github.com/Effect-TS/effect/commit/688d30275ff114e50b4f0d343c6227ee33d67348), [`c7072e4`](https://github.com/Effect-TS/effect/commit/c7072e46356a19615625ae0fa569b1c3a79d50be), [`425cedb`](https://github.com/Effect-TS/effect/commit/425cedb5801c9527c116a64d280d9e7c2820f922), [`88bc680`](https://github.com/Effect-TS/effect/commit/88bc68001c6c70984b03a5500232e9b5aab14085), [`268e09c`](https://github.com/Effect-TS/effect/commit/268e09c94c9e8f46142b5c9f29ce0a22bfbbc3aa), [`7e94505`](https://github.com/Effect-TS/effect/commit/7e94505edfc9d81ea08cb96a7d935806495bf61e), [`afa5da3`](https://github.com/Effect-TS/effect/commit/afa5da3c8a9848215e49535dd3d39179bfd67715), [`c29ddd8`](https://github.com/Effect-TS/effect/commit/c29ddd899c53c5d8194c4a0dfef2134304dda799), [`48102ac`](https://github.com/Effect-TS/effect/commit/48102ac592ef89fed9d5d10afd2daece0a0ee310), [`463eca3`](https://github.com/Effect-TS/effect/commit/463eca3609a6a9681247abbcc71a6f10b261a1d9), [`e7162ea`](https://github.com/Effect-TS/effect/commit/e7162eae7f04af1d583b1f31fdd767f91ec1c82e), [`95752db`](https://github.com/Effect-TS/effect/commit/95752db5eb3b6e59fb8749e9792704f8b167f36c), [`4577a28`](https://github.com/Effect-TS/effect/commit/4577a28bcdb50b0d184775f8c1b7ad890d86e255), [`973d599`](https://github.com/Effect-TS/effect/commit/973d599a3e8611626764a0e3e8f79edbcf30dc7c), [`2f066a5`](https://github.com/Effect-TS/effect/commit/2f066a58e90316dd88a219fcea94200beda54219), [`f0ccbb6`](https://github.com/Effect-TS/effect/commit/f0ccbb6917be5c08364b0b8adbd0c4c19e7f2de1), [`2dda6ce`](https://github.com/Effect-TS/effect/commit/2dda6cefc641dbea2ea3279a9b96c0332b277a40), [`3da7c0a`](https://github.com/Effect-TS/effect/commit/3da7c0aabab07c8fc682aea7559536bf31705853), [`5a67e0b`](https://github.com/Effect-TS/effect/commit/5a67e0b3a8118dd7be709a56d6d7af4f10e9c4f3), [`73b3a6e`](https://github.com/Effect-TS/effect/commit/73b3a6e3c38795400251d9c5d73f05c88acd90cd), [`c6f9e4c`](https://github.com/Effect-TS/effect/commit/c6f9e4cbe6f9c8f0a03fefde18df4d35e98a24df), [`be02676`](https://github.com/Effect-TS/effect/commit/be026768ee29a1b1a860c1fd17f2a0896c01f61e), [`26de861`](https://github.com/Effect-TS/effect/commit/26de86100cda8617e077ce66f0a6bb8dfe738fb6), [`9313799`](https://github.com/Effect-TS/effect/commit/931379911a74d5e2207859c295b1acc6a5e2917e), [`d5c1e5b`](https://github.com/Effect-TS/effect/commit/d5c1e5bd425dfe983dff1f8c853c0b7eaa772f87), [`11bb31f`](https://github.com/Effect-TS/effect/commit/11bb31fec0bf2fdfc025009a04de4c69bffa433a), [`8c23e46`](https://github.com/Effect-TS/effect/commit/8c23e46a8280e3f2c831d1c0ca3f106c92f8e071), [`9c96011`](https://github.com/Effect-TS/effect/commit/9c960111de202c0f1dd6989e5b8836150e0ca694), [`0d50570`](https://github.com/Effect-TS/effect/commit/0d50570bbf078804092fcd4a47ccd2d0019f989c), [`9ed2bf1`](https://github.com/Effect-TS/effect/commit/9ed2bf18da0e1ff4fecc31b7cbac2976ec999e00), [`76ec912`](https://github.com/Effect-TS/effect/commit/76ec9120f43f4fdb08b9c5170ff9e3fd82f22457), [`af2756d`](https://github.com/Effect-TS/effect/commit/af2756dd1d196aa3f199ce21c6d2dc3ada3b67b5), [`3ec8100`](https://github.com/Effect-TS/effect/commit/3ec8100c793b8a22381ad115106b93824d840f64), [`93abf02`](https://github.com/Effect-TS/effect/commit/93abf020876af0a6b6b8b25621c80760b7996762), [`1e94993`](https://github.com/Effect-TS/effect/commit/1e949935a1065e5140245a514ce3a799061143e3), [`99a4616`](https://github.com/Effect-TS/effect/commit/99a4616264e31de65fa8d3dc19576309b1a3f84a), [`0a7978e`](https://github.com/Effect-TS/effect/commit/0a7978ea8e70e51700345b12f8c19952dde05255), [`b4a060b`](https://github.com/Effect-TS/effect/commit/b4a060b95a6473d54e277dc005172b54aab9bddf), [`274f4a4`](https://github.com/Effect-TS/effect/commit/274f4a4dac5ee2d9dbb6358cc3dd3afe345d4705), [`25f29be`](https://github.com/Effect-TS/effect/commit/25f29be77bdcd5c2a7be8e2710e88f5a10c4cb8d), [`9036440`](https://github.com/Effect-TS/effect/commit/9036440edca01c24371d0c2c1044be5e5d3cb65e), [`415f005`](https://github.com/Effect-TS/effect/commit/415f0053eb848d562febcb5b011e15f31a99bead), [`191308b`](https://github.com/Effect-TS/effect/commit/191308bd0f5ee04557570f2f960c6e06c429bcfd), [`4b96570`](https://github.com/Effect-TS/effect/commit/4b96570ed28d93b14e958742933d227dc5e74b15), [`d84ec3f`](https://github.com/Effect-TS/effect/commit/d84ec3fab0346c01bc59f6624d1c046ca4140087), [`392aa1c`](https://github.com/Effect-TS/effect/commit/392aa1cb275ac9fdb9b4b1831d8c06196bba54b9), [`99e9bd9`](https://github.com/Effect-TS/effect/commit/99e9bd98a354d83202e2522f3ade9ee4bf4a862c), [`6f6f45f`](https://github.com/Effect-TS/effect/commit/6f6f45f480a3f191aed6a82ca5c301f60ef2cf84), [`13fc07e`](https://github.com/Effect-TS/effect/commit/13fc07edeb939ca6229b858c9d9a8c5243ac21b5), [`a4b38dd`](https://github.com/Effect-TS/effect/commit/a4b38dde7f3fdc1b54a49bffdaca912b4c47c76e), [`8b18580`](https://github.com/Effect-TS/effect/commit/8b185807bb99f6de3d9564447e2f4e08d2f8863d), [`0c477fe`](https://github.com/Effect-TS/effect/commit/0c477fe417d3ecab86bb8d9afd0c392e9cb8a3da), [`9a5331d`](https://github.com/Effect-TS/effect/commit/9a5331db94bfdca4f48317d062fbeb96054a9049), [`a706d73`](https://github.com/Effect-TS/effect/commit/a706d73db8a8d460877c5d7e8a793a52d45be307), [`c9a1e72`](https://github.com/Effect-TS/effect/commit/c9a1e72c5965aa9fe01a669ebfd523f803f5a189), [`89eaaf8`](https://github.com/Effect-TS/effect/commit/89eaaf8581908e248c0b1e47baa7906e0c541635), [`8215811`](https://github.com/Effect-TS/effect/commit/8215811d51d28d37b4732a6ab03de4047ada62ea), [`ad69018`](https://github.com/Effect-TS/effect/commit/ad6901857e05e7b436a0149b5ece7f3a36d9ff33), [`a733eba`](https://github.com/Effect-TS/effect/commit/a733eba9f699737a80a9e7276521980bd929075a), [`7282d5f`](https://github.com/Effect-TS/effect/commit/7282d5f1bcc06d7065429ef81a3d17c64043fde4), [`da33572`](https://github.com/Effect-TS/effect/commit/da33572776763e45802ce8c2fe14ec101f9a5c34), [`d9e2d12`](https://github.com/Effect-TS/effect/commit/d9e2d1221768e9a3e5e00165cb95b86e0c126cc0), [`dc923b8`](https://github.com/Effect-TS/effect/commit/dc923b847ac7d9673b4f0aa7c136557287b2e5e3), [`1b74086`](https://github.com/Effect-TS/effect/commit/1b740862e3d25ec5d4b6f86b9fe792b6dabbf84c), [`8a5507b`](https://github.com/Effect-TS/effect/commit/8a5507ba00f641edf71227d79b9f90efdb634fe2), [`23460fa`](https://github.com/Effect-TS/effect/commit/23460fad704c730926bffe3cc6874268ef6680ec), [`0dbbea6`](https://github.com/Effect-TS/effect/commit/0dbbea65a93c59147a4bc407cfcda38ea26341b2), [`70aad3f`](https://github.com/Effect-TS/effect/commit/70aad3ffe617984f5b599aeed1bc7f04899aca55), [`0909236`](https://github.com/Effect-TS/effect/commit/090923611b209a19f3281a7b1d0c15a9b653a0c7), [`a9e66d8`](https://github.com/Effect-TS/effect/commit/a9e66d889017b302355ba5b8bc2b2050ca1ea5aa), [`ee5c0d5`](https://github.com/Effect-TS/effect/commit/ee5c0d5f43348747b04b5f7bf1b39355614c2f2c), [`631c5d6`](https://github.com/Effect-TS/effect/commit/631c5d6d9a81e6389964b878764fb9aea1538247), [`7e60c1a`](https://github.com/Effect-TS/effect/commit/7e60c1ae62a96c768f9c67d706f494213947f22f), [`4cddd3c`](https://github.com/Effect-TS/effect/commit/4cddd3c41e71fc4990d37c42b1bcf2ce3780b41d), [`8727e5d`](https://github.com/Effect-TS/effect/commit/8727e5d2bec8e2d729b7aeafb725a4ab21d533d2), [`2bc97fa`](https://github.com/Effect-TS/effect/commit/2bc97faa0e06b3973780dae26029e2c59f5dba38), [`f9a7f05`](https://github.com/Effect-TS/effect/commit/f9a7f056ba31aef435933acf082ec8c01ac7decc), [`d978cca`](https://github.com/Effect-TS/effect/commit/d978cca02914bec0ddf66bb2d68d90b220096fb6), [`b334ccf`](https://github.com/Effect-TS/effect/commit/b334ccf361a0f11362e7aabdbf70b35f53a39d61), [`0b50cfc`](https://github.com/Effect-TS/effect/commit/0b50cfc644313afe4c318073893231ec1946eb5f), [`5b7d2b5`](https://github.com/Effect-TS/effect/commit/5b7d2b5c90bc77eb9f8be1b91166c53f89aa1d0c), [`6ee26b1`](https://github.com/Effect-TS/effect/commit/6ee26b16b51b380c41dbfa84d612aa525b420fce), [`eab7e6a`](https://github.com/Effect-TS/effect/commit/eab7e6acfc7e422857565b98b9eddd3a98de1b92), [`e618f10`](https://github.com/Effect-TS/effect/commit/e618f1050f41424439eb82478fcf1c5da5a3244d), [`da14b1f`](https://github.com/Effect-TS/effect/commit/da14b1f18d71a18fc431b56d82cffe09aeb92708), [`3314fc3`](https://github.com/Effect-TS/effect/commit/3314fc30c22d5c9d737f3df7201f4a41424f701b), [`66db624`](https://github.com/Effect-TS/effect/commit/66db624670032a02144d42b6e64b83b324c813cd), [`4b8a50d`](https://github.com/Effect-TS/effect/commit/4b8a50db9649603f382f33640698020ca79547d0), [`8d52668`](https://github.com/Effect-TS/effect/commit/8d52668533e9d4a6d6b3be9fe58bd5e69fa80b65), [`1de1055`](https://github.com/Effect-TS/effect/commit/1de10559e7263747751bfb032fd4b0fd8aa28724), [`db130e6`](https://github.com/Effect-TS/effect/commit/db130e64557fafe370819d19f78e3b19a21275d6), [`5fa5fa5`](https://github.com/Effect-TS/effect/commit/5fa5fa5b9bff77352fb9ff2d424bf46bd6309986)]: + - effect@4.0.6 + ## 4.0.5 ### Patch Changes diff --git a/repos/effect/packages/ai/openai/package.json b/repos/effect/packages/ai/openai/package.json index f9a91a928c..525c68ace5 100644 --- a/repos/effect/packages/ai/openai/package.json +++ b/repos/effect/packages/ai/openai/package.json @@ -1,6 +1,6 @@ { "name": "@effect/ai-openai", - "version": "4.0.5", + "version": "4.0.6", "type": "module", "license": "MIT", "description": "An OpenAI provider integration for Effect AI SDK", diff --git a/repos/effect/packages/ai/openai/src/OpenAiDecisionModel.ts b/repos/effect/packages/ai/openai/src/OpenAiDecisionModel.ts index 3e5995b2ed..1d150549c3 100644 --- a/repos/effect/packages/ai/openai/src/OpenAiDecisionModel.ts +++ b/repos/effect/packages/ai/openai/src/OpenAiDecisionModel.ts @@ -54,9 +54,8 @@ export const model = ( * * - Remote URLs fail with `AiError.InvalidUserInputError` because OpenAI only * accepts inline images. - * - If OpenAI refuses any decision, the whole call fails with - * `AiError.ContentPolicyError`. The refused names are in - * `metadata.openai.refusedDecisions`. + * - Refusals yield `"Refused"` with `decideSettled`; `decide` fails with + * `AiError.ContentPolicyError` naming the refused decisions. * * @stability unstable * @category constructors @@ -91,11 +90,11 @@ export const make = Effect.fnUntraced( questions }) const answers: Record = Object.create(null) - const refused: Array = [] for (const answer of response.answers) { + if (answers[answer.name]?._tag === "Refusal") continue switch (answer.type) { case "refusal": - refused.push(answer.name) + answers[answer.name] = { _tag: "Refusal" } break case "choice": answers[answer.name] = { @@ -107,10 +106,10 @@ export const make = Effect.fnUntraced( break case "score": { const decision = decisions[answer.name] - if (decision?._tag !== "Rate") break + const levels = decision?._tag === "Rate" ? decision.criteria : [] const probabilities: Record = Object.create(null) for (const { value, probability } of answer.probabilities) { - const level = decision.criteria[value] + const level = levels[value] if (level !== undefined) probabilities[level.label] = probability } answers[answer.name] = { @@ -126,16 +125,6 @@ export const make = Effect.fnUntraced( break } } - if (refused.length > 0) { - return yield* Effect.fail(AiError.make({ - module: "OpenAiDecisionModel", - method: "decide", - reason: new AiError.ContentPolicyError({ - description: `OpenAI refused to answer: ${refused.join(", ")}`, - metadata: { openai: { errorCode: null, errorType: null, requestId: null, refusedDecisions: refused } } - }) - })) - } return { answers, usage: { inputTokens: response.usage?.input_tokens, outputTokens: response.usage?.output_tokens } diff --git a/repos/effect/packages/ai/openai/src/OpenAiEmbeddingModel.ts b/repos/effect/packages/ai/openai/src/OpenAiEmbeddingModel.ts index 4e809ad5a9..f9b84ad36f 100644 --- a/repos/effect/packages/ai/openai/src/OpenAiEmbeddingModel.ts +++ b/repos/effect/packages/ai/openai/src/OpenAiEmbeddingModel.ts @@ -17,6 +17,7 @@ import { dual } from "effect/Function" import * as Layer from "effect/Layer" import type { Simplify } from "effect/Types" import { OpenAiClient } from "./OpenAiClient.ts" +import { OpenAiConfig } from "./OpenAiConfig.ts" import type * as OpenAiSchema from "./OpenAiSchema.ts" /** @@ -116,6 +117,10 @@ export const model = ( * * **Gotchas** * + * Concurrent `embed` calls share a request when their `Config` and + * `OpenAiConfig` values are equal. Other per-call context, such as + * `FetchHttpClient.RequestInit`, takes the first caller's value for the batch. + * * The service expects numeric embedding vectors. It fails with * `InvalidOutputError` when the provider returns base64 embeddings, * out-of-range indexes, duplicate indexes, or an unexpected number of @@ -140,6 +145,7 @@ export const make = Effect.fnUntraced(function*({ model, config: providerConfig ) return yield* EmbeddingModel.make({ + batchKey: (context) => [Context.getOrUndefined(context, Config), Context.getOrUndefined(context, OpenAiConfig)], embedMany: Effect.fnUntraced(function*({ inputs }) { const config = yield* makeConfig const response = yield* client.createEmbedding({ ...config, input: inputs }) diff --git a/repos/effect/packages/ai/openai/src/OpenAiError.ts b/repos/effect/packages/ai/openai/src/OpenAiError.ts index 470a9ef03f..1f777293e9 100644 --- a/repos/effect/packages/ai/openai/src/OpenAiError.ts +++ b/repos/effect/packages/ai/openai/src/OpenAiError.ts @@ -30,25 +30,6 @@ export type OpenAiErrorMetadata = { readonly requestId: string | null } -/** - * OpenAI-specific content policy metadata fields. - * - * **Details** - * - * Extends base error metadata with the names of decisions that OpenAI refused - * to answer. - * - * @stability unstable - * @category models - * @since 4.0.5 - */ -export type OpenAiContentPolicyMetadata = OpenAiErrorMetadata & { - /** - * Names of the decisions that came back as refusals. - */ - readonly refusedDecisions?: Array -} - /** * OpenAI-specific rate limit metadata fields. * @@ -151,7 +132,7 @@ declare module "effect/ai/AiError" { /** * OpenAI-specific details for the content policy response. */ - readonly openai?: OpenAiContentPolicyMetadata | null + readonly openai?: OpenAiErrorMetadata | null } /** diff --git a/repos/effect/packages/ai/openai/src/OpenAiLanguageModel.ts b/repos/effect/packages/ai/openai/src/OpenAiLanguageModel.ts index 8217553d5b..ee506e0907 100644 --- a/repos/effect/packages/ai/openai/src/OpenAiLanguageModel.ts +++ b/repos/effect/packages/ai/openai/src/OpenAiLanguageModel.ts @@ -192,6 +192,13 @@ declare module "effect/ai/Prompt" { * The detail level of the image to be sent to the model. One of `high`, `low`, or `auto`. Defaults to `auto`. */ readonly imageDetail?: ImageDetail | null + /** + * Marks the input image or file as the end of a reusable prompt prefix. + * + * Requires GPT-5.6 or later. OpenAI may reject requests that use this + * option with earlier models. + */ + readonly promptCacheBreakpoint?: PromptCacheBreakpoint | null } | null } @@ -925,12 +932,13 @@ const prepareMessages = Effect.fnUntraced( } case "file": { + const promptCacheBreakpoint = getPromptCacheBreakpoint(part) if (part.mediaType.startsWith("image/")) { const detail = getImageDetail(part) const mediaType = part.mediaType === "image/*" ? "image/jpeg" : part.mediaType if (typeof part.data === "string" && isFileId(part.data, config)) { - content.push({ type: "input_image", file_id: part.data, detail }) + content.push({ type: "input_image", file_id: part.data, detail, ...promptCacheBreakpoint }) } else { const imageUrl = part.data instanceof URL ? part.data.toString() @@ -939,22 +947,27 @@ const prepareMessages = Effect.fnUntraced( : /^(data:|https?:\/\/)/i.test(part.data) ? part.data : `data:${mediaType};base64,${part.data}` - content.push({ type: "input_image", image_url: imageUrl, detail }) + content.push({ type: "input_image", image_url: imageUrl, detail, ...promptCacheBreakpoint }) } } else if (part.mediaType === "application/pdf") { if (typeof part.data === "string" && isFileId(part.data, config)) { - content.push({ type: "input_file", file_id: part.data }) + content.push({ type: "input_file", file_id: part.data, ...promptCacheBreakpoint }) } if (part.data instanceof URL) { - content.push({ type: "input_file", file_url: part.data.toString() }) + content.push({ type: "input_file", file_url: part.data.toString(), ...promptCacheBreakpoint }) } if (part.data instanceof Uint8Array) { const base64 = Base64.encode(part.data) const fileName = part.fileName ?? `part-${index}.pdf` const fileData = `data:application/pdf;base64,${base64}` - content.push({ type: "input_file", filename: fileName, file_data: fileData }) + content.push({ + type: "input_file", + filename: fileName, + file_data: fileData, + ...promptCacheBreakpoint + }) } } else { return yield* AiError.make({ @@ -3133,7 +3146,7 @@ const getEncryptedContent = ( const getImageDetail = (part: Prompt.FilePart): ImageDetail => part.options.openai?.imageDetail ?? "auto" const getPromptCacheBreakpoint = ( - input: Prompt.SystemMessage | Prompt.TextPart + input: Prompt.SystemMessage | Prompt.TextPart | Prompt.FilePart ) => { const promptCacheBreakpoint = input.options.openai?.promptCacheBreakpoint return Predicate.isNotNullish(promptCacheBreakpoint) ? { prompt_cache_breakpoint: promptCacheBreakpoint } : undefined diff --git a/repos/effect/packages/ai/openai/src/OpenAiSchema.ts b/repos/effect/packages/ai/openai/src/OpenAiSchema.ts index 4849960885..2c07429d6a 100644 --- a/repos/effect/packages/ai/openai/src/OpenAiSchema.ts +++ b/repos/effect/packages/ai/openai/src/OpenAiSchema.ts @@ -93,7 +93,8 @@ const InputImageContent = Schema.Struct({ type: Schema.Literal("input_image"), image_url: Schema.optionalKey(Schema.NullOr(Schema.String)), file_id: Schema.optionalKey(Schema.NullOr(Schema.String)), - detail: Schema.optionalKey(Schema.NullOr(ImageDetail)) + detail: Schema.optionalKey(Schema.NullOr(ImageDetail)), + prompt_cache_breakpoint: Schema.optional(PromptCacheBreakpoint) }) const InputFileContent = Schema.Struct({ @@ -101,7 +102,8 @@ const InputFileContent = Schema.Struct({ file_id: Schema.optionalKey(Schema.NullOr(Schema.String)), filename: Schema.optionalKey(Schema.String), file_url: Schema.optionalKey(Schema.String), - file_data: Schema.optionalKey(Schema.String) + file_data: Schema.optionalKey(Schema.String), + prompt_cache_breakpoint: Schema.optional(PromptCacheBreakpoint) }) /** @@ -703,7 +705,8 @@ export type TextResponseFormatConfiguration = typeof TextResponseFormatConfigura * Validates the Responses API request payload, including input content, model * selection, instructions, reasoning options, text output format, tools, * `tool_choice`, streaming, storage, response continuation, sampling options, - * prompt caching, and optional response fields requested through `include`. + * safety identifiers, prompt caching, and optional response fields requested + * through `include`. * * **Gotchas** * @@ -723,7 +726,9 @@ export const CreateResponse = Schema.Struct({ temperature: Schema.optional(Schema.Finite), top_p: Schema.optional(Schema.Finite), user: Schema.optional(Schema.String), + safety_identifier: Schema.optional(Schema.String), prompt_cache_key: Schema.optional(Schema.String), + prompt_cache_retention: Schema.optional(Schema.Literals(["in_memory", "24h"])), prompt_cache_options: Schema.optional(Schema.Struct({ mode: Schema.optional(Schema.Literals(["implicit", "explicit"])), ttl: Schema.optional(Schema.Literal("30m")), diff --git a/repos/effect/packages/ai/openai/src/internal/errors.ts b/repos/effect/packages/ai/openai/src/internal/errors.ts index 4e2ec36aa3..8572fc4d4c 100644 --- a/repos/effect/packages/ai/openai/src/internal/errors.ts +++ b/repos/effect/packages/ai/openai/src/internal/errors.ts @@ -1,9 +1,11 @@ import * as AiError from "effect/ai/AiError" import type * as Response from "effect/ai/Response" -import * as Duration from "effect/Duration" +import * as Clock from "effect/Clock" +import type * as Duration from "effect/Duration" import * as Effect from "effect/Effect" import type * as Sse from "effect/encoding/Sse" import { dual } from "effect/Function" +import * as Headers from "effect/http/Headers" import type * as HttpClientError from "effect/http/HttpClientError" import type * as HttpClientRequest from "effect/http/HttpClientRequest" import type * as HttpClientResponse from "effect/http/HttpClientResponse" @@ -11,6 +13,7 @@ import * as Number from "effect/Number" import * as Option from "effect/Option" import * as Predicate from "effect/Predicate" import * as Redactable from "effect/Redactable" +import * as Result from "effect/Result" import * as Schema from "effect/Schema" import type { OpenAiErrorMetadata } from "../OpenAiError.ts" @@ -45,7 +48,7 @@ export const mapSchemaError = dual< AiError.make({ module: "OpenAiClient", method, - reason: AiError.InvalidOutputError.fromSchemaError(error) + reason: AiError.InvalidResponseError.fromSchemaError(error) })) /** @internal */ @@ -56,7 +59,7 @@ export const mapSseError = dual< AiError.make({ module: "OpenAiClient", method, - reason: new AiError.InvalidOutputError({ description: error.message }) + reason: new AiError.InvalidResponseError({ description: error.message }) })) /** @internal */ @@ -164,7 +167,9 @@ const mapStatusCodeError = Effect.fnUntraced(function*( ? compatibleDecoded.value.code ?? null : null + const now = yield* Clock.currentTimeMillis const reason = mapStatusCodeToReason({ + now, status, headers, message, @@ -183,16 +188,16 @@ const mapStatusCodeError = Effect.fnUntraced(function*( // Rate Limits // ============================================================================= +const retryAfterFromHeader = (raw: string, now: number): Duration.Duration | undefined => { + const retryAfter = Headers.parseRetryAfter(raw) + return Result.isSuccess(retryAfter) ? Headers.retryAfterDelay(retryAfter.success, now) : undefined +} + /** @internal */ -export const parseRateLimitHeaders = (headers: Record) => { +export const parseRateLimitHeaders = (headers: Record, now: number) => { const retryAfterRaw = headers["retry-after"] - let retryAfter: Duration.Duration | undefined - if (Predicate.isNotUndefined(retryAfterRaw)) { - const parsed = Number.parse(retryAfterRaw) - if (Option.isSome(parsed)) { - retryAfter = Duration.seconds(parsed.value) - } - } + // an invalid Retry-After is treated as absent + const retryAfter = retryAfterRaw === undefined ? undefined : retryAfterFromHeader(retryAfterRaw, now) const remainingRaw = headers["x-ratelimit-remaining-requests"] const remaining = Predicate.isNotUndefined(remainingRaw) ? Option.getOrNull(Number.parse(remainingRaw)) @@ -242,8 +247,10 @@ export const buildHttpContext = (params: { // ============================================================================= /** @internal */ -export const mapStatusCodeToReason = ({ status, headers, message, metadata, http }: { +export const mapStatusCodeToReason = ({ status, headers, message, metadata, http, now }: { readonly status: number + /** Current time in epoch milliseconds, used to measure a Retry-After date. */ + readonly now: number readonly headers: Record readonly message: string | undefined readonly metadata: OpenAiErrorMetadata @@ -271,14 +278,14 @@ export const mapStatusCodeToReason = ({ status, headers, message, metadata, http return new AiError.AuthenticationError({ kind: "InvalidKey", description: errorDescription, - metadata, + metadata: { openai: metadata }, http }) case 403: return new AiError.AuthenticationError({ kind: "InsufficientPermissions", description: errorDescription, - metadata, + metadata: { openai: metadata }, http }) case 404: @@ -312,7 +319,7 @@ export const mapStatusCodeToReason = ({ status, headers, message, metadata, http http }) } - const { retryAfter, ...rateLimitMetadata } = parseRateLimitHeaders(headers) + const { retryAfter, ...rateLimitMetadata } = parseRateLimitHeaders(headers, now) return new AiError.RateLimitError({ retryAfter, metadata: { @@ -328,13 +335,13 @@ export const mapStatusCodeToReason = ({ status, headers, message, metadata, http if (status >= 500) { return new AiError.InternalProviderError({ description: message ?? "Server error", - metadata, + metadata: { openai: metadata }, http }) } return new AiError.UnknownError({ description: message, - metadata, + metadata: { openai: metadata }, http }) } diff --git a/repos/effect/packages/ai/openai/test/OpenAiClient.test.ts b/repos/effect/packages/ai/openai/test/OpenAiClient.test.ts index 2a3b3d5310..4e3c643088 100644 --- a/repos/effect/packages/ai/openai/test/OpenAiClient.test.ts +++ b/repos/effect/packages/ai/openai/test/OpenAiClient.test.ts @@ -6,7 +6,7 @@ import * as OpenAiClientGenerated from "@effect/ai-openai/OpenAiClientGenerated" import * as OpenAiConfig from "@effect/ai-openai/OpenAiConfig" import * as OpenAiLanguageModel from "@effect/ai-openai/OpenAiLanguageModel" import { assert, describe, it } from "@effect/vitest" -import { Config, ConfigProvider, Context, Effect, Layer, Redacted, Schema, Stream } from "effect" +import { Config, ConfigProvider, Context, Duration, Effect, Layer, Redacted, Schema, Stream } from "effect" import { Chat } from "effect/ai" import * as HttpClient from "effect/http/HttpClient" import * as HttpClientError from "effect/http/HttpClientError" @@ -309,6 +309,7 @@ describe("OpenAiClient", () => { it("preserves and truncates a fallback HTTP response", () => { const body = `${"a".repeat(200)}b` const reason = Errors.mapStatusCodeToReason({ + now: 0, status: 400, headers: {}, message: undefined, @@ -358,6 +359,28 @@ describe("OpenAiClient", () => { body: { error: { message: "Rate limit exceeded", type: "rate_limit_error", code: null } } })))) + it.effect.each([ + { name: "ignores a negative Retry-After", header: "-5", expected: undefined }, + { name: "ignores a non-finite Retry-After", header: "Infinity", expected: undefined }, + { name: "honors an HTTP-date Retry-After", header: "Thu, 01 Jan 1970 00:00:30 GMT", expected: 30_000 } + ])("$name", ({ expected, header }) => + Effect.gen(function*() { + const client = yield* OpenAiClient.OpenAiClient + const result = yield* client.createResponse({ model: "gpt-4o", input: "test" }).pipe(Effect.flip) + + assert.strictEqual(result.reason._tag, "RateLimitError") + if (result.reason._tag !== "RateLimitError") return + assert.strictEqual( + result.reason.retryAfter === undefined ? undefined : Duration.toMillis(result.reason.retryAfter), + expected + ) + }).pipe(Effect.provide(makeTestLayer(undefined, { + _tag: "Json", + status: 429, + headers: { "retry-after": header }, + body: { error: { message: "Rate limit exceeded", type: "rate_limit_error", code: null } } + })))) + it.effect("maps 429 with insufficient_quota code to QuotaExhaustedError", () => Effect.gen(function*() { const client = yield* OpenAiClient.OpenAiClient @@ -417,6 +440,7 @@ describe("OpenAiClient", () => { } } const reason = Errors.mapStatusCodeToReason({ + now: 0, status: 429, headers: {}, message: "You exceeded your current quota", @@ -458,14 +482,15 @@ describe("OpenAiClient", () => { body: { error: { message: "Internal server error" } } })))) - it.effect("maps schema error to InvalidOutputError reason", () => + it.effect("maps a malformed successful response envelope to a non-retryable InvalidResponseError reason", () => Effect.gen(function*() { const client = yield* OpenAiClient.OpenAiClient const result = yield* client.createResponse({ model: "gpt-4o", input: "test" }).pipe(Effect.flip) assert.strictEqual(result._tag, "AiError") assert.strictEqual(result.method, "createResponse") - assert.strictEqual(result.reason._tag, "InvalidOutputError") + assert.strictEqual(result.reason._tag, "InvalidResponseError") + assert.isFalse(result.isRetryable) }).pipe(Effect.provide(makeTestLayer(undefined, { _tag: "Json", body: { invalid: "response" } as any @@ -961,6 +986,52 @@ describe("OpenAiClient", () => { ] })))) + it.effect("maps a malformed stream event to a non-retryable InvalidResponseError", () => + Effect.gen(function*() { + const client = yield* OpenAiClient.OpenAiClient + const result = yield* client.createResponseStream({ + model: "gpt-4o", + input: "test" + }).pipe( + Effect.andThen(([_, stream]) => Stream.runDrain(stream)), + Effect.flip + ) + + assert.strictEqual(result.reason._tag, "InvalidResponseError") + assert.isFalse(result.isRetryable) + }).pipe(Effect.provide(makeTestLayer(undefined, { + _tag: "Sse", + events: [ + { + type: "response.output_text.delta", + output_index: "zero", + item_id: "msg_123", + content_index: 0, + delta: "hello" + } + ] + })))) + + it.effect("maps an oversized SSE event to a non-retryable InvalidResponseError", () => + Effect.gen(function*() { + const client = yield* OpenAiClient.OpenAiClient + const result = yield* client.createResponseStream({ + model: "gpt-4o", + input: "test" + }).pipe( + Effect.andThen(([_, stream]) => Stream.runDrain(stream)), + Effect.flip + ) + + assert.strictEqual(result.reason._tag, "InvalidResponseError") + assert.isFalse(result.isRetryable) + }).pipe(Effect.provide(makeTestLayer(undefined, { + _tag: "Sse", + events: [], + // Exceed the default 10 MiB SSE limit without completing an event. + trailer: `data: ${"x".repeat(10 * 1024 * 1024 + 1)}` + })))) + it.effect("maps HTTP error before stream starts", () => Effect.gen(function*() { const client = yield* OpenAiClient.OpenAiClient diff --git a/repos/effect/packages/ai/openai/test/OpenAiDecisionModel.test.ts b/repos/effect/packages/ai/openai/test/OpenAiDecisionModel.test.ts index 2b4b425e0b..a18ab03508 100644 --- a/repos/effect/packages/ai/openai/test/OpenAiDecisionModel.test.ts +++ b/repos/effect/packages/ai/openai/test/OpenAiDecisionModel.test.ts @@ -1,7 +1,7 @@ import { OpenAiClient } from "@effect/ai-openai" import * as OpenAiDecisionModel from "@effect/ai-openai/OpenAiDecisionModel" import { assert, describe, it } from "@effect/vitest" -import { Effect, Layer, Redacted, Schema } from "effect" +import { Effect, Layer, Redacted, Result, Schema } from "effect" import { Decision, DecisionModel, Model } from "effect/ai" import { HttpClient, type HttpClientError, type HttpClientRequest, HttpClientResponse } from "effect/http" @@ -250,21 +250,42 @@ describe("OpenAiDecisionModel", () => { assert.strictEqual(error.reason._tag, "InvalidOutputError") }).pipe(Effect.provide(layer(response(answers.slice(0, 2)))))) - it.effect("fails the whole call with ContentPolicyError when any answer is a refusal", () => + it.effect("decide fails with ContentPolicyError when any answer is a refusal, even beside a missing answer", () => Effect.gen(function*() { const error = yield* Effect.flip(DecisionModel.decide(Triage, { input })) assert.strictEqual(error.reason._tag, "ContentPolicyError") if (error.reason._tag === "ContentPolicyError") { assert.include(error.reason.description, "severity") assert.notInclude(error.reason.description, "team") - assert.include(JSON.stringify(error.reason.metadata.openai), "severity") } }).pipe(Effect.provide(layer(response([ - answers[0], { name: "severity", type: "refusal", refusal: "I can't help with that." }, answers[2] ]))))) + it.effect("decideSettled reports a refusal and a mismatched score per decision", () => + Effect.gen(function*() { + const { outcomes } = yield* DecisionModel.decideSettled(Triage, { input }) + assert.isTrue(Result.isFailure(outcomes.team) && outcomes.team.failure.reason === "Invalid") + assert.isTrue(Result.isFailure(outcomes.severity) && outcomes.severity.failure.reason === "Refused") + assert.deepStrictEqual(outcomes.urgent, Result.succeed({ probability: 0.99 })) + }).pipe(Effect.provide(layer(response([ + { ...answers[1], name: "team" }, + { name: "severity", type: "refusal", refusal: "I can't help with that." }, + answers[2] + ]))))) + + it.effect("a refusal outranks a later answer with the same name", () => + Effect.gen(function*() { + const error = yield* Effect.flip(DecisionModel.decide(Urgency, { input: "Checkout is down" })) + assert.strictEqual(error.reason._tag, "ContentPolicyError") + const { outcomes } = yield* DecisionModel.decideSettled(Urgency, { input: "Checkout is down" }) + assert.isTrue(Result.isFailure(outcomes.urgent) && outcomes.urgent.failure.reason === "Refused") + }).pipe(Effect.provide(layer(response([ + { name: "urgent", type: "refusal" }, + { name: "urgent", type: "predicate", probability: 0.99 } + ]))))) + it.effect("sends string and JSON state with byte array, base64 string, and data URL images", () => { const dataUrl = "data:image/webp;base64,UklGRg==" const images = [ diff --git a/repos/effect/packages/ai/openai/test/OpenAiEmbeddingModel.test.ts b/repos/effect/packages/ai/openai/test/OpenAiEmbeddingModel.test.ts index ffe9c4496c..bd76466b0f 100644 --- a/repos/effect/packages/ai/openai/test/OpenAiEmbeddingModel.test.ts +++ b/repos/effect/packages/ai/openai/test/OpenAiEmbeddingModel.test.ts @@ -113,6 +113,47 @@ describe("OpenAiEmbeddingModel", () => { assert.deepStrictEqual(requestBody.input, ["hello"]) })) + it.effect("concurrent embed calls batch by their withConfigOverride", () => + Effect.gen(function*() { + const requests: Array> = [] + const clientLayer = OpenAiClient.layer({ apiKey: Redacted.make("sk-test-key") }).pipe( + Layer.provide(Layer.succeed( + HttpClient.HttpClient, + makeHttpClient((request) => + Effect.gen(function*() { + const body = yield* getRequestBody(request) + requests.push(body.input) + return jsonResponse(request, { + data: body.input.map((_: string, index: number) => ({ + index, + embedding: Array.from({ length: body.dimensions }, () => 1), + object: "embedding" + })), + model: body.model, + object: "list", + usage: { prompt_tokens: 1, total_tokens: 1 } + }) + }) + ) + )) + ) + + const responses = yield* Effect.gen(function*() { + const model = yield* EmbeddingModel.EmbeddingModel + return yield* Effect.all([ + model.embed("a").pipe(OpenAiEmbeddingModel.withConfigOverride({ dimensions: 2 })), + model.embed("b").pipe(OpenAiEmbeddingModel.withConfigOverride({ dimensions: 3 })), + model.embed("c").pipe(OpenAiEmbeddingModel.withConfigOverride({ dimensions: 2 })) + ], { concurrency: "unbounded" }) + }).pipe( + Effect.provide(OpenAiEmbeddingModel.layer({ model: "text-embedding-3-small" })), + Effect.provide(clientLayer) + ) + + assert.deepStrictEqual(responses.map((response) => response.vector), [[1, 1], [1, 1, 1], [1, 1]]) + assert.deepStrictEqual(requests, [["a", "c"], ["b"]]) + })) + it.effect("fails with InvalidOutputError when provider returns duplicate indices", () => Effect.gen(function*() { const clientLayer = OpenAiClient.layer({ apiKey: Redacted.make("sk-test-key") }).pipe( @@ -271,15 +312,14 @@ const jsonResponse = ( }) ) -const getRequestBody = (request: HttpClientRequest.HttpClientRequest) => - Effect.gen(function*() { - const body = request.body - if (body._tag === "Uint8Array") { - const text = new TextDecoder().decode(body.body) - return JSON.parse(text) - } - return yield* Effect.die(new Error("Expected Uint8Array body")) - }) +const getRequestBody = Effect.fnUntraced(function*(request: HttpClientRequest.HttpClientRequest) { + const body = request.body + if (body._tag === "Uint8Array") { + const text = new TextDecoder().decode(body.body) + return JSON.parse(text) + } + return yield* Effect.die(new Error("Expected Uint8Array body")) +}) const noopOpenAiClient: OpenAiClient.Service = { client: undefined as unknown as OpenAiClient.Service["client"], diff --git a/repos/effect/packages/ai/openai/test/OpenAiLanguageModel.test.ts b/repos/effect/packages/ai/openai/test/OpenAiLanguageModel.test.ts index 197f5ff283..4dc0d3130a 100644 --- a/repos/effect/packages/ai/openai/test/OpenAiLanguageModel.test.ts +++ b/repos/effect/packages/ai/openai/test/OpenAiLanguageModel.test.ts @@ -184,7 +184,7 @@ describe("OpenAiLanguageModel", () => { describe("generateText", () => { describe("message preparation", () => { - it.effect("forwards prompt cache configuration and text breakpoints", () => + it.effect("forwards prompt cache, safety configuration and text breakpoints", () => Effect.gen(function*() { const breakpoint = { mode: "explicit" } as const yield* LanguageModel.generateText({ @@ -203,7 +203,9 @@ describe("OpenAiLanguageModel", () => { }).pipe( Effect.provide(OpenAiLanguageModel.model("gpt-5.6", { prompt_cache_key: "assistant:v1", - prompt_cache_options: { mode: "explicit", ttl: "30m", comparison_response_id: "resp_baseline" } + prompt_cache_options: { mode: "explicit", ttl: "30m", comparison_response_id: "resp_baseline" }, + prompt_cache_retention: "24h", + safety_identifier: "user-hash-123" })) ) @@ -211,6 +213,8 @@ describe("OpenAiLanguageModel", () => { const body = yield* getRequestBody(requests[0]) strictEqual(body.prompt_cache_key, "assistant:v1") + strictEqual(body.prompt_cache_retention, "24h") + strictEqual(body.safety_identifier, "user-hash-123") deepStrictEqual(body.prompt_cache_options, { mode: "explicit", ttl: "30m", @@ -233,6 +237,46 @@ describe("OpenAiLanguageModel", () => { }]) }).pipe(Effect.provide(makeTestLayer({ body: { model: "gpt-5.6" as any } })))) + it.effect("forwards image and file breakpoints", () => + Effect.gen(function*() { + const breakpoint = { mode: "explicit" } as const + yield* LanguageModel.generateText({ + prompt: Prompt.make([ + Prompt.userMessage({ + content: [ + Prompt.filePart({ + mediaType: "image/png", + data: new URL("https://example.com/image.png"), + options: { openai: { promptCacheBreakpoint: breakpoint } } + }), + Prompt.filePart({ + mediaType: "application/pdf", + data: new URL("https://example.com/document.pdf"), + options: { openai: { promptCacheBreakpoint: breakpoint } } + }) + ] + }) + ]) + }).pipe(Effect.provide(OpenAiLanguageModel.model("gpt-5.6"))) + + const requests = yield* MockHttpClient.requests + const body = yield* getRequestBody(requests[0]) + + deepStrictEqual(body.input, [{ + role: "user", + content: [{ + type: "input_image", + image_url: "https://example.com/image.png", + detail: "auto", + prompt_cache_breakpoint: breakpoint + }, { + type: "input_file", + file_url: "https://example.com/document.pdf", + prompt_cache_breakpoint: breakpoint + }] + }]) + }).pipe(Effect.provide(makeTestLayer({ body: { model: "gpt-5.6" as any } })))) + it.effect("forwards implicit prompt cache mode without text breakpoints", () => Effect.gen(function*() { yield* LanguageModel.generateText({ prompt: "Stable context" }).pipe( @@ -2754,15 +2798,14 @@ const makeTestLayer = (options: { })) ) -const getRequestBody = (request: HttpClientRequest.HttpClientRequest) => - Effect.gen(function*() { - const body = request.body - if (body._tag === "Uint8Array") { - const text = new TextDecoder().decode(body.body) - return JSON.parse(text) - } - return yield* Effect.die(new Error("Expected Uint8Array body")) - }) +const getRequestBody = Effect.fnUntraced(function*(request: HttpClientRequest.HttpClientRequest) { + const body = request.body + if (body._tag === "Uint8Array") { + const text = new TextDecoder().decode(body.body) + return JSON.parse(text) + } + return yield* Effect.die(new Error("Expected Uint8Array body")) +}) const decodeToolParamsFromStream = ( parts: ReadonlyArray, diff --git a/repos/effect/packages/ai/openai/test/OpenAiSchema.test.ts b/repos/effect/packages/ai/openai/test/OpenAiSchema.test.ts index da4e2bb83a..869d1b2551 100644 --- a/repos/effect/packages/ai/openai/test/OpenAiSchema.test.ts +++ b/repos/effect/packages/ai/openai/test/OpenAiSchema.test.ts @@ -55,6 +55,18 @@ describe("OpenAiSchema", () => { assert.strictEqual(decoded.reasoning?.effort, "max") }) + it("accepts safety and prompt cache retention request options", () => { + const decoded = Schema.decodeUnknownSync(OpenAiSchema.CreateResponse)({ + safety_identifier: "user-hash-123", + prompt_cache_retention: "24h" + }) + + assert.deepStrictEqual(decoded, { + safety_identifier: "user-hash-123", + prompt_cache_retention: "24h" + }) + }) + it("decodes a representative response payload", () => { const decoded = Schema.decodeUnknownSync(OpenAiSchema.Response)({ ...makeResponse(), diff --git a/repos/effect/packages/ai/openrouter/CHANGELOG.md b/repos/effect/packages/ai/openrouter/CHANGELOG.md index 7e564ba4f2..346409adfa 100644 --- a/repos/effect/packages/ai/openrouter/CHANGELOG.md +++ b/repos/effect/packages/ai/openrouter/CHANGELOG.md @@ -1,5 +1,37 @@ # @effect/ai-openrouter +## 4.0.6 + +### Patch Changes + +- [#9437](https://github.com/Effect-TS/effect/pull/9437) [`fed70de`](https://github.com/Effect-TS/effect/commit/fed70de0dff8af24c5f814bd0d985ab70c70420b) Thanks @effect-bot! - Add `DecisionModel.decideSettled` to retain valid answers alongside per-decision `Result` failures (`Missing`, `Refused`, `NoDistribution`, or `Invalid`). Provider failures still fail the effect; `decide` remains all-or-nothing. + + OpenRouter now passes missing distributions to core validation. OpenAI reports refusals per decision and wrong-kind score answers as invalid rather than missing. + + Update hand-built services to implement `decideSettled`, handle optional provider `probabilities` and refusal answers, and use settled outcomes instead of the removed `OpenAiContentPolicyMetadata` and `metadata.openai.refusedDecisions`. + +- [#9422](https://github.com/Effect-TS/effect/pull/9422) [`82648a2`](https://github.com/Effect-TS/effect/commit/82648a21bf228145a14d2d1390006d67f74d89c7) Thanks @candrewlee14! - `DecisionModel.decide` now returns the response `id`, the `modelId` that answered and provider `metadata`. `OpenRouterDecisionModel` reports `cost` and the serving provider under `metadata.openrouter`. + +- [#9709](https://github.com/Effect-TS/effect/pull/9709) [`85e2aee`](https://github.com/Effect-TS/effect/commit/85e2aeee686963c26ca40d204f982deebb9d6117) Thanks @tim-smart! - Add typed HTTP date and `Retry-After` header values to `Headers`, and parse them consistently across the HTTP client, OTLP exporter, GraphQL client, AI clients and static server. + + - `Headers.parseHttpDate` / `Headers.formatHttpDate` read all three RFC 9110 date formats as UTC, whatever the process time zone, and emit IMF-fixdate. + - `Headers.parseRetryAfter`, `Headers.formatRetryAfter` and `Headers.retryAfterDelay` accept whole non-negative seconds of any size, read exactly, or an HTTP date, and the delay is never negative. HTTP dates accept the `23:59:60` leap second, read as the following second. + - `Headers.RetryAfter` holds either a `Duration` (`RetryAfterDuration`) or a `DateTime.Utc` (`RetryAfterDateTime`). `Headers.retryAfterDuration` builds the first from any `Duration.Input`: infinite durations throw, finite negative ones become zero, and fractional seconds are rounded up when formatted. + - Parse failures are reported as `Headers.HeaderParseError`. + - `Schema.DateTimeUtcFromHttpDate` and `Schema.RetryAfterFromString` expose the same parsing as string codecs for HTTP API headers. + - Invalid `Retry-After` values such as `-5`, `1.5`, `1e3` or `Infinity` are now ignored everywhere. `HttpClient.retryTransient` and `HttpClient.withRateLimiter` no longer honour fractional seconds, and the AI clients no longer produce negative or infinite `retryAfter` durations. + - `HttpStaticServer` reads `If-Modified-Since` and `Last-Modified` as UTC; asctime dates were previously shifted by the process time zone. + +- [#9430](https://github.com/Effect-TS/effect/pull/9430) [`48102ac`](https://github.com/Effect-TS/effect/commit/48102ac592ef89fed9d5d10afd2daece0a0ee310) Thanks @effect-bot! - Add non-retryable `AiError.InvalidResponseError` for provider response schema and SSE decoding failures, with optional HTTP context; model-generated output validation remains retryable. Update exhaustive matches on `AiErrorReason` to handle the new tag, and update serialized-error consumers to accept it. + +- [#9502](https://github.com/Effect-TS/effect/pull/9502) [`447fdbf`](https://github.com/Effect-TS/effect/commit/447fdbf27a57d0e9e84b641bb20219941a1790e8) Thanks @typedrat! - Preserve `cacheControl` options on assistant text and tool results when sending messages to OpenRouter. + +- [#9421](https://github.com/Effect-TS/effect/pull/9421) [`e33e867`](https://github.com/Effect-TS/effect/commit/e33e867ef5823f585ba56bd2b258128804e6b212) Thanks @candrewlee14! - Correct the provider name in `OpenRouterLanguageModel.model` metadata and return type from `"openai"` to `"openrouter"`. Update explicit provider type annotations or comparisons accordingly. + +- [#9263](https://github.com/Effect-TS/effect/pull/9263) [`d453790`](https://github.com/Effect-TS/effect/commit/d45379064ace1c497970fdf772719ca3f7026028) Thanks @effect-bot! - Default schema-backed OpenRouter response formats to strict validation for models using the OpenAI schema transformer (`openai/`, `gpt-`, `o1-`, `o3-`, and `o4-` prefixes). Other models retain `strict: null`, and tool strictness is unchanged. Endpoint support still varies; callers using incompatible endpoints can set `strictJsonSchema: false` to opt out, and explicit overrides remain respected. +- Updated dependencies [[`ce13a48`](https://github.com/Effect-TS/effect/commit/ce13a4872e516187851c5d9336d4df309cc8f0a4), [`f3bbbb4`](https://github.com/Effect-TS/effect/commit/f3bbbb448d431a9d1205ff09ba47cb81af0c7199), [`5547315`](https://github.com/Effect-TS/effect/commit/554731500e43eef47ff68b569116b74006e91d87), [`6093cfb`](https://github.com/Effect-TS/effect/commit/6093cfbbb673d36ecd18edea955763db003c8c26), [`30319d5`](https://github.com/Effect-TS/effect/commit/30319d506c9177d1cc1e0b10372e1ddb776aaf0a), [`74a7596`](https://github.com/Effect-TS/effect/commit/74a759699a8582dbe660b21dcda21cc9db08f613), [`bfb32a3`](https://github.com/Effect-TS/effect/commit/bfb32a3cf5f8d0f02f0dfdb63dcf5237fde755b0), [`f8bf7f4`](https://github.com/Effect-TS/effect/commit/f8bf7f4bcc3ad543cd94cbeb010cb164e5dedcb0), [`7f3d3df`](https://github.com/Effect-TS/effect/commit/7f3d3dfbb7b9e6e041c4ad9654c4532f72d2f16e), [`db1046d`](https://github.com/Effect-TS/effect/commit/db1046d1fa0e4b5818c51d28017a89579b6261d6), [`3c4f88a`](https://github.com/Effect-TS/effect/commit/3c4f88aa4d1d4d958b42fe8d3f1195f0ca990f8a), [`916e709`](https://github.com/Effect-TS/effect/commit/916e70968acc4e443885c67012d855b7cc6f9bca), [`916f843`](https://github.com/Effect-TS/effect/commit/916f8436225153a81a049bdc1feaedcafe4d1ded), [`66ec757`](https://github.com/Effect-TS/effect/commit/66ec7574d23b7943584059b41136bc21ee9d4c53), [`daac06f`](https://github.com/Effect-TS/effect/commit/daac06f79a6ab8c7d3e1b66b427c06f7d64dcc8f), [`c2552ec`](https://github.com/Effect-TS/effect/commit/c2552ec2e031b409b2bef67309312b0168480a39), [`b02c1cd`](https://github.com/Effect-TS/effect/commit/b02c1cde36c890941a235224af8ecb035197582a), [`1855f1d`](https://github.com/Effect-TS/effect/commit/1855f1d359535339bc7640b67e6bb7b12752ba19), [`683ed2c`](https://github.com/Effect-TS/effect/commit/683ed2cb491f9529379c1d5c63933689ff57fd2b), [`ac68940`](https://github.com/Effect-TS/effect/commit/ac68940f7e4246b1beb43f9d451a67f41241ab82), [`22a8401`](https://github.com/Effect-TS/effect/commit/22a84017e7222f332611c1bace05f9fa60c7cf67), [`c53c194`](https://github.com/Effect-TS/effect/commit/c53c19451171a00776b400a8548394ac514b628a), [`ca6d1cc`](https://github.com/Effect-TS/effect/commit/ca6d1cc5f803cd501e85431642b79b552322a76a), [`47940c5`](https://github.com/Effect-TS/effect/commit/47940c55c9e0eb855615094a2aca6e0812307bb2), [`e8b24ef`](https://github.com/Effect-TS/effect/commit/e8b24ef9516706971db948efb2d3eab84ad85951), [`2e8aae0`](https://github.com/Effect-TS/effect/commit/2e8aae0080a8d3e6d00c5ee0bc4f49f0b074acc2), [`ad7497b`](https://github.com/Effect-TS/effect/commit/ad7497bbc1bc70935f8f1ae395ff501532e2e770), [`7f504a3`](https://github.com/Effect-TS/effect/commit/7f504a35edcf44952c55ecbb0916726bac4d3e52), [`99e365d`](https://github.com/Effect-TS/effect/commit/99e365d61a1fdaabeb31dc58e5752cade8a09e1f), [`954067d`](https://github.com/Effect-TS/effect/commit/954067da7fbd20a20c10d5d0de6ffe4690e6bdfd), [`3e1a035`](https://github.com/Effect-TS/effect/commit/3e1a035d9c470ced858b7757f6d102b3aa9be008), [`41a1fe7`](https://github.com/Effect-TS/effect/commit/41a1fe74e6338e8d13d6d4f3daa03f71a826c2c4), [`bcb3a86`](https://github.com/Effect-TS/effect/commit/bcb3a86335a8ed18da39300984c0fbae187fdc6b), [`739d3d1`](https://github.com/Effect-TS/effect/commit/739d3d18728cce0b6d907f0a3f24e04ab0b53560), [`ed4da35`](https://github.com/Effect-TS/effect/commit/ed4da351fb9d29e4dfd6c60bd89ba4c007d264dc), [`3d84e19`](https://github.com/Effect-TS/effect/commit/3d84e19f8f14bccf0aa57b11825590c3bae162f4), [`cc00906`](https://github.com/Effect-TS/effect/commit/cc0090657eeb89debdd54af5fcb99d544f46c1bf), [`09ba1f5`](https://github.com/Effect-TS/effect/commit/09ba1f59952c46cd22229d41236742c3e1f53a34), [`9ff1a73`](https://github.com/Effect-TS/effect/commit/9ff1a73d995702f4c4c335888fe6a186f3f710c7), [`06bc532`](https://github.com/Effect-TS/effect/commit/06bc532b71cd3684f010956774356864dd359372), [`97bfd84`](https://github.com/Effect-TS/effect/commit/97bfd84bb147e10ed562e4c1ffa589cdaf83a596), [`25efaef`](https://github.com/Effect-TS/effect/commit/25efaef0d81d25b2651ef4de474e929d3180810a), [`c10335b`](https://github.com/Effect-TS/effect/commit/c10335be125c7846ae7b914eeb138244b660275b), [`a808972`](https://github.com/Effect-TS/effect/commit/a80897262648e4222ce48943e538f39a6b3f69d6), [`20e844c`](https://github.com/Effect-TS/effect/commit/20e844c7ef27b0e4b4f7911d603c45b1de2b0bc3), [`b9f73f3`](https://github.com/Effect-TS/effect/commit/b9f73f37c4ac538e6a327ed80ac5a0708858c540), [`5739878`](https://github.com/Effect-TS/effect/commit/5739878ded09d83d71d657e3b8d4264f4e4936d9), [`28acdf3`](https://github.com/Effect-TS/effect/commit/28acdf3820b66056037bb7a24cd0e8d3de3fb06d), [`59b6396`](https://github.com/Effect-TS/effect/commit/59b63967c1817ae5acb1811037f5aaa964219b7e), [`0bccc9c`](https://github.com/Effect-TS/effect/commit/0bccc9c3dd5349b20f8d2e509ac5482d0c26a616), [`fed70de`](https://github.com/Effect-TS/effect/commit/fed70de0dff8af24c5f814bd0d985ab70c70420b), [`82648a2`](https://github.com/Effect-TS/effect/commit/82648a21bf228145a14d2d1390006d67f74d89c7), [`afb5f32`](https://github.com/Effect-TS/effect/commit/afb5f326cc14c9849d8d7ddefde1039ae036e017), [`28813f5`](https://github.com/Effect-TS/effect/commit/28813f53a5026984fdc13d3971147ea5dc20104c), [`0fae934`](https://github.com/Effect-TS/effect/commit/0fae934a13aac12f9036b0733edd5dde5ce029f3), [`f260830`](https://github.com/Effect-TS/effect/commit/f2608305861269fc2fbedcbbdcd71e4fc5f98ffc), [`19b63eb`](https://github.com/Effect-TS/effect/commit/19b63eb81b2bccf033aadb565f52a6b0375d26a1), [`14db221`](https://github.com/Effect-TS/effect/commit/14db221f8f584100ff50fd1173b1a99a1caa1d06), [`679f956`](https://github.com/Effect-TS/effect/commit/679f956c4a7527854ff78db071dd7aa417211a48), [`b72707c`](https://github.com/Effect-TS/effect/commit/b72707c2937328fd3bc70f29a6792d84c4256484), [`44c8c1d`](https://github.com/Effect-TS/effect/commit/44c8c1d1c6f01ba9887a051b365e9fa89fcfc685), [`1e7909b`](https://github.com/Effect-TS/effect/commit/1e7909beb05bc740dbd330509e60bb185d7eeed1), [`0611a65`](https://github.com/Effect-TS/effect/commit/0611a657f76836f9ca275f64764b82270b4eda10), [`2fadc65`](https://github.com/Effect-TS/effect/commit/2fadc654aa655fbd1912ce51eb1b764a757ac362), [`feb6ce5`](https://github.com/Effect-TS/effect/commit/feb6ce59a38ab15fc6f89545bd5f0992dd58a06a), [`5a2ed53`](https://github.com/Effect-TS/effect/commit/5a2ed5323884a435bfcfb5e3a6f2ae29547458ad), [`c8d26a4`](https://github.com/Effect-TS/effect/commit/c8d26a46731cb26a9b9612d9fde823d8813cfc04), [`0c8b446`](https://github.com/Effect-TS/effect/commit/0c8b446709c737c10d0e11af4313bb29fdff2588), [`05c3409`](https://github.com/Effect-TS/effect/commit/05c340955944374e3ae47a1de0f7cb9b41a3b0e6), [`b04191b`](https://github.com/Effect-TS/effect/commit/b04191b2941534754282c51ef2df2f965ac43a5c), [`05c3409`](https://github.com/Effect-TS/effect/commit/05c340955944374e3ae47a1de0f7cb9b41a3b0e6), [`4067e64`](https://github.com/Effect-TS/effect/commit/4067e6428776ae573c8cc72f7eaebd078a4ff8c7), [`e04563d`](https://github.com/Effect-TS/effect/commit/e04563dcd6ddeea74c088a12194c5a73e597ba2f), [`02d7fc5`](https://github.com/Effect-TS/effect/commit/02d7fc583053c47dea3b5e1d9d6be164aeb321b6), [`561d2a1`](https://github.com/Effect-TS/effect/commit/561d2a1ddfa3c4340510f23da7adc50c89d97863), [`8fe2dcc`](https://github.com/Effect-TS/effect/commit/8fe2dcc2137788a71177d42a9f67fb7003224868), [`a8086fa`](https://github.com/Effect-TS/effect/commit/a8086fa26a2c9a9f5f7f1d7590bbc494a0feb02a), [`88a33b4`](https://github.com/Effect-TS/effect/commit/88a33b4122f7ffbbc1423add793a79abffd99c75), [`e77372a`](https://github.com/Effect-TS/effect/commit/e77372a8a2e7caec11999636e04c6e31c52e315f), [`3dd4388`](https://github.com/Effect-TS/effect/commit/3dd438810df34ed3b3d2e3e35ccf84ccb00feb85), [`ba2b00c`](https://github.com/Effect-TS/effect/commit/ba2b00c0658b65775809e0052ec9612432600c31), [`a049d4a`](https://github.com/Effect-TS/effect/commit/a049d4ad25f17cf4985284a8d15b07aea2a33242), [`84aeede`](https://github.com/Effect-TS/effect/commit/84aeede040460a2b24bcf0db7a5d0a9b0e3c42d4), [`43e5e19`](https://github.com/Effect-TS/effect/commit/43e5e198d16df038b60764f5dd0218cdcf828f16), [`77bbf16`](https://github.com/Effect-TS/effect/commit/77bbf16e50d6b0ea090288cc5d3f11b20706a027), [`6af3a47`](https://github.com/Effect-TS/effect/commit/6af3a47d5b5d698c3ba896c37684266cd465bef1), [`aac8d21`](https://github.com/Effect-TS/effect/commit/aac8d21911333dedb649f4f9d302c28d14d46369), [`84c8dc4`](https://github.com/Effect-TS/effect/commit/84c8dc47b09067362749620f75068d6f28044660), [`78d1988`](https://github.com/Effect-TS/effect/commit/78d198818d4fd547ce2e9431302f66cb00dc7ad3), [`5676a2a`](https://github.com/Effect-TS/effect/commit/5676a2ac5a918f8f91fc7838d7adf1b6f5e86a6d), [`4b437bf`](https://github.com/Effect-TS/effect/commit/4b437bf4645fabc8aaeaa3b5a88e8c9048b9e4f5), [`1af11fd`](https://github.com/Effect-TS/effect/commit/1af11fda1bccf823f74121b5125aa3befa084b18), [`716f8df`](https://github.com/Effect-TS/effect/commit/716f8dfa178f849e5b288464373bd132860c2e19), [`ee27718`](https://github.com/Effect-TS/effect/commit/ee27718a02125328a7478812b1b7d7c1d8532578), [`4d86d1b`](https://github.com/Effect-TS/effect/commit/4d86d1b8887d48fa24acf6ad54f33ef3d55a7cd8), [`5462ed7`](https://github.com/Effect-TS/effect/commit/5462ed7b3a4c37206c47e401d6be293dc540350f), [`658837c`](https://github.com/Effect-TS/effect/commit/658837c3b249d982df56f2990247499746e36672), [`00261cc`](https://github.com/Effect-TS/effect/commit/00261cc12c31ecc7878f80670439731c905dc2bd), [`7cccb93`](https://github.com/Effect-TS/effect/commit/7cccb9385729bc0aed6630edea95d7db15439ac4), [`7c98228`](https://github.com/Effect-TS/effect/commit/7c982281cf73958833054175fc86db385e043cf6), [`5a8e248`](https://github.com/Effect-TS/effect/commit/5a8e2481546352e2ad6233dad49420b14c59f624), [`8037a65`](https://github.com/Effect-TS/effect/commit/8037a65f40cae9163ab9e0e687e289f717ae570b), [`25912db`](https://github.com/Effect-TS/effect/commit/25912db9d493227a2d6fa45cebcfc7507e638d8a), [`7f32606`](https://github.com/Effect-TS/effect/commit/7f32606baac43a67d171ae93b77c7b29791c9692), [`0386f88`](https://github.com/Effect-TS/effect/commit/0386f88295ac70cbc1f90ab8cceb0981d848c628), [`403298a`](https://github.com/Effect-TS/effect/commit/403298ac457fb634eca2c013fed26d37fc80445f), [`5a3ebc0`](https://github.com/Effect-TS/effect/commit/5a3ebc082124f9e10106b2347736e66fc74ebc36), [`fac2191`](https://github.com/Effect-TS/effect/commit/fac21911964b742a38e1d4e73bd4f95aec4911dc), [`fdb12dc`](https://github.com/Effect-TS/effect/commit/fdb12dcd61b6505eed02e6f7a3c88a85d4c48e8d), [`97909b8`](https://github.com/Effect-TS/effect/commit/97909b89b167da54a56e6f82f1c4db8bfedae04f), [`c3983f4`](https://github.com/Effect-TS/effect/commit/c3983f4db2b3052a58e8045cce270fd7ccc4e2b2), [`3a5e65e`](https://github.com/Effect-TS/effect/commit/3a5e65e409c9569e23bb7af61df401347e9ec420), [`8b40fbc`](https://github.com/Effect-TS/effect/commit/8b40fbc93caf52f3a4c69020849f0051f100be65), [`3f3e119`](https://github.com/Effect-TS/effect/commit/3f3e119ad61c2167bc6c7170c3fb68cd1eb3421e), [`7c7b8ed`](https://github.com/Effect-TS/effect/commit/7c7b8eda54e44f9cddc1188f994f7fdddf901055), [`d4f71bc`](https://github.com/Effect-TS/effect/commit/d4f71bc42507b638d149fb8806d26ea626d17956), [`c66241e`](https://github.com/Effect-TS/effect/commit/c66241e791c4b51d8a65051142d01682590f9bf8), [`f0e941e`](https://github.com/Effect-TS/effect/commit/f0e941ee2f90f0a891d1228a90da91386f0258cb), [`5f8a6e5`](https://github.com/Effect-TS/effect/commit/5f8a6e5108aa6c6cd4043a3c9f4dc93d43369623), [`2800a09`](https://github.com/Effect-TS/effect/commit/2800a0910f7f4ad7cfa21dfc840edf8fe6e67ca7), [`e9749b0`](https://github.com/Effect-TS/effect/commit/e9749b04c10badabcdf71e6958de5ac56a5316e9), [`eaef09c`](https://github.com/Effect-TS/effect/commit/eaef09c11b6ed2029c513eea0aa3256dcc6f517f), [`a3c56e5`](https://github.com/Effect-TS/effect/commit/a3c56e5b5e756e879f1cfb90230e8eae620e33df), [`bb61152`](https://github.com/Effect-TS/effect/commit/bb611522fcfb4d317b33acdf5ee24a480d042435), [`e9c842b`](https://github.com/Effect-TS/effect/commit/e9c842bc294abf658d698bb17732ae13465b3817), [`6d8c8e2`](https://github.com/Effect-TS/effect/commit/6d8c8e2075b15ae84ae96e93a0d295cb9221b496), [`1f28187`](https://github.com/Effect-TS/effect/commit/1f281878ef559ef356af2d0e11b381aba857cae1), [`0b9842b`](https://github.com/Effect-TS/effect/commit/0b9842b76bbc1b016c8abc46432052e1f60b2fa3), [`c29ddd8`](https://github.com/Effect-TS/effect/commit/c29ddd899c53c5d8194c4a0dfef2134304dda799), [`84c3405`](https://github.com/Effect-TS/effect/commit/84c34057ceab9fe6782b73f4a04a31304e0d14de), [`3934c0c`](https://github.com/Effect-TS/effect/commit/3934c0c61857c03468c08c2477f49a6b977d28ff), [`048e7c8`](https://github.com/Effect-TS/effect/commit/048e7c87125c4ace5a61895feac3d049a317bbfc), [`1121334`](https://github.com/Effect-TS/effect/commit/1121334ce68c379051011a2ffea37b0a30ea9a47), [`9a680d6`](https://github.com/Effect-TS/effect/commit/9a680d6985c6a27d7ebb29cd9eb674f3fe524505), [`85e2aee`](https://github.com/Effect-TS/effect/commit/85e2aeee686963c26ca40d204f982deebb9d6117), [`34ec440`](https://github.com/Effect-TS/effect/commit/34ec4407a14ce045ca14cc76c77582884a7e1ebe), [`343a149`](https://github.com/Effect-TS/effect/commit/343a149fa7d5a9a0a45841b1b6800b0ecfa16a32), [`915117b`](https://github.com/Effect-TS/effect/commit/915117b523cb6e5cbc98b5ff1c119a8d0d1f6b9a), [`7f823c1`](https://github.com/Effect-TS/effect/commit/7f823c10d20ef53a1081e0d0e2008a4116140799), [`6e614e6`](https://github.com/Effect-TS/effect/commit/6e614e6feb4281a9e81f5ec3be48e7905ff4292c), [`9990b39`](https://github.com/Effect-TS/effect/commit/9990b3977a544247cd4123edde8353a55bf77192), [`22bf781`](https://github.com/Effect-TS/effect/commit/22bf7812aa8a20c676efade943c342cde4ef1cbd), [`41f08b7`](https://github.com/Effect-TS/effect/commit/41f08b75aba40e95cd6c743ec6fcf1f5022f7b5b), [`e8482ca`](https://github.com/Effect-TS/effect/commit/e8482cad9651fa8fe8036fe6292a286994897d0d), [`5ae85fa`](https://github.com/Effect-TS/effect/commit/5ae85fa8ac1fc5e12d5fe8fe7388ac504788ae3b), [`c477a0a`](https://github.com/Effect-TS/effect/commit/c477a0a8010bd0a988221bc00c446848314e1c3c), [`8fbb39e`](https://github.com/Effect-TS/effect/commit/8fbb39eb92577915831f19a83b51be376e844613), [`1c70828`](https://github.com/Effect-TS/effect/commit/1c7082847453e9c2be0cc267a39ca554c52f4013), [`8207d1e`](https://github.com/Effect-TS/effect/commit/8207d1e91ead8aac8f01d9e13b49624b64237c8b), [`7886b84`](https://github.com/Effect-TS/effect/commit/7886b847658664f64392a14efda86f44b3d1e170), [`4839c60`](https://github.com/Effect-TS/effect/commit/4839c601bdda845d9a54d77f578f64372475ab04), [`2595d83`](https://github.com/Effect-TS/effect/commit/2595d831f702d4a748f97e33e71fd5250f58af56), [`f5ae278`](https://github.com/Effect-TS/effect/commit/f5ae27827a82db3f8973b70745fc6f14e7d8fa80), [`7a8ce00`](https://github.com/Effect-TS/effect/commit/7a8ce0098a5f22cbb6c110b16e850fb3fcbcadf2), [`688d302`](https://github.com/Effect-TS/effect/commit/688d30275ff114e50b4f0d343c6227ee33d67348), [`c7072e4`](https://github.com/Effect-TS/effect/commit/c7072e46356a19615625ae0fa569b1c3a79d50be), [`425cedb`](https://github.com/Effect-TS/effect/commit/425cedb5801c9527c116a64d280d9e7c2820f922), [`88bc680`](https://github.com/Effect-TS/effect/commit/88bc68001c6c70984b03a5500232e9b5aab14085), [`268e09c`](https://github.com/Effect-TS/effect/commit/268e09c94c9e8f46142b5c9f29ce0a22bfbbc3aa), [`7e94505`](https://github.com/Effect-TS/effect/commit/7e94505edfc9d81ea08cb96a7d935806495bf61e), [`afa5da3`](https://github.com/Effect-TS/effect/commit/afa5da3c8a9848215e49535dd3d39179bfd67715), [`c29ddd8`](https://github.com/Effect-TS/effect/commit/c29ddd899c53c5d8194c4a0dfef2134304dda799), [`48102ac`](https://github.com/Effect-TS/effect/commit/48102ac592ef89fed9d5d10afd2daece0a0ee310), [`463eca3`](https://github.com/Effect-TS/effect/commit/463eca3609a6a9681247abbcc71a6f10b261a1d9), [`e7162ea`](https://github.com/Effect-TS/effect/commit/e7162eae7f04af1d583b1f31fdd767f91ec1c82e), [`95752db`](https://github.com/Effect-TS/effect/commit/95752db5eb3b6e59fb8749e9792704f8b167f36c), [`4577a28`](https://github.com/Effect-TS/effect/commit/4577a28bcdb50b0d184775f8c1b7ad890d86e255), [`973d599`](https://github.com/Effect-TS/effect/commit/973d599a3e8611626764a0e3e8f79edbcf30dc7c), [`2f066a5`](https://github.com/Effect-TS/effect/commit/2f066a58e90316dd88a219fcea94200beda54219), [`f0ccbb6`](https://github.com/Effect-TS/effect/commit/f0ccbb6917be5c08364b0b8adbd0c4c19e7f2de1), [`2dda6ce`](https://github.com/Effect-TS/effect/commit/2dda6cefc641dbea2ea3279a9b96c0332b277a40), [`3da7c0a`](https://github.com/Effect-TS/effect/commit/3da7c0aabab07c8fc682aea7559536bf31705853), [`5a67e0b`](https://github.com/Effect-TS/effect/commit/5a67e0b3a8118dd7be709a56d6d7af4f10e9c4f3), [`73b3a6e`](https://github.com/Effect-TS/effect/commit/73b3a6e3c38795400251d9c5d73f05c88acd90cd), [`c6f9e4c`](https://github.com/Effect-TS/effect/commit/c6f9e4cbe6f9c8f0a03fefde18df4d35e98a24df), [`be02676`](https://github.com/Effect-TS/effect/commit/be026768ee29a1b1a860c1fd17f2a0896c01f61e), [`26de861`](https://github.com/Effect-TS/effect/commit/26de86100cda8617e077ce66f0a6bb8dfe738fb6), [`9313799`](https://github.com/Effect-TS/effect/commit/931379911a74d5e2207859c295b1acc6a5e2917e), [`d5c1e5b`](https://github.com/Effect-TS/effect/commit/d5c1e5bd425dfe983dff1f8c853c0b7eaa772f87), [`11bb31f`](https://github.com/Effect-TS/effect/commit/11bb31fec0bf2fdfc025009a04de4c69bffa433a), [`8c23e46`](https://github.com/Effect-TS/effect/commit/8c23e46a8280e3f2c831d1c0ca3f106c92f8e071), [`9c96011`](https://github.com/Effect-TS/effect/commit/9c960111de202c0f1dd6989e5b8836150e0ca694), [`0d50570`](https://github.com/Effect-TS/effect/commit/0d50570bbf078804092fcd4a47ccd2d0019f989c), [`9ed2bf1`](https://github.com/Effect-TS/effect/commit/9ed2bf18da0e1ff4fecc31b7cbac2976ec999e00), [`76ec912`](https://github.com/Effect-TS/effect/commit/76ec9120f43f4fdb08b9c5170ff9e3fd82f22457), [`af2756d`](https://github.com/Effect-TS/effect/commit/af2756dd1d196aa3f199ce21c6d2dc3ada3b67b5), [`3ec8100`](https://github.com/Effect-TS/effect/commit/3ec8100c793b8a22381ad115106b93824d840f64), [`93abf02`](https://github.com/Effect-TS/effect/commit/93abf020876af0a6b6b8b25621c80760b7996762), [`1e94993`](https://github.com/Effect-TS/effect/commit/1e949935a1065e5140245a514ce3a799061143e3), [`99a4616`](https://github.com/Effect-TS/effect/commit/99a4616264e31de65fa8d3dc19576309b1a3f84a), [`0a7978e`](https://github.com/Effect-TS/effect/commit/0a7978ea8e70e51700345b12f8c19952dde05255), [`b4a060b`](https://github.com/Effect-TS/effect/commit/b4a060b95a6473d54e277dc005172b54aab9bddf), [`274f4a4`](https://github.com/Effect-TS/effect/commit/274f4a4dac5ee2d9dbb6358cc3dd3afe345d4705), [`25f29be`](https://github.com/Effect-TS/effect/commit/25f29be77bdcd5c2a7be8e2710e88f5a10c4cb8d), [`9036440`](https://github.com/Effect-TS/effect/commit/9036440edca01c24371d0c2c1044be5e5d3cb65e), [`415f005`](https://github.com/Effect-TS/effect/commit/415f0053eb848d562febcb5b011e15f31a99bead), [`191308b`](https://github.com/Effect-TS/effect/commit/191308bd0f5ee04557570f2f960c6e06c429bcfd), [`4b96570`](https://github.com/Effect-TS/effect/commit/4b96570ed28d93b14e958742933d227dc5e74b15), [`d84ec3f`](https://github.com/Effect-TS/effect/commit/d84ec3fab0346c01bc59f6624d1c046ca4140087), [`392aa1c`](https://github.com/Effect-TS/effect/commit/392aa1cb275ac9fdb9b4b1831d8c06196bba54b9), [`99e9bd9`](https://github.com/Effect-TS/effect/commit/99e9bd98a354d83202e2522f3ade9ee4bf4a862c), [`6f6f45f`](https://github.com/Effect-TS/effect/commit/6f6f45f480a3f191aed6a82ca5c301f60ef2cf84), [`13fc07e`](https://github.com/Effect-TS/effect/commit/13fc07edeb939ca6229b858c9d9a8c5243ac21b5), [`a4b38dd`](https://github.com/Effect-TS/effect/commit/a4b38dde7f3fdc1b54a49bffdaca912b4c47c76e), [`8b18580`](https://github.com/Effect-TS/effect/commit/8b185807bb99f6de3d9564447e2f4e08d2f8863d), [`0c477fe`](https://github.com/Effect-TS/effect/commit/0c477fe417d3ecab86bb8d9afd0c392e9cb8a3da), [`9a5331d`](https://github.com/Effect-TS/effect/commit/9a5331db94bfdca4f48317d062fbeb96054a9049), [`a706d73`](https://github.com/Effect-TS/effect/commit/a706d73db8a8d460877c5d7e8a793a52d45be307), [`c9a1e72`](https://github.com/Effect-TS/effect/commit/c9a1e72c5965aa9fe01a669ebfd523f803f5a189), [`89eaaf8`](https://github.com/Effect-TS/effect/commit/89eaaf8581908e248c0b1e47baa7906e0c541635), [`8215811`](https://github.com/Effect-TS/effect/commit/8215811d51d28d37b4732a6ab03de4047ada62ea), [`ad69018`](https://github.com/Effect-TS/effect/commit/ad6901857e05e7b436a0149b5ece7f3a36d9ff33), [`a733eba`](https://github.com/Effect-TS/effect/commit/a733eba9f699737a80a9e7276521980bd929075a), [`7282d5f`](https://github.com/Effect-TS/effect/commit/7282d5f1bcc06d7065429ef81a3d17c64043fde4), [`da33572`](https://github.com/Effect-TS/effect/commit/da33572776763e45802ce8c2fe14ec101f9a5c34), [`d9e2d12`](https://github.com/Effect-TS/effect/commit/d9e2d1221768e9a3e5e00165cb95b86e0c126cc0), [`dc923b8`](https://github.com/Effect-TS/effect/commit/dc923b847ac7d9673b4f0aa7c136557287b2e5e3), [`1b74086`](https://github.com/Effect-TS/effect/commit/1b740862e3d25ec5d4b6f86b9fe792b6dabbf84c), [`8a5507b`](https://github.com/Effect-TS/effect/commit/8a5507ba00f641edf71227d79b9f90efdb634fe2), [`23460fa`](https://github.com/Effect-TS/effect/commit/23460fad704c730926bffe3cc6874268ef6680ec), [`0dbbea6`](https://github.com/Effect-TS/effect/commit/0dbbea65a93c59147a4bc407cfcda38ea26341b2), [`70aad3f`](https://github.com/Effect-TS/effect/commit/70aad3ffe617984f5b599aeed1bc7f04899aca55), [`0909236`](https://github.com/Effect-TS/effect/commit/090923611b209a19f3281a7b1d0c15a9b653a0c7), [`a9e66d8`](https://github.com/Effect-TS/effect/commit/a9e66d889017b302355ba5b8bc2b2050ca1ea5aa), [`ee5c0d5`](https://github.com/Effect-TS/effect/commit/ee5c0d5f43348747b04b5f7bf1b39355614c2f2c), [`631c5d6`](https://github.com/Effect-TS/effect/commit/631c5d6d9a81e6389964b878764fb9aea1538247), [`7e60c1a`](https://github.com/Effect-TS/effect/commit/7e60c1ae62a96c768f9c67d706f494213947f22f), [`4cddd3c`](https://github.com/Effect-TS/effect/commit/4cddd3c41e71fc4990d37c42b1bcf2ce3780b41d), [`8727e5d`](https://github.com/Effect-TS/effect/commit/8727e5d2bec8e2d729b7aeafb725a4ab21d533d2), [`2bc97fa`](https://github.com/Effect-TS/effect/commit/2bc97faa0e06b3973780dae26029e2c59f5dba38), [`f9a7f05`](https://github.com/Effect-TS/effect/commit/f9a7f056ba31aef435933acf082ec8c01ac7decc), [`d978cca`](https://github.com/Effect-TS/effect/commit/d978cca02914bec0ddf66bb2d68d90b220096fb6), [`b334ccf`](https://github.com/Effect-TS/effect/commit/b334ccf361a0f11362e7aabdbf70b35f53a39d61), [`0b50cfc`](https://github.com/Effect-TS/effect/commit/0b50cfc644313afe4c318073893231ec1946eb5f), [`5b7d2b5`](https://github.com/Effect-TS/effect/commit/5b7d2b5c90bc77eb9f8be1b91166c53f89aa1d0c), [`6ee26b1`](https://github.com/Effect-TS/effect/commit/6ee26b16b51b380c41dbfa84d612aa525b420fce), [`eab7e6a`](https://github.com/Effect-TS/effect/commit/eab7e6acfc7e422857565b98b9eddd3a98de1b92), [`e618f10`](https://github.com/Effect-TS/effect/commit/e618f1050f41424439eb82478fcf1c5da5a3244d), [`da14b1f`](https://github.com/Effect-TS/effect/commit/da14b1f18d71a18fc431b56d82cffe09aeb92708), [`3314fc3`](https://github.com/Effect-TS/effect/commit/3314fc30c22d5c9d737f3df7201f4a41424f701b), [`66db624`](https://github.com/Effect-TS/effect/commit/66db624670032a02144d42b6e64b83b324c813cd), [`4b8a50d`](https://github.com/Effect-TS/effect/commit/4b8a50db9649603f382f33640698020ca79547d0), [`8d52668`](https://github.com/Effect-TS/effect/commit/8d52668533e9d4a6d6b3be9fe58bd5e69fa80b65), [`1de1055`](https://github.com/Effect-TS/effect/commit/1de10559e7263747751bfb032fd4b0fd8aa28724), [`db130e6`](https://github.com/Effect-TS/effect/commit/db130e64557fafe370819d19f78e3b19a21275d6), [`5fa5fa5`](https://github.com/Effect-TS/effect/commit/5fa5fa5b9bff77352fb9ff2d424bf46bd6309986)]: + - effect@4.0.6 + ## 4.0.5 ### Patch Changes diff --git a/repos/effect/packages/ai/openrouter/package.json b/repos/effect/packages/ai/openrouter/package.json index 0d63a00849..76de6e1bb4 100644 --- a/repos/effect/packages/ai/openrouter/package.json +++ b/repos/effect/packages/ai/openrouter/package.json @@ -1,6 +1,6 @@ { "name": "@effect/ai-openrouter", - "version": "4.0.5", + "version": "4.0.6", "type": "module", "license": "MIT", "description": "An OpenRouter provider integration for Effect AI SDK", diff --git a/repos/effect/packages/ai/openrouter/src/OpenRouterClient.ts b/repos/effect/packages/ai/openrouter/src/OpenRouterClient.ts index 67b4a92625..869c82e99c 100644 --- a/repos/effect/packages/ai/openrouter/src/OpenRouterClient.ts +++ b/repos/effect/packages/ai/openrouter/src/OpenRouterClient.ts @@ -238,20 +238,20 @@ export const make = Effect.fnUntraced( const createChatCompletion: Service["createChatCompletion"] = (payload) => client.sendChatCompletionRequest({ payload, config: { includeResponse: true } }).pipe( Effect.catchTags({ - SendChatCompletionRequest400: (error) => Effect.fail(Errors.mapClientError(error, "createChatCompletion")), - SendChatCompletionRequest401: (error) => Effect.fail(Errors.mapClientError(error, "createChatCompletion")), - SendChatCompletionRequest402: (error) => Effect.fail(Errors.mapClientError(error, "createChatCompletion")), - SendChatCompletionRequest403: (error) => Effect.fail(Errors.mapClientError(error, "createChatCompletion")), - SendChatCompletionRequest404: (error) => Effect.fail(Errors.mapClientError(error, "createChatCompletion")), - SendChatCompletionRequest408: (error) => Effect.fail(Errors.mapClientError(error, "createChatCompletion")), - SendChatCompletionRequest413: (error) => Effect.fail(Errors.mapClientError(error, "createChatCompletion")), - SendChatCompletionRequest422: (error) => Effect.fail(Errors.mapClientError(error, "createChatCompletion")), - SendChatCompletionRequest429: (error) => Effect.fail(Errors.mapClientError(error, "createChatCompletion")), - SendChatCompletionRequest500: (error) => Effect.fail(Errors.mapClientError(error, "createChatCompletion")), - SendChatCompletionRequest502: (error) => Effect.fail(Errors.mapClientError(error, "createChatCompletion")), - SendChatCompletionRequest503: (error) => Effect.fail(Errors.mapClientError(error, "createChatCompletion")), - SendChatCompletionRequest524: (error) => Effect.fail(Errors.mapClientError(error, "createChatCompletion")), - SendChatCompletionRequest529: (error) => Effect.fail(Errors.mapClientError(error, "createChatCompletion")), + SendChatCompletionRequest400: (error) => Errors.mapClientError(error, "createChatCompletion"), + SendChatCompletionRequest401: (error) => Errors.mapClientError(error, "createChatCompletion"), + SendChatCompletionRequest402: (error) => Errors.mapClientError(error, "createChatCompletion"), + SendChatCompletionRequest403: (error) => Errors.mapClientError(error, "createChatCompletion"), + SendChatCompletionRequest404: (error) => Errors.mapClientError(error, "createChatCompletion"), + SendChatCompletionRequest408: (error) => Errors.mapClientError(error, "createChatCompletion"), + SendChatCompletionRequest413: (error) => Errors.mapClientError(error, "createChatCompletion"), + SendChatCompletionRequest422: (error) => Errors.mapClientError(error, "createChatCompletion"), + SendChatCompletionRequest429: (error) => Errors.mapClientError(error, "createChatCompletion"), + SendChatCompletionRequest500: (error) => Errors.mapClientError(error, "createChatCompletion"), + SendChatCompletionRequest502: (error) => Errors.mapClientError(error, "createChatCompletion"), + SendChatCompletionRequest503: (error) => Errors.mapClientError(error, "createChatCompletion"), + SendChatCompletionRequest524: (error) => Errors.mapClientError(error, "createChatCompletion"), + SendChatCompletionRequest529: (error) => Errors.mapClientError(error, "createChatCompletion"), HttpClientError: (error) => Errors.mapHttpClientError(error, "createChatCompletion"), SchemaError: (error) => Effect.fail(Errors.mapSchemaError(error, "createChatCompletion")) }) diff --git a/repos/effect/packages/ai/openrouter/src/OpenRouterDecisionModel.ts b/repos/effect/packages/ai/openrouter/src/OpenRouterDecisionModel.ts index b548ed5d6e..683f91d392 100644 --- a/repos/effect/packages/ai/openrouter/src/OpenRouterDecisionModel.ts +++ b/repos/effect/packages/ai/openrouter/src/OpenRouterDecisionModel.ts @@ -13,6 +13,28 @@ import * as Layer from "effect/Layer" import { OpenRouterClient } from "./OpenRouterClient.ts" import type * as OpenRouterSchema from "./OpenRouterSchema.ts" +declare module "effect/ai/DecisionModel" { + /** + * OpenRouter metadata returned with a decision response. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ + export interface DecideResponseMetadata { + readonly openrouter?: { + /** + * Credits charged for the request. + */ + readonly cost?: number + /** + * Upstream provider that served the request. + */ + readonly provider?: string + } + } +} + /** * Request options that override model defaults at call time. * @@ -42,8 +64,11 @@ export const model = ( Model.make("openrouter", model, layer({ model, config })) /** - * Creates a decision service that requires full choice and score distributions. - * Score indices map to criteria labels; cost, id, and provider metadata are omitted. + * Creates a decision service that validates full choice and score distributions. + * Missing distributions yield `"NoDistribution"` with `decideSettled` or + * `AiError.InvalidOutputError` with `decide`. + * Score indices map to criteria labels. Reported `cost` and the serving + * provider are returned under `metadata.openrouter`. * Probabilities arrive rounded to two decimals, so small sum drift is rescaled. * * @stability unstable @@ -95,15 +120,6 @@ export const make = Effect.fnUntraced(function*(options: { answers[key] = { _tag: "Probability", probability: answer.noul } continue } - if (answer.probabilities === undefined) { - return yield* AiError.make({ - module: "OpenRouterDecisionModel", - method: "decide", - reason: new AiError.InvalidOutputError({ - description: `Provider returned no probabilities for decision "${key}"` - }) - }) - } if (answer.type === "choice") { answers[key] = { _tag: "Classify", @@ -116,18 +132,31 @@ export const make = Effect.fnUntraced(function*(options: { const probabilities: Record = Object.create(null) if (decision?._tag === "Rate") { for (const [index, { label }] of decision.criteria.entries()) { - const probability = answer.probabilities[String(index)] + const probability = answer.probabilities?.[String(index)] if (probability !== undefined) { probabilities[label] = probability } } } - answers[key] = { _tag: "Rate", rating: answer.score, probabilities, confidence: answer.confidence } + answers[key] = { + _tag: "Rate", + rating: answer.score, + probabilities: answer.probabilities === undefined ? undefined : probabilities, + confidence: answer.confidence + } } } return { answers, - usage: { inputTokens: response.usage.input_tokens, outputTokens: response.usage.output_tokens } + usage: { inputTokens: response.usage.input_tokens, outputTokens: response.usage.output_tokens }, + id: response.id, + modelId: response.model, + metadata: { + openrouter: { + ...(response.usage.cost === undefined ? undefined : { cost: response.usage.cost }), + ...(response.provider === undefined ? undefined : { provider: response.provider }) + } + } } }) }) diff --git a/repos/effect/packages/ai/openrouter/src/OpenRouterLanguageModel.ts b/repos/effect/packages/ai/openrouter/src/OpenRouterLanguageModel.ts index 0952a3504c..667a76d5b4 100644 --- a/repos/effect/packages/ai/openrouter/src/OpenRouterLanguageModel.ts +++ b/repos/effect/packages/ai/openrouter/src/OpenRouterLanguageModel.ts @@ -72,8 +72,20 @@ export class Config extends Context.Service< /** * Whether to use strict JSON schema validation for structured outputs. * - * Only applies to models that support structured outputs. Defaults to - * `true` when structured outputs are supported. + * For schema-backed response formats, defaults to `true` when the selected + * schema transformer is the OpenAI transformer (`openai/`, `gpt-`, `o1-`, + * `o3-`, and `o4-` model prefixes). Other models, including Anthropic models, + * send `strict: null` by default, leaving strictness unspecified. Explicit + * `true` or `false` overrides either default. This does not affect tool + * strictness. + * + * **Gotchas** + * + * Transformer selection is not an endpoint-capability check. Support and + * enforcement depend on the selected model and provider. For incompatible + * endpoints, set `false` to opt out of the strict default. Strict mode may + * restrict supported JSON Schema features; see OpenRouter + * [structured output guidance](https://openrouter.ai/docs/guides/features/structured-outputs). */ readonly strictJsonSchema?: boolean | undefined } @@ -527,8 +539,8 @@ declare module "effect/ai/Response" { export const model = ( model: string, config?: Omit -): AiModel.Model<"openai", LanguageModel.LanguageModel, OpenRouterClient> => - AiModel.make("openai", model, layer({ model, config })) +): AiModel.Model<"openrouter", LanguageModel.LanguageModel, OpenRouterClient> => + AiModel.make("openrouter", model, layer({ model, config })) /** * Creates an OpenRouter `LanguageModel` service from a model identifier and @@ -845,14 +857,16 @@ const prepareMessages = Effect.fnUntraced( } case "assistant": { - let text = "" let reasoning = "" + const textParts: Array = [] const toolCalls: Array = [] for (const part of message.content) { switch (part.type) { case "text": { - text += part.text + if (part.text.length > 0) { + textParts.push(part) + } break } @@ -902,9 +916,22 @@ const prepareMessages = Effect.fnUntraced( } } + // Tool calls cannot carry cache breakpoints; use the last text part. + const messageCacheControl = getCacheControl(message) + const content = textParts.map((part, i) => { + const cache_control = getCacheControl(part) ?? (i === textParts.length - 1 ? messageCacheControl : null) + return { + type: "text", + text: part.text, + ...(Predicate.isNotNull(cache_control) ? { cache_control } : undefined) + } + }) + messages.push({ role: "assistant", - content: text, + content: content.some((part) => Predicate.isNotNullish(part.cache_control)) + ? content + : content.map((part) => part.text).join(""), reasoning: reasoning.length > 0 ? reasoning : null, ...(Predicate.isNotNull(reasoningDetails) ? { reasoning_details: reasoningDetails } : undefined), ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : undefined) @@ -914,16 +941,20 @@ const prepareMessages = Effect.fnUntraced( } case "tool": { - for (const part of message.content) { - // Skip tool approval parts - if (part.type === "tool-approval-response") { - continue - } + const results = message.content.filter((part) => part.type === "tool-result") + const messageCacheControl = getCacheControl(message) + + for (let i = 0; i < results.length; i++) { + const part = results[i] + const text = typeof part.result === "string" ? part.result : JSON.stringify(part.result) + const cache_control = getCacheControl(part) ?? (i === results.length - 1 ? messageCacheControl : null) messages.push({ role: "tool", tool_call_id: part.id, - content: typeof part.result === "string" ? part.result : JSON.stringify(part.result) + content: Predicate.isNotNull(cache_control) && typeof text === "string" && text.length > 0 + ? [{ type: "text", text, cache_control }] + : text }) } @@ -1811,7 +1842,7 @@ const getResponseFormat = Effect.fnUntraced(function*({ config, options, transfo json_schema: { name: options.responseFormat.objectName, schema: jsonSchema, - strict: config.strictJsonSchema ?? null, + strict: config.strictJsonSchema ?? (transformer === toCodecOpenAI ? true : null), ...(Predicate.isNotUndefined(description) ? { description } : undefined) } } diff --git a/repos/effect/packages/ai/openrouter/src/internal/errors.ts b/repos/effect/packages/ai/openrouter/src/internal/errors.ts index 618c2c800d..6cbef40048 100644 --- a/repos/effect/packages/ai/openrouter/src/internal/errors.ts +++ b/repos/effect/packages/ai/openrouter/src/internal/errors.ts @@ -1,9 +1,11 @@ import * as AiError from "effect/ai/AiError" import type * as Response from "effect/ai/Response" -import * as Duration from "effect/Duration" +import * as Clock from "effect/Clock" +import type * as Duration from "effect/Duration" import * as Effect from "effect/Effect" import type * as Sse from "effect/encoding/Sse" import { dual } from "effect/Function" +import * as Headers from "effect/http/Headers" import type * as HttpClientError from "effect/http/HttpClientError" import type * as HttpClientRequest from "effect/http/HttpClientRequest" import type * as HttpClientResponse from "effect/http/HttpClientResponse" @@ -11,6 +13,7 @@ import * as Number from "effect/Number" import * as Option from "effect/Option" import * as Predicate from "effect/Predicate" import * as Redactable from "effect/Redactable" +import * as Result from "effect/Result" import * as Schema from "effect/Schema" import type * as Generated from "../Generated.ts" import type { OpenRouterErrorMetadata } from "../OpenRouterError.ts" @@ -50,7 +53,7 @@ export const mapSchemaError = dual< AiError.make({ module: "OpenRouterClient", method, - reason: AiError.InvalidOutputError.fromSchemaError(error) + reason: AiError.InvalidResponseError.fromSchemaError(error) })) /** @internal */ @@ -61,32 +64,39 @@ export const mapSseError = dual< AiError.make({ module: "OpenRouterClient", method, - reason: new AiError.InvalidOutputError({ description: error.message }) + reason: new AiError.InvalidResponseError({ description: error.message }) })) +type ClientError = Generated.OpenRouterClientError + /** @internal */ export const mapClientError = dual< - (method: string) => (error: Generated.OpenRouterClientError) => AiError.AiError, - (error: Generated.OpenRouterClientError, method: string) => AiError.AiError ->(2, (error, method) => { - const { request, response, cause } = error - const status = response.status - const headers = response.headers as Record - const metadata: OpenRouterErrorMetadata = { - errorCode: cause.error.code ?? null, - errorType: cause.error.type ?? null, - requestId: headers["x-request-id"] ?? null - } - const http = buildHttpContext({ request, response, body: JSON.stringify(cause) }) - const reason = mapStatusCodeToReason({ - status, - headers, - message: cause.error.message, - metadata, - http + (method: string) => (error: ClientError) => Effect.Effect, + (error: ClientError, method: string) => Effect.Effect +>( + 2, + Effect.fnUntraced(function*(error: ClientError, method: string) { + const now = yield* Clock.currentTimeMillis + const { request, response, cause } = error + const status = response.status + const headers = response.headers as Record + const metadata: OpenRouterErrorMetadata = { + errorCode: cause.error.code ?? null, + errorType: cause.error.type ?? null, + requestId: headers["x-request-id"] ?? null + } + const http = buildHttpContext({ request, response, body: JSON.stringify(cause) }) + const reason = mapStatusCodeToReason({ + now, + status, + headers, + message: cause.error.message, + metadata, + http + }) + return yield* AiError.make({ module: "OpenRouterClient", method, reason }) }) - return AiError.make({ module: "OpenRouterClient", method, reason }) -}) +) /** @internal */ export const mapHttpClientError = dual< @@ -179,7 +189,9 @@ const mapStatusCodeError = Effect.fnUntraced(function*( } const decoded = Schema.decodeUnknownOption(OpenRouterErrorBody)(json) + const now = yield* Clock.currentTimeMillis const reason = mapStatusCodeToReason({ + now, status, headers, message: Option.isSome(decoded) ? decoded.value.error.message : undefined, @@ -198,16 +210,16 @@ const mapStatusCodeError = Effect.fnUntraced(function*( // Rate Limits // ============================================================================= +const retryAfterFromHeader = (raw: string, now: number): Duration.Duration | undefined => { + const retryAfter = Headers.parseRetryAfter(raw) + return Result.isSuccess(retryAfter) ? Headers.retryAfterDelay(retryAfter.success, now) : undefined +} + /** @internal */ -export const parseRateLimitHeaders = (headers: Record) => { +export const parseRateLimitHeaders = (headers: Record, now: number) => { const retryAfterRaw = headers["retry-after"] - let retryAfter: Duration.Duration | undefined - if (Predicate.isNotUndefined(retryAfterRaw)) { - const parsed = Number.parse(retryAfterRaw) - if (Option.isSome(parsed)) { - retryAfter = Duration.seconds(parsed.value) - } - } + // an invalid Retry-After is treated as absent + const retryAfter = retryAfterRaw === undefined ? undefined : retryAfterFromHeader(retryAfterRaw, now) const remainingRaw = headers["x-ratelimit-remaining-requests"] const remaining = Predicate.isNotUndefined(remainingRaw) ? Option.getOrNull(Number.parse(remainingRaw)) @@ -257,8 +269,10 @@ export const buildHttpContext = (params: { // ============================================================================= /** @internal */ -export const mapStatusCodeToReason = ({ status, headers, message, metadata, http }: { +export const mapStatusCodeToReason = ({ status, headers, message, metadata, http, now }: { readonly status: number + /** Current time in epoch milliseconds, used to measure a Retry-After date. */ + readonly now: number readonly headers: Record readonly message: string | undefined readonly metadata: OpenRouterErrorMetadata @@ -314,7 +328,7 @@ export const mapStatusCodeToReason = ({ status, headers, message, metadata, http http }) } - const { retryAfter, ...rateLimitMetadata } = parseRateLimitHeaders(headers) + const { retryAfter, ...rateLimitMetadata } = parseRateLimitHeaders(headers, now) return new AiError.RateLimitError({ retryAfter, metadata: { diff --git a/repos/effect/packages/ai/openrouter/test/OpenRouterClient.test.ts b/repos/effect/packages/ai/openrouter/test/OpenRouterClient.test.ts index c2fba232dc..660c7a79d4 100644 --- a/repos/effect/packages/ai/openrouter/test/OpenRouterClient.test.ts +++ b/repos/effect/packages/ai/openrouter/test/OpenRouterClient.test.ts @@ -97,6 +97,7 @@ describe("OpenRouterClient", () => { it("preserves and truncates a fallback HTTP response", () => { const body = `${"a".repeat(200)}b` const reason = Errors.mapStatusCodeToReason({ + now: 0, status: 400, headers: {}, message: undefined, diff --git a/repos/effect/packages/ai/openrouter/test/OpenRouterDecisionModel.test.ts b/repos/effect/packages/ai/openrouter/test/OpenRouterDecisionModel.test.ts index c3da61a013..af01f1bbc9 100644 --- a/repos/effect/packages/ai/openrouter/test/OpenRouterDecisionModel.test.ts +++ b/repos/effect/packages/ai/openrouter/test/OpenRouterDecisionModel.test.ts @@ -408,7 +408,7 @@ describe("OpenRouterDecisionModel", () => { assert.strictEqual(error.reason._tag, "InvalidUserInputError") })) - it.effect("maps a malformed successful response to a client InvalidOutputError", () => + it.effect("maps a malformed successful response to a non-retryable client InvalidResponseError", () => Effect.gen(function*() { const error = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe( Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })), @@ -423,7 +423,7 @@ describe("OpenRouterDecisionModel", () => { assert.strictEqual(error._tag, "AiError") assert.strictEqual(error.module, "OpenRouterClient") assert.strictEqual(error.method, "createDecisions") - assert.strictEqual(error.reason._tag, "InvalidOutputError") + assert.strictEqual(error.reason._tag, "InvalidResponseError") })) it.effect("make constructs a DecisionModel service", () => @@ -462,7 +462,7 @@ describe("OpenRouterDecisionModel", () => { Effect.gen(function*() { const { confidence: _choiceConfidence, ...department } = decisionsResponse.answers.department const { confidence: _scoreConfidence, legend: _legend, ...frustration } = decisionsResponse.answers.frustration - const { answers, usage } = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe( + const { answers, id, metadata, usage } = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe( Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })), Effect.provide(makeClientLayer((request) => Effect.succeed(jsonResponse(request, { @@ -476,15 +476,18 @@ describe("OpenRouterDecisionModel", () => { assert.strictEqual(answers.frustration.label, "Very angry, strong language") assert.strictEqual(answers.urgent.probability, 0.999) assert.deepStrictEqual({ ...usage }, { inputTokens: 312, outputTokens: 48 }) + assert.isUndefined(id) + assert.deepStrictEqual(metadata.openrouter, {}) })) - it.effect("drops provider-specific cost, id and provider metadata", () => + it.effect("returns the response id, resolved model, cost and serving provider", () => Effect.gen(function*() { const result = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe( - Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })), + Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model-latest" })), Effect.provide(makeClientLayer((request) => Effect.succeed(jsonResponse(request, { ...decisionsResponse, + model: "test/decision-model-1.2", id: "decision-1", provider: "test-provider", usage: { ...decisionsResponse.usage, cost: 0.001 } @@ -492,8 +495,9 @@ describe("OpenRouterDecisionModel", () => { )) ) assert.deepStrictEqual({ ...result.usage }, { inputTokens: 312, outputTokens: 48 }) - assert.isFalse("id" in result) - assert.isFalse("provider" in result) + assert.strictEqual(result.id, "decision-1") + assert.strictEqual(result.modelId, "test/decision-model-1.2") + assert.deepStrictEqual(result.metadata.openrouter, { cost: 0.001, provider: "test-provider" }) assert.isFalse("cost" in result.answers.department) })) @@ -771,6 +775,27 @@ describe("OpenRouterDecisionModel", () => { assert.strictEqual(error.reason._tag, "InvalidOutputError") })) + it.effect("decideSettled reports choice and score answers without probabilities as NoDistribution", () => + Effect.gen(function*() { + const { probabilities: _choice, ...department } = decisionsResponse.answers.department + const { probabilities: _score, ...frustration } = decisionsResponse.answers.frustration + const { outcomes } = yield* DecisionModel.decideSettled(TicketTriage, { input: ticket }).pipe( + Effect.provide(OpenRouterDecisionModel.layer({ model: "test/decision-model" })), + Effect.provide(makeClientLayer((request) => + Effect.succeed(jsonResponse(request, { + ...decisionsResponse, + answers: { ...decisionsResponse.answers, department, frustration } + })) + )) + ) + + assert.isTrue(Result.isFailure(outcomes.department) && outcomes.department.failure.reason === "NoDistribution") + assert.isTrue( + Result.isFailure(outcomes.frustration) && outcomes.frustration.failure.reason === "NoDistribution" + ) + assert.isTrue(Result.isSuccess(outcomes.urgent)) + })) + it.effect("fails with InvalidOutputError when a score distribution skips a level index", () => Effect.gen(function*() { const error = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe( @@ -902,12 +927,11 @@ const jsonResponse = ( }) ) -const getRequestBody = (request: HttpClientRequest.HttpClientRequest) => - Effect.gen(function*() { - const body = request.body - if (body._tag === "Uint8Array") { - const text = new TextDecoder().decode(body.body) - return JSON.parse(text) - } - return yield* Effect.die(new Error("Expected Uint8Array body")) - }) +const getRequestBody = Effect.fnUntraced(function*(request: HttpClientRequest.HttpClientRequest) { + const body = request.body + if (body._tag === "Uint8Array") { + const text = new TextDecoder().decode(body.body) + return JSON.parse(text) + } + return yield* Effect.die(new Error("Expected Uint8Array body")) +}) diff --git a/repos/effect/packages/ai/openrouter/test/OpenRouterLanguageModel.test.ts b/repos/effect/packages/ai/openrouter/test/OpenRouterLanguageModel.test.ts index 4d0723c5fb..04b6c228c7 100644 --- a/repos/effect/packages/ai/openrouter/test/OpenRouterLanguageModel.test.ts +++ b/repos/effect/packages/ai/openrouter/test/OpenRouterLanguageModel.test.ts @@ -2,11 +2,38 @@ import { Generated, OpenRouterClient, OpenRouterLanguageModel } from "@effect/ai import { assert, describe, it } from "@effect/vitest" import { deepStrictEqual, strictEqual } from "@effect/vitest/utils" import { Array, Context, Effect, Layer, Redacted, Ref, Schema, Stream } from "effect" -import { LanguageModel, Prompt, Tool, Toolkit } from "effect/ai" +import { LanguageModel, Model, Prompt, Tool, Toolkit } from "effect/ai" import { HttpClient, type HttpClientError, type HttpClientRequest, HttpClientResponse } from "effect/http" describe("OpenRouterLanguageModel", () => { describe("strictJsonSchema", () => { + for ( + const [model, strict] of [ + ["openai/gpt-4o-mini", true], + ["google/gemini-2.5-flash", null] + ] as const + ) { + it.effect(`defaults response strictness to ${strict} for ${model}`, () => + Effect.gen(function*() { + yield* LanguageModel.generateObject({ + prompt: "Give me a name", + schema: Schema.Struct({ name: Schema.String }) + }).pipe(Effect.provide(OpenRouterLanguageModel.model(model))) + + const requests = yield* MockHttpClient.requests + const body = yield* getRequestBody(requests[0]) + strictEqual(body.response_format.json_schema.strict, strict) + }).pipe(Effect.provide(makeTestLayer({ + body: { + choices: [{ + finish_reason: "stop", + index: 0, + message: { role: "assistant", content: JSON.stringify({ name: "Alice" }) } + }] + } + })))) + } + it.effect("omits false from requests while preserving response strictness", () => Effect.gen(function*() { yield* LanguageModel.generateObject({ @@ -49,6 +76,14 @@ describe("OpenRouterLanguageModel", () => { }).pipe(Effect.provide(makeStreamTestLayer([])))) }) + it.effect("model provides openrouter provider metadata", () => + Effect.gen(function*() { + strictEqual(yield* Model.ProviderName, "openrouter") + }).pipe( + Effect.provide(OpenRouterLanguageModel.model("google/gemini-2.5-flash")), + Effect.provide(makeTestLayer()) + )) + describe("generateText", () => { describe("message preparation", () => { describe("audio file parts", () => { @@ -250,6 +285,113 @@ describe("OpenRouterLanguageModel", () => { assert.isDefined(toolResult) strictEqual(toolResult.content, "PLAIN_TEXT_SENTINEL\n") }).pipe(Effect.provide(makeTestLayer()))) + + describe("cache control", () => { + const ephemeral = { type: "ephemeral" } as const + const oneHour = { type: "ephemeral", ttl: "1h" } as const + const cached = { openrouter: { cacheControl: ephemeral } } + const partCached = { openrouter: { cacheControl: oneHour } } + + it.effect("sends cached void tool results without throwing", () => + Effect.gen(function*() { + yield* LanguageModel.generateText({ + prompt: Prompt.make([{ + role: "tool", + content: [Prompt.toolResultPart({ + id: "call_void", + name: "void_tool", + result: undefined, + isFailure: false, + providerExecuted: false + })], + options: cached + }]), + disableToolCallResolution: true + }).pipe(Effect.provide(OpenRouterLanguageModel.model("anthropic/claude-sonnet-4.5"))) + + const requests = yield* MockHttpClient.requests + const body = yield* getRequestBody(requests[0]) + const toolResult = body.messages.find((message: any) => message.role === "tool") + strictEqual(toolResult.tool_call_id, "call_void") + }).pipe(Effect.provide(makeTestLayer()))) + + for (const partLevel of [false, true]) { + it.effect( + partLevel + ? "preserves part breakpoints and prefers them to message breakpoints" + : "puts message breakpoints on the last assistant text and tool result", + () => + Effect.gen(function*() { + yield* LanguageModel.generateText({ + prompt: Prompt.make([ + { role: "user", content: "Look up both" }, + { + role: "assistant", + content: [ + Prompt.reasoningPart({ text: "Thinking" }), + Prompt.textPart({ text: "First", options: partLevel ? partCached : undefined }), + Prompt.textPart({ text: "Second" }), + Prompt.textPart({ text: "Last", options: partLevel ? partCached : undefined }), + ...["a", "b", "c"].map((id) => + Prompt.toolCallPart({ id, name: "lookup", params: {}, providerExecuted: false }) + ) + ], + options: cached + }, + { + role: "tool", + content: [ + ...["a", "b", "c"].map((id) => + Prompt.toolResultPart({ + id, + name: "lookup", + result: id === "c" ? { value: 42 } : id, + isFailure: false, + providerExecuted: false, + options: partLevel && id !== "b" ? partCached : undefined + }) + ), + Prompt.toolApprovalResponsePart({ approvalId: "approval", approved: true }) + ], + options: cached + } + ]), + disableToolCallResolution: true + }).pipe(Effect.provide(OpenRouterLanguageModel.model("anthropic/claude-sonnet-4.5"))) + + const requests = yield* MockHttpClient.requests + const body = yield* getRequestBody(requests[0]) + const assistant = body.messages.find((message: any) => message.role === "assistant") + deepStrictEqual(assistant.content, [ + partLevel + ? { type: "text", text: "First", cache_control: oneHour } + : { type: "text", text: "First" }, + { type: "text", text: "Second" }, + { type: "text", text: "Last", cache_control: partLevel ? oneHour : ephemeral } + ]) + strictEqual(assistant.reasoning, "Thinking") + deepStrictEqual(assistant.tool_calls, [ + { id: "a", type: "function", function: { name: "lookup", arguments: "{}" } }, + { id: "b", type: "function", function: { name: "lookup", arguments: "{}" } }, + { id: "c", type: "function", function: { name: "lookup", arguments: "{}" } } + ]) + deepStrictEqual(body.messages.filter((message: any) => message.role === "tool"), [ + { + role: "tool", + tool_call_id: "a", + content: partLevel ? [{ type: "text", text: "a", cache_control: oneHour }] : "a" + }, + { role: "tool", tool_call_id: "b", content: "b" }, + { + role: "tool", + tool_call_id: "c", + content: [{ type: "text", text: "{\"value\":42}", cache_control: partLevel ? oneHour : ephemeral }] + } + ]) + }).pipe(Effect.provide(makeTestLayer())) + ) + } + }) }) describe("tool preparation", () => { @@ -626,15 +768,14 @@ const makeTestLayer = (options: { })) ) -const getRequestBody = (request: HttpClientRequest.HttpClientRequest) => - Effect.gen(function*() { - const body = request.body - if (body._tag === "Uint8Array") { - const text = new TextDecoder().decode(body.body) - return JSON.parse(text) - } - return yield* Effect.die(new Error("Expected Uint8Array body")) - }) +const getRequestBody = Effect.fnUntraced(function*(request: HttpClientRequest.HttpClientRequest) { + const body = request.body + if (body._tag === "Uint8Array") { + const text = new TextDecoder().decode(body.body) + return JSON.parse(text) + } + return yield* Effect.die(new Error("Expected Uint8Array body")) +}) const makeStreamTestLayer = (events: ReadonlyArray) => { const body = events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join("") + "data: [DONE]\n\n" diff --git a/repos/effect/packages/ai/typesafe/CHANGELOG.md b/repos/effect/packages/ai/typesafe/CHANGELOG.md index b8eae25715..ab018bc339 100644 --- a/repos/effect/packages/ai/typesafe/CHANGELOG.md +++ b/repos/effect/packages/ai/typesafe/CHANGELOG.md @@ -1,5 +1,23 @@ # @effect/ai-typesafe +## 4.0.6 + +### Patch Changes + +- [#9709](https://github.com/Effect-TS/effect/pull/9709) [`85e2aee`](https://github.com/Effect-TS/effect/commit/85e2aeee686963c26ca40d204f982deebb9d6117) Thanks @tim-smart! - Add typed HTTP date and `Retry-After` header values to `Headers`, and parse them consistently across the HTTP client, OTLP exporter, GraphQL client, AI clients and static server. + + - `Headers.parseHttpDate` / `Headers.formatHttpDate` read all three RFC 9110 date formats as UTC, whatever the process time zone, and emit IMF-fixdate. + - `Headers.parseRetryAfter`, `Headers.formatRetryAfter` and `Headers.retryAfterDelay` accept whole non-negative seconds of any size, read exactly, or an HTTP date, and the delay is never negative. HTTP dates accept the `23:59:60` leap second, read as the following second. + - `Headers.RetryAfter` holds either a `Duration` (`RetryAfterDuration`) or a `DateTime.Utc` (`RetryAfterDateTime`). `Headers.retryAfterDuration` builds the first from any `Duration.Input`: infinite durations throw, finite negative ones become zero, and fractional seconds are rounded up when formatted. + - Parse failures are reported as `Headers.HeaderParseError`. + - `Schema.DateTimeUtcFromHttpDate` and `Schema.RetryAfterFromString` expose the same parsing as string codecs for HTTP API headers. + - Invalid `Retry-After` values such as `-5`, `1.5`, `1e3` or `Infinity` are now ignored everywhere. `HttpClient.retryTransient` and `HttpClient.withRateLimiter` no longer honour fractional seconds, and the AI clients no longer produce negative or infinite `retryAfter` durations. + - `HttpStaticServer` reads `If-Modified-Since` and `Last-Modified` as UTC; asctime dates were previously shifted by the process time zone. + +- [#9430](https://github.com/Effect-TS/effect/pull/9430) [`48102ac`](https://github.com/Effect-TS/effect/commit/48102ac592ef89fed9d5d10afd2daece0a0ee310) Thanks @effect-bot! - Add non-retryable `AiError.InvalidResponseError` for provider response schema and SSE decoding failures, with optional HTTP context; model-generated output validation remains retryable. Update exhaustive matches on `AiErrorReason` to handle the new tag, and update serialized-error consumers to accept it. +- Updated dependencies [[`ce13a48`](https://github.com/Effect-TS/effect/commit/ce13a4872e516187851c5d9336d4df309cc8f0a4), [`f3bbbb4`](https://github.com/Effect-TS/effect/commit/f3bbbb448d431a9d1205ff09ba47cb81af0c7199), [`5547315`](https://github.com/Effect-TS/effect/commit/554731500e43eef47ff68b569116b74006e91d87), [`6093cfb`](https://github.com/Effect-TS/effect/commit/6093cfbbb673d36ecd18edea955763db003c8c26), [`30319d5`](https://github.com/Effect-TS/effect/commit/30319d506c9177d1cc1e0b10372e1ddb776aaf0a), [`74a7596`](https://github.com/Effect-TS/effect/commit/74a759699a8582dbe660b21dcda21cc9db08f613), [`bfb32a3`](https://github.com/Effect-TS/effect/commit/bfb32a3cf5f8d0f02f0dfdb63dcf5237fde755b0), [`f8bf7f4`](https://github.com/Effect-TS/effect/commit/f8bf7f4bcc3ad543cd94cbeb010cb164e5dedcb0), [`7f3d3df`](https://github.com/Effect-TS/effect/commit/7f3d3dfbb7b9e6e041c4ad9654c4532f72d2f16e), [`db1046d`](https://github.com/Effect-TS/effect/commit/db1046d1fa0e4b5818c51d28017a89579b6261d6), [`3c4f88a`](https://github.com/Effect-TS/effect/commit/3c4f88aa4d1d4d958b42fe8d3f1195f0ca990f8a), [`916e709`](https://github.com/Effect-TS/effect/commit/916e70968acc4e443885c67012d855b7cc6f9bca), [`916f843`](https://github.com/Effect-TS/effect/commit/916f8436225153a81a049bdc1feaedcafe4d1ded), [`66ec757`](https://github.com/Effect-TS/effect/commit/66ec7574d23b7943584059b41136bc21ee9d4c53), [`daac06f`](https://github.com/Effect-TS/effect/commit/daac06f79a6ab8c7d3e1b66b427c06f7d64dcc8f), [`c2552ec`](https://github.com/Effect-TS/effect/commit/c2552ec2e031b409b2bef67309312b0168480a39), [`b02c1cd`](https://github.com/Effect-TS/effect/commit/b02c1cde36c890941a235224af8ecb035197582a), [`1855f1d`](https://github.com/Effect-TS/effect/commit/1855f1d359535339bc7640b67e6bb7b12752ba19), [`683ed2c`](https://github.com/Effect-TS/effect/commit/683ed2cb491f9529379c1d5c63933689ff57fd2b), [`ac68940`](https://github.com/Effect-TS/effect/commit/ac68940f7e4246b1beb43f9d451a67f41241ab82), [`22a8401`](https://github.com/Effect-TS/effect/commit/22a84017e7222f332611c1bace05f9fa60c7cf67), [`c53c194`](https://github.com/Effect-TS/effect/commit/c53c19451171a00776b400a8548394ac514b628a), [`ca6d1cc`](https://github.com/Effect-TS/effect/commit/ca6d1cc5f803cd501e85431642b79b552322a76a), [`47940c5`](https://github.com/Effect-TS/effect/commit/47940c55c9e0eb855615094a2aca6e0812307bb2), [`e8b24ef`](https://github.com/Effect-TS/effect/commit/e8b24ef9516706971db948efb2d3eab84ad85951), [`2e8aae0`](https://github.com/Effect-TS/effect/commit/2e8aae0080a8d3e6d00c5ee0bc4f49f0b074acc2), [`ad7497b`](https://github.com/Effect-TS/effect/commit/ad7497bbc1bc70935f8f1ae395ff501532e2e770), [`7f504a3`](https://github.com/Effect-TS/effect/commit/7f504a35edcf44952c55ecbb0916726bac4d3e52), [`99e365d`](https://github.com/Effect-TS/effect/commit/99e365d61a1fdaabeb31dc58e5752cade8a09e1f), [`954067d`](https://github.com/Effect-TS/effect/commit/954067da7fbd20a20c10d5d0de6ffe4690e6bdfd), [`3e1a035`](https://github.com/Effect-TS/effect/commit/3e1a035d9c470ced858b7757f6d102b3aa9be008), [`41a1fe7`](https://github.com/Effect-TS/effect/commit/41a1fe74e6338e8d13d6d4f3daa03f71a826c2c4), [`bcb3a86`](https://github.com/Effect-TS/effect/commit/bcb3a86335a8ed18da39300984c0fbae187fdc6b), [`739d3d1`](https://github.com/Effect-TS/effect/commit/739d3d18728cce0b6d907f0a3f24e04ab0b53560), [`ed4da35`](https://github.com/Effect-TS/effect/commit/ed4da351fb9d29e4dfd6c60bd89ba4c007d264dc), [`3d84e19`](https://github.com/Effect-TS/effect/commit/3d84e19f8f14bccf0aa57b11825590c3bae162f4), [`cc00906`](https://github.com/Effect-TS/effect/commit/cc0090657eeb89debdd54af5fcb99d544f46c1bf), [`09ba1f5`](https://github.com/Effect-TS/effect/commit/09ba1f59952c46cd22229d41236742c3e1f53a34), [`9ff1a73`](https://github.com/Effect-TS/effect/commit/9ff1a73d995702f4c4c335888fe6a186f3f710c7), [`06bc532`](https://github.com/Effect-TS/effect/commit/06bc532b71cd3684f010956774356864dd359372), [`97bfd84`](https://github.com/Effect-TS/effect/commit/97bfd84bb147e10ed562e4c1ffa589cdaf83a596), [`25efaef`](https://github.com/Effect-TS/effect/commit/25efaef0d81d25b2651ef4de474e929d3180810a), [`c10335b`](https://github.com/Effect-TS/effect/commit/c10335be125c7846ae7b914eeb138244b660275b), [`a808972`](https://github.com/Effect-TS/effect/commit/a80897262648e4222ce48943e538f39a6b3f69d6), [`20e844c`](https://github.com/Effect-TS/effect/commit/20e844c7ef27b0e4b4f7911d603c45b1de2b0bc3), [`b9f73f3`](https://github.com/Effect-TS/effect/commit/b9f73f37c4ac538e6a327ed80ac5a0708858c540), [`5739878`](https://github.com/Effect-TS/effect/commit/5739878ded09d83d71d657e3b8d4264f4e4936d9), [`28acdf3`](https://github.com/Effect-TS/effect/commit/28acdf3820b66056037bb7a24cd0e8d3de3fb06d), [`59b6396`](https://github.com/Effect-TS/effect/commit/59b63967c1817ae5acb1811037f5aaa964219b7e), [`0bccc9c`](https://github.com/Effect-TS/effect/commit/0bccc9c3dd5349b20f8d2e509ac5482d0c26a616), [`fed70de`](https://github.com/Effect-TS/effect/commit/fed70de0dff8af24c5f814bd0d985ab70c70420b), [`82648a2`](https://github.com/Effect-TS/effect/commit/82648a21bf228145a14d2d1390006d67f74d89c7), [`afb5f32`](https://github.com/Effect-TS/effect/commit/afb5f326cc14c9849d8d7ddefde1039ae036e017), [`28813f5`](https://github.com/Effect-TS/effect/commit/28813f53a5026984fdc13d3971147ea5dc20104c), [`0fae934`](https://github.com/Effect-TS/effect/commit/0fae934a13aac12f9036b0733edd5dde5ce029f3), [`f260830`](https://github.com/Effect-TS/effect/commit/f2608305861269fc2fbedcbbdcd71e4fc5f98ffc), [`19b63eb`](https://github.com/Effect-TS/effect/commit/19b63eb81b2bccf033aadb565f52a6b0375d26a1), [`14db221`](https://github.com/Effect-TS/effect/commit/14db221f8f584100ff50fd1173b1a99a1caa1d06), [`679f956`](https://github.com/Effect-TS/effect/commit/679f956c4a7527854ff78db071dd7aa417211a48), [`b72707c`](https://github.com/Effect-TS/effect/commit/b72707c2937328fd3bc70f29a6792d84c4256484), [`44c8c1d`](https://github.com/Effect-TS/effect/commit/44c8c1d1c6f01ba9887a051b365e9fa89fcfc685), [`1e7909b`](https://github.com/Effect-TS/effect/commit/1e7909beb05bc740dbd330509e60bb185d7eeed1), [`0611a65`](https://github.com/Effect-TS/effect/commit/0611a657f76836f9ca275f64764b82270b4eda10), [`2fadc65`](https://github.com/Effect-TS/effect/commit/2fadc654aa655fbd1912ce51eb1b764a757ac362), [`feb6ce5`](https://github.com/Effect-TS/effect/commit/feb6ce59a38ab15fc6f89545bd5f0992dd58a06a), [`5a2ed53`](https://github.com/Effect-TS/effect/commit/5a2ed5323884a435bfcfb5e3a6f2ae29547458ad), [`c8d26a4`](https://github.com/Effect-TS/effect/commit/c8d26a46731cb26a9b9612d9fde823d8813cfc04), [`0c8b446`](https://github.com/Effect-TS/effect/commit/0c8b446709c737c10d0e11af4313bb29fdff2588), [`05c3409`](https://github.com/Effect-TS/effect/commit/05c340955944374e3ae47a1de0f7cb9b41a3b0e6), [`b04191b`](https://github.com/Effect-TS/effect/commit/b04191b2941534754282c51ef2df2f965ac43a5c), [`05c3409`](https://github.com/Effect-TS/effect/commit/05c340955944374e3ae47a1de0f7cb9b41a3b0e6), [`4067e64`](https://github.com/Effect-TS/effect/commit/4067e6428776ae573c8cc72f7eaebd078a4ff8c7), [`e04563d`](https://github.com/Effect-TS/effect/commit/e04563dcd6ddeea74c088a12194c5a73e597ba2f), [`02d7fc5`](https://github.com/Effect-TS/effect/commit/02d7fc583053c47dea3b5e1d9d6be164aeb321b6), [`561d2a1`](https://github.com/Effect-TS/effect/commit/561d2a1ddfa3c4340510f23da7adc50c89d97863), [`8fe2dcc`](https://github.com/Effect-TS/effect/commit/8fe2dcc2137788a71177d42a9f67fb7003224868), [`a8086fa`](https://github.com/Effect-TS/effect/commit/a8086fa26a2c9a9f5f7f1d7590bbc494a0feb02a), [`88a33b4`](https://github.com/Effect-TS/effect/commit/88a33b4122f7ffbbc1423add793a79abffd99c75), [`e77372a`](https://github.com/Effect-TS/effect/commit/e77372a8a2e7caec11999636e04c6e31c52e315f), [`3dd4388`](https://github.com/Effect-TS/effect/commit/3dd438810df34ed3b3d2e3e35ccf84ccb00feb85), [`ba2b00c`](https://github.com/Effect-TS/effect/commit/ba2b00c0658b65775809e0052ec9612432600c31), [`a049d4a`](https://github.com/Effect-TS/effect/commit/a049d4ad25f17cf4985284a8d15b07aea2a33242), [`84aeede`](https://github.com/Effect-TS/effect/commit/84aeede040460a2b24bcf0db7a5d0a9b0e3c42d4), [`43e5e19`](https://github.com/Effect-TS/effect/commit/43e5e198d16df038b60764f5dd0218cdcf828f16), [`77bbf16`](https://github.com/Effect-TS/effect/commit/77bbf16e50d6b0ea090288cc5d3f11b20706a027), [`6af3a47`](https://github.com/Effect-TS/effect/commit/6af3a47d5b5d698c3ba896c37684266cd465bef1), [`aac8d21`](https://github.com/Effect-TS/effect/commit/aac8d21911333dedb649f4f9d302c28d14d46369), [`84c8dc4`](https://github.com/Effect-TS/effect/commit/84c8dc47b09067362749620f75068d6f28044660), [`78d1988`](https://github.com/Effect-TS/effect/commit/78d198818d4fd547ce2e9431302f66cb00dc7ad3), [`5676a2a`](https://github.com/Effect-TS/effect/commit/5676a2ac5a918f8f91fc7838d7adf1b6f5e86a6d), [`4b437bf`](https://github.com/Effect-TS/effect/commit/4b437bf4645fabc8aaeaa3b5a88e8c9048b9e4f5), [`1af11fd`](https://github.com/Effect-TS/effect/commit/1af11fda1bccf823f74121b5125aa3befa084b18), [`716f8df`](https://github.com/Effect-TS/effect/commit/716f8dfa178f849e5b288464373bd132860c2e19), [`ee27718`](https://github.com/Effect-TS/effect/commit/ee27718a02125328a7478812b1b7d7c1d8532578), [`4d86d1b`](https://github.com/Effect-TS/effect/commit/4d86d1b8887d48fa24acf6ad54f33ef3d55a7cd8), [`5462ed7`](https://github.com/Effect-TS/effect/commit/5462ed7b3a4c37206c47e401d6be293dc540350f), [`658837c`](https://github.com/Effect-TS/effect/commit/658837c3b249d982df56f2990247499746e36672), [`00261cc`](https://github.com/Effect-TS/effect/commit/00261cc12c31ecc7878f80670439731c905dc2bd), [`7cccb93`](https://github.com/Effect-TS/effect/commit/7cccb9385729bc0aed6630edea95d7db15439ac4), [`7c98228`](https://github.com/Effect-TS/effect/commit/7c982281cf73958833054175fc86db385e043cf6), [`5a8e248`](https://github.com/Effect-TS/effect/commit/5a8e2481546352e2ad6233dad49420b14c59f624), [`8037a65`](https://github.com/Effect-TS/effect/commit/8037a65f40cae9163ab9e0e687e289f717ae570b), [`25912db`](https://github.com/Effect-TS/effect/commit/25912db9d493227a2d6fa45cebcfc7507e638d8a), [`7f32606`](https://github.com/Effect-TS/effect/commit/7f32606baac43a67d171ae93b77c7b29791c9692), [`0386f88`](https://github.com/Effect-TS/effect/commit/0386f88295ac70cbc1f90ab8cceb0981d848c628), [`403298a`](https://github.com/Effect-TS/effect/commit/403298ac457fb634eca2c013fed26d37fc80445f), [`5a3ebc0`](https://github.com/Effect-TS/effect/commit/5a3ebc082124f9e10106b2347736e66fc74ebc36), [`fac2191`](https://github.com/Effect-TS/effect/commit/fac21911964b742a38e1d4e73bd4f95aec4911dc), [`fdb12dc`](https://github.com/Effect-TS/effect/commit/fdb12dcd61b6505eed02e6f7a3c88a85d4c48e8d), [`97909b8`](https://github.com/Effect-TS/effect/commit/97909b89b167da54a56e6f82f1c4db8bfedae04f), [`c3983f4`](https://github.com/Effect-TS/effect/commit/c3983f4db2b3052a58e8045cce270fd7ccc4e2b2), [`3a5e65e`](https://github.com/Effect-TS/effect/commit/3a5e65e409c9569e23bb7af61df401347e9ec420), [`8b40fbc`](https://github.com/Effect-TS/effect/commit/8b40fbc93caf52f3a4c69020849f0051f100be65), [`3f3e119`](https://github.com/Effect-TS/effect/commit/3f3e119ad61c2167bc6c7170c3fb68cd1eb3421e), [`7c7b8ed`](https://github.com/Effect-TS/effect/commit/7c7b8eda54e44f9cddc1188f994f7fdddf901055), [`d4f71bc`](https://github.com/Effect-TS/effect/commit/d4f71bc42507b638d149fb8806d26ea626d17956), [`c66241e`](https://github.com/Effect-TS/effect/commit/c66241e791c4b51d8a65051142d01682590f9bf8), [`f0e941e`](https://github.com/Effect-TS/effect/commit/f0e941ee2f90f0a891d1228a90da91386f0258cb), [`5f8a6e5`](https://github.com/Effect-TS/effect/commit/5f8a6e5108aa6c6cd4043a3c9f4dc93d43369623), [`2800a09`](https://github.com/Effect-TS/effect/commit/2800a0910f7f4ad7cfa21dfc840edf8fe6e67ca7), [`e9749b0`](https://github.com/Effect-TS/effect/commit/e9749b04c10badabcdf71e6958de5ac56a5316e9), [`eaef09c`](https://github.com/Effect-TS/effect/commit/eaef09c11b6ed2029c513eea0aa3256dcc6f517f), [`a3c56e5`](https://github.com/Effect-TS/effect/commit/a3c56e5b5e756e879f1cfb90230e8eae620e33df), [`bb61152`](https://github.com/Effect-TS/effect/commit/bb611522fcfb4d317b33acdf5ee24a480d042435), [`e9c842b`](https://github.com/Effect-TS/effect/commit/e9c842bc294abf658d698bb17732ae13465b3817), [`6d8c8e2`](https://github.com/Effect-TS/effect/commit/6d8c8e2075b15ae84ae96e93a0d295cb9221b496), [`1f28187`](https://github.com/Effect-TS/effect/commit/1f281878ef559ef356af2d0e11b381aba857cae1), [`0b9842b`](https://github.com/Effect-TS/effect/commit/0b9842b76bbc1b016c8abc46432052e1f60b2fa3), [`c29ddd8`](https://github.com/Effect-TS/effect/commit/c29ddd899c53c5d8194c4a0dfef2134304dda799), [`84c3405`](https://github.com/Effect-TS/effect/commit/84c34057ceab9fe6782b73f4a04a31304e0d14de), [`3934c0c`](https://github.com/Effect-TS/effect/commit/3934c0c61857c03468c08c2477f49a6b977d28ff), [`048e7c8`](https://github.com/Effect-TS/effect/commit/048e7c87125c4ace5a61895feac3d049a317bbfc), [`1121334`](https://github.com/Effect-TS/effect/commit/1121334ce68c379051011a2ffea37b0a30ea9a47), [`9a680d6`](https://github.com/Effect-TS/effect/commit/9a680d6985c6a27d7ebb29cd9eb674f3fe524505), [`85e2aee`](https://github.com/Effect-TS/effect/commit/85e2aeee686963c26ca40d204f982deebb9d6117), [`34ec440`](https://github.com/Effect-TS/effect/commit/34ec4407a14ce045ca14cc76c77582884a7e1ebe), [`343a149`](https://github.com/Effect-TS/effect/commit/343a149fa7d5a9a0a45841b1b6800b0ecfa16a32), [`915117b`](https://github.com/Effect-TS/effect/commit/915117b523cb6e5cbc98b5ff1c119a8d0d1f6b9a), [`7f823c1`](https://github.com/Effect-TS/effect/commit/7f823c10d20ef53a1081e0d0e2008a4116140799), [`6e614e6`](https://github.com/Effect-TS/effect/commit/6e614e6feb4281a9e81f5ec3be48e7905ff4292c), [`9990b39`](https://github.com/Effect-TS/effect/commit/9990b3977a544247cd4123edde8353a55bf77192), [`22bf781`](https://github.com/Effect-TS/effect/commit/22bf7812aa8a20c676efade943c342cde4ef1cbd), [`41f08b7`](https://github.com/Effect-TS/effect/commit/41f08b75aba40e95cd6c743ec6fcf1f5022f7b5b), [`e8482ca`](https://github.com/Effect-TS/effect/commit/e8482cad9651fa8fe8036fe6292a286994897d0d), [`5ae85fa`](https://github.com/Effect-TS/effect/commit/5ae85fa8ac1fc5e12d5fe8fe7388ac504788ae3b), [`c477a0a`](https://github.com/Effect-TS/effect/commit/c477a0a8010bd0a988221bc00c446848314e1c3c), [`8fbb39e`](https://github.com/Effect-TS/effect/commit/8fbb39eb92577915831f19a83b51be376e844613), [`1c70828`](https://github.com/Effect-TS/effect/commit/1c7082847453e9c2be0cc267a39ca554c52f4013), [`8207d1e`](https://github.com/Effect-TS/effect/commit/8207d1e91ead8aac8f01d9e13b49624b64237c8b), [`7886b84`](https://github.com/Effect-TS/effect/commit/7886b847658664f64392a14efda86f44b3d1e170), [`4839c60`](https://github.com/Effect-TS/effect/commit/4839c601bdda845d9a54d77f578f64372475ab04), [`2595d83`](https://github.com/Effect-TS/effect/commit/2595d831f702d4a748f97e33e71fd5250f58af56), [`f5ae278`](https://github.com/Effect-TS/effect/commit/f5ae27827a82db3f8973b70745fc6f14e7d8fa80), [`7a8ce00`](https://github.com/Effect-TS/effect/commit/7a8ce0098a5f22cbb6c110b16e850fb3fcbcadf2), [`688d302`](https://github.com/Effect-TS/effect/commit/688d30275ff114e50b4f0d343c6227ee33d67348), [`c7072e4`](https://github.com/Effect-TS/effect/commit/c7072e46356a19615625ae0fa569b1c3a79d50be), [`425cedb`](https://github.com/Effect-TS/effect/commit/425cedb5801c9527c116a64d280d9e7c2820f922), [`88bc680`](https://github.com/Effect-TS/effect/commit/88bc68001c6c70984b03a5500232e9b5aab14085), [`268e09c`](https://github.com/Effect-TS/effect/commit/268e09c94c9e8f46142b5c9f29ce0a22bfbbc3aa), [`7e94505`](https://github.com/Effect-TS/effect/commit/7e94505edfc9d81ea08cb96a7d935806495bf61e), [`afa5da3`](https://github.com/Effect-TS/effect/commit/afa5da3c8a9848215e49535dd3d39179bfd67715), [`c29ddd8`](https://github.com/Effect-TS/effect/commit/c29ddd899c53c5d8194c4a0dfef2134304dda799), [`48102ac`](https://github.com/Effect-TS/effect/commit/48102ac592ef89fed9d5d10afd2daece0a0ee310), [`463eca3`](https://github.com/Effect-TS/effect/commit/463eca3609a6a9681247abbcc71a6f10b261a1d9), [`e7162ea`](https://github.com/Effect-TS/effect/commit/e7162eae7f04af1d583b1f31fdd767f91ec1c82e), [`95752db`](https://github.com/Effect-TS/effect/commit/95752db5eb3b6e59fb8749e9792704f8b167f36c), [`4577a28`](https://github.com/Effect-TS/effect/commit/4577a28bcdb50b0d184775f8c1b7ad890d86e255), [`973d599`](https://github.com/Effect-TS/effect/commit/973d599a3e8611626764a0e3e8f79edbcf30dc7c), [`2f066a5`](https://github.com/Effect-TS/effect/commit/2f066a58e90316dd88a219fcea94200beda54219), [`f0ccbb6`](https://github.com/Effect-TS/effect/commit/f0ccbb6917be5c08364b0b8adbd0c4c19e7f2de1), [`2dda6ce`](https://github.com/Effect-TS/effect/commit/2dda6cefc641dbea2ea3279a9b96c0332b277a40), [`3da7c0a`](https://github.com/Effect-TS/effect/commit/3da7c0aabab07c8fc682aea7559536bf31705853), [`5a67e0b`](https://github.com/Effect-TS/effect/commit/5a67e0b3a8118dd7be709a56d6d7af4f10e9c4f3), [`73b3a6e`](https://github.com/Effect-TS/effect/commit/73b3a6e3c38795400251d9c5d73f05c88acd90cd), [`c6f9e4c`](https://github.com/Effect-TS/effect/commit/c6f9e4cbe6f9c8f0a03fefde18df4d35e98a24df), [`be02676`](https://github.com/Effect-TS/effect/commit/be026768ee29a1b1a860c1fd17f2a0896c01f61e), [`26de861`](https://github.com/Effect-TS/effect/commit/26de86100cda8617e077ce66f0a6bb8dfe738fb6), [`9313799`](https://github.com/Effect-TS/effect/commit/931379911a74d5e2207859c295b1acc6a5e2917e), [`d5c1e5b`](https://github.com/Effect-TS/effect/commit/d5c1e5bd425dfe983dff1f8c853c0b7eaa772f87), [`11bb31f`](https://github.com/Effect-TS/effect/commit/11bb31fec0bf2fdfc025009a04de4c69bffa433a), [`8c23e46`](https://github.com/Effect-TS/effect/commit/8c23e46a8280e3f2c831d1c0ca3f106c92f8e071), [`9c96011`](https://github.com/Effect-TS/effect/commit/9c960111de202c0f1dd6989e5b8836150e0ca694), [`0d50570`](https://github.com/Effect-TS/effect/commit/0d50570bbf078804092fcd4a47ccd2d0019f989c), [`9ed2bf1`](https://github.com/Effect-TS/effect/commit/9ed2bf18da0e1ff4fecc31b7cbac2976ec999e00), [`76ec912`](https://github.com/Effect-TS/effect/commit/76ec9120f43f4fdb08b9c5170ff9e3fd82f22457), [`af2756d`](https://github.com/Effect-TS/effect/commit/af2756dd1d196aa3f199ce21c6d2dc3ada3b67b5), [`3ec8100`](https://github.com/Effect-TS/effect/commit/3ec8100c793b8a22381ad115106b93824d840f64), [`93abf02`](https://github.com/Effect-TS/effect/commit/93abf020876af0a6b6b8b25621c80760b7996762), [`1e94993`](https://github.com/Effect-TS/effect/commit/1e949935a1065e5140245a514ce3a799061143e3), [`99a4616`](https://github.com/Effect-TS/effect/commit/99a4616264e31de65fa8d3dc19576309b1a3f84a), [`0a7978e`](https://github.com/Effect-TS/effect/commit/0a7978ea8e70e51700345b12f8c19952dde05255), [`b4a060b`](https://github.com/Effect-TS/effect/commit/b4a060b95a6473d54e277dc005172b54aab9bddf), [`274f4a4`](https://github.com/Effect-TS/effect/commit/274f4a4dac5ee2d9dbb6358cc3dd3afe345d4705), [`25f29be`](https://github.com/Effect-TS/effect/commit/25f29be77bdcd5c2a7be8e2710e88f5a10c4cb8d), [`9036440`](https://github.com/Effect-TS/effect/commit/9036440edca01c24371d0c2c1044be5e5d3cb65e), [`415f005`](https://github.com/Effect-TS/effect/commit/415f0053eb848d562febcb5b011e15f31a99bead), [`191308b`](https://github.com/Effect-TS/effect/commit/191308bd0f5ee04557570f2f960c6e06c429bcfd), [`4b96570`](https://github.com/Effect-TS/effect/commit/4b96570ed28d93b14e958742933d227dc5e74b15), [`d84ec3f`](https://github.com/Effect-TS/effect/commit/d84ec3fab0346c01bc59f6624d1c046ca4140087), [`392aa1c`](https://github.com/Effect-TS/effect/commit/392aa1cb275ac9fdb9b4b1831d8c06196bba54b9), [`99e9bd9`](https://github.com/Effect-TS/effect/commit/99e9bd98a354d83202e2522f3ade9ee4bf4a862c), [`6f6f45f`](https://github.com/Effect-TS/effect/commit/6f6f45f480a3f191aed6a82ca5c301f60ef2cf84), [`13fc07e`](https://github.com/Effect-TS/effect/commit/13fc07edeb939ca6229b858c9d9a8c5243ac21b5), [`a4b38dd`](https://github.com/Effect-TS/effect/commit/a4b38dde7f3fdc1b54a49bffdaca912b4c47c76e), [`8b18580`](https://github.com/Effect-TS/effect/commit/8b185807bb99f6de3d9564447e2f4e08d2f8863d), [`0c477fe`](https://github.com/Effect-TS/effect/commit/0c477fe417d3ecab86bb8d9afd0c392e9cb8a3da), [`9a5331d`](https://github.com/Effect-TS/effect/commit/9a5331db94bfdca4f48317d062fbeb96054a9049), [`a706d73`](https://github.com/Effect-TS/effect/commit/a706d73db8a8d460877c5d7e8a793a52d45be307), [`c9a1e72`](https://github.com/Effect-TS/effect/commit/c9a1e72c5965aa9fe01a669ebfd523f803f5a189), [`89eaaf8`](https://github.com/Effect-TS/effect/commit/89eaaf8581908e248c0b1e47baa7906e0c541635), [`8215811`](https://github.com/Effect-TS/effect/commit/8215811d51d28d37b4732a6ab03de4047ada62ea), [`ad69018`](https://github.com/Effect-TS/effect/commit/ad6901857e05e7b436a0149b5ece7f3a36d9ff33), [`a733eba`](https://github.com/Effect-TS/effect/commit/a733eba9f699737a80a9e7276521980bd929075a), [`7282d5f`](https://github.com/Effect-TS/effect/commit/7282d5f1bcc06d7065429ef81a3d17c64043fde4), [`da33572`](https://github.com/Effect-TS/effect/commit/da33572776763e45802ce8c2fe14ec101f9a5c34), [`d9e2d12`](https://github.com/Effect-TS/effect/commit/d9e2d1221768e9a3e5e00165cb95b86e0c126cc0), [`dc923b8`](https://github.com/Effect-TS/effect/commit/dc923b847ac7d9673b4f0aa7c136557287b2e5e3), [`1b74086`](https://github.com/Effect-TS/effect/commit/1b740862e3d25ec5d4b6f86b9fe792b6dabbf84c), [`8a5507b`](https://github.com/Effect-TS/effect/commit/8a5507ba00f641edf71227d79b9f90efdb634fe2), [`23460fa`](https://github.com/Effect-TS/effect/commit/23460fad704c730926bffe3cc6874268ef6680ec), [`0dbbea6`](https://github.com/Effect-TS/effect/commit/0dbbea65a93c59147a4bc407cfcda38ea26341b2), [`70aad3f`](https://github.com/Effect-TS/effect/commit/70aad3ffe617984f5b599aeed1bc7f04899aca55), [`0909236`](https://github.com/Effect-TS/effect/commit/090923611b209a19f3281a7b1d0c15a9b653a0c7), [`a9e66d8`](https://github.com/Effect-TS/effect/commit/a9e66d889017b302355ba5b8bc2b2050ca1ea5aa), [`ee5c0d5`](https://github.com/Effect-TS/effect/commit/ee5c0d5f43348747b04b5f7bf1b39355614c2f2c), [`631c5d6`](https://github.com/Effect-TS/effect/commit/631c5d6d9a81e6389964b878764fb9aea1538247), [`7e60c1a`](https://github.com/Effect-TS/effect/commit/7e60c1ae62a96c768f9c67d706f494213947f22f), [`4cddd3c`](https://github.com/Effect-TS/effect/commit/4cddd3c41e71fc4990d37c42b1bcf2ce3780b41d), [`8727e5d`](https://github.com/Effect-TS/effect/commit/8727e5d2bec8e2d729b7aeafb725a4ab21d533d2), [`2bc97fa`](https://github.com/Effect-TS/effect/commit/2bc97faa0e06b3973780dae26029e2c59f5dba38), [`f9a7f05`](https://github.com/Effect-TS/effect/commit/f9a7f056ba31aef435933acf082ec8c01ac7decc), [`d978cca`](https://github.com/Effect-TS/effect/commit/d978cca02914bec0ddf66bb2d68d90b220096fb6), [`b334ccf`](https://github.com/Effect-TS/effect/commit/b334ccf361a0f11362e7aabdbf70b35f53a39d61), [`0b50cfc`](https://github.com/Effect-TS/effect/commit/0b50cfc644313afe4c318073893231ec1946eb5f), [`5b7d2b5`](https://github.com/Effect-TS/effect/commit/5b7d2b5c90bc77eb9f8be1b91166c53f89aa1d0c), [`6ee26b1`](https://github.com/Effect-TS/effect/commit/6ee26b16b51b380c41dbfa84d612aa525b420fce), [`eab7e6a`](https://github.com/Effect-TS/effect/commit/eab7e6acfc7e422857565b98b9eddd3a98de1b92), [`e618f10`](https://github.com/Effect-TS/effect/commit/e618f1050f41424439eb82478fcf1c5da5a3244d), [`da14b1f`](https://github.com/Effect-TS/effect/commit/da14b1f18d71a18fc431b56d82cffe09aeb92708), [`3314fc3`](https://github.com/Effect-TS/effect/commit/3314fc30c22d5c9d737f3df7201f4a41424f701b), [`66db624`](https://github.com/Effect-TS/effect/commit/66db624670032a02144d42b6e64b83b324c813cd), [`4b8a50d`](https://github.com/Effect-TS/effect/commit/4b8a50db9649603f382f33640698020ca79547d0), [`8d52668`](https://github.com/Effect-TS/effect/commit/8d52668533e9d4a6d6b3be9fe58bd5e69fa80b65), [`1de1055`](https://github.com/Effect-TS/effect/commit/1de10559e7263747751bfb032fd4b0fd8aa28724), [`db130e6`](https://github.com/Effect-TS/effect/commit/db130e64557fafe370819d19f78e3b19a21275d6), [`5fa5fa5`](https://github.com/Effect-TS/effect/commit/5fa5fa5b9bff77352fb9ff2d424bf46bd6309986)]: + - effect@4.0.6 + ## 4.0.5 ### Patch Changes diff --git a/repos/effect/packages/ai/typesafe/package.json b/repos/effect/packages/ai/typesafe/package.json index 0c89df7646..1b01bedce0 100644 --- a/repos/effect/packages/ai/typesafe/package.json +++ b/repos/effect/packages/ai/typesafe/package.json @@ -1,6 +1,6 @@ { "name": "@effect/ai-typesafe", - "version": "4.0.5", + "version": "4.0.6", "type": "module", "license": "MIT", "description": "A TypeSafe DecisionModel provider for Effect", diff --git a/repos/effect/packages/ai/typesafe/src/internal/errors.ts b/repos/effect/packages/ai/typesafe/src/internal/errors.ts index 388bcdb186..2aa0eb9317 100644 --- a/repos/effect/packages/ai/typesafe/src/internal/errors.ts +++ b/repos/effect/packages/ai/typesafe/src/internal/errors.ts @@ -1,11 +1,14 @@ import * as AiError from "effect/ai/AiError" +import * as Clock from "effect/Clock" import * as Duration from "effect/Duration" import * as Effect from "effect/Effect" +import * as Headers from "effect/http/Headers" import type * as HttpClientError from "effect/http/HttpClientError" import type * as HttpClientRequest from "effect/http/HttpClientRequest" import * as Num from "effect/Number" import * as Option from "effect/Option" import * as Redactable from "effect/Redactable" +import * as Result from "effect/Result" import * as Schema from "effect/Schema" const requestDetails = (request: HttpClientRequest.HttpClientRequest) => ({ @@ -30,22 +33,22 @@ const nonNegative = (value: string | undefined): number | undefined => { return parsed !== undefined && Number.isFinite(parsed) && parsed >= 0 ? parsed : undefined } -const retryAfter = (headers: Readonly>) => { +// `retry-after-ms` is TypeSafe-specific and wins over the standard header; +// an invalid Retry-After is treated as absent +const retryAfter = (headers: Readonly>, now: number) => { const ms = nonNegative(headers["retry-after-ms"]) if (ms !== undefined) return Duration.millis(ms) const raw = headers["retry-after"] - if (raw === undefined || raw.trim() === "") return undefined - const seconds = nonNegative(raw) - if (seconds !== undefined) return Duration.seconds(seconds) - const date = Date.parse(raw) - return Number.isNaN(date) ? undefined : Duration.millis(Math.max(0, date - Date.now())) + if (raw === undefined) return undefined + const retryAfter = Headers.parseRetryAfter(raw) + return Result.isSuccess(retryAfter) ? Headers.retryAfterDelay(retryAfter.success, now) : undefined } export const mapSchemaError = (error: Schema.SchemaError, method: string): AiError.AiError => AiError.make({ module: "TypeSafeClient", method, - reason: AiError.InvalidOutputError.fromSchemaError(error) + reason: AiError.InvalidResponseError.fromSchemaError(error) }) export const mapHttpClientError = Effect.fnUntraced( @@ -90,7 +93,7 @@ export const mapHttpClientError = Effect.fnUntraced( }) if (response.status === 429) { reason = new AiError.RateLimitError({ - retryAfter: retryAfter(response.headers), + retryAfter: retryAfter(response.headers, yield* Clock.currentTimeMillis), http, metadata: { typesafe: { diff --git a/repos/effect/packages/ai/typesafe/test/TypeSafeClient.test.ts b/repos/effect/packages/ai/typesafe/test/TypeSafeClient.test.ts index c3340a5a97..e526dbc626 100644 --- a/repos/effect/packages/ai/typesafe/test/TypeSafeClient.test.ts +++ b/repos/effect/packages/ai/typesafe/test/TypeSafeClient.test.ts @@ -6,7 +6,7 @@ import * as HttpClient from "effect/http/HttpClient" import * as HttpClientError from "effect/http/HttpClientError" import * as HttpClientRequest from "effect/http/HttpClientRequest" import * as HttpClientResponse from "effect/http/HttpClientResponse" -import { vi } from "vitest" +import { TestClock } from "effect/testing" const systemOneRequest = { state: "My card was charged twice.", model: "jev-latest", @@ -259,18 +259,14 @@ describe("TypeSafeClient", () => { ) { it.effect("honors a " + name + " HTTP-date Retry-After", () => Effect.gen(function*() { - const now = vi.spyOn(Date, "now").mockReturnValue(Date.parse("2026-09-18T12:00:00Z")) - try { - const client = yield* TypeSafeClient.TypeSafeClient - const result = yield* client.systemOne(systemOneRequest).pipe(Effect.flip) - - assert.strictEqual(result.reason._tag, "RateLimitError") - if (result.reason._tag === "RateLimitError") { - assert.isDefined(result.reason.retryAfter) - assert.strictEqual(Duration.toMillis(result.reason.retryAfter!), expected) - } - } finally { - now.mockRestore() + yield* TestClock.setTime(Date.parse("2026-09-18T12:00:00Z")) + const client = yield* TypeSafeClient.TypeSafeClient + const result = yield* client.systemOne(systemOneRequest).pipe(Effect.flip) + + assert.strictEqual(result.reason._tag, "RateLimitError") + if (result.reason._tag === "RateLimitError") { + assert.isDefined(result.reason.retryAfter) + assert.strictEqual(Duration.toMillis(result.reason.retryAfter!), expected) } }).pipe(Effect.provide(makeTestLayer(undefined, { status: 429, @@ -435,14 +431,14 @@ describe("TypeSafeClient", () => { body: { message: "Internal error" } })))) - it.effect("maps a malformed success body to InvalidOutputError reason", () => + it.effect("maps a malformed success body to a non-retryable InvalidResponseError reason", () => Effect.gen(function*() { const client = yield* TypeSafeClient.TypeSafeClient const result = yield* client.systemOne(systemOneRequest).pipe(Effect.flip) assert.strictEqual(result._tag, "AiError") assert.strictEqual(result.method, "systemOne") - assert.strictEqual(result.reason._tag, "InvalidOutputError") + assert.strictEqual(result.reason._tag, "InvalidResponseError") }).pipe(Effect.provide(makeTestLayer(undefined, { status: 200, body: { model: "jev-latest", answers: { urgent: { type: "noul" } } } @@ -557,12 +553,11 @@ const makeResponse = ( }) ) -const getRequestBody = (request: HttpClientRequest.HttpClientRequest) => - Effect.gen(function*() { - const body = request.body - if (body._tag === "Uint8Array") { - const text = new TextDecoder().decode(body.body) - return JSON.parse(text) - } - return yield* Effect.die(new Error("Expected Uint8Array body")) - }) +const getRequestBody = Effect.fnUntraced(function*(request: HttpClientRequest.HttpClientRequest) { + const body = request.body + if (body._tag === "Uint8Array") { + const text = new TextDecoder().decode(body.body) + return JSON.parse(text) + } + return yield* Effect.die(new Error("Expected Uint8Array body")) +}) diff --git a/repos/effect/packages/ai/typesafe/test/TypeSafeDecisionModel.test.ts b/repos/effect/packages/ai/typesafe/test/TypeSafeDecisionModel.test.ts index 071451bd04..4d1e5d2c42 100644 --- a/repos/effect/packages/ai/typesafe/test/TypeSafeDecisionModel.test.ts +++ b/repos/effect/packages/ai/typesafe/test/TypeSafeDecisionModel.test.ts @@ -152,11 +152,12 @@ describe("TypeSafeDecisionModel", () => { })) for ( - const [name, probabilities] of [ - ["out-of-range", { a: 1.01, b: -0.02 }], - ["all-zero", { a: 0, b: 0 }], - ["missing label", { a: 0.5 }], - ["non-finite", { a: Infinity, b: 0.5 }] + const [name, probabilities, reason] of [ + ["out-of-range", { a: 1.01, b: -0.02 }, "InvalidOutputError"], + ["all-zero", { a: 0, b: 0 }, "InvalidOutputError"], + ["missing label", { a: 0.5 }, "InvalidOutputError"], + // `Infinity` serializes as `null`, which the client response schema rejects + ["non-finite", { a: Infinity, b: 0.5 }, "InvalidResponseError"] ] as const ) { it.effect("rejects " + name + " choice probabilities", () => @@ -175,7 +176,7 @@ describe("TypeSafeDecisionModel", () => { )), Effect.flip ) - assert.strictEqual(error.reason._tag, "InvalidOutputError") + assert.strictEqual(error.reason._tag, reason) })) } @@ -603,7 +604,7 @@ describe("TypeSafeDecisionModel", () => { assert.strictEqual(error.reason._tag, "InvalidOutputError") })) - it.effect("fails with InvalidOutputError when a score answer omits probabilities", () => + it.effect("fails with a non-retryable InvalidResponseError when a score answer omits probabilities", () => Effect.gen(function*() { const { probabilities: _probabilities, ...frustration } = systemOneResponse.answers.frustration const error = yield* DecisionModel.decide(TicketTriage, { input: ticket }).pipe( @@ -618,7 +619,7 @@ describe("TypeSafeDecisionModel", () => { ) assert.strictEqual(error._tag, "AiError") - assert.strictEqual(error.reason._tag, "InvalidOutputError") + assert.strictEqual(error.reason._tag, "InvalidResponseError") })) it.effect("fails with InvalidOutputError when a score distribution skips a level index", () => @@ -731,15 +732,14 @@ const jsonResponse = ( }) ) -const getRequestBody = (request: HttpClientRequest.HttpClientRequest) => - Effect.gen(function*() { - const body = request.body - if (body._tag === "Uint8Array") { - const text = new TextDecoder().decode(body.body) - return JSON.parse(text) - } - return yield* Effect.die(new Error("Expected Uint8Array body")) - }) +const getRequestBody = Effect.fnUntraced(function*(request: HttpClientRequest.HttpClientRequest) { + const body = request.body + if (body._tag === "Uint8Array") { + const text = new TextDecoder().decode(body.body) + return JSON.parse(text) + } + return yield* Effect.die(new Error("Expected Uint8Array body")) +}) const noopTypeSafeClient: TypeSafeClient.Service = { client: undefined as unknown as TypeSafeClient.Service["client"], diff --git a/repos/effect/packages/atom/react/CHANGELOG.md b/repos/effect/packages/atom/react/CHANGELOG.md index 1d20aef210..7888a04266 100644 --- a/repos/effect/packages/atom/react/CHANGELOG.md +++ b/repos/effect/packages/atom/react/CHANGELOG.md @@ -1,5 +1,13 @@ # @effect/atom-react +## 4.0.6 + +### Patch Changes + +- [#9522](https://github.com/Effect-TS/effect/pull/9522) [`f8bf7f4`](https://github.com/Effect-TS/effect/commit/f8bf7f4bcc3ad543cd94cbeb010cb164e5dedcb0) Thanks @tim-smart! - Add a `skipInitialRun` option to `Atom.serializable` and `Atom.initialValue`. A seeded effect or stream atom then keeps its hydrated or initial value instead of running on mount, until it is refreshed or a dependency changes. Factory `get(...)` calls and `withReactivity` keys are still tracked; dependencies read inside the effect are tracked from its first real run. +- Updated dependencies [[`ce13a48`](https://github.com/Effect-TS/effect/commit/ce13a4872e516187851c5d9336d4df309cc8f0a4), [`f3bbbb4`](https://github.com/Effect-TS/effect/commit/f3bbbb448d431a9d1205ff09ba47cb81af0c7199), [`5547315`](https://github.com/Effect-TS/effect/commit/554731500e43eef47ff68b569116b74006e91d87), [`6093cfb`](https://github.com/Effect-TS/effect/commit/6093cfbbb673d36ecd18edea955763db003c8c26), [`30319d5`](https://github.com/Effect-TS/effect/commit/30319d506c9177d1cc1e0b10372e1ddb776aaf0a), [`74a7596`](https://github.com/Effect-TS/effect/commit/74a759699a8582dbe660b21dcda21cc9db08f613), [`bfb32a3`](https://github.com/Effect-TS/effect/commit/bfb32a3cf5f8d0f02f0dfdb63dcf5237fde755b0), [`f8bf7f4`](https://github.com/Effect-TS/effect/commit/f8bf7f4bcc3ad543cd94cbeb010cb164e5dedcb0), [`7f3d3df`](https://github.com/Effect-TS/effect/commit/7f3d3dfbb7b9e6e041c4ad9654c4532f72d2f16e), [`db1046d`](https://github.com/Effect-TS/effect/commit/db1046d1fa0e4b5818c51d28017a89579b6261d6), [`3c4f88a`](https://github.com/Effect-TS/effect/commit/3c4f88aa4d1d4d958b42fe8d3f1195f0ca990f8a), [`916e709`](https://github.com/Effect-TS/effect/commit/916e70968acc4e443885c67012d855b7cc6f9bca), [`916f843`](https://github.com/Effect-TS/effect/commit/916f8436225153a81a049bdc1feaedcafe4d1ded), [`66ec757`](https://github.com/Effect-TS/effect/commit/66ec7574d23b7943584059b41136bc21ee9d4c53), [`daac06f`](https://github.com/Effect-TS/effect/commit/daac06f79a6ab8c7d3e1b66b427c06f7d64dcc8f), [`c2552ec`](https://github.com/Effect-TS/effect/commit/c2552ec2e031b409b2bef67309312b0168480a39), [`b02c1cd`](https://github.com/Effect-TS/effect/commit/b02c1cde36c890941a235224af8ecb035197582a), [`1855f1d`](https://github.com/Effect-TS/effect/commit/1855f1d359535339bc7640b67e6bb7b12752ba19), [`683ed2c`](https://github.com/Effect-TS/effect/commit/683ed2cb491f9529379c1d5c63933689ff57fd2b), [`ac68940`](https://github.com/Effect-TS/effect/commit/ac68940f7e4246b1beb43f9d451a67f41241ab82), [`22a8401`](https://github.com/Effect-TS/effect/commit/22a84017e7222f332611c1bace05f9fa60c7cf67), [`c53c194`](https://github.com/Effect-TS/effect/commit/c53c19451171a00776b400a8548394ac514b628a), [`ca6d1cc`](https://github.com/Effect-TS/effect/commit/ca6d1cc5f803cd501e85431642b79b552322a76a), [`47940c5`](https://github.com/Effect-TS/effect/commit/47940c55c9e0eb855615094a2aca6e0812307bb2), [`e8b24ef`](https://github.com/Effect-TS/effect/commit/e8b24ef9516706971db948efb2d3eab84ad85951), [`2e8aae0`](https://github.com/Effect-TS/effect/commit/2e8aae0080a8d3e6d00c5ee0bc4f49f0b074acc2), [`ad7497b`](https://github.com/Effect-TS/effect/commit/ad7497bbc1bc70935f8f1ae395ff501532e2e770), [`7f504a3`](https://github.com/Effect-TS/effect/commit/7f504a35edcf44952c55ecbb0916726bac4d3e52), [`99e365d`](https://github.com/Effect-TS/effect/commit/99e365d61a1fdaabeb31dc58e5752cade8a09e1f), [`954067d`](https://github.com/Effect-TS/effect/commit/954067da7fbd20a20c10d5d0de6ffe4690e6bdfd), [`3e1a035`](https://github.com/Effect-TS/effect/commit/3e1a035d9c470ced858b7757f6d102b3aa9be008), [`41a1fe7`](https://github.com/Effect-TS/effect/commit/41a1fe74e6338e8d13d6d4f3daa03f71a826c2c4), [`bcb3a86`](https://github.com/Effect-TS/effect/commit/bcb3a86335a8ed18da39300984c0fbae187fdc6b), [`739d3d1`](https://github.com/Effect-TS/effect/commit/739d3d18728cce0b6d907f0a3f24e04ab0b53560), [`ed4da35`](https://github.com/Effect-TS/effect/commit/ed4da351fb9d29e4dfd6c60bd89ba4c007d264dc), [`3d84e19`](https://github.com/Effect-TS/effect/commit/3d84e19f8f14bccf0aa57b11825590c3bae162f4), [`cc00906`](https://github.com/Effect-TS/effect/commit/cc0090657eeb89debdd54af5fcb99d544f46c1bf), [`09ba1f5`](https://github.com/Effect-TS/effect/commit/09ba1f59952c46cd22229d41236742c3e1f53a34), [`9ff1a73`](https://github.com/Effect-TS/effect/commit/9ff1a73d995702f4c4c335888fe6a186f3f710c7), [`06bc532`](https://github.com/Effect-TS/effect/commit/06bc532b71cd3684f010956774356864dd359372), [`97bfd84`](https://github.com/Effect-TS/effect/commit/97bfd84bb147e10ed562e4c1ffa589cdaf83a596), [`25efaef`](https://github.com/Effect-TS/effect/commit/25efaef0d81d25b2651ef4de474e929d3180810a), [`c10335b`](https://github.com/Effect-TS/effect/commit/c10335be125c7846ae7b914eeb138244b660275b), [`a808972`](https://github.com/Effect-TS/effect/commit/a80897262648e4222ce48943e538f39a6b3f69d6), [`20e844c`](https://github.com/Effect-TS/effect/commit/20e844c7ef27b0e4b4f7911d603c45b1de2b0bc3), [`b9f73f3`](https://github.com/Effect-TS/effect/commit/b9f73f37c4ac538e6a327ed80ac5a0708858c540), [`5739878`](https://github.com/Effect-TS/effect/commit/5739878ded09d83d71d657e3b8d4264f4e4936d9), [`28acdf3`](https://github.com/Effect-TS/effect/commit/28acdf3820b66056037bb7a24cd0e8d3de3fb06d), [`59b6396`](https://github.com/Effect-TS/effect/commit/59b63967c1817ae5acb1811037f5aaa964219b7e), [`0bccc9c`](https://github.com/Effect-TS/effect/commit/0bccc9c3dd5349b20f8d2e509ac5482d0c26a616), [`fed70de`](https://github.com/Effect-TS/effect/commit/fed70de0dff8af24c5f814bd0d985ab70c70420b), [`82648a2`](https://github.com/Effect-TS/effect/commit/82648a21bf228145a14d2d1390006d67f74d89c7), [`afb5f32`](https://github.com/Effect-TS/effect/commit/afb5f326cc14c9849d8d7ddefde1039ae036e017), [`28813f5`](https://github.com/Effect-TS/effect/commit/28813f53a5026984fdc13d3971147ea5dc20104c), [`0fae934`](https://github.com/Effect-TS/effect/commit/0fae934a13aac12f9036b0733edd5dde5ce029f3), [`f260830`](https://github.com/Effect-TS/effect/commit/f2608305861269fc2fbedcbbdcd71e4fc5f98ffc), [`19b63eb`](https://github.com/Effect-TS/effect/commit/19b63eb81b2bccf033aadb565f52a6b0375d26a1), [`14db221`](https://github.com/Effect-TS/effect/commit/14db221f8f584100ff50fd1173b1a99a1caa1d06), [`679f956`](https://github.com/Effect-TS/effect/commit/679f956c4a7527854ff78db071dd7aa417211a48), [`b72707c`](https://github.com/Effect-TS/effect/commit/b72707c2937328fd3bc70f29a6792d84c4256484), [`44c8c1d`](https://github.com/Effect-TS/effect/commit/44c8c1d1c6f01ba9887a051b365e9fa89fcfc685), [`1e7909b`](https://github.com/Effect-TS/effect/commit/1e7909beb05bc740dbd330509e60bb185d7eeed1), [`0611a65`](https://github.com/Effect-TS/effect/commit/0611a657f76836f9ca275f64764b82270b4eda10), [`2fadc65`](https://github.com/Effect-TS/effect/commit/2fadc654aa655fbd1912ce51eb1b764a757ac362), [`feb6ce5`](https://github.com/Effect-TS/effect/commit/feb6ce59a38ab15fc6f89545bd5f0992dd58a06a), [`5a2ed53`](https://github.com/Effect-TS/effect/commit/5a2ed5323884a435bfcfb5e3a6f2ae29547458ad), [`c8d26a4`](https://github.com/Effect-TS/effect/commit/c8d26a46731cb26a9b9612d9fde823d8813cfc04), [`0c8b446`](https://github.com/Effect-TS/effect/commit/0c8b446709c737c10d0e11af4313bb29fdff2588), [`05c3409`](https://github.com/Effect-TS/effect/commit/05c340955944374e3ae47a1de0f7cb9b41a3b0e6), [`b04191b`](https://github.com/Effect-TS/effect/commit/b04191b2941534754282c51ef2df2f965ac43a5c), [`05c3409`](https://github.com/Effect-TS/effect/commit/05c340955944374e3ae47a1de0f7cb9b41a3b0e6), [`4067e64`](https://github.com/Effect-TS/effect/commit/4067e6428776ae573c8cc72f7eaebd078a4ff8c7), [`e04563d`](https://github.com/Effect-TS/effect/commit/e04563dcd6ddeea74c088a12194c5a73e597ba2f), [`02d7fc5`](https://github.com/Effect-TS/effect/commit/02d7fc583053c47dea3b5e1d9d6be164aeb321b6), [`561d2a1`](https://github.com/Effect-TS/effect/commit/561d2a1ddfa3c4340510f23da7adc50c89d97863), [`8fe2dcc`](https://github.com/Effect-TS/effect/commit/8fe2dcc2137788a71177d42a9f67fb7003224868), [`a8086fa`](https://github.com/Effect-TS/effect/commit/a8086fa26a2c9a9f5f7f1d7590bbc494a0feb02a), [`88a33b4`](https://github.com/Effect-TS/effect/commit/88a33b4122f7ffbbc1423add793a79abffd99c75), [`e77372a`](https://github.com/Effect-TS/effect/commit/e77372a8a2e7caec11999636e04c6e31c52e315f), [`3dd4388`](https://github.com/Effect-TS/effect/commit/3dd438810df34ed3b3d2e3e35ccf84ccb00feb85), [`ba2b00c`](https://github.com/Effect-TS/effect/commit/ba2b00c0658b65775809e0052ec9612432600c31), [`a049d4a`](https://github.com/Effect-TS/effect/commit/a049d4ad25f17cf4985284a8d15b07aea2a33242), [`84aeede`](https://github.com/Effect-TS/effect/commit/84aeede040460a2b24bcf0db7a5d0a9b0e3c42d4), [`43e5e19`](https://github.com/Effect-TS/effect/commit/43e5e198d16df038b60764f5dd0218cdcf828f16), [`77bbf16`](https://github.com/Effect-TS/effect/commit/77bbf16e50d6b0ea090288cc5d3f11b20706a027), [`6af3a47`](https://github.com/Effect-TS/effect/commit/6af3a47d5b5d698c3ba896c37684266cd465bef1), [`aac8d21`](https://github.com/Effect-TS/effect/commit/aac8d21911333dedb649f4f9d302c28d14d46369), [`84c8dc4`](https://github.com/Effect-TS/effect/commit/84c8dc47b09067362749620f75068d6f28044660), [`78d1988`](https://github.com/Effect-TS/effect/commit/78d198818d4fd547ce2e9431302f66cb00dc7ad3), [`5676a2a`](https://github.com/Effect-TS/effect/commit/5676a2ac5a918f8f91fc7838d7adf1b6f5e86a6d), [`4b437bf`](https://github.com/Effect-TS/effect/commit/4b437bf4645fabc8aaeaa3b5a88e8c9048b9e4f5), [`1af11fd`](https://github.com/Effect-TS/effect/commit/1af11fda1bccf823f74121b5125aa3befa084b18), [`716f8df`](https://github.com/Effect-TS/effect/commit/716f8dfa178f849e5b288464373bd132860c2e19), [`ee27718`](https://github.com/Effect-TS/effect/commit/ee27718a02125328a7478812b1b7d7c1d8532578), [`4d86d1b`](https://github.com/Effect-TS/effect/commit/4d86d1b8887d48fa24acf6ad54f33ef3d55a7cd8), [`5462ed7`](https://github.com/Effect-TS/effect/commit/5462ed7b3a4c37206c47e401d6be293dc540350f), [`658837c`](https://github.com/Effect-TS/effect/commit/658837c3b249d982df56f2990247499746e36672), [`00261cc`](https://github.com/Effect-TS/effect/commit/00261cc12c31ecc7878f80670439731c905dc2bd), [`7cccb93`](https://github.com/Effect-TS/effect/commit/7cccb9385729bc0aed6630edea95d7db15439ac4), [`7c98228`](https://github.com/Effect-TS/effect/commit/7c982281cf73958833054175fc86db385e043cf6), [`5a8e248`](https://github.com/Effect-TS/effect/commit/5a8e2481546352e2ad6233dad49420b14c59f624), [`8037a65`](https://github.com/Effect-TS/effect/commit/8037a65f40cae9163ab9e0e687e289f717ae570b), [`25912db`](https://github.com/Effect-TS/effect/commit/25912db9d493227a2d6fa45cebcfc7507e638d8a), [`7f32606`](https://github.com/Effect-TS/effect/commit/7f32606baac43a67d171ae93b77c7b29791c9692), [`0386f88`](https://github.com/Effect-TS/effect/commit/0386f88295ac70cbc1f90ab8cceb0981d848c628), [`403298a`](https://github.com/Effect-TS/effect/commit/403298ac457fb634eca2c013fed26d37fc80445f), [`5a3ebc0`](https://github.com/Effect-TS/effect/commit/5a3ebc082124f9e10106b2347736e66fc74ebc36), [`fac2191`](https://github.com/Effect-TS/effect/commit/fac21911964b742a38e1d4e73bd4f95aec4911dc), [`fdb12dc`](https://github.com/Effect-TS/effect/commit/fdb12dcd61b6505eed02e6f7a3c88a85d4c48e8d), [`97909b8`](https://github.com/Effect-TS/effect/commit/97909b89b167da54a56e6f82f1c4db8bfedae04f), [`c3983f4`](https://github.com/Effect-TS/effect/commit/c3983f4db2b3052a58e8045cce270fd7ccc4e2b2), [`3a5e65e`](https://github.com/Effect-TS/effect/commit/3a5e65e409c9569e23bb7af61df401347e9ec420), [`8b40fbc`](https://github.com/Effect-TS/effect/commit/8b40fbc93caf52f3a4c69020849f0051f100be65), [`3f3e119`](https://github.com/Effect-TS/effect/commit/3f3e119ad61c2167bc6c7170c3fb68cd1eb3421e), [`7c7b8ed`](https://github.com/Effect-TS/effect/commit/7c7b8eda54e44f9cddc1188f994f7fdddf901055), [`d4f71bc`](https://github.com/Effect-TS/effect/commit/d4f71bc42507b638d149fb8806d26ea626d17956), [`c66241e`](https://github.com/Effect-TS/effect/commit/c66241e791c4b51d8a65051142d01682590f9bf8), [`f0e941e`](https://github.com/Effect-TS/effect/commit/f0e941ee2f90f0a891d1228a90da91386f0258cb), [`5f8a6e5`](https://github.com/Effect-TS/effect/commit/5f8a6e5108aa6c6cd4043a3c9f4dc93d43369623), [`2800a09`](https://github.com/Effect-TS/effect/commit/2800a0910f7f4ad7cfa21dfc840edf8fe6e67ca7), [`e9749b0`](https://github.com/Effect-TS/effect/commit/e9749b04c10badabcdf71e6958de5ac56a5316e9), [`eaef09c`](https://github.com/Effect-TS/effect/commit/eaef09c11b6ed2029c513eea0aa3256dcc6f517f), [`a3c56e5`](https://github.com/Effect-TS/effect/commit/a3c56e5b5e756e879f1cfb90230e8eae620e33df), [`bb61152`](https://github.com/Effect-TS/effect/commit/bb611522fcfb4d317b33acdf5ee24a480d042435), [`e9c842b`](https://github.com/Effect-TS/effect/commit/e9c842bc294abf658d698bb17732ae13465b3817), [`6d8c8e2`](https://github.com/Effect-TS/effect/commit/6d8c8e2075b15ae84ae96e93a0d295cb9221b496), [`1f28187`](https://github.com/Effect-TS/effect/commit/1f281878ef559ef356af2d0e11b381aba857cae1), [`0b9842b`](https://github.com/Effect-TS/effect/commit/0b9842b76bbc1b016c8abc46432052e1f60b2fa3), [`c29ddd8`](https://github.com/Effect-TS/effect/commit/c29ddd899c53c5d8194c4a0dfef2134304dda799), [`84c3405`](https://github.com/Effect-TS/effect/commit/84c34057ceab9fe6782b73f4a04a31304e0d14de), [`3934c0c`](https://github.com/Effect-TS/effect/commit/3934c0c61857c03468c08c2477f49a6b977d28ff), [`048e7c8`](https://github.com/Effect-TS/effect/commit/048e7c87125c4ace5a61895feac3d049a317bbfc), [`1121334`](https://github.com/Effect-TS/effect/commit/1121334ce68c379051011a2ffea37b0a30ea9a47), [`9a680d6`](https://github.com/Effect-TS/effect/commit/9a680d6985c6a27d7ebb29cd9eb674f3fe524505), [`85e2aee`](https://github.com/Effect-TS/effect/commit/85e2aeee686963c26ca40d204f982deebb9d6117), [`34ec440`](https://github.com/Effect-TS/effect/commit/34ec4407a14ce045ca14cc76c77582884a7e1ebe), [`343a149`](https://github.com/Effect-TS/effect/commit/343a149fa7d5a9a0a45841b1b6800b0ecfa16a32), [`915117b`](https://github.com/Effect-TS/effect/commit/915117b523cb6e5cbc98b5ff1c119a8d0d1f6b9a), [`7f823c1`](https://github.com/Effect-TS/effect/commit/7f823c10d20ef53a1081e0d0e2008a4116140799), [`6e614e6`](https://github.com/Effect-TS/effect/commit/6e614e6feb4281a9e81f5ec3be48e7905ff4292c), [`9990b39`](https://github.com/Effect-TS/effect/commit/9990b3977a544247cd4123edde8353a55bf77192), [`22bf781`](https://github.com/Effect-TS/effect/commit/22bf7812aa8a20c676efade943c342cde4ef1cbd), [`41f08b7`](https://github.com/Effect-TS/effect/commit/41f08b75aba40e95cd6c743ec6fcf1f5022f7b5b), [`e8482ca`](https://github.com/Effect-TS/effect/commit/e8482cad9651fa8fe8036fe6292a286994897d0d), [`5ae85fa`](https://github.com/Effect-TS/effect/commit/5ae85fa8ac1fc5e12d5fe8fe7388ac504788ae3b), [`c477a0a`](https://github.com/Effect-TS/effect/commit/c477a0a8010bd0a988221bc00c446848314e1c3c), [`8fbb39e`](https://github.com/Effect-TS/effect/commit/8fbb39eb92577915831f19a83b51be376e844613), [`1c70828`](https://github.com/Effect-TS/effect/commit/1c7082847453e9c2be0cc267a39ca554c52f4013), [`8207d1e`](https://github.com/Effect-TS/effect/commit/8207d1e91ead8aac8f01d9e13b49624b64237c8b), [`7886b84`](https://github.com/Effect-TS/effect/commit/7886b847658664f64392a14efda86f44b3d1e170), [`4839c60`](https://github.com/Effect-TS/effect/commit/4839c601bdda845d9a54d77f578f64372475ab04), [`2595d83`](https://github.com/Effect-TS/effect/commit/2595d831f702d4a748f97e33e71fd5250f58af56), [`f5ae278`](https://github.com/Effect-TS/effect/commit/f5ae27827a82db3f8973b70745fc6f14e7d8fa80), [`7a8ce00`](https://github.com/Effect-TS/effect/commit/7a8ce0098a5f22cbb6c110b16e850fb3fcbcadf2), [`688d302`](https://github.com/Effect-TS/effect/commit/688d30275ff114e50b4f0d343c6227ee33d67348), [`c7072e4`](https://github.com/Effect-TS/effect/commit/c7072e46356a19615625ae0fa569b1c3a79d50be), [`425cedb`](https://github.com/Effect-TS/effect/commit/425cedb5801c9527c116a64d280d9e7c2820f922), [`88bc680`](https://github.com/Effect-TS/effect/commit/88bc68001c6c70984b03a5500232e9b5aab14085), [`268e09c`](https://github.com/Effect-TS/effect/commit/268e09c94c9e8f46142b5c9f29ce0a22bfbbc3aa), [`7e94505`](https://github.com/Effect-TS/effect/commit/7e94505edfc9d81ea08cb96a7d935806495bf61e), [`afa5da3`](https://github.com/Effect-TS/effect/commit/afa5da3c8a9848215e49535dd3d39179bfd67715), [`c29ddd8`](https://github.com/Effect-TS/effect/commit/c29ddd899c53c5d8194c4a0dfef2134304dda799), [`48102ac`](https://github.com/Effect-TS/effect/commit/48102ac592ef89fed9d5d10afd2daece0a0ee310), [`463eca3`](https://github.com/Effect-TS/effect/commit/463eca3609a6a9681247abbcc71a6f10b261a1d9), [`e7162ea`](https://github.com/Effect-TS/effect/commit/e7162eae7f04af1d583b1f31fdd767f91ec1c82e), [`95752db`](https://github.com/Effect-TS/effect/commit/95752db5eb3b6e59fb8749e9792704f8b167f36c), [`4577a28`](https://github.com/Effect-TS/effect/commit/4577a28bcdb50b0d184775f8c1b7ad890d86e255), [`973d599`](https://github.com/Effect-TS/effect/commit/973d599a3e8611626764a0e3e8f79edbcf30dc7c), [`2f066a5`](https://github.com/Effect-TS/effect/commit/2f066a58e90316dd88a219fcea94200beda54219), [`f0ccbb6`](https://github.com/Effect-TS/effect/commit/f0ccbb6917be5c08364b0b8adbd0c4c19e7f2de1), [`2dda6ce`](https://github.com/Effect-TS/effect/commit/2dda6cefc641dbea2ea3279a9b96c0332b277a40), [`3da7c0a`](https://github.com/Effect-TS/effect/commit/3da7c0aabab07c8fc682aea7559536bf31705853), [`5a67e0b`](https://github.com/Effect-TS/effect/commit/5a67e0b3a8118dd7be709a56d6d7af4f10e9c4f3), [`73b3a6e`](https://github.com/Effect-TS/effect/commit/73b3a6e3c38795400251d9c5d73f05c88acd90cd), [`c6f9e4c`](https://github.com/Effect-TS/effect/commit/c6f9e4cbe6f9c8f0a03fefde18df4d35e98a24df), [`be02676`](https://github.com/Effect-TS/effect/commit/be026768ee29a1b1a860c1fd17f2a0896c01f61e), [`26de861`](https://github.com/Effect-TS/effect/commit/26de86100cda8617e077ce66f0a6bb8dfe738fb6), [`9313799`](https://github.com/Effect-TS/effect/commit/931379911a74d5e2207859c295b1acc6a5e2917e), [`d5c1e5b`](https://github.com/Effect-TS/effect/commit/d5c1e5bd425dfe983dff1f8c853c0b7eaa772f87), [`11bb31f`](https://github.com/Effect-TS/effect/commit/11bb31fec0bf2fdfc025009a04de4c69bffa433a), [`8c23e46`](https://github.com/Effect-TS/effect/commit/8c23e46a8280e3f2c831d1c0ca3f106c92f8e071), [`9c96011`](https://github.com/Effect-TS/effect/commit/9c960111de202c0f1dd6989e5b8836150e0ca694), [`0d50570`](https://github.com/Effect-TS/effect/commit/0d50570bbf078804092fcd4a47ccd2d0019f989c), [`9ed2bf1`](https://github.com/Effect-TS/effect/commit/9ed2bf18da0e1ff4fecc31b7cbac2976ec999e00), [`76ec912`](https://github.com/Effect-TS/effect/commit/76ec9120f43f4fdb08b9c5170ff9e3fd82f22457), [`af2756d`](https://github.com/Effect-TS/effect/commit/af2756dd1d196aa3f199ce21c6d2dc3ada3b67b5), [`3ec8100`](https://github.com/Effect-TS/effect/commit/3ec8100c793b8a22381ad115106b93824d840f64), [`93abf02`](https://github.com/Effect-TS/effect/commit/93abf020876af0a6b6b8b25621c80760b7996762), [`1e94993`](https://github.com/Effect-TS/effect/commit/1e949935a1065e5140245a514ce3a799061143e3), [`99a4616`](https://github.com/Effect-TS/effect/commit/99a4616264e31de65fa8d3dc19576309b1a3f84a), [`0a7978e`](https://github.com/Effect-TS/effect/commit/0a7978ea8e70e51700345b12f8c19952dde05255), [`b4a060b`](https://github.com/Effect-TS/effect/commit/b4a060b95a6473d54e277dc005172b54aab9bddf), [`274f4a4`](https://github.com/Effect-TS/effect/commit/274f4a4dac5ee2d9dbb6358cc3dd3afe345d4705), [`25f29be`](https://github.com/Effect-TS/effect/commit/25f29be77bdcd5c2a7be8e2710e88f5a10c4cb8d), [`9036440`](https://github.com/Effect-TS/effect/commit/9036440edca01c24371d0c2c1044be5e5d3cb65e), [`415f005`](https://github.com/Effect-TS/effect/commit/415f0053eb848d562febcb5b011e15f31a99bead), [`191308b`](https://github.com/Effect-TS/effect/commit/191308bd0f5ee04557570f2f960c6e06c429bcfd), [`4b96570`](https://github.com/Effect-TS/effect/commit/4b96570ed28d93b14e958742933d227dc5e74b15), [`d84ec3f`](https://github.com/Effect-TS/effect/commit/d84ec3fab0346c01bc59f6624d1c046ca4140087), [`392aa1c`](https://github.com/Effect-TS/effect/commit/392aa1cb275ac9fdb9b4b1831d8c06196bba54b9), [`99e9bd9`](https://github.com/Effect-TS/effect/commit/99e9bd98a354d83202e2522f3ade9ee4bf4a862c), [`6f6f45f`](https://github.com/Effect-TS/effect/commit/6f6f45f480a3f191aed6a82ca5c301f60ef2cf84), [`13fc07e`](https://github.com/Effect-TS/effect/commit/13fc07edeb939ca6229b858c9d9a8c5243ac21b5), [`a4b38dd`](https://github.com/Effect-TS/effect/commit/a4b38dde7f3fdc1b54a49bffdaca912b4c47c76e), [`8b18580`](https://github.com/Effect-TS/effect/commit/8b185807bb99f6de3d9564447e2f4e08d2f8863d), [`0c477fe`](https://github.com/Effect-TS/effect/commit/0c477fe417d3ecab86bb8d9afd0c392e9cb8a3da), [`9a5331d`](https://github.com/Effect-TS/effect/commit/9a5331db94bfdca4f48317d062fbeb96054a9049), [`a706d73`](https://github.com/Effect-TS/effect/commit/a706d73db8a8d460877c5d7e8a793a52d45be307), [`c9a1e72`](https://github.com/Effect-TS/effect/commit/c9a1e72c5965aa9fe01a669ebfd523f803f5a189), [`89eaaf8`](https://github.com/Effect-TS/effect/commit/89eaaf8581908e248c0b1e47baa7906e0c541635), [`8215811`](https://github.com/Effect-TS/effect/commit/8215811d51d28d37b4732a6ab03de4047ada62ea), [`ad69018`](https://github.com/Effect-TS/effect/commit/ad6901857e05e7b436a0149b5ece7f3a36d9ff33), [`a733eba`](https://github.com/Effect-TS/effect/commit/a733eba9f699737a80a9e7276521980bd929075a), [`7282d5f`](https://github.com/Effect-TS/effect/commit/7282d5f1bcc06d7065429ef81a3d17c64043fde4), [`da33572`](https://github.com/Effect-TS/effect/commit/da33572776763e45802ce8c2fe14ec101f9a5c34), [`d9e2d12`](https://github.com/Effect-TS/effect/commit/d9e2d1221768e9a3e5e00165cb95b86e0c126cc0), [`dc923b8`](https://github.com/Effect-TS/effect/commit/dc923b847ac7d9673b4f0aa7c136557287b2e5e3), [`1b74086`](https://github.com/Effect-TS/effect/commit/1b740862e3d25ec5d4b6f86b9fe792b6dabbf84c), [`8a5507b`](https://github.com/Effect-TS/effect/commit/8a5507ba00f641edf71227d79b9f90efdb634fe2), [`23460fa`](https://github.com/Effect-TS/effect/commit/23460fad704c730926bffe3cc6874268ef6680ec), [`0dbbea6`](https://github.com/Effect-TS/effect/commit/0dbbea65a93c59147a4bc407cfcda38ea26341b2), [`70aad3f`](https://github.com/Effect-TS/effect/commit/70aad3ffe617984f5b599aeed1bc7f04899aca55), [`0909236`](https://github.com/Effect-TS/effect/commit/090923611b209a19f3281a7b1d0c15a9b653a0c7), [`a9e66d8`](https://github.com/Effect-TS/effect/commit/a9e66d889017b302355ba5b8bc2b2050ca1ea5aa), [`ee5c0d5`](https://github.com/Effect-TS/effect/commit/ee5c0d5f43348747b04b5f7bf1b39355614c2f2c), [`631c5d6`](https://github.com/Effect-TS/effect/commit/631c5d6d9a81e6389964b878764fb9aea1538247), [`7e60c1a`](https://github.com/Effect-TS/effect/commit/7e60c1ae62a96c768f9c67d706f494213947f22f), [`4cddd3c`](https://github.com/Effect-TS/effect/commit/4cddd3c41e71fc4990d37c42b1bcf2ce3780b41d), [`8727e5d`](https://github.com/Effect-TS/effect/commit/8727e5d2bec8e2d729b7aeafb725a4ab21d533d2), [`2bc97fa`](https://github.com/Effect-TS/effect/commit/2bc97faa0e06b3973780dae26029e2c59f5dba38), [`f9a7f05`](https://github.com/Effect-TS/effect/commit/f9a7f056ba31aef435933acf082ec8c01ac7decc), [`d978cca`](https://github.com/Effect-TS/effect/commit/d978cca02914bec0ddf66bb2d68d90b220096fb6), [`b334ccf`](https://github.com/Effect-TS/effect/commit/b334ccf361a0f11362e7aabdbf70b35f53a39d61), [`0b50cfc`](https://github.com/Effect-TS/effect/commit/0b50cfc644313afe4c318073893231ec1946eb5f), [`5b7d2b5`](https://github.com/Effect-TS/effect/commit/5b7d2b5c90bc77eb9f8be1b91166c53f89aa1d0c), [`6ee26b1`](https://github.com/Effect-TS/effect/commit/6ee26b16b51b380c41dbfa84d612aa525b420fce), [`eab7e6a`](https://github.com/Effect-TS/effect/commit/eab7e6acfc7e422857565b98b9eddd3a98de1b92), [`e618f10`](https://github.com/Effect-TS/effect/commit/e618f1050f41424439eb82478fcf1c5da5a3244d), [`da14b1f`](https://github.com/Effect-TS/effect/commit/da14b1f18d71a18fc431b56d82cffe09aeb92708), [`3314fc3`](https://github.com/Effect-TS/effect/commit/3314fc30c22d5c9d737f3df7201f4a41424f701b), [`66db624`](https://github.com/Effect-TS/effect/commit/66db624670032a02144d42b6e64b83b324c813cd), [`4b8a50d`](https://github.com/Effect-TS/effect/commit/4b8a50db9649603f382f33640698020ca79547d0), [`8d52668`](https://github.com/Effect-TS/effect/commit/8d52668533e9d4a6d6b3be9fe58bd5e69fa80b65), [`1de1055`](https://github.com/Effect-TS/effect/commit/1de10559e7263747751bfb032fd4b0fd8aa28724), [`db130e6`](https://github.com/Effect-TS/effect/commit/db130e64557fafe370819d19f78e3b19a21275d6), [`5fa5fa5`](https://github.com/Effect-TS/effect/commit/5fa5fa5b9bff77352fb9ff2d424bf46bd6309986)]: + - effect@4.0.6 + ## 4.0.5 ### Patch Changes diff --git a/repos/effect/packages/atom/react/package.json b/repos/effect/packages/atom/react/package.json index 543733154d..1995def592 100644 --- a/repos/effect/packages/atom/react/package.json +++ b/repos/effect/packages/atom/react/package.json @@ -1,6 +1,6 @@ { "name": "@effect/atom-react", - "version": "4.0.5", + "version": "4.0.6", "type": "module", "license": "MIT", "description": "React bindings for the Effect Atom modules", @@ -73,8 +73,8 @@ "@types/scheduler": "^0.28.0", "effect": "workspace:^", "jsdom": "~30.0.1", - "react": "^19.3.0", - "react-dom": "^19.3.0", + "react": "^19.3.1", + "react-dom": "^19.3.1", "react-error-boundary": "^6.1.6", "scheduler": "^0.28.0" } diff --git a/repos/effect/packages/atom/react/src/Hooks.ts b/repos/effect/packages/atom/react/src/Hooks.ts index 87b753565e..1087081a0b 100644 --- a/repos/effect/packages/atom/react/src/Hooks.ts +++ b/repos/effect/packages/atom/react/src/Hooks.ts @@ -244,6 +244,13 @@ export const useAtomMount = (atom: Atom.Atom): void => { * accepts a write value or updater function; for `AsyncResult` atoms, `promise` * and `promiseExit` modes return a promise for the success value or full `Exit`. * + * **Gotchas** + * + * In `promise` and `promiseExit` modes, the setter observes the atom's next + * settled result, not a result correlated with that write. Overlapping writes + * can resolve with another write's result. Serialize writes when each caller + * needs its own result. + * * @see {@link useAtom} for reading and updating the same writable atom * * @stability unstable @@ -309,6 +316,13 @@ export const useAtomRefresh = (atom: Atom.Atom): () => void => { * Use when a React component needs both to render the current value of a * writable atom and update it from the same component. * + * **Gotchas** + * + * In `promise` and `promiseExit` modes, the setter observes the atom's next + * settled result, not a result correlated with that write. Overlapping writes + * can resolve with another write's result. Serialize writes when each caller + * needs its own result. + * * @see {@link useAtomValue} for subscribing to an atom without a setter * @see {@link useAtomSet} for updating a writable atom without subscribing to its value * diff --git a/repos/effect/packages/atom/react/src/RegistryContext.ts b/repos/effect/packages/atom/react/src/RegistryContext.ts index 3509dccffe..d6aab5ae08 100644 --- a/repos/effect/packages/atom/react/src/RegistryContext.ts +++ b/repos/effect/packages/atom/react/src/RegistryContext.ts @@ -78,7 +78,7 @@ export const RegistryContext = React.createContext(At */ export const RegistryProvider = (options: { readonly children?: React.ReactNode | undefined - readonly initialValues?: Iterable, any]> | undefined + readonly initialValues?: Iterable, any, Atom.InitialValueOptions?]> | undefined readonly scheduleTask?: ((f: () => void) => () => void) | undefined readonly timeoutResolution?: number | undefined readonly defaultIdleTTL?: number | undefined diff --git a/repos/effect/packages/atom/react/test/index.test.tsx b/repos/effect/packages/atom/react/test/index.test.tsx index aae21c6139..bf5ef24e75 100644 --- a/repos/effect/packages/atom/react/test/index.test.tsx +++ b/repos/effect/packages/atom/react/test/index.test.tsx @@ -1,4 +1,5 @@ // +import { assert } from "@effect/vitest" import { act, render, screen, waitFor } from "@testing-library/react" import { Cause, Context, Effect, Latch, Layer } from "effect" import * as AsyncResult from "effect/reactivity/AsyncResult" @@ -482,6 +483,9 @@ describe("atom-react", { concurrent: false }, () => { ) expect(screen.getByTestId("value")).toHaveTextContent("Initial") + // Hydration starts a client effect while the server effect is still pending. + expect(start).toBe(2) + expect(stop).toBe(0) act(() => { Effect.runSync(latch.open) @@ -495,8 +499,44 @@ describe("atom-react", { concurrent: false }, () => { } expect(screen.getByTestId("value")).toHaveTextContent("Success") - expect(start).toBe(1) - expect(stop).toBe(1) + expect(start).toBe(2) + expect(stop).toBe(2) + }) + + test("HydrationBoundary updates existing readers after commit", () => { + const count = Atom.make(0).pipe( + Atom.serializable({ key: "existing-count", schema: Schema.Number }) + ) + const server = AtomRegistry.make() + try { + server.set(count, 5) + const state = Hydration.dehydrate(server) + + function Count({ id }: { id: string }) { + return
{useAtomValue(count)}
+ } + + const { rerender } = render( + + + + ) + assert.strictEqual(screen.getByTestId("header").textContent, "0") + + rerender( + + + + + + + ) + + assert.strictEqual(screen.getByTestId("header").textContent, "5") + assert.strictEqual(screen.getByTestId("page").textContent, "5") + } finally { + server.dispose() + } }) test("HydrationBoundary splits new vs existing atoms", () => { diff --git a/repos/effect/packages/atom/solid/CHANGELOG.md b/repos/effect/packages/atom/solid/CHANGELOG.md index 3b20ba862a..43f5b2ebf9 100644 --- a/repos/effect/packages/atom/solid/CHANGELOG.md +++ b/repos/effect/packages/atom/solid/CHANGELOG.md @@ -1,5 +1,13 @@ # @effect/atom-solid +## 4.0.6 + +### Patch Changes + +- [#9522](https://github.com/Effect-TS/effect/pull/9522) [`f8bf7f4`](https://github.com/Effect-TS/effect/commit/f8bf7f4bcc3ad543cd94cbeb010cb164e5dedcb0) Thanks @tim-smart! - Add a `skipInitialRun` option to `Atom.serializable` and `Atom.initialValue`. A seeded effect or stream atom then keeps its hydrated or initial value instead of running on mount, until it is refreshed or a dependency changes. Factory `get(...)` calls and `withReactivity` keys are still tracked; dependencies read inside the effect are tracked from its first real run. +- Updated dependencies [[`ce13a48`](https://github.com/Effect-TS/effect/commit/ce13a4872e516187851c5d9336d4df309cc8f0a4), [`f3bbbb4`](https://github.com/Effect-TS/effect/commit/f3bbbb448d431a9d1205ff09ba47cb81af0c7199), [`5547315`](https://github.com/Effect-TS/effect/commit/554731500e43eef47ff68b569116b74006e91d87), [`6093cfb`](https://github.com/Effect-TS/effect/commit/6093cfbbb673d36ecd18edea955763db003c8c26), [`30319d5`](https://github.com/Effect-TS/effect/commit/30319d506c9177d1cc1e0b10372e1ddb776aaf0a), [`74a7596`](https://github.com/Effect-TS/effect/commit/74a759699a8582dbe660b21dcda21cc9db08f613), [`bfb32a3`](https://github.com/Effect-TS/effect/commit/bfb32a3cf5f8d0f02f0dfdb63dcf5237fde755b0), [`f8bf7f4`](https://github.com/Effect-TS/effect/commit/f8bf7f4bcc3ad543cd94cbeb010cb164e5dedcb0), [`7f3d3df`](https://github.com/Effect-TS/effect/commit/7f3d3dfbb7b9e6e041c4ad9654c4532f72d2f16e), [`db1046d`](https://github.com/Effect-TS/effect/commit/db1046d1fa0e4b5818c51d28017a89579b6261d6), [`3c4f88a`](https://github.com/Effect-TS/effect/commit/3c4f88aa4d1d4d958b42fe8d3f1195f0ca990f8a), [`916e709`](https://github.com/Effect-TS/effect/commit/916e70968acc4e443885c67012d855b7cc6f9bca), [`916f843`](https://github.com/Effect-TS/effect/commit/916f8436225153a81a049bdc1feaedcafe4d1ded), [`66ec757`](https://github.com/Effect-TS/effect/commit/66ec7574d23b7943584059b41136bc21ee9d4c53), [`daac06f`](https://github.com/Effect-TS/effect/commit/daac06f79a6ab8c7d3e1b66b427c06f7d64dcc8f), [`c2552ec`](https://github.com/Effect-TS/effect/commit/c2552ec2e031b409b2bef67309312b0168480a39), [`b02c1cd`](https://github.com/Effect-TS/effect/commit/b02c1cde36c890941a235224af8ecb035197582a), [`1855f1d`](https://github.com/Effect-TS/effect/commit/1855f1d359535339bc7640b67e6bb7b12752ba19), [`683ed2c`](https://github.com/Effect-TS/effect/commit/683ed2cb491f9529379c1d5c63933689ff57fd2b), [`ac68940`](https://github.com/Effect-TS/effect/commit/ac68940f7e4246b1beb43f9d451a67f41241ab82), [`22a8401`](https://github.com/Effect-TS/effect/commit/22a84017e7222f332611c1bace05f9fa60c7cf67), [`c53c194`](https://github.com/Effect-TS/effect/commit/c53c19451171a00776b400a8548394ac514b628a), [`ca6d1cc`](https://github.com/Effect-TS/effect/commit/ca6d1cc5f803cd501e85431642b79b552322a76a), [`47940c5`](https://github.com/Effect-TS/effect/commit/47940c55c9e0eb855615094a2aca6e0812307bb2), [`e8b24ef`](https://github.com/Effect-TS/effect/commit/e8b24ef9516706971db948efb2d3eab84ad85951), [`2e8aae0`](https://github.com/Effect-TS/effect/commit/2e8aae0080a8d3e6d00c5ee0bc4f49f0b074acc2), [`ad7497b`](https://github.com/Effect-TS/effect/commit/ad7497bbc1bc70935f8f1ae395ff501532e2e770), [`7f504a3`](https://github.com/Effect-TS/effect/commit/7f504a35edcf44952c55ecbb0916726bac4d3e52), [`99e365d`](https://github.com/Effect-TS/effect/commit/99e365d61a1fdaabeb31dc58e5752cade8a09e1f), [`954067d`](https://github.com/Effect-TS/effect/commit/954067da7fbd20a20c10d5d0de6ffe4690e6bdfd), [`3e1a035`](https://github.com/Effect-TS/effect/commit/3e1a035d9c470ced858b7757f6d102b3aa9be008), [`41a1fe7`](https://github.com/Effect-TS/effect/commit/41a1fe74e6338e8d13d6d4f3daa03f71a826c2c4), [`bcb3a86`](https://github.com/Effect-TS/effect/commit/bcb3a86335a8ed18da39300984c0fbae187fdc6b), [`739d3d1`](https://github.com/Effect-TS/effect/commit/739d3d18728cce0b6d907f0a3f24e04ab0b53560), [`ed4da35`](https://github.com/Effect-TS/effect/commit/ed4da351fb9d29e4dfd6c60bd89ba4c007d264dc), [`3d84e19`](https://github.com/Effect-TS/effect/commit/3d84e19f8f14bccf0aa57b11825590c3bae162f4), [`cc00906`](https://github.com/Effect-TS/effect/commit/cc0090657eeb89debdd54af5fcb99d544f46c1bf), [`09ba1f5`](https://github.com/Effect-TS/effect/commit/09ba1f59952c46cd22229d41236742c3e1f53a34), [`9ff1a73`](https://github.com/Effect-TS/effect/commit/9ff1a73d995702f4c4c335888fe6a186f3f710c7), [`06bc532`](https://github.com/Effect-TS/effect/commit/06bc532b71cd3684f010956774356864dd359372), [`97bfd84`](https://github.com/Effect-TS/effect/commit/97bfd84bb147e10ed562e4c1ffa589cdaf83a596), [`25efaef`](https://github.com/Effect-TS/effect/commit/25efaef0d81d25b2651ef4de474e929d3180810a), [`c10335b`](https://github.com/Effect-TS/effect/commit/c10335be125c7846ae7b914eeb138244b660275b), [`a808972`](https://github.com/Effect-TS/effect/commit/a80897262648e4222ce48943e538f39a6b3f69d6), [`20e844c`](https://github.com/Effect-TS/effect/commit/20e844c7ef27b0e4b4f7911d603c45b1de2b0bc3), [`b9f73f3`](https://github.com/Effect-TS/effect/commit/b9f73f37c4ac538e6a327ed80ac5a0708858c540), [`5739878`](https://github.com/Effect-TS/effect/commit/5739878ded09d83d71d657e3b8d4264f4e4936d9), [`28acdf3`](https://github.com/Effect-TS/effect/commit/28acdf3820b66056037bb7a24cd0e8d3de3fb06d), [`59b6396`](https://github.com/Effect-TS/effect/commit/59b63967c1817ae5acb1811037f5aaa964219b7e), [`0bccc9c`](https://github.com/Effect-TS/effect/commit/0bccc9c3dd5349b20f8d2e509ac5482d0c26a616), [`fed70de`](https://github.com/Effect-TS/effect/commit/fed70de0dff8af24c5f814bd0d985ab70c70420b), [`82648a2`](https://github.com/Effect-TS/effect/commit/82648a21bf228145a14d2d1390006d67f74d89c7), [`afb5f32`](https://github.com/Effect-TS/effect/commit/afb5f326cc14c9849d8d7ddefde1039ae036e017), [`28813f5`](https://github.com/Effect-TS/effect/commit/28813f53a5026984fdc13d3971147ea5dc20104c), [`0fae934`](https://github.com/Effect-TS/effect/commit/0fae934a13aac12f9036b0733edd5dde5ce029f3), [`f260830`](https://github.com/Effect-TS/effect/commit/f2608305861269fc2fbedcbbdcd71e4fc5f98ffc), [`19b63eb`](https://github.com/Effect-TS/effect/commit/19b63eb81b2bccf033aadb565f52a6b0375d26a1), [`14db221`](https://github.com/Effect-TS/effect/commit/14db221f8f584100ff50fd1173b1a99a1caa1d06), [`679f956`](https://github.com/Effect-TS/effect/commit/679f956c4a7527854ff78db071dd7aa417211a48), [`b72707c`](https://github.com/Effect-TS/effect/commit/b72707c2937328fd3bc70f29a6792d84c4256484), [`44c8c1d`](https://github.com/Effect-TS/effect/commit/44c8c1d1c6f01ba9887a051b365e9fa89fcfc685), [`1e7909b`](https://github.com/Effect-TS/effect/commit/1e7909beb05bc740dbd330509e60bb185d7eeed1), [`0611a65`](https://github.com/Effect-TS/effect/commit/0611a657f76836f9ca275f64764b82270b4eda10), [`2fadc65`](https://github.com/Effect-TS/effect/commit/2fadc654aa655fbd1912ce51eb1b764a757ac362), [`feb6ce5`](https://github.com/Effect-TS/effect/commit/feb6ce59a38ab15fc6f89545bd5f0992dd58a06a), [`5a2ed53`](https://github.com/Effect-TS/effect/commit/5a2ed5323884a435bfcfb5e3a6f2ae29547458ad), [`c8d26a4`](https://github.com/Effect-TS/effect/commit/c8d26a46731cb26a9b9612d9fde823d8813cfc04), [`0c8b446`](https://github.com/Effect-TS/effect/commit/0c8b446709c737c10d0e11af4313bb29fdff2588), [`05c3409`](https://github.com/Effect-TS/effect/commit/05c340955944374e3ae47a1de0f7cb9b41a3b0e6), [`b04191b`](https://github.com/Effect-TS/effect/commit/b04191b2941534754282c51ef2df2f965ac43a5c), [`05c3409`](https://github.com/Effect-TS/effect/commit/05c340955944374e3ae47a1de0f7cb9b41a3b0e6), [`4067e64`](https://github.com/Effect-TS/effect/commit/4067e6428776ae573c8cc72f7eaebd078a4ff8c7), [`e04563d`](https://github.com/Effect-TS/effect/commit/e04563dcd6ddeea74c088a12194c5a73e597ba2f), [`02d7fc5`](https://github.com/Effect-TS/effect/commit/02d7fc583053c47dea3b5e1d9d6be164aeb321b6), [`561d2a1`](https://github.com/Effect-TS/effect/commit/561d2a1ddfa3c4340510f23da7adc50c89d97863), [`8fe2dcc`](https://github.com/Effect-TS/effect/commit/8fe2dcc2137788a71177d42a9f67fb7003224868), [`a8086fa`](https://github.com/Effect-TS/effect/commit/a8086fa26a2c9a9f5f7f1d7590bbc494a0feb02a), [`88a33b4`](https://github.com/Effect-TS/effect/commit/88a33b4122f7ffbbc1423add793a79abffd99c75), [`e77372a`](https://github.com/Effect-TS/effect/commit/e77372a8a2e7caec11999636e04c6e31c52e315f), [`3dd4388`](https://github.com/Effect-TS/effect/commit/3dd438810df34ed3b3d2e3e35ccf84ccb00feb85), [`ba2b00c`](https://github.com/Effect-TS/effect/commit/ba2b00c0658b65775809e0052ec9612432600c31), [`a049d4a`](https://github.com/Effect-TS/effect/commit/a049d4ad25f17cf4985284a8d15b07aea2a33242), [`84aeede`](https://github.com/Effect-TS/effect/commit/84aeede040460a2b24bcf0db7a5d0a9b0e3c42d4), [`43e5e19`](https://github.com/Effect-TS/effect/commit/43e5e198d16df038b60764f5dd0218cdcf828f16), [`77bbf16`](https://github.com/Effect-TS/effect/commit/77bbf16e50d6b0ea090288cc5d3f11b20706a027), [`6af3a47`](https://github.com/Effect-TS/effect/commit/6af3a47d5b5d698c3ba896c37684266cd465bef1), [`aac8d21`](https://github.com/Effect-TS/effect/commit/aac8d21911333dedb649f4f9d302c28d14d46369), [`84c8dc4`](https://github.com/Effect-TS/effect/commit/84c8dc47b09067362749620f75068d6f28044660), [`78d1988`](https://github.com/Effect-TS/effect/commit/78d198818d4fd547ce2e9431302f66cb00dc7ad3), [`5676a2a`](https://github.com/Effect-TS/effect/commit/5676a2ac5a918f8f91fc7838d7adf1b6f5e86a6d), [`4b437bf`](https://github.com/Effect-TS/effect/commit/4b437bf4645fabc8aaeaa3b5a88e8c9048b9e4f5), [`1af11fd`](https://github.com/Effect-TS/effect/commit/1af11fda1bccf823f74121b5125aa3befa084b18), [`716f8df`](https://github.com/Effect-TS/effect/commit/716f8dfa178f849e5b288464373bd132860c2e19), [`ee27718`](https://github.com/Effect-TS/effect/commit/ee27718a02125328a7478812b1b7d7c1d8532578), [`4d86d1b`](https://github.com/Effect-TS/effect/commit/4d86d1b8887d48fa24acf6ad54f33ef3d55a7cd8), [`5462ed7`](https://github.com/Effect-TS/effect/commit/5462ed7b3a4c37206c47e401d6be293dc540350f), [`658837c`](https://github.com/Effect-TS/effect/commit/658837c3b249d982df56f2990247499746e36672), [`00261cc`](https://github.com/Effect-TS/effect/commit/00261cc12c31ecc7878f80670439731c905dc2bd), [`7cccb93`](https://github.com/Effect-TS/effect/commit/7cccb9385729bc0aed6630edea95d7db15439ac4), [`7c98228`](https://github.com/Effect-TS/effect/commit/7c982281cf73958833054175fc86db385e043cf6), [`5a8e248`](https://github.com/Effect-TS/effect/commit/5a8e2481546352e2ad6233dad49420b14c59f624), [`8037a65`](https://github.com/Effect-TS/effect/commit/8037a65f40cae9163ab9e0e687e289f717ae570b), [`25912db`](https://github.com/Effect-TS/effect/commit/25912db9d493227a2d6fa45cebcfc7507e638d8a), [`7f32606`](https://github.com/Effect-TS/effect/commit/7f32606baac43a67d171ae93b77c7b29791c9692), [`0386f88`](https://github.com/Effect-TS/effect/commit/0386f88295ac70cbc1f90ab8cceb0981d848c628), [`403298a`](https://github.com/Effect-TS/effect/commit/403298ac457fb634eca2c013fed26d37fc80445f), [`5a3ebc0`](https://github.com/Effect-TS/effect/commit/5a3ebc082124f9e10106b2347736e66fc74ebc36), [`fac2191`](https://github.com/Effect-TS/effect/commit/fac21911964b742a38e1d4e73bd4f95aec4911dc), [`fdb12dc`](https://github.com/Effect-TS/effect/commit/fdb12dcd61b6505eed02e6f7a3c88a85d4c48e8d), [`97909b8`](https://github.com/Effect-TS/effect/commit/97909b89b167da54a56e6f82f1c4db8bfedae04f), [`c3983f4`](https://github.com/Effect-TS/effect/commit/c3983f4db2b3052a58e8045cce270fd7ccc4e2b2), [`3a5e65e`](https://github.com/Effect-TS/effect/commit/3a5e65e409c9569e23bb7af61df401347e9ec420), [`8b40fbc`](https://github.com/Effect-TS/effect/commit/8b40fbc93caf52f3a4c69020849f0051f100be65), [`3f3e119`](https://github.com/Effect-TS/effect/commit/3f3e119ad61c2167bc6c7170c3fb68cd1eb3421e), [`7c7b8ed`](https://github.com/Effect-TS/effect/commit/7c7b8eda54e44f9cddc1188f994f7fdddf901055), [`d4f71bc`](https://github.com/Effect-TS/effect/commit/d4f71bc42507b638d149fb8806d26ea626d17956), [`c66241e`](https://github.com/Effect-TS/effect/commit/c66241e791c4b51d8a65051142d01682590f9bf8), [`f0e941e`](https://github.com/Effect-TS/effect/commit/f0e941ee2f90f0a891d1228a90da91386f0258cb), [`5f8a6e5`](https://github.com/Effect-TS/effect/commit/5f8a6e5108aa6c6cd4043a3c9f4dc93d43369623), [`2800a09`](https://github.com/Effect-TS/effect/commit/2800a0910f7f4ad7cfa21dfc840edf8fe6e67ca7), [`e9749b0`](https://github.com/Effect-TS/effect/commit/e9749b04c10badabcdf71e6958de5ac56a5316e9), [`eaef09c`](https://github.com/Effect-TS/effect/commit/eaef09c11b6ed2029c513eea0aa3256dcc6f517f), [`a3c56e5`](https://github.com/Effect-TS/effect/commit/a3c56e5b5e756e879f1cfb90230e8eae620e33df), [`bb61152`](https://github.com/Effect-TS/effect/commit/bb611522fcfb4d317b33acdf5ee24a480d042435), [`e9c842b`](https://github.com/Effect-TS/effect/commit/e9c842bc294abf658d698bb17732ae13465b3817), [`6d8c8e2`](https://github.com/Effect-TS/effect/commit/6d8c8e2075b15ae84ae96e93a0d295cb9221b496), [`1f28187`](https://github.com/Effect-TS/effect/commit/1f281878ef559ef356af2d0e11b381aba857cae1), [`0b9842b`](https://github.com/Effect-TS/effect/commit/0b9842b76bbc1b016c8abc46432052e1f60b2fa3), [`c29ddd8`](https://github.com/Effect-TS/effect/commit/c29ddd899c53c5d8194c4a0dfef2134304dda799), [`84c3405`](https://github.com/Effect-TS/effect/commit/84c34057ceab9fe6782b73f4a04a31304e0d14de), [`3934c0c`](https://github.com/Effect-TS/effect/commit/3934c0c61857c03468c08c2477f49a6b977d28ff), [`048e7c8`](https://github.com/Effect-TS/effect/commit/048e7c87125c4ace5a61895feac3d049a317bbfc), [`1121334`](https://github.com/Effect-TS/effect/commit/1121334ce68c379051011a2ffea37b0a30ea9a47), [`9a680d6`](https://github.com/Effect-TS/effect/commit/9a680d6985c6a27d7ebb29cd9eb674f3fe524505), [`85e2aee`](https://github.com/Effect-TS/effect/commit/85e2aeee686963c26ca40d204f982deebb9d6117), [`34ec440`](https://github.com/Effect-TS/effect/commit/34ec4407a14ce045ca14cc76c77582884a7e1ebe), [`343a149`](https://github.com/Effect-TS/effect/commit/343a149fa7d5a9a0a45841b1b6800b0ecfa16a32), [`915117b`](https://github.com/Effect-TS/effect/commit/915117b523cb6e5cbc98b5ff1c119a8d0d1f6b9a), [`7f823c1`](https://github.com/Effect-TS/effect/commit/7f823c10d20ef53a1081e0d0e2008a4116140799), [`6e614e6`](https://github.com/Effect-TS/effect/commit/6e614e6feb4281a9e81f5ec3be48e7905ff4292c), [`9990b39`](https://github.com/Effect-TS/effect/commit/9990b3977a544247cd4123edde8353a55bf77192), [`22bf781`](https://github.com/Effect-TS/effect/commit/22bf7812aa8a20c676efade943c342cde4ef1cbd), [`41f08b7`](https://github.com/Effect-TS/effect/commit/41f08b75aba40e95cd6c743ec6fcf1f5022f7b5b), [`e8482ca`](https://github.com/Effect-TS/effect/commit/e8482cad9651fa8fe8036fe6292a286994897d0d), [`5ae85fa`](https://github.com/Effect-TS/effect/commit/5ae85fa8ac1fc5e12d5fe8fe7388ac504788ae3b), [`c477a0a`](https://github.com/Effect-TS/effect/commit/c477a0a8010bd0a988221bc00c446848314e1c3c), [`8fbb39e`](https://github.com/Effect-TS/effect/commit/8fbb39eb92577915831f19a83b51be376e844613), [`1c70828`](https://github.com/Effect-TS/effect/commit/1c7082847453e9c2be0cc267a39ca554c52f4013), [`8207d1e`](https://github.com/Effect-TS/effect/commit/8207d1e91ead8aac8f01d9e13b49624b64237c8b), [`7886b84`](https://github.com/Effect-TS/effect/commit/7886b847658664f64392a14efda86f44b3d1e170), [`4839c60`](https://github.com/Effect-TS/effect/commit/4839c601bdda845d9a54d77f578f64372475ab04), [`2595d83`](https://github.com/Effect-TS/effect/commit/2595d831f702d4a748f97e33e71fd5250f58af56), [`f5ae278`](https://github.com/Effect-TS/effect/commit/f5ae27827a82db3f8973b70745fc6f14e7d8fa80), [`7a8ce00`](https://github.com/Effect-TS/effect/commit/7a8ce0098a5f22cbb6c110b16e850fb3fcbcadf2), [`688d302`](https://github.com/Effect-TS/effect/commit/688d30275ff114e50b4f0d343c6227ee33d67348), [`c7072e4`](https://github.com/Effect-TS/effect/commit/c7072e46356a19615625ae0fa569b1c3a79d50be), [`425cedb`](https://github.com/Effect-TS/effect/commit/425cedb5801c9527c116a64d280d9e7c2820f922), [`88bc680`](https://github.com/Effect-TS/effect/commit/88bc68001c6c70984b03a5500232e9b5aab14085), [`268e09c`](https://github.com/Effect-TS/effect/commit/268e09c94c9e8f46142b5c9f29ce0a22bfbbc3aa), [`7e94505`](https://github.com/Effect-TS/effect/commit/7e94505edfc9d81ea08cb96a7d935806495bf61e), [`afa5da3`](https://github.com/Effect-TS/effect/commit/afa5da3c8a9848215e49535dd3d39179bfd67715), [`c29ddd8`](https://github.com/Effect-TS/effect/commit/c29ddd899c53c5d8194c4a0dfef2134304dda799), [`48102ac`](https://github.com/Effect-TS/effect/commit/48102ac592ef89fed9d5d10afd2daece0a0ee310), [`463eca3`](https://github.com/Effect-TS/effect/commit/463eca3609a6a9681247abbcc71a6f10b261a1d9), [`e7162ea`](https://github.com/Effect-TS/effect/commit/e7162eae7f04af1d583b1f31fdd767f91ec1c82e), [`95752db`](https://github.com/Effect-TS/effect/commit/95752db5eb3b6e59fb8749e9792704f8b167f36c), [`4577a28`](https://github.com/Effect-TS/effect/commit/4577a28bcdb50b0d184775f8c1b7ad890d86e255), [`973d599`](https://github.com/Effect-TS/effect/commit/973d599a3e8611626764a0e3e8f79edbcf30dc7c), [`2f066a5`](https://github.com/Effect-TS/effect/commit/2f066a58e90316dd88a219fcea94200beda54219), [`f0ccbb6`](https://github.com/Effect-TS/effect/commit/f0ccbb6917be5c08364b0b8adbd0c4c19e7f2de1), [`2dda6ce`](https://github.com/Effect-TS/effect/commit/2dda6cefc641dbea2ea3279a9b96c0332b277a40), [`3da7c0a`](https://github.com/Effect-TS/effect/commit/3da7c0aabab07c8fc682aea7559536bf31705853), [`5a67e0b`](https://github.com/Effect-TS/effect/commit/5a67e0b3a8118dd7be709a56d6d7af4f10e9c4f3), [`73b3a6e`](https://github.com/Effect-TS/effect/commit/73b3a6e3c38795400251d9c5d73f05c88acd90cd), [`c6f9e4c`](https://github.com/Effect-TS/effect/commit/c6f9e4cbe6f9c8f0a03fefde18df4d35e98a24df), [`be02676`](https://github.com/Effect-TS/effect/commit/be026768ee29a1b1a860c1fd17f2a0896c01f61e), [`26de861`](https://github.com/Effect-TS/effect/commit/26de86100cda8617e077ce66f0a6bb8dfe738fb6), [`9313799`](https://github.com/Effect-TS/effect/commit/931379911a74d5e2207859c295b1acc6a5e2917e), [`d5c1e5b`](https://github.com/Effect-TS/effect/commit/d5c1e5bd425dfe983dff1f8c853c0b7eaa772f87), [`11bb31f`](https://github.com/Effect-TS/effect/commit/11bb31fec0bf2fdfc025009a04de4c69bffa433a), [`8c23e46`](https://github.com/Effect-TS/effect/commit/8c23e46a8280e3f2c831d1c0ca3f106c92f8e071), [`9c96011`](https://github.com/Effect-TS/effect/commit/9c960111de202c0f1dd6989e5b8836150e0ca694), [`0d50570`](https://github.com/Effect-TS/effect/commit/0d50570bbf078804092fcd4a47ccd2d0019f989c), [`9ed2bf1`](https://github.com/Effect-TS/effect/commit/9ed2bf18da0e1ff4fecc31b7cbac2976ec999e00), [`76ec912`](https://github.com/Effect-TS/effect/commit/76ec9120f43f4fdb08b9c5170ff9e3fd82f22457), [`af2756d`](https://github.com/Effect-TS/effect/commit/af2756dd1d196aa3f199ce21c6d2dc3ada3b67b5), [`3ec8100`](https://github.com/Effect-TS/effect/commit/3ec8100c793b8a22381ad115106b93824d840f64), [`93abf02`](https://github.com/Effect-TS/effect/commit/93abf020876af0a6b6b8b25621c80760b7996762), [`1e94993`](https://github.com/Effect-TS/effect/commit/1e949935a1065e5140245a514ce3a799061143e3), [`99a4616`](https://github.com/Effect-TS/effect/commit/99a4616264e31de65fa8d3dc19576309b1a3f84a), [`0a7978e`](https://github.com/Effect-TS/effect/commit/0a7978ea8e70e51700345b12f8c19952dde05255), [`b4a060b`](https://github.com/Effect-TS/effect/commit/b4a060b95a6473d54e277dc005172b54aab9bddf), [`274f4a4`](https://github.com/Effect-TS/effect/commit/274f4a4dac5ee2d9dbb6358cc3dd3afe345d4705), [`25f29be`](https://github.com/Effect-TS/effect/commit/25f29be77bdcd5c2a7be8e2710e88f5a10c4cb8d), [`9036440`](https://github.com/Effect-TS/effect/commit/9036440edca01c24371d0c2c1044be5e5d3cb65e), [`415f005`](https://github.com/Effect-TS/effect/commit/415f0053eb848d562febcb5b011e15f31a99bead), [`191308b`](https://github.com/Effect-TS/effect/commit/191308bd0f5ee04557570f2f960c6e06c429bcfd), [`4b96570`](https://github.com/Effect-TS/effect/commit/4b96570ed28d93b14e958742933d227dc5e74b15), [`d84ec3f`](https://github.com/Effect-TS/effect/commit/d84ec3fab0346c01bc59f6624d1c046ca4140087), [`392aa1c`](https://github.com/Effect-TS/effect/commit/392aa1cb275ac9fdb9b4b1831d8c06196bba54b9), [`99e9bd9`](https://github.com/Effect-TS/effect/commit/99e9bd98a354d83202e2522f3ade9ee4bf4a862c), [`6f6f45f`](https://github.com/Effect-TS/effect/commit/6f6f45f480a3f191aed6a82ca5c301f60ef2cf84), [`13fc07e`](https://github.com/Effect-TS/effect/commit/13fc07edeb939ca6229b858c9d9a8c5243ac21b5), [`a4b38dd`](https://github.com/Effect-TS/effect/commit/a4b38dde7f3fdc1b54a49bffdaca912b4c47c76e), [`8b18580`](https://github.com/Effect-TS/effect/commit/8b185807bb99f6de3d9564447e2f4e08d2f8863d), [`0c477fe`](https://github.com/Effect-TS/effect/commit/0c477fe417d3ecab86bb8d9afd0c392e9cb8a3da), [`9a5331d`](https://github.com/Effect-TS/effect/commit/9a5331db94bfdca4f48317d062fbeb96054a9049), [`a706d73`](https://github.com/Effect-TS/effect/commit/a706d73db8a8d460877c5d7e8a793a52d45be307), [`c9a1e72`](https://github.com/Effect-TS/effect/commit/c9a1e72c5965aa9fe01a669ebfd523f803f5a189), [`89eaaf8`](https://github.com/Effect-TS/effect/commit/89eaaf8581908e248c0b1e47baa7906e0c541635), [`8215811`](https://github.com/Effect-TS/effect/commit/8215811d51d28d37b4732a6ab03de4047ada62ea), [`ad69018`](https://github.com/Effect-TS/effect/commit/ad6901857e05e7b436a0149b5ece7f3a36d9ff33), [`a733eba`](https://github.com/Effect-TS/effect/commit/a733eba9f699737a80a9e7276521980bd929075a), [`7282d5f`](https://github.com/Effect-TS/effect/commit/7282d5f1bcc06d7065429ef81a3d17c64043fde4), [`da33572`](https://github.com/Effect-TS/effect/commit/da33572776763e45802ce8c2fe14ec101f9a5c34), [`d9e2d12`](https://github.com/Effect-TS/effect/commit/d9e2d1221768e9a3e5e00165cb95b86e0c126cc0), [`dc923b8`](https://github.com/Effect-TS/effect/commit/dc923b847ac7d9673b4f0aa7c136557287b2e5e3), [`1b74086`](https://github.com/Effect-TS/effect/commit/1b740862e3d25ec5d4b6f86b9fe792b6dabbf84c), [`8a5507b`](https://github.com/Effect-TS/effect/commit/8a5507ba00f641edf71227d79b9f90efdb634fe2), [`23460fa`](https://github.com/Effect-TS/effect/commit/23460fad704c730926bffe3cc6874268ef6680ec), [`0dbbea6`](https://github.com/Effect-TS/effect/commit/0dbbea65a93c59147a4bc407cfcda38ea26341b2), [`70aad3f`](https://github.com/Effect-TS/effect/commit/70aad3ffe617984f5b599aeed1bc7f04899aca55), [`0909236`](https://github.com/Effect-TS/effect/commit/090923611b209a19f3281a7b1d0c15a9b653a0c7), [`a9e66d8`](https://github.com/Effect-TS/effect/commit/a9e66d889017b302355ba5b8bc2b2050ca1ea5aa), [`ee5c0d5`](https://github.com/Effect-TS/effect/commit/ee5c0d5f43348747b04b5f7bf1b39355614c2f2c), [`631c5d6`](https://github.com/Effect-TS/effect/commit/631c5d6d9a81e6389964b878764fb9aea1538247), [`7e60c1a`](https://github.com/Effect-TS/effect/commit/7e60c1ae62a96c768f9c67d706f494213947f22f), [`4cddd3c`](https://github.com/Effect-TS/effect/commit/4cddd3c41e71fc4990d37c42b1bcf2ce3780b41d), [`8727e5d`](https://github.com/Effect-TS/effect/commit/8727e5d2bec8e2d729b7aeafb725a4ab21d533d2), [`2bc97fa`](https://github.com/Effect-TS/effect/commit/2bc97faa0e06b3973780dae26029e2c59f5dba38), [`f9a7f05`](https://github.com/Effect-TS/effect/commit/f9a7f056ba31aef435933acf082ec8c01ac7decc), [`d978cca`](https://github.com/Effect-TS/effect/commit/d978cca02914bec0ddf66bb2d68d90b220096fb6), [`b334ccf`](https://github.com/Effect-TS/effect/commit/b334ccf361a0f11362e7aabdbf70b35f53a39d61), [`0b50cfc`](https://github.com/Effect-TS/effect/commit/0b50cfc644313afe4c318073893231ec1946eb5f), [`5b7d2b5`](https://github.com/Effect-TS/effect/commit/5b7d2b5c90bc77eb9f8be1b91166c53f89aa1d0c), [`6ee26b1`](https://github.com/Effect-TS/effect/commit/6ee26b16b51b380c41dbfa84d612aa525b420fce), [`eab7e6a`](https://github.com/Effect-TS/effect/commit/eab7e6acfc7e422857565b98b9eddd3a98de1b92), [`e618f10`](https://github.com/Effect-TS/effect/commit/e618f1050f41424439eb82478fcf1c5da5a3244d), [`da14b1f`](https://github.com/Effect-TS/effect/commit/da14b1f18d71a18fc431b56d82cffe09aeb92708), [`3314fc3`](https://github.com/Effect-TS/effect/commit/3314fc30c22d5c9d737f3df7201f4a41424f701b), [`66db624`](https://github.com/Effect-TS/effect/commit/66db624670032a02144d42b6e64b83b324c813cd), [`4b8a50d`](https://github.com/Effect-TS/effect/commit/4b8a50db9649603f382f33640698020ca79547d0), [`8d52668`](https://github.com/Effect-TS/effect/commit/8d52668533e9d4a6d6b3be9fe58bd5e69fa80b65), [`1de1055`](https://github.com/Effect-TS/effect/commit/1de10559e7263747751bfb032fd4b0fd8aa28724), [`db130e6`](https://github.com/Effect-TS/effect/commit/db130e64557fafe370819d19f78e3b19a21275d6), [`5fa5fa5`](https://github.com/Effect-TS/effect/commit/5fa5fa5b9bff77352fb9ff2d424bf46bd6309986)]: + - effect@4.0.6 + ## 4.0.5 ### Patch Changes diff --git a/repos/effect/packages/atom/solid/package.json b/repos/effect/packages/atom/solid/package.json index 9f614f0cd9..41f3227d6b 100644 --- a/repos/effect/packages/atom/solid/package.json +++ b/repos/effect/packages/atom/solid/package.json @@ -1,6 +1,6 @@ { "name": "@effect/atom-solid", - "version": "4.0.5", + "version": "4.0.6", "type": "module", "license": "MIT", "description": "SolidJS bindings for the Effect Atom modules", diff --git a/repos/effect/packages/atom/solid/src/Hooks.ts b/repos/effect/packages/atom/solid/src/Hooks.ts index 7c0877fa6b..97c11d3dfe 100644 --- a/repos/effect/packages/atom/solid/src/Hooks.ts +++ b/repos/effect/packages/atom/solid/src/Hooks.ts @@ -151,6 +151,13 @@ export const useAtomMount = (atom: () => Atom.Atom): void => { /** * Returns a setter for a writable atom without subscribing to its value. * + * **Gotchas** + * + * In `promise` and `promiseExit` modes, the setter observes the atom's next + * settled result, not a result correlated with that write. Overlapping writes + * can resolve with another write's result. Serialize writes when each caller + * needs its own result. + * * @stability unstable * @category hooks * @since 4.0.0 @@ -206,6 +213,13 @@ export const useAtomRefresh = (atom: () => Atom.Atom): () => void => { * `AsyncResult` atoms, `promise` and `promiseExit` modes return promises for the * success value or full `Exit`. * + * **Gotchas** + * + * In `promise` and `promiseExit` modes, the setter observes the atom's next + * settled result, not a result correlated with that write. Overlapping writes + * can resolve with another write's result. Serialize writes when each caller + * needs its own result. + * * @see {@link useAtomValue} for subscribing to an atom without a setter * @see {@link useAtomSet} for updating a writable atom without subscribing to its value * diff --git a/repos/effect/packages/atom/solid/src/RegistryContext.ts b/repos/effect/packages/atom/solid/src/RegistryContext.ts index ab327f76f4..b67adecd61 100644 --- a/repos/effect/packages/atom/solid/src/RegistryContext.ts +++ b/repos/effect/packages/atom/solid/src/RegistryContext.ts @@ -62,7 +62,7 @@ export const RegistryContext = createContext(AtomRegi */ export const RegistryProvider = (options: { readonly children?: JSX.Element | undefined - readonly initialValues?: Iterable, any]> | undefined + readonly initialValues?: Iterable, any, Atom.InitialValueOptions?]> | undefined readonly scheduleTask?: ((f: () => void) => () => void) | undefined readonly timeoutResolution?: number | undefined readonly defaultIdleTTL?: number | undefined diff --git a/repos/effect/packages/atom/vue/CHANGELOG.md b/repos/effect/packages/atom/vue/CHANGELOG.md index 1cf87edc13..b7df3fe560 100644 --- a/repos/effect/packages/atom/vue/CHANGELOG.md +++ b/repos/effect/packages/atom/vue/CHANGELOG.md @@ -1,5 +1,12 @@ # @effect/atom-vue +## 4.0.6 + +### Patch Changes + +- Updated dependencies [[`ce13a48`](https://github.com/Effect-TS/effect/commit/ce13a4872e516187851c5d9336d4df309cc8f0a4), [`f3bbbb4`](https://github.com/Effect-TS/effect/commit/f3bbbb448d431a9d1205ff09ba47cb81af0c7199), [`5547315`](https://github.com/Effect-TS/effect/commit/554731500e43eef47ff68b569116b74006e91d87), [`6093cfb`](https://github.com/Effect-TS/effect/commit/6093cfbbb673d36ecd18edea955763db003c8c26), [`30319d5`](https://github.com/Effect-TS/effect/commit/30319d506c9177d1cc1e0b10372e1ddb776aaf0a), [`74a7596`](https://github.com/Effect-TS/effect/commit/74a759699a8582dbe660b21dcda21cc9db08f613), [`bfb32a3`](https://github.com/Effect-TS/effect/commit/bfb32a3cf5f8d0f02f0dfdb63dcf5237fde755b0), [`f8bf7f4`](https://github.com/Effect-TS/effect/commit/f8bf7f4bcc3ad543cd94cbeb010cb164e5dedcb0), [`7f3d3df`](https://github.com/Effect-TS/effect/commit/7f3d3dfbb7b9e6e041c4ad9654c4532f72d2f16e), [`db1046d`](https://github.com/Effect-TS/effect/commit/db1046d1fa0e4b5818c51d28017a89579b6261d6), [`3c4f88a`](https://github.com/Effect-TS/effect/commit/3c4f88aa4d1d4d958b42fe8d3f1195f0ca990f8a), [`916e709`](https://github.com/Effect-TS/effect/commit/916e70968acc4e443885c67012d855b7cc6f9bca), [`916f843`](https://github.com/Effect-TS/effect/commit/916f8436225153a81a049bdc1feaedcafe4d1ded), [`66ec757`](https://github.com/Effect-TS/effect/commit/66ec7574d23b7943584059b41136bc21ee9d4c53), [`daac06f`](https://github.com/Effect-TS/effect/commit/daac06f79a6ab8c7d3e1b66b427c06f7d64dcc8f), [`c2552ec`](https://github.com/Effect-TS/effect/commit/c2552ec2e031b409b2bef67309312b0168480a39), [`b02c1cd`](https://github.com/Effect-TS/effect/commit/b02c1cde36c890941a235224af8ecb035197582a), [`1855f1d`](https://github.com/Effect-TS/effect/commit/1855f1d359535339bc7640b67e6bb7b12752ba19), [`683ed2c`](https://github.com/Effect-TS/effect/commit/683ed2cb491f9529379c1d5c63933689ff57fd2b), [`ac68940`](https://github.com/Effect-TS/effect/commit/ac68940f7e4246b1beb43f9d451a67f41241ab82), [`22a8401`](https://github.com/Effect-TS/effect/commit/22a84017e7222f332611c1bace05f9fa60c7cf67), [`c53c194`](https://github.com/Effect-TS/effect/commit/c53c19451171a00776b400a8548394ac514b628a), [`ca6d1cc`](https://github.com/Effect-TS/effect/commit/ca6d1cc5f803cd501e85431642b79b552322a76a), [`47940c5`](https://github.com/Effect-TS/effect/commit/47940c55c9e0eb855615094a2aca6e0812307bb2), [`e8b24ef`](https://github.com/Effect-TS/effect/commit/e8b24ef9516706971db948efb2d3eab84ad85951), [`2e8aae0`](https://github.com/Effect-TS/effect/commit/2e8aae0080a8d3e6d00c5ee0bc4f49f0b074acc2), [`ad7497b`](https://github.com/Effect-TS/effect/commit/ad7497bbc1bc70935f8f1ae395ff501532e2e770), [`7f504a3`](https://github.com/Effect-TS/effect/commit/7f504a35edcf44952c55ecbb0916726bac4d3e52), [`99e365d`](https://github.com/Effect-TS/effect/commit/99e365d61a1fdaabeb31dc58e5752cade8a09e1f), [`954067d`](https://github.com/Effect-TS/effect/commit/954067da7fbd20a20c10d5d0de6ffe4690e6bdfd), [`3e1a035`](https://github.com/Effect-TS/effect/commit/3e1a035d9c470ced858b7757f6d102b3aa9be008), [`41a1fe7`](https://github.com/Effect-TS/effect/commit/41a1fe74e6338e8d13d6d4f3daa03f71a826c2c4), [`bcb3a86`](https://github.com/Effect-TS/effect/commit/bcb3a86335a8ed18da39300984c0fbae187fdc6b), [`739d3d1`](https://github.com/Effect-TS/effect/commit/739d3d18728cce0b6d907f0a3f24e04ab0b53560), [`ed4da35`](https://github.com/Effect-TS/effect/commit/ed4da351fb9d29e4dfd6c60bd89ba4c007d264dc), [`3d84e19`](https://github.com/Effect-TS/effect/commit/3d84e19f8f14bccf0aa57b11825590c3bae162f4), [`cc00906`](https://github.com/Effect-TS/effect/commit/cc0090657eeb89debdd54af5fcb99d544f46c1bf), [`09ba1f5`](https://github.com/Effect-TS/effect/commit/09ba1f59952c46cd22229d41236742c3e1f53a34), [`9ff1a73`](https://github.com/Effect-TS/effect/commit/9ff1a73d995702f4c4c335888fe6a186f3f710c7), [`06bc532`](https://github.com/Effect-TS/effect/commit/06bc532b71cd3684f010956774356864dd359372), [`97bfd84`](https://github.com/Effect-TS/effect/commit/97bfd84bb147e10ed562e4c1ffa589cdaf83a596), [`25efaef`](https://github.com/Effect-TS/effect/commit/25efaef0d81d25b2651ef4de474e929d3180810a), [`c10335b`](https://github.com/Effect-TS/effect/commit/c10335be125c7846ae7b914eeb138244b660275b), [`a808972`](https://github.com/Effect-TS/effect/commit/a80897262648e4222ce48943e538f39a6b3f69d6), [`20e844c`](https://github.com/Effect-TS/effect/commit/20e844c7ef27b0e4b4f7911d603c45b1de2b0bc3), [`b9f73f3`](https://github.com/Effect-TS/effect/commit/b9f73f37c4ac538e6a327ed80ac5a0708858c540), [`5739878`](https://github.com/Effect-TS/effect/commit/5739878ded09d83d71d657e3b8d4264f4e4936d9), [`28acdf3`](https://github.com/Effect-TS/effect/commit/28acdf3820b66056037bb7a24cd0e8d3de3fb06d), [`59b6396`](https://github.com/Effect-TS/effect/commit/59b63967c1817ae5acb1811037f5aaa964219b7e), [`0bccc9c`](https://github.com/Effect-TS/effect/commit/0bccc9c3dd5349b20f8d2e509ac5482d0c26a616), [`fed70de`](https://github.com/Effect-TS/effect/commit/fed70de0dff8af24c5f814bd0d985ab70c70420b), [`82648a2`](https://github.com/Effect-TS/effect/commit/82648a21bf228145a14d2d1390006d67f74d89c7), [`afb5f32`](https://github.com/Effect-TS/effect/commit/afb5f326cc14c9849d8d7ddefde1039ae036e017), [`28813f5`](https://github.com/Effect-TS/effect/commit/28813f53a5026984fdc13d3971147ea5dc20104c), [`0fae934`](https://github.com/Effect-TS/effect/commit/0fae934a13aac12f9036b0733edd5dde5ce029f3), [`f260830`](https://github.com/Effect-TS/effect/commit/f2608305861269fc2fbedcbbdcd71e4fc5f98ffc), [`19b63eb`](https://github.com/Effect-TS/effect/commit/19b63eb81b2bccf033aadb565f52a6b0375d26a1), [`14db221`](https://github.com/Effect-TS/effect/commit/14db221f8f584100ff50fd1173b1a99a1caa1d06), [`679f956`](https://github.com/Effect-TS/effect/commit/679f956c4a7527854ff78db071dd7aa417211a48), [`b72707c`](https://github.com/Effect-TS/effect/commit/b72707c2937328fd3bc70f29a6792d84c4256484), [`44c8c1d`](https://github.com/Effect-TS/effect/commit/44c8c1d1c6f01ba9887a051b365e9fa89fcfc685), [`1e7909b`](https://github.com/Effect-TS/effect/commit/1e7909beb05bc740dbd330509e60bb185d7eeed1), [`0611a65`](https://github.com/Effect-TS/effect/commit/0611a657f76836f9ca275f64764b82270b4eda10), [`2fadc65`](https://github.com/Effect-TS/effect/commit/2fadc654aa655fbd1912ce51eb1b764a757ac362), [`feb6ce5`](https://github.com/Effect-TS/effect/commit/feb6ce59a38ab15fc6f89545bd5f0992dd58a06a), [`5a2ed53`](https://github.com/Effect-TS/effect/commit/5a2ed5323884a435bfcfb5e3a6f2ae29547458ad), [`c8d26a4`](https://github.com/Effect-TS/effect/commit/c8d26a46731cb26a9b9612d9fde823d8813cfc04), [`0c8b446`](https://github.com/Effect-TS/effect/commit/0c8b446709c737c10d0e11af4313bb29fdff2588), [`05c3409`](https://github.com/Effect-TS/effect/commit/05c340955944374e3ae47a1de0f7cb9b41a3b0e6), [`b04191b`](https://github.com/Effect-TS/effect/commit/b04191b2941534754282c51ef2df2f965ac43a5c), [`05c3409`](https://github.com/Effect-TS/effect/commit/05c340955944374e3ae47a1de0f7cb9b41a3b0e6), [`4067e64`](https://github.com/Effect-TS/effect/commit/4067e6428776ae573c8cc72f7eaebd078a4ff8c7), [`e04563d`](https://github.com/Effect-TS/effect/commit/e04563dcd6ddeea74c088a12194c5a73e597ba2f), [`02d7fc5`](https://github.com/Effect-TS/effect/commit/02d7fc583053c47dea3b5e1d9d6be164aeb321b6), [`561d2a1`](https://github.com/Effect-TS/effect/commit/561d2a1ddfa3c4340510f23da7adc50c89d97863), [`8fe2dcc`](https://github.com/Effect-TS/effect/commit/8fe2dcc2137788a71177d42a9f67fb7003224868), [`a8086fa`](https://github.com/Effect-TS/effect/commit/a8086fa26a2c9a9f5f7f1d7590bbc494a0feb02a), [`88a33b4`](https://github.com/Effect-TS/effect/commit/88a33b4122f7ffbbc1423add793a79abffd99c75), [`e77372a`](https://github.com/Effect-TS/effect/commit/e77372a8a2e7caec11999636e04c6e31c52e315f), [`3dd4388`](https://github.com/Effect-TS/effect/commit/3dd438810df34ed3b3d2e3e35ccf84ccb00feb85), [`ba2b00c`](https://github.com/Effect-TS/effect/commit/ba2b00c0658b65775809e0052ec9612432600c31), [`a049d4a`](https://github.com/Effect-TS/effect/commit/a049d4ad25f17cf4985284a8d15b07aea2a33242), [`84aeede`](https://github.com/Effect-TS/effect/commit/84aeede040460a2b24bcf0db7a5d0a9b0e3c42d4), [`43e5e19`](https://github.com/Effect-TS/effect/commit/43e5e198d16df038b60764f5dd0218cdcf828f16), [`77bbf16`](https://github.com/Effect-TS/effect/commit/77bbf16e50d6b0ea090288cc5d3f11b20706a027), [`6af3a47`](https://github.com/Effect-TS/effect/commit/6af3a47d5b5d698c3ba896c37684266cd465bef1), [`aac8d21`](https://github.com/Effect-TS/effect/commit/aac8d21911333dedb649f4f9d302c28d14d46369), [`84c8dc4`](https://github.com/Effect-TS/effect/commit/84c8dc47b09067362749620f75068d6f28044660), [`78d1988`](https://github.com/Effect-TS/effect/commit/78d198818d4fd547ce2e9431302f66cb00dc7ad3), [`5676a2a`](https://github.com/Effect-TS/effect/commit/5676a2ac5a918f8f91fc7838d7adf1b6f5e86a6d), [`4b437bf`](https://github.com/Effect-TS/effect/commit/4b437bf4645fabc8aaeaa3b5a88e8c9048b9e4f5), [`1af11fd`](https://github.com/Effect-TS/effect/commit/1af11fda1bccf823f74121b5125aa3befa084b18), [`716f8df`](https://github.com/Effect-TS/effect/commit/716f8dfa178f849e5b288464373bd132860c2e19), [`ee27718`](https://github.com/Effect-TS/effect/commit/ee27718a02125328a7478812b1b7d7c1d8532578), [`4d86d1b`](https://github.com/Effect-TS/effect/commit/4d86d1b8887d48fa24acf6ad54f33ef3d55a7cd8), [`5462ed7`](https://github.com/Effect-TS/effect/commit/5462ed7b3a4c37206c47e401d6be293dc540350f), [`658837c`](https://github.com/Effect-TS/effect/commit/658837c3b249d982df56f2990247499746e36672), [`00261cc`](https://github.com/Effect-TS/effect/commit/00261cc12c31ecc7878f80670439731c905dc2bd), [`7cccb93`](https://github.com/Effect-TS/effect/commit/7cccb9385729bc0aed6630edea95d7db15439ac4), [`7c98228`](https://github.com/Effect-TS/effect/commit/7c982281cf73958833054175fc86db385e043cf6), [`5a8e248`](https://github.com/Effect-TS/effect/commit/5a8e2481546352e2ad6233dad49420b14c59f624), [`8037a65`](https://github.com/Effect-TS/effect/commit/8037a65f40cae9163ab9e0e687e289f717ae570b), [`25912db`](https://github.com/Effect-TS/effect/commit/25912db9d493227a2d6fa45cebcfc7507e638d8a), [`7f32606`](https://github.com/Effect-TS/effect/commit/7f32606baac43a67d171ae93b77c7b29791c9692), [`0386f88`](https://github.com/Effect-TS/effect/commit/0386f88295ac70cbc1f90ab8cceb0981d848c628), [`403298a`](https://github.com/Effect-TS/effect/commit/403298ac457fb634eca2c013fed26d37fc80445f), [`5a3ebc0`](https://github.com/Effect-TS/effect/commit/5a3ebc082124f9e10106b2347736e66fc74ebc36), [`fac2191`](https://github.com/Effect-TS/effect/commit/fac21911964b742a38e1d4e73bd4f95aec4911dc), [`fdb12dc`](https://github.com/Effect-TS/effect/commit/fdb12dcd61b6505eed02e6f7a3c88a85d4c48e8d), [`97909b8`](https://github.com/Effect-TS/effect/commit/97909b89b167da54a56e6f82f1c4db8bfedae04f), [`c3983f4`](https://github.com/Effect-TS/effect/commit/c3983f4db2b3052a58e8045cce270fd7ccc4e2b2), [`3a5e65e`](https://github.com/Effect-TS/effect/commit/3a5e65e409c9569e23bb7af61df401347e9ec420), [`8b40fbc`](https://github.com/Effect-TS/effect/commit/8b40fbc93caf52f3a4c69020849f0051f100be65), [`3f3e119`](https://github.com/Effect-TS/effect/commit/3f3e119ad61c2167bc6c7170c3fb68cd1eb3421e), [`7c7b8ed`](https://github.com/Effect-TS/effect/commit/7c7b8eda54e44f9cddc1188f994f7fdddf901055), [`d4f71bc`](https://github.com/Effect-TS/effect/commit/d4f71bc42507b638d149fb8806d26ea626d17956), [`c66241e`](https://github.com/Effect-TS/effect/commit/c66241e791c4b51d8a65051142d01682590f9bf8), [`f0e941e`](https://github.com/Effect-TS/effect/commit/f0e941ee2f90f0a891d1228a90da91386f0258cb), [`5f8a6e5`](https://github.com/Effect-TS/effect/commit/5f8a6e5108aa6c6cd4043a3c9f4dc93d43369623), [`2800a09`](https://github.com/Effect-TS/effect/commit/2800a0910f7f4ad7cfa21dfc840edf8fe6e67ca7), [`e9749b0`](https://github.com/Effect-TS/effect/commit/e9749b04c10badabcdf71e6958de5ac56a5316e9), [`eaef09c`](https://github.com/Effect-TS/effect/commit/eaef09c11b6ed2029c513eea0aa3256dcc6f517f), [`a3c56e5`](https://github.com/Effect-TS/effect/commit/a3c56e5b5e756e879f1cfb90230e8eae620e33df), [`bb61152`](https://github.com/Effect-TS/effect/commit/bb611522fcfb4d317b33acdf5ee24a480d042435), [`e9c842b`](https://github.com/Effect-TS/effect/commit/e9c842bc294abf658d698bb17732ae13465b3817), [`6d8c8e2`](https://github.com/Effect-TS/effect/commit/6d8c8e2075b15ae84ae96e93a0d295cb9221b496), [`1f28187`](https://github.com/Effect-TS/effect/commit/1f281878ef559ef356af2d0e11b381aba857cae1), [`0b9842b`](https://github.com/Effect-TS/effect/commit/0b9842b76bbc1b016c8abc46432052e1f60b2fa3), [`c29ddd8`](https://github.com/Effect-TS/effect/commit/c29ddd899c53c5d8194c4a0dfef2134304dda799), [`84c3405`](https://github.com/Effect-TS/effect/commit/84c34057ceab9fe6782b73f4a04a31304e0d14de), [`3934c0c`](https://github.com/Effect-TS/effect/commit/3934c0c61857c03468c08c2477f49a6b977d28ff), [`048e7c8`](https://github.com/Effect-TS/effect/commit/048e7c87125c4ace5a61895feac3d049a317bbfc), [`1121334`](https://github.com/Effect-TS/effect/commit/1121334ce68c379051011a2ffea37b0a30ea9a47), [`9a680d6`](https://github.com/Effect-TS/effect/commit/9a680d6985c6a27d7ebb29cd9eb674f3fe524505), [`85e2aee`](https://github.com/Effect-TS/effect/commit/85e2aeee686963c26ca40d204f982deebb9d6117), [`34ec440`](https://github.com/Effect-TS/effect/commit/34ec4407a14ce045ca14cc76c77582884a7e1ebe), [`343a149`](https://github.com/Effect-TS/effect/commit/343a149fa7d5a9a0a45841b1b6800b0ecfa16a32), [`915117b`](https://github.com/Effect-TS/effect/commit/915117b523cb6e5cbc98b5ff1c119a8d0d1f6b9a), [`7f823c1`](https://github.com/Effect-TS/effect/commit/7f823c10d20ef53a1081e0d0e2008a4116140799), [`6e614e6`](https://github.com/Effect-TS/effect/commit/6e614e6feb4281a9e81f5ec3be48e7905ff4292c), [`9990b39`](https://github.com/Effect-TS/effect/commit/9990b3977a544247cd4123edde8353a55bf77192), [`22bf781`](https://github.com/Effect-TS/effect/commit/22bf7812aa8a20c676efade943c342cde4ef1cbd), [`41f08b7`](https://github.com/Effect-TS/effect/commit/41f08b75aba40e95cd6c743ec6fcf1f5022f7b5b), [`e8482ca`](https://github.com/Effect-TS/effect/commit/e8482cad9651fa8fe8036fe6292a286994897d0d), [`5ae85fa`](https://github.com/Effect-TS/effect/commit/5ae85fa8ac1fc5e12d5fe8fe7388ac504788ae3b), [`c477a0a`](https://github.com/Effect-TS/effect/commit/c477a0a8010bd0a988221bc00c446848314e1c3c), [`8fbb39e`](https://github.com/Effect-TS/effect/commit/8fbb39eb92577915831f19a83b51be376e844613), [`1c70828`](https://github.com/Effect-TS/effect/commit/1c7082847453e9c2be0cc267a39ca554c52f4013), [`8207d1e`](https://github.com/Effect-TS/effect/commit/8207d1e91ead8aac8f01d9e13b49624b64237c8b), [`7886b84`](https://github.com/Effect-TS/effect/commit/7886b847658664f64392a14efda86f44b3d1e170), [`4839c60`](https://github.com/Effect-TS/effect/commit/4839c601bdda845d9a54d77f578f64372475ab04), [`2595d83`](https://github.com/Effect-TS/effect/commit/2595d831f702d4a748f97e33e71fd5250f58af56), [`f5ae278`](https://github.com/Effect-TS/effect/commit/f5ae27827a82db3f8973b70745fc6f14e7d8fa80), [`7a8ce00`](https://github.com/Effect-TS/effect/commit/7a8ce0098a5f22cbb6c110b16e850fb3fcbcadf2), [`688d302`](https://github.com/Effect-TS/effect/commit/688d30275ff114e50b4f0d343c6227ee33d67348), [`c7072e4`](https://github.com/Effect-TS/effect/commit/c7072e46356a19615625ae0fa569b1c3a79d50be), [`425cedb`](https://github.com/Effect-TS/effect/commit/425cedb5801c9527c116a64d280d9e7c2820f922), [`88bc680`](https://github.com/Effect-TS/effect/commit/88bc68001c6c70984b03a5500232e9b5aab14085), [`268e09c`](https://github.com/Effect-TS/effect/commit/268e09c94c9e8f46142b5c9f29ce0a22bfbbc3aa), [`7e94505`](https://github.com/Effect-TS/effect/commit/7e94505edfc9d81ea08cb96a7d935806495bf61e), [`afa5da3`](https://github.com/Effect-TS/effect/commit/afa5da3c8a9848215e49535dd3d39179bfd67715), [`c29ddd8`](https://github.com/Effect-TS/effect/commit/c29ddd899c53c5d8194c4a0dfef2134304dda799), [`48102ac`](https://github.com/Effect-TS/effect/commit/48102ac592ef89fed9d5d10afd2daece0a0ee310), [`463eca3`](https://github.com/Effect-TS/effect/commit/463eca3609a6a9681247abbcc71a6f10b261a1d9), [`e7162ea`](https://github.com/Effect-TS/effect/commit/e7162eae7f04af1d583b1f31fdd767f91ec1c82e), [`95752db`](https://github.com/Effect-TS/effect/commit/95752db5eb3b6e59fb8749e9792704f8b167f36c), [`4577a28`](https://github.com/Effect-TS/effect/commit/4577a28bcdb50b0d184775f8c1b7ad890d86e255), [`973d599`](https://github.com/Effect-TS/effect/commit/973d599a3e8611626764a0e3e8f79edbcf30dc7c), [`2f066a5`](https://github.com/Effect-TS/effect/commit/2f066a58e90316dd88a219fcea94200beda54219), [`f0ccbb6`](https://github.com/Effect-TS/effect/commit/f0ccbb6917be5c08364b0b8adbd0c4c19e7f2de1), [`2dda6ce`](https://github.com/Effect-TS/effect/commit/2dda6cefc641dbea2ea3279a9b96c0332b277a40), [`3da7c0a`](https://github.com/Effect-TS/effect/commit/3da7c0aabab07c8fc682aea7559536bf31705853), [`5a67e0b`](https://github.com/Effect-TS/effect/commit/5a67e0b3a8118dd7be709a56d6d7af4f10e9c4f3), [`73b3a6e`](https://github.com/Effect-TS/effect/commit/73b3a6e3c38795400251d9c5d73f05c88acd90cd), [`c6f9e4c`](https://github.com/Effect-TS/effect/commit/c6f9e4cbe6f9c8f0a03fefde18df4d35e98a24df), [`be02676`](https://github.com/Effect-TS/effect/commit/be026768ee29a1b1a860c1fd17f2a0896c01f61e), [`26de861`](https://github.com/Effect-TS/effect/commit/26de86100cda8617e077ce66f0a6bb8dfe738fb6), [`9313799`](https://github.com/Effect-TS/effect/commit/931379911a74d5e2207859c295b1acc6a5e2917e), [`d5c1e5b`](https://github.com/Effect-TS/effect/commit/d5c1e5bd425dfe983dff1f8c853c0b7eaa772f87), [`11bb31f`](https://github.com/Effect-TS/effect/commit/11bb31fec0bf2fdfc025009a04de4c69bffa433a), [`8c23e46`](https://github.com/Effect-TS/effect/commit/8c23e46a8280e3f2c831d1c0ca3f106c92f8e071), [`9c96011`](https://github.com/Effect-TS/effect/commit/9c960111de202c0f1dd6989e5b8836150e0ca694), [`0d50570`](https://github.com/Effect-TS/effect/commit/0d50570bbf078804092fcd4a47ccd2d0019f989c), [`9ed2bf1`](https://github.com/Effect-TS/effect/commit/9ed2bf18da0e1ff4fecc31b7cbac2976ec999e00), [`76ec912`](https://github.com/Effect-TS/effect/commit/76ec9120f43f4fdb08b9c5170ff9e3fd82f22457), [`af2756d`](https://github.com/Effect-TS/effect/commit/af2756dd1d196aa3f199ce21c6d2dc3ada3b67b5), [`3ec8100`](https://github.com/Effect-TS/effect/commit/3ec8100c793b8a22381ad115106b93824d840f64), [`93abf02`](https://github.com/Effect-TS/effect/commit/93abf020876af0a6b6b8b25621c80760b7996762), [`1e94993`](https://github.com/Effect-TS/effect/commit/1e949935a1065e5140245a514ce3a799061143e3), [`99a4616`](https://github.com/Effect-TS/effect/commit/99a4616264e31de65fa8d3dc19576309b1a3f84a), [`0a7978e`](https://github.com/Effect-TS/effect/commit/0a7978ea8e70e51700345b12f8c19952dde05255), [`b4a060b`](https://github.com/Effect-TS/effect/commit/b4a060b95a6473d54e277dc005172b54aab9bddf), [`274f4a4`](https://github.com/Effect-TS/effect/commit/274f4a4dac5ee2d9dbb6358cc3dd3afe345d4705), [`25f29be`](https://github.com/Effect-TS/effect/commit/25f29be77bdcd5c2a7be8e2710e88f5a10c4cb8d), [`9036440`](https://github.com/Effect-TS/effect/commit/9036440edca01c24371d0c2c1044be5e5d3cb65e), [`415f005`](https://github.com/Effect-TS/effect/commit/415f0053eb848d562febcb5b011e15f31a99bead), [`191308b`](https://github.com/Effect-TS/effect/commit/191308bd0f5ee04557570f2f960c6e06c429bcfd), [`4b96570`](https://github.com/Effect-TS/effect/commit/4b96570ed28d93b14e958742933d227dc5e74b15), [`d84ec3f`](https://github.com/Effect-TS/effect/commit/d84ec3fab0346c01bc59f6624d1c046ca4140087), [`392aa1c`](https://github.com/Effect-TS/effect/commit/392aa1cb275ac9fdb9b4b1831d8c06196bba54b9), [`99e9bd9`](https://github.com/Effect-TS/effect/commit/99e9bd98a354d83202e2522f3ade9ee4bf4a862c), [`6f6f45f`](https://github.com/Effect-TS/effect/commit/6f6f45f480a3f191aed6a82ca5c301f60ef2cf84), [`13fc07e`](https://github.com/Effect-TS/effect/commit/13fc07edeb939ca6229b858c9d9a8c5243ac21b5), [`a4b38dd`](https://github.com/Effect-TS/effect/commit/a4b38dde7f3fdc1b54a49bffdaca912b4c47c76e), [`8b18580`](https://github.com/Effect-TS/effect/commit/8b185807bb99f6de3d9564447e2f4e08d2f8863d), [`0c477fe`](https://github.com/Effect-TS/effect/commit/0c477fe417d3ecab86bb8d9afd0c392e9cb8a3da), [`9a5331d`](https://github.com/Effect-TS/effect/commit/9a5331db94bfdca4f48317d062fbeb96054a9049), [`a706d73`](https://github.com/Effect-TS/effect/commit/a706d73db8a8d460877c5d7e8a793a52d45be307), [`c9a1e72`](https://github.com/Effect-TS/effect/commit/c9a1e72c5965aa9fe01a669ebfd523f803f5a189), [`89eaaf8`](https://github.com/Effect-TS/effect/commit/89eaaf8581908e248c0b1e47baa7906e0c541635), [`8215811`](https://github.com/Effect-TS/effect/commit/8215811d51d28d37b4732a6ab03de4047ada62ea), [`ad69018`](https://github.com/Effect-TS/effect/commit/ad6901857e05e7b436a0149b5ece7f3a36d9ff33), [`a733eba`](https://github.com/Effect-TS/effect/commit/a733eba9f699737a80a9e7276521980bd929075a), [`7282d5f`](https://github.com/Effect-TS/effect/commit/7282d5f1bcc06d7065429ef81a3d17c64043fde4), [`da33572`](https://github.com/Effect-TS/effect/commit/da33572776763e45802ce8c2fe14ec101f9a5c34), [`d9e2d12`](https://github.com/Effect-TS/effect/commit/d9e2d1221768e9a3e5e00165cb95b86e0c126cc0), [`dc923b8`](https://github.com/Effect-TS/effect/commit/dc923b847ac7d9673b4f0aa7c136557287b2e5e3), [`1b74086`](https://github.com/Effect-TS/effect/commit/1b740862e3d25ec5d4b6f86b9fe792b6dabbf84c), [`8a5507b`](https://github.com/Effect-TS/effect/commit/8a5507ba00f641edf71227d79b9f90efdb634fe2), [`23460fa`](https://github.com/Effect-TS/effect/commit/23460fad704c730926bffe3cc6874268ef6680ec), [`0dbbea6`](https://github.com/Effect-TS/effect/commit/0dbbea65a93c59147a4bc407cfcda38ea26341b2), [`70aad3f`](https://github.com/Effect-TS/effect/commit/70aad3ffe617984f5b599aeed1bc7f04899aca55), [`0909236`](https://github.com/Effect-TS/effect/commit/090923611b209a19f3281a7b1d0c15a9b653a0c7), [`a9e66d8`](https://github.com/Effect-TS/effect/commit/a9e66d889017b302355ba5b8bc2b2050ca1ea5aa), [`ee5c0d5`](https://github.com/Effect-TS/effect/commit/ee5c0d5f43348747b04b5f7bf1b39355614c2f2c), [`631c5d6`](https://github.com/Effect-TS/effect/commit/631c5d6d9a81e6389964b878764fb9aea1538247), [`7e60c1a`](https://github.com/Effect-TS/effect/commit/7e60c1ae62a96c768f9c67d706f494213947f22f), [`4cddd3c`](https://github.com/Effect-TS/effect/commit/4cddd3c41e71fc4990d37c42b1bcf2ce3780b41d), [`8727e5d`](https://github.com/Effect-TS/effect/commit/8727e5d2bec8e2d729b7aeafb725a4ab21d533d2), [`2bc97fa`](https://github.com/Effect-TS/effect/commit/2bc97faa0e06b3973780dae26029e2c59f5dba38), [`f9a7f05`](https://github.com/Effect-TS/effect/commit/f9a7f056ba31aef435933acf082ec8c01ac7decc), [`d978cca`](https://github.com/Effect-TS/effect/commit/d978cca02914bec0ddf66bb2d68d90b220096fb6), [`b334ccf`](https://github.com/Effect-TS/effect/commit/b334ccf361a0f11362e7aabdbf70b35f53a39d61), [`0b50cfc`](https://github.com/Effect-TS/effect/commit/0b50cfc644313afe4c318073893231ec1946eb5f), [`5b7d2b5`](https://github.com/Effect-TS/effect/commit/5b7d2b5c90bc77eb9f8be1b91166c53f89aa1d0c), [`6ee26b1`](https://github.com/Effect-TS/effect/commit/6ee26b16b51b380c41dbfa84d612aa525b420fce), [`eab7e6a`](https://github.com/Effect-TS/effect/commit/eab7e6acfc7e422857565b98b9eddd3a98de1b92), [`e618f10`](https://github.com/Effect-TS/effect/commit/e618f1050f41424439eb82478fcf1c5da5a3244d), [`da14b1f`](https://github.com/Effect-TS/effect/commit/da14b1f18d71a18fc431b56d82cffe09aeb92708), [`3314fc3`](https://github.com/Effect-TS/effect/commit/3314fc30c22d5c9d737f3df7201f4a41424f701b), [`66db624`](https://github.com/Effect-TS/effect/commit/66db624670032a02144d42b6e64b83b324c813cd), [`4b8a50d`](https://github.com/Effect-TS/effect/commit/4b8a50db9649603f382f33640698020ca79547d0), [`8d52668`](https://github.com/Effect-TS/effect/commit/8d52668533e9d4a6d6b3be9fe58bd5e69fa80b65), [`1de1055`](https://github.com/Effect-TS/effect/commit/1de10559e7263747751bfb032fd4b0fd8aa28724), [`db130e6`](https://github.com/Effect-TS/effect/commit/db130e64557fafe370819d19f78e3b19a21275d6), [`5fa5fa5`](https://github.com/Effect-TS/effect/commit/5fa5fa5b9bff77352fb9ff2d424bf46bd6309986)]: + - effect@4.0.6 + ## 4.0.5 ### Patch Changes diff --git a/repos/effect/packages/atom/vue/package.json b/repos/effect/packages/atom/vue/package.json index d983ef8baa..ca77f48e65 100644 --- a/repos/effect/packages/atom/vue/package.json +++ b/repos/effect/packages/atom/vue/package.json @@ -1,6 +1,6 @@ { "name": "@effect/atom-vue", - "version": "4.0.5", + "version": "4.0.6", "type": "module", "license": "MIT", "description": "Vue bindings for the Effect Atom modules", @@ -61,7 +61,7 @@ }, "devDependencies": { "effect": "workspace:^", - "vue": "^3.5.43" + "vue": "^3.5.44" }, "peerDependencies": { "effect": "workspace:^", diff --git a/repos/effect/packages/atom/vue/src/index.ts b/repos/effect/packages/atom/vue/src/index.ts index 39e0a210fc..ce72794e07 100644 --- a/repos/effect/packages/atom/vue/src/index.ts +++ b/repos/effect/packages/atom/vue/src/index.ts @@ -111,6 +111,13 @@ const useAtomValueRef = >(atom: () => A) => { /** * Returns a reactive atom value and a setter with the requested write mode. * + * **Gotchas** + * + * In `promise` and `promiseExit` modes, the setter observes the atom's next + * settled result, not a result correlated with that write. Overlapping writes + * can resolve with another write's result. Serialize writes when each caller + * needs its own result. + * * @stability unstable * @category composables * @since 4.0.0 @@ -194,6 +201,13 @@ function setAtom /** * Returns a setter for an atom, mounting it for the lifetime of the composable. * + * **Gotchas** + * + * In `promise` and `promiseExit` modes, the setter observes the atom's next + * settled result, not a result correlated with that write. Overlapping writes + * can resolve with another write's result. Serialize writes when each caller + * needs its own result. + * * @stability unstable * @category composables * @since 4.0.0 diff --git a/repos/effect/packages/effect/CHANGELOG.md b/repos/effect/packages/effect/CHANGELOG.md index fb6f29ad07..84c56cc33a 100644 --- a/repos/effect/packages/effect/CHANGELOG.md +++ b/repos/effect/packages/effect/CHANGELOG.md @@ -1,5 +1,599 @@ # effect +## 4.0.6 + +### Patch Changes + +- [#9674](https://github.com/Effect-TS/effect/pull/9674) [`ce13a48`](https://github.com/Effect-TS/effect/commit/ce13a4872e516187851c5d9336d4df309cc8f0a4) Thanks @effect-bot! - Allow `Response.AllParts`, `Response.Part` and `Response.StreamPart` to be called without a toolkit, decoding tool call and tool result parts for any tool name with `unknown` parameters and results. + +- [#9696](https://github.com/Effect-TS/effect/pull/9696) [`f3bbbb4`](https://github.com/Effect-TS/effect/commit/f3bbbb448d431a9d1205ff09ba47cb81af0c7199) Thanks @gcanti! - Fix `Array.unionWith` calling custom equivalence callbacks with values from the wrong input or in the wrong order. + +- [#9511](https://github.com/Effect-TS/effect/pull/9511) [`5547315`](https://github.com/Effect-TS/effect/commit/554731500e43eef47ff68b569116b74006e91d87) Thanks @sledorze! - Speed up field copying in `Data.Class`, `Data.Error`, `Request.Class` and `Persistable` constructors. + +- [#9743](https://github.com/Effect-TS/effect/pull/9743) [`6093cfb`](https://github.com/Effect-TS/effect/commit/6093cfbbb673d36ecd18edea955763db003c8c26) Thanks @effect-bot! - Fix `AsyncResult` equality treating unrelated `Equal` values with a matching tag and hash as equal. + +- [#9607](https://github.com/Effect-TS/effect/pull/9607) [`30319d5`](https://github.com/Effect-TS/effect/commit/30319d506c9177d1cc1e0b10372e1ddb776aaf0a) Thanks @effect-bot! - Fix Effect atoms dropping the latest exposed success when an asynchronous run fails, so an `initialValue` or a success published with `get.setSelf` remains available as the failure's previous success. + +- [#9313](https://github.com/Effect-TS/effect/pull/9313) [`74a7596`](https://github.com/Effect-TS/effect/commit/74a759699a8582dbe660b21dcda21cc9db08f613) Thanks @effect-bot! - Preserve Reactivity and global layers when replacing an atom runtime layer through registry initial values, so injected test layers retain shared invalidation and global services. + +- [#9190](https://github.com/Effect-TS/effect/pull/9190) [`bfb32a3`](https://github.com/Effect-TS/effect/commit/bfb32a3cf5f8d0f02f0dfdb63dcf5237fde755b0) Thanks @danieljvdm! - Log Atom scope finalizer failures at `References.UnhandledLogLevel` when an atom is disposed, using the atom runtime's logger. Interruption-only cleanup is not logged. + +- [#9522](https://github.com/Effect-TS/effect/pull/9522) [`f8bf7f4`](https://github.com/Effect-TS/effect/commit/f8bf7f4bcc3ad543cd94cbeb010cb164e5dedcb0) Thanks @tim-smart! - Add a `skipInitialRun` option to `Atom.serializable` and `Atom.initialValue`. A seeded effect or stream atom then keeps its hydrated or initial value instead of running on mount, until it is refreshed or a dependency changes. Factory `get(...)` calls and `withReactivity` keys are still tracked; dependencies read inside the effect are tracked from its first real run. + +- [#9718](https://github.com/Effect-TS/effect/pull/9718) [`7f3d3df`](https://github.com/Effect-TS/effect/commit/7f3d3dfbb7b9e6e041c4ad9654c4532f72d2f16e) Thanks @fubhy! - Add `Base64Url.encodePadded` and a `padding` option to `SchemaGetter.encodeBase64Url` for consumers that require padded URL-safe Base64. + +- [#9698](https://github.com/Effect-TS/effect/pull/9698) [`db1046d`](https://github.com/Effect-TS/effect/commit/db1046d1fa0e4b5818c51d28017a89579b6261d6) Thanks @effect-bot! - Encode `HttpClientRequest.basicAuth` credentials as UTF-8 before base64 encoding, so non-ASCII usernames and passwords no longer throw or get mangled. + +- [#9326](https://github.com/Effect-TS/effect/pull/9326) [`3c4f88a`](https://github.com/Effect-TS/effect/commit/3c4f88aa4d1d4d958b42fe8d3f1195f0ca990f8a) Thanks @effect-bot! - Block imports of nested internal modules in the development and published package exports. These implementation details are not public APIs; public entry points remain available. + +- [#9441](https://github.com/Effect-TS/effect/pull/9441) [`916e709`](https://github.com/Effect-TS/effect/commit/916e70968acc4e443885c67012d855b7cc6f9bca) Thanks @fubhy! - Add the unstable `Bytes` module with `concat`, `Equivalence`, and `Order` for `Uint8Array` values. `Stream.mkUint8Array` now returns `Uint8Array`. Browser HTTP clients collect streamed request bodies without repeatedly copying accumulated bytes. + +- [#9470](https://github.com/Effect-TS/effect/pull/9470) [`916f843`](https://github.com/Effect-TS/effect/commit/916f8436225153a81a049bdc1feaedcafe4d1ded) Thanks @tim-smart! - Reduce EventLog remote replay overhead by reusing decoded conflict payloads within each batch. + +- [#9617](https://github.com/Effect-TS/effect/pull/9617) [`66ec757`](https://github.com/Effect-TS/effect/commit/66ec7574d23b7943584059b41136bc21ee9d4c53) Thanks @Marve10s! - Fix `Effect.cachedInvalidateWithTTL` caching a stale result when invalidated while the effect is running. + +- [#9499](https://github.com/Effect-TS/effect/pull/9499) [`daac06f`](https://github.com/Effect-TS/effect/commit/daac06f79a6ab8c7d3e1b66b427c06f7d64dcc8f) Thanks @schutzelaars! - Fix `Effect.callback` retaining completed suspension state, such as values received by `Deferred.await` or `PubSub.take`, until the next full GC. + +- [#9727](https://github.com/Effect-TS/effect/pull/9727) [`c2552ec`](https://github.com/Effect-TS/effect/commit/c2552ec2e031b409b2bef67309312b0168480a39) Thanks @Marve10s! - Fix `Cause.pretty`, `Cause.prettyErrors`, loggers and `runMain` overflowing the stack when an error's `cause` chain is cyclic. + +- [#9540](https://github.com/Effect-TS/effect/pull/9540) [`b02c1cd`](https://github.com/Effect-TS/effect/commit/b02c1cde36c890941a235224af8ecb035197582a) Thanks @Marve10s! - Preserve falsy nested error causes (`0`, `false`, `""`, `null`) in `Cause.prettyErrors` and `Cause.pretty`. + +- [#9418](https://github.com/Effect-TS/effect/pull/9418) [`1855f1d`](https://github.com/Effect-TS/effect/commit/1855f1d359535339bc7640b67e6bb7b12752ba19) Thanks @jpowersdev! - Fix `Channel.fromTransformStream` so an upstream failure is reported when a read is pending, instead of the error raised by aborting the transform stream. + +- [#9602](https://github.com/Effect-TS/effect/pull/9602) [`683ed2c`](https://github.com/Effect-TS/effect/commit/683ed2cb491f9529379c1d5c63933689ff57fd2b) Thanks @effect-bot! - Preserve `Prompt.FilePart` byte and URL data when chat history is exported or persisted as JSON. Encoding now produces tagged `{ _tag: "Uint8Array", base64 }` and `{ _tag: "URL", href }` objects that decode back to their original types, and `Chat.Persistence` reports undecodable stored history as a `PersistenceError` instead of a defect. + +- [#9238](https://github.com/Effect-TS/effect/pull/9238) [`ac68940`](https://github.com/Effect-TS/effect/commit/ac68940f7e4246b1beb43f9d451a67f41241ab82) Thanks @effect-bot! - Fix child-process cleanup without `forceKillAfter` to give remaining process-group members up to one second to exit after the leader exits. Clarify that waiting for the leader has no time limit and never escalates to `SIGKILL` when `forceKillAfter` is omitted. + +- [#9611](https://github.com/Effect-TS/effect/pull/9611) [`22a8401`](https://github.com/Effect-TS/effect/commit/22a84017e7222f332611c1bace05f9fa60c7cf67) Thanks @itisrmk! - Fix `Chunk.get` and `Chunk.getUnsafe` throwing on in-range fractional indexes of one-element chunks, and treat `NaN` and infinite indexes as out of bounds. + +- [#9287](https://github.com/Effect-TS/effect/pull/9287) [`c53c194`](https://github.com/Effect-TS/effect/commit/c53c19451171a00776b400a8548394ac514b628a) Thanks @devangpratap! - Fix the CLI lexer treating a short option cluster with an inline value, such as `-vn=3`, as one unknown flag instead of `-v -n=3`. A single dash followed by a multi-character name, such as `-num=3`, is now also split into separate flags; use `--num=3` instead. + +- [#9655](https://github.com/Effect-TS/effect/pull/9655) [`ca6d1cc`](https://github.com/Effect-TS/effect/commit/ca6d1cc5f803cd501e85431642b79b552322a76a) Thanks @effect-bot! - Add `Command.validate` and `CliError.ValidationError` to the unstable CLI. `validate` checks the parsed input before the handler runs, for rules such as mutually exclusive flags. A failure is shown with the command help, like any parse error. + +- [#9650](https://github.com/Effect-TS/effect/pull/9650) [`47940c5`](https://github.com/Effect-TS/effect/commit/47940c55c9e0eb855615094a2aca6e0812307bb2) Thanks @effect-bot! - Add `Flag.withNegation` and `Flag.withoutNegation` (with `Param` counterparts) to rename or remove the `--no-` form of boolean flags in parsing and shell completions. + +- [#9285](https://github.com/Effect-TS/effect/pull/9285) [`e8b24ef`](https://github.com/Effect-TS/effect/commit/e8b24ef9516706971db948efb2d3eab84ad85951) Thanks @okarin1511! - Fix the CLI treating `--=value` as a flag with an empty name. The argument is now passed through unchanged as a plain value, so it no longer fails with `Unrecognized flag: --`. + +- [#9666](https://github.com/Effect-TS/effect/pull/9666) [`2e8aae0`](https://github.com/Effect-TS/effect/commit/2e8aae0080a8d3e6d00c5ee0bc4f49f0b074acc2) Thanks @effect-bot! - Allow `Command.withExamples` to take a function that receives `{ commandPath }`, so help examples match wherever a command is mounted. + +- [#8350](https://github.com/Effect-TS/effect/pull/8350) [`ad7497b`](https://github.com/Effect-TS/effect/commit/ad7497bbc1bc70935f8f1ae395ff501532e2e770) Thanks @ArjunCodess! - Add a `TerminalCapabilities` reference and layer to `effect/cli`; built-in `Command` help, version, and error output now follows it, so `FORCE_COLOR` enables color off a TTY and `TERM=dumb` disables it. + +- [#9248](https://github.com/Effect-TS/effect/pull/9248) [`7f504a3`](https://github.com/Effect-TS/effect/commit/7f504a35edcf44952c55ecbb0916726bac4d3e52) Thanks @bezff! - Fix the ClickHouse migrator creating its ledger with SQLite syntax and wrapping the run in `BEGIN TRANSACTION`. + +- [#9395](https://github.com/Effect-TS/effect/pull/9395) [`99e365d`](https://github.com/Effect-TS/effect/commit/99e365d61a1fdaabeb31dc58e5752cade8a09e1f) Thanks @effect-bot! - Prevent `Effect.forkIn` and `Effect.forkScoped` from running work in an already-closed scope, even when `startImmediately` or `uninterruptible` is enabled. They now return an interrupted fiber without executing the supplied effect. + +- [#9407](https://github.com/Effect-TS/effect/pull/9407) [`954067d`](https://github.com/Effect-TS/effect/commit/954067da7fbd20a20c10d5d0de6ffe4690e6bdfd) Thanks @effect-bot! - Log and retry failures when scheduling the initial ClusterCron run so transient failures no longer leave the cron stopped. + +- [#9677](https://github.com/Effect-TS/effect/pull/9677) [`3e1a035`](https://github.com/Effect-TS/effect/commit/3e1a035d9c470ced858b7757f6d102b3aa9be008) Thanks @effect-bot! - Fix `ClusterCron` with `calculateNextRunFromPrevious: true` not resuming after its pending run was lost. + + In this mode every run is now processed by a single `executor` entity per cron job, so a restart no longer starts a second chain: runs never overlap and missed runs are caught up in order. Pending runs scheduled by earlier releases join the executor's chain. As a result, `Entity.CurrentAddress` inside `execute` no longer identifies the scheduled time; use the new `ClusterCron.CurrentScheduledTime` service instead, which is available in both modes. + + Failed runs are still retried until they succeed or are older than `skipIfOlderThan`, but in this mode a failing run now blocks later runs while it is retried, instead of later runs executing alongside it. + +- [#9577](https://github.com/Effect-TS/effect/pull/9577) [`41a1fe7`](https://github.com/Effect-TS/effect/commit/41a1fe74e6338e8d13d6d4f3daa03f71a826c2c4) Thanks @tim-smart! - `ClusterMetrics.entities` now reads the active entity count when metrics are snapshotted, instead of being updated by a loop once a second. The series is recorded in the `MetricRegistry` of the entity manager's context and removed when the entity manager shuts down. + +- [#9475](https://github.com/Effect-TS/effect/pull/9475) [`bcb3a86`](https://github.com/Effect-TS/effect/commit/bcb3a86335a8ed18da39300984c0fbae187fdc6b) Thanks @effect-bot! - Warn when a forced cluster shard release remains pending beyond `shardLockExpiration`, including the affected shards and pending entity types and singletons. Release failures now emit rate-limited warnings while preserving the requirement that teardown and lock release finish before reacquisition. + +- [#9336](https://github.com/Effect-TS/effect/pull/9336) [`739d3d1`](https://github.com/Effect-TS/effect/commit/739d3d18728cce0b6d907f0a3f24e04ab0b53560) Thanks @effect-bot! - Answer fresh, locally owned persisted cluster requests from a reply handler registered before the request is saved, instead of reading every local reply back from storage or polling for it. Duplicates and streams still read their replies from storage, and a dropped reply handler now falls back to storage immediately instead of waiting for the next poll. `MessageStorage.registerReplyHandler` must register its handler as soon as the effect starts. + +- [#9732](https://github.com/Effect-TS/effect/pull/9732) [`ed4da35`](https://github.com/Effect-TS/effect/commit/ed4da351fb9d29e4dfd6c60bd89ba4c007d264dc) Thanks @effect-bot! - Read persisted cluster messages from storage as soon as their entity type registers, instead of waiting for the next `entityMessagePollInterval`. + +- [#9716](https://github.com/Effect-TS/effect/pull/9716) [`3d84e19`](https://github.com/Effect-TS/effect/commit/3d84e19f8f14bccf0aa57b11825590c3bae162f4) Thanks @tim-smart! - Fix cluster shard releases waiting out `entityTerminationTimeout` when the shard moves while an entity is waiting to restart after a defect. + +- [#9256](https://github.com/Effect-TS/effect/pull/9256) [`cc00906`](https://github.com/Effect-TS/effect/commit/cc0090657eeb89debdd54af5fcb99d544f46c1bf) Thanks @Tanner-Scadden! - Add `ClusterMetrics.shardReleaseDuration`, a timer recording how long each completed shard release takes. + +- [#9244](https://github.com/Effect-TS/effect/pull/9244) [`09ba1f5`](https://github.com/Effect-TS/effect/commit/09ba1f59952c46cd22229d41236742c3e1f53a34) Thanks @effect-bot! - Keep the cluster shards gauge accurate when shards are released or shard locks are lost. + +- [#9513](https://github.com/Effect-TS/effect/pull/9513) [`9ff1a73`](https://github.com/Effect-TS/effect/commit/9ff1a73d995702f4c4c335888fe6a186f3f710c7) Thanks @effect-bot! - Repeat the stalled forced shard release warning every `max(shardLockExpiration, 30 seconds)` while the release stays pending, and add the `ClusterMetrics.shardsReleasing` gauge (`effect_cluster_shards_releasing`) counting the shards a runner is still releasing. + +- [#9443](https://github.com/Effect-TS/effect/pull/9443) [`06bc532`](https://github.com/Effect-TS/effect/commit/06bc532b71cd3684f010956774356864dd359372) Thanks @effect-bot! - Fix a Sharding startup race that could access singleton synchronization before initialization, causing a ReferenceError and delaying shutdown. + +- [#6441](https://github.com/Effect-TS/effect/pull/6441) [`97bfd84`](https://github.com/Effect-TS/effect/commit/97bfd84bb147e10ed562e4c1ffa589cdaf83a596) Thanks @sbking! - Fix SQL Server support in cluster `SqlMessageStorage`. Message claims and duplicate checks now use valid T-SQL. The SQL Server tables use filtered unique indexes instead of nullable `UNIQUE` constraints, and `NVARCHAR(MAX)` instead of `TEXT`, so keyless messages and chunked replies can be stored. + +- [#9645](https://github.com/Effect-TS/effect/pull/9645) [`25efaef`](https://github.com/Effect-TS/effect/commit/25efaef0d81d25b2651ef4de474e929d3180810a) Thanks @effect-bot! - Fix `ClusterWorkflowEngine` losing a `Workflow.interrupt` sent while a run was still saving its `Suspended` reply. + +- [#9394](https://github.com/Effect-TS/effect/pull/9394) [`c10335b`](https://github.com/Effect-TS/effect/commit/c10335be125c7846ae7b914eeb138244b660275b) Thanks @effect-bot! - Fix suspended cluster workflow scopes not being released when an execution completes. Earlier runs retain their resources through suspension and release them on completion or owner teardown, without running durable workflow finalizers on abandonment. + +- [#9490](https://github.com/Effect-TS/effect/pull/9490) [`a808972`](https://github.com/Effect-TS/effect/commit/a80897262648e4222ce48943e538f39a6b3f69d6) Thanks @tim-smart! - Remove regex lookbehind from ConfigProvider variable expansion to prevent module-loading syntax errors on Safari versions before 16.4, while preserving interpolation and escaping behavior. + +- [#9621](https://github.com/Effect-TS/effect/pull/9621) [`20e844c`](https://github.com/Effect-TS/effect/commit/20e844c7ef27b0e4b4f7911d603c45b1de2b0bc3) Thanks @Marve10s! - Fix `Config.schema` dropping record entries for numeric key schemas such as `Schema.Int` or `Schema.Number`. + +- [#9539](https://github.com/Effect-TS/effect/pull/9539) [`b9f73f3`](https://github.com/Effect-TS/effect/commit/b9f73f37c4ac538e6a327ed80ac5a0708858c540) Thanks @Marve10s! - Fix `Cron.prev` skipping an occurrence earlier in the same second as the input. + +- [#9719](https://github.com/Effect-TS/effect/pull/9719) [`5739878`](https://github.com/Effect-TS/effect/commit/5739878ded09d83d71d657e3b8d4264f4e4936d9) Thanks @Marve10s! - Fix `DateTime.formatIsoDate` and `DateTime.formatIsoDateUtc` dropping the day for years before 0000 or after 9999. + +- [#9654](https://github.com/Effect-TS/effect/pull/9654) [`28acdf3`](https://github.com/Effect-TS/effect/commit/28acdf3820b66056037bb7a24cd0e8d3de3fb06d) Thanks @okarin1511! - Fix DateTime.zoneFromString rejecting "GMT" and accept compact UTC offsets without colon + +- [#9582](https://github.com/Effect-TS/effect/pull/9582) [`59b6396`](https://github.com/Effect-TS/effect/commit/59b63967c1817ae5acb1811037f5aaa964219b7e) Thanks @itisrmk! - Accept lowercase `z` and `gmt` zone designators in `DateTime.make` and `DateTime.makeUnsafe`. These strings previously had a second `Z` appended and were rejected. + +- [#9725](https://github.com/Effect-TS/effect/pull/9725) [`0bccc9c`](https://github.com/Effect-TS/effect/commit/0bccc9c3dd5349b20f8d2e509ac5482d0c26a616) Thanks @Marve10s! - Fix `DateTime` values in named time zones turning year zero and BCE dates into CE years. + +- [#9437](https://github.com/Effect-TS/effect/pull/9437) [`fed70de`](https://github.com/Effect-TS/effect/commit/fed70de0dff8af24c5f814bd0d985ab70c70420b) Thanks @effect-bot! - Add `DecisionModel.decideSettled` to retain valid answers alongside per-decision `Result` failures (`Missing`, `Refused`, `NoDistribution`, or `Invalid`). Provider failures still fail the effect; `decide` remains all-or-nothing. + + OpenRouter now passes missing distributions to core validation. OpenAI reports refusals per decision and wrong-kind score answers as invalid rather than missing. + + Update hand-built services to implement `decideSettled`, handle optional provider `probabilities` and refusal answers, and use settled outcomes instead of the removed `OpenAiContentPolicyMetadata` and `metadata.openai.refusedDecisions`. + +- [#9422](https://github.com/Effect-TS/effect/pull/9422) [`82648a2`](https://github.com/Effect-TS/effect/commit/82648a21bf228145a14d2d1390006d67f74d89c7) Thanks @candrewlee14! - `DecisionModel.decide` now returns the response `id`, the `modelId` that answered and provider `metadata`. `OpenRouterDecisionModel` reports `cost` and the serving provider under `metadata.openrouter`. + +- [#9436](https://github.com/Effect-TS/effect/pull/9436) [`afb5f32`](https://github.com/Effect-TS/effect/commit/afb5f326cc14c9849d8d7ddefde1039ae036e017) Thanks @jpowersdev! - Fix `Channel.decodeText` and `Stream.decodeText` dropping incomplete trailing characters. Flush buffered bytes on normal completion, emitting U+FFFD by default or a defect with `fatal: true`. + +- [#9325](https://github.com/Effect-TS/effect/pull/9325) [`28813f5`](https://github.com/Effect-TS/effect/commit/28813f53a5026984fdc13d3971147ea5dc20104c) Thanks @effect-bot! - Rerun reactive queries only once per invalidation or batch flush when multiple keys match the same query, avoiding duplicate query results and atom refreshes. + +- [#9692](https://github.com/Effect-TS/effect/pull/9692) [`0fae934`](https://github.com/Effect-TS/effect/commit/0fae934a13aac12f9036b0733edd5dde5ce029f3) Thanks @effect-bot! - Destroy the `AsyncResource` a fiber captures when it yields once the fiber resumes, so tools like vitest's `detectAsyncLeaks` no longer report `effect/Fiber` leaks. + +- [#9410](https://github.com/Effect-TS/effect/pull/9410) [`f260830`](https://github.com/Effect-TS/effect/commit/f2608305861269fc2fbedcbbdcd71e4fc5f98ffc) Thanks @effect-bot! - Send an initial DevTools Ping immediately so a server that only responds to client messages can complete startup without waiting for the connection timeout. + +- [#9329](https://github.com/Effect-TS/effect/pull/9329) [`19b63eb`](https://github.com/Effect-TS/effect/commit/19b63eb81b2bccf033aadb565f52a6b0375d26a1) Thanks @effect-bot! - Include disposal library types in the Effect and ManagedRuntime declarations so non-Node consumers can type-check imports without explicitly enabling `ESNext.Disposable`. + +- [#9258](https://github.com/Effect-TS/effect/pull/9258) [`14db221`](https://github.com/Effect-TS/effect/commit/14db221f8f584100ff50fd1173b1a99a1caa1d06) Thanks @devangpratap! - Fix `Duration.fromInputUnsafe` losing nanosecond precision for large integer tuple and object inputs. + +- [#9506](https://github.com/Effect-TS/effect/pull/9506) [`679f956`](https://github.com/Effect-TS/effect/commit/679f956c4a7527854ff78db071dd7aa417211a48) Thanks @tim-smart! - Use `Effect.fnUntraced` for internal functions that returned `Effect.gen`. + +- [#9604](https://github.com/Effect-TS/effect/pull/9604) [`b72707c`](https://github.com/Effect-TS/effect/commit/b72707c2937328fd3bc70f29a6792d84c4256484) Thanks @effect-bot! - Document how `Effect.result` and `Effect.option` handle causes that mix typed errors with interruptions or defects, and the parameter-count rule that decides whether `Effect.promise`, `Effect.tryPromise` and `Effect.callback` provide an `AbortSignal`. + +- [#9552](https://github.com/Effect-TS/effect/pull/9552) [`44c8c1d`](https://github.com/Effect-TS/effect/commit/44c8c1d1c6f01ba9887a051b365e9fa89fcfc685) Thanks @wmaurer! - Add an optional `batchKey` to `EmbeddingModel.make` to group concurrent `embed` calls by provider-relevant context. OpenAI and OpenAI-compatible providers now separate batches with different configuration overrides or client transforms. Undeclared context, such as per-call `FetchHttpClient.RequestInit`, still uses the first caller's value. + +- [#9148](https://github.com/Effect-TS/effect/pull/9148) [`1e7909b`](https://github.com/Effect-TS/effect/commit/1e7909beb05bc740dbd330509e60bb185d7eeed1) Thanks @fubhy! - Add `effect/encoding/Tar` for streaming tar decoding and encoding. Decode v7, USTAR, PAX and non-sparse GNU archives; encode USTAR with PAX headers as needed. Entry constructors use reproducible defaults. + +- [#9420](https://github.com/Effect-TS/effect/pull/9420) [`0611a65`](https://github.com/Effect-TS/effect/commit/0611a657f76836f9ca275f64764b82270b4eda10) Thanks @jpowersdev! - Add the experimental `Xml` module to `effect/encoding` for streaming XML 1.0 decoding and encoding over text or UTF-8 bytes. Decoding checks well-formedness, enforces size and depth limits, and never reads external resources or expands DOCTYPE entities. `elements` collects repeated elements, such as feed items, into trees. + +- [#9419](https://github.com/Effect-TS/effect/pull/9419) [`2fadc65`](https://github.com/Effect-TS/effect/commit/2fadc654aa655fbd1912ce51eb1b764a757ac362) Thanks @jpowersdev! - Add experimental `effect/encoding/Zip` for streaming zip encoding and decoding. Encode and decode stored and deflated entries, with ZIP64 when needed; decoding reads entries as the archive arrives and checks each body's CRC-32 and sizes. This API may receive breaking changes in patch releases. + +- [#9578](https://github.com/Effect-TS/effect/pull/9578) [`feb6ce5`](https://github.com/Effect-TS/effect/commit/feb6ce59a38ab15fc6f89545bd5f0992dd58a06a) Thanks @tim-smart! - Add the always-on `ClusterMetrics.entityMailboxSizeMax` gauge, which reports the largest mailbox size among the resident entities of each type, read when metrics are snapshotted. + +- [#9572](https://github.com/Effect-TS/effect/pull/9572) [`5a2ed53`](https://github.com/Effect-TS/effect/commit/5a2ed5323884a435bfcfb5e3a6f2ae29547458ad) Thanks @tim-smart! - Add the unstable `Metric.remove`, `Metric.removeUnsafe` and `Metric.registerUnsafe` APIs for removing series and registering series whose value is read on snapshot. The OTLP and OpenTelemetry exporters now drop delta state for series that are no longer registered. Add the opt-in `ClusterMetrics.entityMailboxSize` gauge, enabled with `ShardingConfig.entityMailboxMetrics` or the `mailboxMetrics` entity option, which reports the mailbox size of each resident entity and removes the series when the entity is released. + +- [#9721](https://github.com/Effect-TS/effect/pull/9721) [`c8d26a4`](https://github.com/Effect-TS/effect/commit/c8d26a46731cb26a9b9612d9fde823d8813cfc04) Thanks @Marve10s! - Fix `Equivalence.combineAll` losing its criteria after the first comparison when given a one-shot iterable. + +- [#9482](https://github.com/Effect-TS/effect/pull/9482) [`0c8b446`](https://github.com/Effect-TS/effect/commit/0c8b446709c737c10d0e11af4313bb29fdff2588) Thanks @tim-smart! - Track the source remote of each imported entry in `SqlEventJournal` and the IndexedDB `EventJournal`, instead of one row per entry per remote. + + - Storage grows with entries rather than entries times remotes. An imported entry is still relayed to other remotes and never uploaded back to its source. + - Journals from earlier releases migrate on first open: SQL drops the remotes table and IndexedDB upgrades to version 2. SQL needs a writable connection. + - An entry acknowledged by a remote, or known to several remotes before migrating, may be uploaded to a remote once more. The server treats it as a duplicate. + +- [#9448](https://github.com/Effect-TS/effect/pull/9448) [`05c3409`](https://github.com/Effect-TS/effect/commit/05c340955944374e3ae47a1de0f7cb9b41a3b0e6) Thanks @tim-smart! - Fix the memory and IndexedDB `EventJournal` advancing a remote's sequence for a batch whose callback failed, and the memory journal missing conflicts while a clock-skewed local write leaves it unordered. + +- [#9479](https://github.com/Effect-TS/effect/pull/9479) [`b04191b`](https://github.com/Effect-TS/effect/commit/b04191b2941534754282c51ef2df2f965ac43a5c) Thanks @tim-smart! - Encrypted event log replication now encrypts entries in batches of up to 100 entries or about 256 KB of payload, one AES-GCM call per batch, and encrypted servers store and stream those batches. Each batch authenticates its store id and plaintext entry ids. This changes the encrypted protocol, APIs and server tables without migration: upgrade encrypted clients and servers together, and start encrypted servers on fresh storage because existing per-entry rows are not read. + + ### Breaking changes + + - `EncryptedEntry` and `EncryptedRemoteEntry` are replaced by `EncryptedBatch` and `EncryptedRemoteBatch`, and `WriteEntries.encryptedEntries` becomes `batches`. + - `EventLogEncryption.encrypt` and `decrypt` take the store id and work on batches, and the `decodeChanges` option of `EventLogRemote.makeWith` receives the store id. + - `EventLogServerEncrypted.Storage.write` and `changes` use batches, `PersistedEntry` is removed, and `SqlEventLogServerEncrypted` no longer accepts `insertBatchSize`. + + Event journals now keep the first copy of an entry that is repeated within one remote delivery, and `SqlEventJournal` advances an entry's remote sequence when it arrives again later, so `nextRemoteSequence` passes it. + +- [#9448](https://github.com/Effect-TS/effect/pull/9448) [`05c3409`](https://github.com/Effect-TS/effect/commit/05c340955944374e3ae47a1de0f7cb9b41a3b0e6) Thanks @tim-smart! - `EventJournal.writeFromRemote` now hands its `effect` the whole batch with conflicts read up front, and `EventLog.HandlerTransaction` runs a replayed batch's handlers in one transaction. + +- [#9445](https://github.com/Effect-TS/effect/pull/9445) [`4067e64`](https://github.com/Effect-TS/effect/commit/4067e6428776ae573c8cc72f7eaebd078a4ff8c7) Thanks @tim-smart! - Flush EventLog reactivity invalidations once per replayed remote batch instead of once per entry. + +- [#9471](https://github.com/Effect-TS/effect/pull/9471) [`e04563d`](https://github.com/Effect-TS/effect/commit/e04563dcd6ddeea74c088a12194c5a73e597ba2f) Thanks @tim-smart! - Speed up UUID formatting and memoize `EventJournal.Entry.idString`. + +- [#9491](https://github.com/Effect-TS/effect/pull/9491) [`02d7fc5`](https://github.com/Effect-TS/effect/commit/02d7fc583053c47dea3b5e1d9d6be164aeb321b6) Thanks @tim-smart! - Add `EventLog.groupBatchCompaction` for compaction across primary keys, with explicit forwarding or dropping of undecodable entries. Both compaction APIs now provide `forward(entry)` to preserve entries unchanged at their original batch positions. Align server compaction output ordering with client replay. + +- [#9521](https://github.com/Effect-TS/effect/pull/9521) [`561d2a1`](https://github.com/Effect-TS/effect/commit/561d2a1ddfa3c4340510f23da7adc50c89d97863) Thanks @tim-smart! - Generate EventLog entry ids from a hybrid logical clock, so a local write always sorts after every entry committed to its journal, even when the local clock is behind. + + - The memory, IndexedDB and SQL journals generate each local id after the latest id in their store, without adopting it into the process's id sequence. The IndexedDB and SQL journals read it from the store, because other tabs or processes sharing the store keep their own sequence. This adds one read per `writeMany`. Inside an enclosing SQL transaction, local ids also sort after its uncommitted rows, and a rollback leaves nothing behind. + - Clocks that run ahead are not clamped. A warning is logged when an imported id is more than 24 hours ahead of the local clock, read from the `Clock` service. + - If a store holds an id at the end of the UUID v7 timestamp range, local writes to that store fail with an `EventJournalError` instead of committing ids that sort before it. `makeEntryIdUnsafe` throws a `RangeError` once its own sequence passes the range. Ids created with an explicit `msecs` still clamp. + +- [#9520](https://github.com/Effect-TS/effect/pull/9520) [`8fe2dcc`](https://github.com/Effect-TS/effect/commit/8fe2dcc2137788a71177d42a9f67fb7003224868) Thanks @tim-smart! - Order EventLog conflicts by entry id in every journal and the unencrypted server, so replicas converge. A conflict is now a stored entry for the same event and primary key with a strictly greater id, remote batches replay in id order after compaction, and compaction replacements are ordered and conflict-checked by the source entry they are anchored to. + +- [#9507](https://github.com/Effect-TS/effect/pull/9507) [`a8086fa`](https://github.com/Effect-TS/effect/commit/a8086fa26a2c9a9f5f7f1d7590bbc494a0feb02a) Thanks @tim-smart! - Fix event log server compaction failing with an argument-limit error on large groups, and stop scanning the whole in-memory journal on replay. + +- [#9444](https://github.com/Effect-TS/effect/pull/9444) [`88a33b4`](https://github.com/Effect-TS/effect/commit/88a33b4122f7ffbbc1423add793a79abffd99c75) Thanks @tim-smart! - Decode `EventJournal` entry timestamps without allocating, and merge remote imports into the in-memory journal instead of re-sorting it on every batch. + +- [#9508](https://github.com/Effect-TS/effect/pull/9508) [`e77372a`](https://github.com/Effect-TS/effect/commit/e77372a8a2e7caec11999636e04c6e31c52e315f) Thanks @tim-smart! - Send chunked event-log remote writes with up to four parts in flight instead of one at a time. + +- [#9480](https://github.com/Effect-TS/effect/pull/9480) [`3dd4388`](https://github.com/Effect-TS/effect/commit/3dd438810df34ed3b3d2e3e35ccf84ccb00feb85) Thanks @tim-smart! - Reduce per-entry overhead in `EventLog` remote replay by caching payload decoders per handler and merging handler contexts once per batch. + +- [#9523](https://github.com/Effect-TS/effect/pull/9523) [`ba2b00c`](https://github.com/Effect-TS/effect/commit/ba2b00c0658b65775809e0052ec9612432600c31) Thanks @tim-smart! - Add the unstable `effect/eventlog/EventRow` module for convergent row updates with `set`, `patch` and `remove` events. It supports single-field, composite and derived keys, per-field conflict resolution through typed storage callbacks, and compaction that drops superseded entries. The constructors return `{ event, payload }`, which spreads into `EventLog.write` and works as a `writeMany` item or client argument. + + Breaking: the function returned by `EventLog.makeClient` now takes one `{ event, payload }` object instead of positional `event` and `payload` arguments. + +- [#9224](https://github.com/Effect-TS/effect/pull/9224) [`a049d4a`](https://github.com/Effect-TS/effect/commit/a049d4ad25f17cf4985284a8d15b07aea2a33242) Thanks @effect-bot! - Allow event-log authentication to retry failed session-auth binding lookups after storage recovers, while continuing to cache successful lookups. + +- [#9478](https://github.com/Effect-TS/effect/pull/9478) [`84aeede`](https://github.com/Effect-TS/effect/commit/84aeede040460a2b24bcf0db7a5d0a9b0e3c42d4) Thanks @tim-smart! - Skip decrypting remote changes the local event journal already stores, and pass entry bytes to WebCrypto without copying them. `EventJournal` gains a required `entriesById` method, which custom journal implementations must add. + +- [#9526](https://github.com/Effect-TS/effect/pull/9526) [`43e5e19`](https://github.com/Effect-TS/effect/commit/43e5e198d16df038b60764f5dd0218cdcf828f16) Thanks @tim-smart! - Fail and retry an EventLog remote replay batch when an entry fails to decode or its handler fails, instead of committing the entry without applying it. + +- [#9473](https://github.com/Effect-TS/effect/pull/9473) [`77bbf16`](https://github.com/Effect-TS/effect/commit/77bbf16e50d6b0ea090288cc5d3f11b20706a027) Thanks @tim-smart! - Speed up decoding of eventlog entries and unencrypted write frames by building instances from already validated fields. The wire format is unchanged. + +- [#9477](https://github.com/Effect-TS/effect/pull/9477) [`6af3a47`](https://github.com/Effect-TS/effect/commit/6af3a47d5b5d698c3ba896c37684266cd465bef1) Thanks @tim-smart! - Acknowledge event log writes instead of echoing them to the writing connection. Breaking protocol change: upgrade clients and servers together; mixed versions are unsupported. + + Custom implementations must add `EventJournal.writeRemoteAcks`, add encrypted `Storage.entrySequences` (the sequence each entry was first stored at, used to acknowledge skipped batches), and return `RemoteEntry` values from unencrypted `Storage.entriesAfter`. Set `firstSequence` when sequences start above 0. Custom remotes can still return `void` from `write` to confirm uploads through the change stream. + + `SqlEventLogServerEncrypted` adds a sequence column to its entry id tables, so encrypted servers still need fresh storage. + +- [#9452](https://github.com/Effect-TS/effect/pull/9452) [`aac8d21`](https://github.com/Effect-TS/effect/commit/aac8d21911333dedb649f4f9d302c28d14d46369) Thanks @tim-smart! - Add `EventLog.writeMany` and `EventJournal.writeMany`, which write a batch of local events under one lock, one handler transaction and one journal commit. Clock-based entry ids are now strictly ascending within a process. + +- [#9690](https://github.com/Effect-TS/effect/pull/9690) [`84c8dc4`](https://github.com/Effect-TS/effect/commit/84c8dc47b09067362749620f75068d6f28044660) Thanks @effect-bot! - Document that `HttpClient.followRedirects` needs `FetchHttpClient.RequestInit` with `{ redirect: "manual" }` when used with `FetchHttpClient`. + +- [#9603](https://github.com/Effect-TS/effect/pull/9603) [`78d1988`](https://github.com/Effect-TS/effect/commit/78d198818d4fd547ce2e9431302f66cb00dc7ad3) Thanks @effect-bot! - Wait for replaced `FiberMap` and `FiberHandle` fibers to finish their interruption cleanup before the owning scope finishes closing. + +- [#9729](https://github.com/Effect-TS/effect/pull/9729) [`5676a2a`](https://github.com/Effect-TS/effect/commit/5676a2ac5a918f8f91fc7838d7adf1b6f5e86a6d) Thanks @Marve10s! - Fix `Fiber.runIn` and `Effect.forkIn` hanging when the attached fiber closes its own scope. + +- [#9433](https://github.com/Effect-TS/effect/pull/9433) [`4b437bf`](https://github.com/Effect-TS/effect/commit/4b437bf4645fabc8aaeaa3b5a88e8c9048b9e4f5) Thanks @effect-bot! - Give defects precedence when the runtime combines a typed failure with a defect, so typed-error recovery can no longer swallow a crashing finalizer or a dying concurrent sibling. + +- [#9290](https://github.com/Effect-TS/effect/pull/9290) [`1af11fd`](https://github.com/Effect-TS/effect/commit/1af11fda1bccf823f74121b5125aa3befa084b18) Thanks @effect-bot! - Fix `AtomHttpApi` query response mode inference so literal and union response modes are accepted and produce the corresponding success types. + +- [#9524](https://github.com/Effect-TS/effect/pull/9524) [`716f8df`](https://github.com/Effect-TS/effect/commit/716f8dfa178f849e5b288464373bd132860c2e19) Thanks @tim-smart! - Fix `Atom.Serializable` encode and decode types to use `Schema.Json`, matching the JSON codec used at runtime. + +- [#9404](https://github.com/Effect-TS/effect/pull/9404) [`ee27718`](https://github.com/Effect-TS/effect/commit/ee27718a02125328a7478812b1b7d7c1d8532578) Thanks @effect-bot! - Fix `SchemaGetter.BigInt` to return `SchemaIssue.InvalidValue` for inputs that cannot be converted to a bigint. This also makes `SchemaTransformation.bigintFromString` and `Snowflake.SnowflakeFromString` reject invalid strings with typed schema errors instead of defects. + +- [#9411](https://github.com/Effect-TS/effect/pull/9411) [`4d86d1b`](https://github.com/Effect-TS/effect/commit/4d86d1b8887d48fa24acf6ad54f33ef3d55a7cd8) Thanks @effect-bot! - Start `Stream.callback` and `Channel.callback` registration before the first pull, so synchronous listener setup no longer misses early events. Browser HTTP response stream listeners now stay installed until the stream is finalized. + +- [#9323](https://github.com/Effect-TS/effect/pull/9323) [`5462ed7`](https://github.com/Effect-TS/effect/commit/5462ed7b3a4c37206c47e401d6be293dc540350f) Thanks @effect-bot! - Fix Cause runtime JSON and inspection output to use `reasons` instead of `failures`, including within failed Exits. Schema validation error paths now also use `reasons`; Cause and Exit schema encodings are unchanged. + +- [#9400](https://github.com/Effect-TS/effect/pull/9400) [`658837c`](https://github.com/Effect-TS/effect/commit/658837c3b249d982df56f2990247499746e36672) Thanks @effect-bot! - Preserve input order in concurrent `Effect.filter` results and record `Effect.all` keys, including `mode: "result"`. + +- [#9226](https://github.com/Effect-TS/effect/pull/9226) [`00261cc`](https://github.com/Effect-TS/effect/commit/00261cc12c31ecc7878f80670439731c905dc2bd) Thanks @effect-bot! - Reconnect the DevTools client after startup connection failures or dropped connections, and log connection failures at Debug level. Stop failed or pending connections on shutdown while preserving queued telemetry flushing for healthy connections. + +- [#9261](https://github.com/Effect-TS/effect/pull/9261) [`7cccb93`](https://github.com/Effect-TS/effect/commit/7cccb9385729bc0aed6630edea95d7db15439ac4) Thanks @effect-bot! - Preserve nanosecond precision when parsing large decimal Duration strings in units from milliseconds through weeks, and avoid intermediate overflow when converting large finite durations to numeric units. + +- [#9403](https://github.com/Effect-TS/effect/pull/9403) [`7c98228`](https://github.com/Effect-TS/effect/commit/7c982281cf73958833054175fc86db385e043cf6) Thanks @effect-bot! - Fix `Effect.effectify` discarding synchronous throws after a synchronous callback. These throws now become defects by default, or typed failures when `onSyncError` is provided. + +- [#9469](https://github.com/Effect-TS/effect/pull/9469) [`5a8e248`](https://github.com/Effect-TS/effect/commit/5a8e2481546352e2ad6233dad49420b14c59f624) Thanks @tim-smart! - Fix encrypted SQL event-log storage to return and publish only newly inserted entries, avoiding duplicate broadcasts when clients re-upload a batch. + +- [#9227](https://github.com/Effect-TS/effect/pull/9227) [`8037a65`](https://github.com/Effect-TS/effect/commit/8037a65f40cae9163ab9e0e687e289f717ae570b) Thanks @effect-bot! - Preserve undecodable event payloads unchanged during group compaction so they do not stall replication or prevent valid events from being compacted. + +- [#9393](https://github.com/Effect-TS/effect/pull/9393) [`25912db`](https://github.com/Effect-TS/effect/commit/25912db9d493227a2d6fa45cebcfc7507e638d8a) Thanks @effect-bot! - Preserve remote replay order for unaffected events when compaction is enabled, and replay replacements in compactor write order at their source position. + +- [#9457](https://github.com/Effect-TS/effect/pull/9457) [`7f32606`](https://github.com/Effect-TS/effect/commit/7f32606baac43a67d171ae93b77c7b29791c9692) Thanks @tim-smart! - Fix IndexedDB event journal uploads resending entries the remote has not echoed back. Upload progress is now checkpointed by insertion order, so entries imported later with older ids are still forwarded. Existing databases upgrade automatically; entries the remote has not confirmed may be uploaded once more after the upgrade. + +- [#9239](https://github.com/Effect-TS/effect/pull/9239) [`0386f88`](https://github.com/Effect-TS/effect/commit/0386f88295ac70cbc1f90ab8cceb0981d848c628) Thanks @effect-bot! - Prevent `WorkflowEngine.layerMemory` from running duplicate compensations after workflow replays. Each replay now starts with a fresh workflow scope, as with the cluster engine, so only the final run's compensations run. + + Finalizers registered in runs that suspended still run once the execution finishes, in the same order, but they now observe `Exit.void` instead of the execution's final exit. + +- [#9237](https://github.com/Effect-TS/effect/pull/9237) [`403298a`](https://github.com/Effect-TS/effect/commit/403298ac457fb634eca2c013fed26d37fc80445f) Thanks @effect-bot! - Align safe interruption in `WorkflowEngine.layerMemory` with the cluster engine: workflows that finish without suspending now complete normally instead of being interrupted and running compensation. Preserve unsafe interruption across suspended replays. + +- [#9203](https://github.com/Effect-TS/effect/pull/9203) [`5a3ebc0`](https://github.com/Effect-TS/effect/commit/5a3ebc082124f9e10106b2347736e66fc74ebc36) Thanks @effect-bot! - Allow absolute HTTP URLs in opaque-origin documents, such as sandboxed srcdoc iframes, by ignoring an invalid inferred base URL. Preserve relative URL resolution when the document provides a valid base. + +- [#9390](https://github.com/Effect-TS/effect/pull/9390) [`fac2191`](https://github.com/Effect-TS/effect/commit/fac21911964b742a38e1d4e73bd4f95aec4911dc) Thanks @effect-bot! - Fix rate limiter delay mode to charge oversized token requests against the shared store, preserving queue ordering and delay/reset timing for fixed-window and token-bucket limits. + +- [#9600](https://github.com/Effect-TS/effect/pull/9600) [`fdb12dc`](https://github.com/Effect-TS/effect/commit/fdb12dcd61b6505eed02e6f7a3c88a85d4c48e8d) Thanks @effect-bot! - Fix `RequestResolver.persisted` to persist only refresh results when using `staleWhileRevalidate`. Whole-batch refresh failures now replace stale values, while refreshes that complete no entries leave the original expiry unchanged. + +- [#9519](https://github.com/Effect-TS/effect/pull/9519) [`97909b8`](https://github.com/Effect-TS/effect/commit/97909b89b167da54a56e6f82f1c4db8bfedae04f) Thanks @tim-smart! - Fix dependency tracking for hydrated plain serializable atoms. They now build on first read or mount like wrapped atoms, so hydrated effect and stream atoms run on mount after SSR instead of skipping that initial run. + +- [#9241](https://github.com/Effect-TS/effect/pull/9241) [`c3983f4`](https://github.com/Effect-TS/effect/commit/c3983f4db2b3052a58e8045cce270fd7ccc4e2b2) Thanks @effect-bot! - Preserve fractional token counts in the Redis fixed-window rate limiter so remaining capacity, excess-request rejection, and delay reservations are calculated correctly. + +- [#9568](https://github.com/Effect-TS/effect/pull/9568) [`3a5e65e`](https://github.com/Effect-TS/effect/commit/3a5e65e409c9569e23bb7af61df401347e9ec420) Thanks @tim-smart! - Run each `RequestResolver.fromEffect` entry with its own caller context, without inheriting another caller's services. + +- [#9386](https://github.com/Effect-TS/effect/pull/9386) [`8b40fbc`](https://github.com/Effect-TS/effect/commit/8b40fbc93caf52f3a4c69020849f0051f100be65) Thanks @effect-bot! - Fix `Resource.auto` to wait for the first scheduled run before refreshing, avoiding an extra acquisition immediately after construction. + +- [#9379](https://github.com/Effect-TS/effect/pull/9379) [`3f3e119`](https://github.com/Effect-TS/effect/commit/3f3e119ad61c2167bc6c7170c3fb68cd1eb3421e) Thanks @boozedog! - Fix TestSchema round-trip assertions to use schema equivalence, avoiding false failures for object prototypes and signed zeros. + +- [#9317](https://github.com/Effect-TS/effect/pull/9317) [`7c7b8ed`](https://github.com/Effect-TS/effect/commit/7c7b8eda54e44f9cddc1188f994f7fdddf901055) Thanks @effect-bot! - Fix hooks and parse options being ignored or reused across calls to `Schema.toStandardSchemaV1`. Both `Schema.toStandardSchemaV1` and `Schema.toStandardJSONSchemaV1` now return fresh shallow copies and compose in either order. Access statics that use `this.#field` through the original class. + +- [#9383](https://github.com/Effect-TS/effect/pull/9383) [`d4f71bc`](https://github.com/Effect-TS/effect/commit/d4f71bc42507b638d149fb8806d26ea626d17956) Thanks @effect-bot! - Fix unordered `Stream.mapEffect`, `Stream.flattenEffect`, and `Stream.bindEffect` dropping in-flight results when concurrency is `Infinity` or an oversized positive number. These concurrency values now use the same safe bound as `"unbounded"`, so successful completion waits for pending effects. + +- [#9397](https://github.com/Effect-TS/effect/pull/9397) [`c66241e`](https://github.com/Effect-TS/effect/commit/c66241e791c4b51d8a65051142d01682590f9bf8) Thanks @effect-bot! - Invalidate atom Stream mutation reactivity keys only after successful completion, rather than on failure, defects, interruption, or replacement by a newer write. + +- [#9225](https://github.com/Effect-TS/effect/pull/9225) [`f0e941e`](https://github.com/Effect-TS/effect/commit/f0e941ee2f90f0a891d1228a90da91386f0258cb) Thanks @effect-bot! - Fix unencrypted event-log replication to acknowledge already committed entries without re-running their handlers, even when storage order differs from timestamp order or entries share a timestamp. Exclude older entries from handler conflicts regardless of storage order. + +- [#9466](https://github.com/Effect-TS/effect/pull/9466) [`5f8a6e5`](https://github.com/Effect-TS/effect/commit/5f8a6e5108aa6c6cd4043a3c9f4dc93d43369623) Thanks @effect-bot! - Prevent unhandled errors from Node ws WebSockets when a pending handshake is closed after an open timeout or fiber interruption. + +- [#9549](https://github.com/Effect-TS/effect/pull/9549) [`2800a09`](https://github.com/Effect-TS/effect/commit/2800a0910f7f4ad7cfa21dfc840edf8fe6e67ca7) Thanks @effect-bot! - Flatten redundant nested `anyOf` alternatives in `Schema.toJsonSchemaDocument` output while preserving annotations, references, and `oneOf` boundaries. + +- [#9319](https://github.com/Effect-TS/effect/pull/9319) [`e9749b0`](https://github.com/Effect-TS/effect/commit/e9749b04c10badabcdf71e6958de5ac56a5316e9) Thanks @effect-bot! - Prevent atom hydration from replacing newer AsyncResult values with older dehydrated state. + +- [#9468](https://github.com/Effect-TS/effect/pull/9468) [`eaef09c`](https://github.com/Effect-TS/effect/commit/eaef09c11b6ed2029c513eea0aa3256dcc6f517f) Thanks @tim-smart! - Fix `EventLog` remote reconnects resuming from the sequence captured at startup instead of the journal's current sequence, which replayed already-written entries and re-triggered reactivity invalidations. + +- [#9569](https://github.com/Effect-TS/effect/pull/9569) [`a3c56e5`](https://github.com/Effect-TS/effect/commit/a3c56e5b5e756e879f1cfb90230e8eae620e33df) Thanks @tim-smart! - Pre-response handlers and `HttpMiddleware.withLoggerDisabled` now apply to exactly one server request. `HttpApiTest` no longer accumulates handlers across retry attempts that reuse the same client request. `HttpEffect.appendPreResponseHandlerUnsafe` now takes the fiber context instead of the request. + +- [#9385](https://github.com/Effect-TS/effect/pull/9385) [`bb61152`](https://github.com/Effect-TS/effect/commit/bb611522fcfb4d317b33acdf5ee24a480d042435) Thanks @effect-bot! - Restart the random sequence from the seed on each evaluation of an effect wrapped with `Random.withSeed`. This also ensures repeated evaluations of `TestCrypto.make` and independent builds of `TestCrypto.layer` start fresh deterministic sequences. + +- [#9240](https://github.com/Effect-TS/effect/pull/9240) [`e9c842b`](https://github.com/Effect-TS/effect/commit/e9c842bc294abf658d698bb17732ae13465b3817) Thanks @effect-bot! - Fix SQL persistence reads to check expiry on every execution, so reused `get` and `getMany` effects stop returning expired entries in both SQL layouts. + +- [#9401](https://github.com/Effect-TS/effect/pull/9401) [`6d8c8e2`](https://github.com/Effect-TS/effect/commit/6d8c8e2075b15ae84ae96e93a0d295cb9221b496) Thanks @effect-bot! - Apply GraphQL resolver batch limits independently per protocol URL and `batchKey`, so interleaved lookups for different tenants or servers do not fragment each other's batches. `GraphQLResolver.request` resolves each lookup's URL in the calling fiber, so lookups that rely on a `Context.Reference` default are no longer routed with another lookup's value. + +- [#9536](https://github.com/Effect-TS/effect/pull/9536) [`1f28187`](https://github.com/Effect-TS/effect/commit/1f281878ef559ef356af2d0e11b381aba857cae1) Thanks @schutzelaars! - Fix `HashMap.set` discarding equal-but-distinct replacement values. + +- [#9548](https://github.com/Effect-TS/effect/pull/9548) [`0b9842b`](https://github.com/Effect-TS/effect/commit/0b9842b76bbc1b016c8abc46432052e1f60b2fa3) Thanks @effect-bot! - Load the bundled Scalar and Swagger UI assets when `HttpApiScalar.layer` or `HttpApiSwagger.layer` is built instead of when `effect/http-api` is imported. Importing the module no longer evaluates about 5 MB of embedded JavaScript and CSS. + +- [#9575](https://github.com/Effect-TS/effect/pull/9575) [`c29ddd8`](https://github.com/Effect-TS/effect/commit/c29ddd899c53c5d8194c4a0dfef2134304dda799) Thanks @tim-smart! - Add streaming request payloads to `HttpApi`. An endpoint can declare `HttpApiSchema.StreamUint8Array` or the new `HttpApiSchema.StreamNdjson({ schema })` as its `payload`, and the handler receives a lazy `Stream` that reads the body as it pulls. Bad NDJSON lines fail the stream with `HttpApiSchemaError` (400). The new `maxBytes` and `maxElementBytes` limits fail it with the new `HttpApiPayloadTooLarge` (413), and a `content-length` over `maxBytes` is rejected before the handler runs. `StreamNdjson` also works as a success response. + + `HttpApiClient` sends a `Stream` payload without buffering it and infers the stream's requirements. `HttpApiTest` passes stream payloads through unbuffered. `OpenApi.fromApi` marks streaming request bodies with `x-effect-stream`, and `openapi-generator` maps them, and NDJSON or JSON Lines bodies with an `itemSchema`, back to `StreamUint8Array` and `StreamNdjson`. + + The Node server drains up to 256 KiB of a body the handler left unread, then closes the connection, instead of reading the whole upload or leaving the socket stuck. + +- [#9563](https://github.com/Effect-TS/effect/pull/9563) [`84c3405`](https://github.com/Effect-TS/effect/commit/84c34057ceab9fe6782b73f4a04a31304e0d14de) Thanks @tim-smart! - Allow undeclared `HttpServerRespondable` errors in `HttpApiBuilder` to render their own responses when error schema encoding fails. Oversized multipart streaming uploads now return 413 instead of 500, while declared error schemas retain precedence. + +- [`3934c0c`](https://github.com/Effect-TS/effect/commit/3934c0c61857c03468c08c2477f49a6b977d28ff) Thanks @tim-smart! - Return 413 instead of 400 when buffered or decompressed request bodies exceed their size limit, while preserving 400 for other parse failures. Node HTTP servers drain oversized bodies so the response can reach the client without destroying the socket. Node stream readers expose `MaxBytesError` and an optional `destroyOnOverflow` setting; their default destruction behavior is unchanged. + +- [#9728](https://github.com/Effect-TS/effect/pull/9728) [`048e7c8`](https://github.com/Effect-TS/effect/commit/048e7c87125c4ace5a61895feac3d049a317bbfc) Thanks @Marve10s! - Fix `HttpClient.followRedirects` dying on an invalid `Location` header; it now returns the redirect response unchanged. + +- [#9673](https://github.com/Effect-TS/effect/pull/9673) [`1121334`](https://github.com/Effect-TS/effect/commit/1121334ce68c379051011a2ffea37b0a30ea9a47) Thanks @effect-bot! - Add `HttpClientResponse.toWeb`, which returns the original Web `Response` of a fetch-backed response as `Some`, and `None` for other backends. The original body keeps runtime capabilities such as workerd's known length, so it can be written to Cloudflare R2 without copying. + +- [#9339](https://github.com/Effect-TS/effect/pull/9339) [`9a680d6`](https://github.com/Effect-TS/effect/commit/9a680d6985c6a27d7ebb29cd9eb674f3fe524505) Thanks @fubhy! - Add raw HTTP/1.1 upgrades through `HttpClientRequest.upgrade`, `HttpClientResponse.upgrade`, the Node HTTP server, and `HttpApiSchema.Upgrade`. Node clients expose the upgraded connection as a `Socket`; HttpApi clients return `{ headers, socket }` and OpenAPI documents the 101 response. + + Custom `HttpClientResponse` implementations must add `upgrade`. Fetch and XMLHttpRequest reject upgrade requests with a `TransportError` before anything is sent; Bun and Deno servers still support only WebSocket upgrades. + +- [#9709](https://github.com/Effect-TS/effect/pull/9709) [`85e2aee`](https://github.com/Effect-TS/effect/commit/85e2aeee686963c26ca40d204f982deebb9d6117) Thanks @tim-smart! - Add typed HTTP date and `Retry-After` header values to `Headers`, and parse them consistently across the HTTP client, OTLP exporter, GraphQL client, AI clients and static server. + + - `Headers.parseHttpDate` / `Headers.formatHttpDate` read all three RFC 9110 date formats as UTC, whatever the process time zone, and emit IMF-fixdate. + - `Headers.parseRetryAfter`, `Headers.formatRetryAfter` and `Headers.retryAfterDelay` accept whole non-negative seconds of any size, read exactly, or an HTTP date, and the delay is never negative. HTTP dates accept the `23:59:60` leap second, read as the following second. + - `Headers.RetryAfter` holds either a `Duration` (`RetryAfterDuration`) or a `DateTime.Utc` (`RetryAfterDateTime`). `Headers.retryAfterDuration` builds the first from any `Duration.Input`: infinite durations throw, finite negative ones become zero, and fractional seconds are rounded up when formatted. + - Parse failures are reported as `Headers.HeaderParseError`. + - `Schema.DateTimeUtcFromHttpDate` and `Schema.RetryAfterFromString` expose the same parsing as string codecs for HTTP API headers. + - Invalid `Retry-After` values such as `-5`, `1.5`, `1e3` or `Infinity` are now ignored everywhere. `HttpClient.retryTransient` and `HttpClient.withRateLimiter` no longer honour fractional seconds, and the AI clients no longer produce negative or infinite `retryAfter` durations. + - `HttpStaticServer` reads `If-Modified-Since` and `Last-Modified` as UTC; asctime dates were previously shifted by the process time zone. + +- [#9293](https://github.com/Effect-TS/effect/pull/9293) [`34ec440`](https://github.com/Effect-TS/effect/commit/34ec4407a14ce045ca14cc76c77582884a7e1ebe) Thanks @bezff! - Keep HTTP message bodies unread during inspection. + +- [#8691](https://github.com/Effect-TS/effect/pull/8691) [`343a149`](https://github.com/Effect-TS/effect/commit/343a149fa7d5a9a0a45841b1b6800b0ecfa16a32) Thanks @Marve10s! - Document how to define `HttpMiddleware.make` middleware that preserves app error and service types, including with `Effect.fn`. + +- [#9447](https://github.com/Effect-TS/effect/pull/9447) [`915117b`](https://github.com/Effect-TS/effect/commit/915117b523cb6e5cbc98b5ff1c119a8d0d1f6b9a) Thanks @bezff! - Describe `HttpRouter` route misses caused by a path parameter longer than `maxParamLength`. + +- [#9417](https://github.com/Effect-TS/effect/pull/9417) [`7f823c1`](https://github.com/Effect-TS/effect/commit/7f823c10d20ef53a1081e0d0e2008a4116140799) Thanks @fubhy! - Fix HTTP response framing: omit bodies for 204, 205 and 304, and remove `content-length` for 204 and 205. Node now aborts responses whose bodies do not match `content-length`. Deno and Bun preserve raw Web `Response` objects when merging headers, and Deno preserves their status and headers for HEAD requests. + +- [#9415](https://github.com/Effect-TS/effect/pull/9415) [`6e614e6`](https://github.com/Effect-TS/effect/commit/6e614e6feb4281a9e81f5ec3be48e7905ff4292c) Thanks @fubhy! - - Fix HTTPS detection in `HttpServerRequest.toURL` and tracing. An exact `http` or `https` in `x-forwarded-proto` takes precedence; otherwise, infer HTTPS from `originalUrl`. Paths starting with `//` preserve the request host. + - Node server requests now expose an absolute `originalUrl` when a Host header is present, fixing `HttpMiddleware.searchParamsParser`. Use `url` instead of `originalUrl` to read the path. Hostless requests are unchanged. + - For absolute-form Node requests, derive `url` and `headers.host` from the request target, ignoring a conflicting Host header. + +- [#9726](https://github.com/Effect-TS/effect/pull/9726) [`9990b39`](https://github.com/Effect-TS/effect/commit/9990b3977a544247cd4123edde8353a55bf77192) Thanks @Marve10s! - Remove path parameter regex constraints from `HttpApiClient` URLs and `OpenApi` paths. + +- [#6837](https://github.com/Effect-TS/effect/pull/6837) [`22bf781`](https://github.com/Effect-TS/effect/commit/22bf7812aa8a20c676efade943c342cde4ef1cbd) Thanks @aryasaatvik! - Fix `HttpApiSchema.status` forking named schemas, so a schema reused across response statuses emits a single OpenAPI component. + +- [#9518](https://github.com/Effect-TS/effect/pull/9518) [`41f08b7`](https://github.com/Effect-TS/effect/commit/41f08b75aba40e95cd6c743ec6fcf1f5022f7b5b) Thanks @pujitha24! - Fix `Hydration.hydrate` to notify existing atom subscribers immediately instead of on the next read. + +- [#9476](https://github.com/Effect-TS/effect/pull/9476) [`e8482ca`](https://github.com/Effect-TS/effect/commit/e8482cad9651fa8fe8036fe6292a286994897d0d) Thanks @tim-smart! - Index duplicate and conflict detection in the unencrypted event-log server once per incoming batch instead of rescanning stored history for every entry. `EventLogServerUnencrypted.Storage.entriesAfter` now receives the batch as an optional third argument. The built-in storages use it to read only matching entries, and SQL storage creates a `_event_key_idx` index on startup for SQLite, Postgres and MySQL. Custom storages that ignore the argument keep working. + +- [#9474](https://github.com/Effect-TS/effect/pull/9474) [`5ae85fa`](https://github.com/Effect-TS/effect/commit/5ae85fa8ac1fc5e12d5fe8fe7388ac504788ae3b) Thanks @tim-smart! - Speed up conflict detection in the IndexedDB event journal. Remote writes now read only the stored entries that share an event and primary key with the batch, through a new `[event, primaryKey, id]` index, and no longer re-validate stored rows. Existing databases upgrade to version 3, which only adds the index. + +- [#9609](https://github.com/Effect-TS/effect/pull/9609) [`c477a0a`](https://github.com/Effect-TS/effect/commit/c477a0a8010bd0a988221bc00c446848314e1c3c) Thanks @itisrmk! - Fix `Ini.parse` dropping a trailing backslash from unquoted keys, values and section names. + +- [#9267](https://github.com/Effect-TS/effect/pull/9267) [`8fbb39e`](https://github.com/Effect-TS/effect/commit/8fbb39eb92577915831f19a83b51be376e844613) Thanks @effect-bot! - Isolate `AtomHttpApi` and `AtomRpc` mutations by service ID so services sharing endpoint names or RPC tags invoke their own clients with their own arguments. + +- [#9493](https://github.com/Effect-TS/effect/pull/9493) [`1c70828`](https://github.com/Effect-TS/effect/commit/1c7082847453e9c2be0cc267a39ca554c52f4013) Thanks @Marve10s! - Fix `Iterable.dedupeAdjacent` and `Iterable.dedupeAdjacentWith` overflowing the stack on long runs of equal elements. + +- [#9431](https://github.com/Effect-TS/effect/pull/9431) [`8207d1e`](https://github.com/Effect-TS/effect/commit/8207d1e91ead8aac8f01d9e13b49624b64237c8b) Thanks @jpowersdev! - Add `JsonArray` to `effect/encoding`. Its decoders stream the elements of one JSON array, at the root or selected by a JSON Pointer, without buffering the whole document; its encoders write a stream of values as one JSON array. + +- [#9247](https://github.com/Effect-TS/effect/pull/9247) [`7886b84`](https://github.com/Effect-TS/effect/commit/7886b847658664f64392a14efda86f44b3d1e170) Thanks @gcanti! - Add `includeAnnotationKey` to `SchemaRepresentation.fromJsonSchemaDocument` and `fromJsonSchemaMultiDocument` to preserve selected custom JSON metadata as schema annotations. Selected annotations survive normalization of retained values and can be re-exported with the exporter's matching option. Discarded annotated references no longer force resolution, while surviving annotated circular aliases are consistently rejected, including annotations introduced through `allOf`. + +- [#9252](https://github.com/Effect-TS/effect/pull/9252) [`4839c60`](https://github.com/Effect-TS/effect/commit/4839c601bdda845d9a54d77f578f64372475ab04) Thanks @nikelborm! - Made catchIf not put Refinement function arg as generic Err param + +- [#9324](https://github.com/Effect-TS/effect/pull/9324) [`2595d83`](https://github.com/Effect-TS/effect/commit/2595d831f702d4a748f97e33e71fd5250f58af56) Thanks @effect-bot! - Fix `Effect.fromOption` to evaluate a custom `onNone` error factory on each execution rather than during construction. Exceptions thrown by the factory now become runtime defects. + +- [#9723](https://github.com/Effect-TS/effect/pull/9723) [`f5ae278`](https://github.com/Effect-TS/effect/commit/f5ae27827a82db3f8973b70745fc6f14e7d8fa80) Thanks @Marve10s! - Fix `ManagedRuntime` interrupting requests that dispose it before their first async step. + +- [#8862](https://github.com/Effect-TS/effect/pull/8862) [`7a8ce00`](https://github.com/Effect-TS/effect/commit/7a8ce0098a5f22cbb6c110b16e850fb3fcbcadf2) Thanks @lloydrichards! - Add the experimental `McpClient` and `McpClientError` modules, a scoped MCP client for the `2025-11-25` and `2026-07-28` protocols over stdio or Streamable HTTP. It discovers tools, prompts and resources, calls tools with optional schema decoding, and exposes discovered tools to a language model through `McpClient.toolkit`. + + MCP schemas now check `_meta`, `structuredContent`, capability records and other JSON-valued fields only one level deep when decoding, and type them as `unknown` or `Readonly>`. `McpServer` still rejects non-JSON tool results when it serializes 2026-07-28 responses. + +- [#8553](https://github.com/Effect-TS/effect/pull/8553) [`688d302`](https://github.com/Effect-TS/effect/commit/688d30275ff114e50b4f0d343c6227ee33d67348) Thanks @Floriferous! - Stop publishing a non-recursive MCP tool schema root in `$defs` as well as inline, so `tools/list` no longer sends it twice. A custom `toJsonSchema` `$ref` into the root's own definition, such as `#/$defs/Note/properties/title` when `Note` is the tool schema, no longer resolves. + +- [#9634](https://github.com/Effect-TS/effect/pull/9634) [`c7072e4`](https://github.com/Effect-TS/effect/commit/c7072e46356a19615625ae0fa569b1c3a79d50be) Thanks @tim-smart! - Memoize composed layers by identity, as in v3. `Layer.provide`, `Layer.provideMerge` and `Layer.mergeAll` (and the operators built on them) now check the `MemoMap`, so a composed layer referenced from several parents is built once per memo map instead of once per path. Deeply shared `.Default`-style layers no longer rebuild their whole chain on every path, which could leave millions of scopes open at startup. + + A composed layer that is shared now shares everything inside it, including dependencies wrapped in `Layer.fresh`. Wrap the composed layer itself in `Layer.fresh` to get a separate copy. As with leaf layers, a shared build runs on the first requester's fiber, so if that fiber is interrupted, every requester waiting on the shared build sees the interruption. + +- [#9234](https://github.com/Effect-TS/effect/pull/9234) [`425cedb`](https://github.com/Effect-TS/effect/commit/425cedb5801c9527c116a64d280d9e7c2820f922) Thanks @effect-bot! - Fix `WorkflowEngine.layerMemory` so `interruptUnsafe` completes running and suspended workflows with an interrupted exit without requiring an external wake. + +- [#9598](https://github.com/Effect-TS/effect/pull/9598) [`88bc680`](https://github.com/Effect-TS/effect/commit/88bc68001c6c70984b03a5500232e9b5aab14085) Thanks @effect-bot! - Fail SQL migrations with a typed `MigrationError` instead of a defect, so `Effect.catchTag("MigrationError")` and retries can handle them. + +- [#9668](https://github.com/Effect-TS/effect/pull/9668) [`268e09c`](https://github.com/Effect-TS/effect/commit/268e09c94c9e8f46142b5c9f29ce0a22bfbbc3aa) Thanks @effect-bot! - Fix `Model.Class` and `VariantSchema` class `.extend` to accept variant fields and regenerate every variant schema from the merged fields. + +- [#9715](https://github.com/Effect-TS/effect/pull/9715) [`7e94505`](https://github.com/Effect-TS/effect/commit/7e94505edfc9d81ea08cb96a7d935806495bf61e) Thanks @tim-smart! - Fix SQL Server deadlocks in the SQL `PersistedQueue` store. Offers no longer use a serializable `MERGE`, and claims skip rows locked by other claims or settlements. + +- [#9699](https://github.com/Effect-TS/effect/pull/9699) [`afa5da3`](https://github.com/Effect-TS/effect/commit/afa5da3c8a9848215e49535dd3d39179bfd67715) Thanks @effect-bot! - Fix `MultipartParser` handling of `Content-Disposition`: decode RFC 8187 `filename*` values with an optional language tag in UTF-8 or ISO-8859-1, and report malformed disposition headers as `InvalidDisposition` instead of silently producing a nameless field. + +- [#9575](https://github.com/Effect-TS/effect/pull/9575) [`c29ddd8`](https://github.com/Effect-TS/effect/commit/c29ddd899c53c5d8194c4a0dfef2134304dda799) Thanks @tim-smart! - `Ndjson` byte decoders split lines on raw bytes and decode each line as strict UTF-8. They accept `\n`, `\r\n` and a bare `\r`, skip a leading BOM, and take a `maxLineBytes` option. `NdjsonError` now has a `reason` (`"InvalidJson"`, `"InvalidUtf8"` or `"LineTooLarge"`) and an `unterminated` flag. Byte and string decoders emit the lines before the first bad line, then fail. + +- [#9430](https://github.com/Effect-TS/effect/pull/9430) [`48102ac`](https://github.com/Effect-TS/effect/commit/48102ac592ef89fed9d5d10afd2daece0a0ee310) Thanks @effect-bot! - Add non-retryable `AiError.InvalidResponseError` for provider response schema and SSE decoding failures, with optional HTTP context; model-generated output validation remains retryable. Update exhaustive matches on `AiErrorReason` to handle the new tag, and update serialized-error consumers to accept it. + +- [#9270](https://github.com/Effect-TS/effect/pull/9270) [`463eca3`](https://github.com/Effect-TS/effect/commit/463eca3609a6a9681247abbcc71a6f10b261a1d9) Thanks @Marve10s! - Fix `Number.nextPow2` returning the wrong power of two for some large inputs, such as `2 ** 29` and `2 ** 52 + 1`. + +- [#9694](https://github.com/Effect-TS/effect/pull/9694) [`e7162ea`](https://github.com/Effect-TS/effect/commit/e7162eae7f04af1d583b1f31fdd767f91ec1c82e) Thanks @effect-bot! - Render bare numbers with a standard leading integer check as `Schema.Int`, preserving custom annotations and other checks. Only emit the `decodeError` helper in generated HTTP clients when an operation has error responses. + +- [#7340](https://github.com/Effect-TS/effect/pull/7340) [`95752db`](https://github.com/Effect-TS/effect/commit/95752db5eb3b6e59fb8749e9792704f8b167f36c) Thanks @astahmer! - Add an `includeAnnotationKey` option to `OpenApi.fromApi` that selects custom schema annotation keys, such as `x-*` vendor extensions, to include in the generated OpenAPI schemas. It mirrors the `Schema.toJsonSchemaDocument` option of the same name, which `fromApi` previously did not forward. + +- [#9462](https://github.com/Effect-TS/effect/pull/9462) [`4577a28`](https://github.com/Effect-TS/effect/commit/4577a28bcdb50b0d184775f8c1b7ad890d86e255) Thanks @tobimori! - Support the nullable single-member `allOf` tooling convention in `JsonSchema.fromSchemaOpenApi3_0` when no string `type` or `anyOf` is present. Sibling constraints are preserved and may still reject null. + +- [#9199](https://github.com/Effect-TS/effect/pull/9199) [`973d599`](https://github.com/Effect-TS/effect/commit/973d599a3e8611626764a0e3e8f79edbcf30dc7c) Thanks @jayhemnani9910! - Fix OTLP integer attributes outside the signed 64-bit range causing collectors to reject the export. They are now sent as `doubleValue` for numbers and `stringValue` for bigints, and large in-range numbers keep their exact value in JSON. + +- [#9198](https://github.com/Effect-TS/effect/pull/9198) [`2f066a5`](https://github.com/Effect-TS/effect/commit/2f066a58e90316dd88a219fcea94200beda54219) Thanks @jayhemnani9910! - Fix `OtlpSerialization.layerJson` writing `NaN` and `Infinity` as `null`, which made OTLP collectors reject the whole export. + +- [#9334](https://github.com/Effect-TS/effect/pull/9334) [`f0ccbb6`](https://github.com/Effect-TS/effect/commit/f0ccbb6917be5c08364b0b8adbd0c4c19e7f2de1) Thanks @effect-bot! - Make overlapping `ManagedRuntime.dispose` and `ManagedRuntime.disposeEffect` calls wait for the disposal in progress, and make the `dispose` function returned by `HttpEffect.toWebHandlerLayerWith` (and `HttpRouter.toWebHandler`) return the same promise on every call. Fibers started by the runtime, their child fibers and its finalizers still return immediately when they dispose it. + +- [#9615](https://github.com/Effect-TS/effect/pull/9615) [`2dda6ce`](https://github.com/Effect-TS/effect/commit/2dda6cefc641dbea2ea3279a9b96c0332b277a40) Thanks @Marve10s! - Fix `Path.fromFileUrl` throwing a `URIError` instead of failing with `BadArgument` when the URL contains a malformed percent-encoding. + +- [#8676](https://github.com/Effect-TS/effect/pull/8676) [`3da7c0a`](https://github.com/Effect-TS/effect/commit/3da7c0aabab07c8fc682aea7559536bf31705853) Thanks @rchasman! - With `SpanPropagationEnabled`, statement spans record `db.client.connection.wait_time_ms`, and in `@effect/sql-pg` a statement that waited for a new session gets a `db.connect` child span covering the part of the connect it waited for. + +- [#9222](https://github.com/Effect-TS/effect/pull/9222) [`5a67e0b`](https://github.com/Effect-TS/effect/commit/5a67e0b3a8118dd7be709a56d6d7af4f10e9c4f3) Thanks @effect-bot! - Fix `SqlRunnerStorage` hanging on PGlite. + + `SqlClient` now has a `driver` field identifying the driver that created it, such as `"pg"`, `"pglite"` or `"sqlite-node"`. Every Effect SQL driver sets it. On PGlite, runner storage no longer reserves the only connection and uses lock table rows instead of advisory locks, so runner and application queries are no longer blocked. + + Third-party drivers must pass a `driver` name to `SqlClient.make`, and hand-written `SqlClient` implementations must provide a `driver` field: + + ```ts + const make = Effect.gen(function*() { + return yield* SqlClient.make({ + driver: "my-driver", + acquirer, + compiler, + spanAttributes: [] + }) + }) + ``` + +- [#9498](https://github.com/Effect-TS/effect/pull/9498) [`73b3a6e`](https://github.com/Effect-TS/effect/commit/73b3a6e3c38795400251d9c5d73f05c88acd90cd) Thanks @schutzelaars! - Fix unnecessary value retention in `PubSub` replay buffers and `SubscriptionRef`. + +- [#9446](https://github.com/Effect-TS/effect/pull/9446) [`c6f9e4c`](https://github.com/Effect-TS/effect/commit/c6f9e4cbe6f9c8f0a03fefde18df4d35e98a24df) Thanks @jayhemnani9910! - Fix `PubSub.takeBetween` throwing `RangeError` on large backlogs. + +- [#9495](https://github.com/Effect-TS/effect/pull/9495) [`be02676`](https://github.com/Effect-TS/effect/commit/be026768ee29a1b1a860c1fd17f2a0896c01f61e) Thanks @Marve10s! - Fix reused `PubSub.takeUpTo` effects returning fewer messages after consuming replayed messages. + +- [#9458](https://github.com/Effect-TS/effect/pull/9458) [`26de861`](https://github.com/Effect-TS/effect/commit/26de86100cda8617e077ce66f0a6bb8dfe738fb6) Thanks @tim-smart! - Avoid loading Node's internal undici on cold start when importing `effect/Schema` or starting a Node HTTP server before any web API is used. `Schema.FormData` resolves the global constructor during validation, and `@effect/platform-node` loads `node:http` and `node:https` without evaluating their lazy web API exports. + +- [#9322](https://github.com/Effect-TS/effect/pull/9322) [`9313799`](https://github.com/Effect-TS/effect/commit/931379911a74d5e2207859c295b1acc6a5e2917e) Thanks @effect-bot! - Stop failed `Reactivity.query` and `SqlClient.reactiveMailbox` queries from rerunning on later invalidations, and discard pending reruns when a query fails. + +- [#9328](https://github.com/Effect-TS/effect/pull/9328) [`d5c1e5b`](https://github.com/Effect-TS/effect/commit/d5c1e5bd425dfe983dff1f8c853c0b7eaa772f87) Thanks @effect-bot! - Propagate all entry release defects when an `RcMap` owner scope closes, after attempting every release and clearing the map. This also applies to resources managed through `LayerMap`. + +- [#9191](https://github.com/Effect-TS/effect/pull/9191) [`11bb31f`](https://github.com/Effect-TS/effect/commit/11bb31fec0bf2fdfc025009a04de4c69bffa433a) Thanks @danieljvdm! - Fix `Channel.fromReadableStream`, `Channel.fromTransformStream` and `Stream.fromReadableStream` discarding rejected cancel hooks and leaving the reader locked after finalization. A rejected cancel hook is now a defect, and an already errored stream is not reported twice. + +- [#9396](https://github.com/Effect-TS/effect/pull/9396) [`8c23e46`](https://github.com/Effect-TS/effect/commit/8c23e46a8280e3f2c831d1c0ca3f106c92f8e071) Thanks @effect-bot! - Fix `Stream.toReadableStream` to wait for Web Stream demand before pulling the next upstream chunk, avoiding premature upstream side effects. + +- [#9235](https://github.com/Effect-TS/effect/pull/9235) [`9c96011`](https://github.com/Effect-TS/effect/commit/9c960111de202c0f1dd6989e5b8836150e0ca694) Thanks @effect-bot! - Fix the Redis PersistedQueue store dropping entries when a payload or custom id contains a lone UTF-16 surrogate. + +- [#9618](https://github.com/Effect-TS/effect/pull/9618) [`0d50570`](https://github.com/Effect-TS/effect/commit/0d50570bbf078804092fcd4a47ccd2d0019f989c) Thanks @Marve10s! - Fix `Effect.repeat`, `repeatOrElse`, `schedule` and `scheduleFrom` treating a `Cause.Done` failure from the repeated effect as schedule completion. + +- [#9542](https://github.com/Effect-TS/effect/pull/9542) [`9ed2bf1`](https://github.com/Effect-TS/effect/commit/9ed2bf18da0e1ff4fecc31b7cbac2976ec999e00) Thanks @Marve10s! - Fix `RequestResolver.withCache` to call the wrapped resolver's `preCheck` on cache misses, preserving hook completion and nested cache reuse. + +- [#9541](https://github.com/Effect-TS/effect/pull/9541) [`76ec912`](https://github.com/Effect-TS/effect/commit/76ec9120f43f4fdb08b9c5170ff9e3fd82f22457) Thanks @Marve10s! - Fix `RequestResolver.withSpan` dropping every span option except `attributes` and `links` when the options are returned by a function. + +- [#9439](https://github.com/Effect-TS/effect/pull/9439) [`af2756d`](https://github.com/Effect-TS/effect/commit/af2756dd1d196aa3f199ce21c6d2dc3ada3b67b5) Thanks @tim-smart! - Fix `Effect.request` failing when a resolver's batch delay completes synchronously, such as `RequestResolver.setDelay(0)` under `TestClock`, by registering the request before the delay starts. + +- [#9230](https://github.com/Effect-TS/effect/pull/9230) [`3ec8100`](https://github.com/Effect-TS/effect/commit/3ec8100c793b8a22381ad115106b93824d840f64) Thanks @effect-bot! - Keep `Resource.auto` refreshing according to its schedule after a failed refresh, while preserving the previously cached value. + +- [#9460](https://github.com/Effect-TS/effect/pull/9460) [`93abf02`](https://github.com/Effect-TS/effect/commit/93abf020876af0a6b6b8b25621c80760b7996762) Thanks @schutzelaars! - Fix `RpcClient` retaining request state when a request fails without a reply. + +- [#9232](https://github.com/Effect-TS/effect/pull/9232) [`1e94993`](https://github.com/Effect-TS/effect/commit/1e949935a1065e5140245a514ce3a799061143e3) Thanks @effect-bot! - Fix RPC connection defects failing new requests started synchronously by pending callers. Only requests pending when defect delivery begins are failed; new requests await their own responses. Ensure all pending calls are released even if defect delivery is interrupted. + +- [#9656](https://github.com/Effect-TS/effect/pull/9656) [`99a4616`](https://github.com/Effect-TS/effect/commit/99a4616264e31de65fa8d3dc19576309b1a3f84a) Thanks @effect-bot! - Add an opt-in `batch` option to `RpcClient.layerProtocolHttp` and `RpcClient.makeProtocolHttp` that sends concurrent RPC requests from the same client in one HTTP request. + +- [#9297](https://github.com/Effect-TS/effect/pull/9297) [`0a7978e`](https://github.com/Effect-TS/effect/commit/0a7978ea8e70e51700345b12f8c19952dde05255) Thanks @tim-smart! - Fix HTTP RPC cancellation before the first response chunk and during buffered response collection so client aborts interrupt server-side requests and release their resources. + +- [#9562](https://github.com/Effect-TS/effect/pull/9562) [`b4a060b`](https://github.com/Effect-TS/effect/commit/b4a060b95a6473d54e277dc005172b54aab9bddf) Thanks @tim-smart! - Change RPC NDJSON's content type from `application/ndjson` to `application/x-ndjson`; update any exact media-type checks in custom integrations. Reject malformed non-empty lines with `RpcSerialization.NdjsonDecodeError` (preserving the parse error as `cause`) instead of silently dropping messages, while explicitly skipping empty LF and CRLF lines. Existing RPC HTTP clients and servers remain compatible across the header change. + +- [#9665](https://github.com/Effect-TS/effect/pull/9665) [`274f4a4`](https://github.com/Effect-TS/effect/commit/274f4a4dac5ee2d9dbb6358cc3dd3afe345d4705) Thanks @effect-bot! - Add the `Rpc.ParseOptions` annotation, so RPC payloads and responses can be decoded and encoded with schema parse options such as `concurrency`. + +- [#9601](https://github.com/Effect-TS/effect/pull/9601) [`25f29be`](https://github.com/Effect-TS/effect/commit/25f29be77bdcd5c2a7be8e2710e88f5a10c4cb8d) Thanks @effect-bot! - Fix `runMain` skipping finalizers when `SIGINT` or `SIGTERM` arrives during synchronous startup. + +- [#9547](https://github.com/Effect-TS/effect/pull/9547) [`9036440`](https://github.com/Effect-TS/effect/commit/9036440edca01c24371d0c2c1044be5e5d3cb65e) Thanks @effect-bot! - Prevent uncaught errors when unread HTTP responses are garbage-collected in runtimes such as Cloudflare Workers that restrict aborting request I/O outside its request context. + +- [#9593](https://github.com/Effect-TS/effect/pull/9593) [`415f005`](https://github.com/Effect-TS/effect/commit/415f0053eb848d562febcb5b011e15f31a99bead) Thanks @magoz! - Fix `Schema.isBase64` and `Schema.isBase64Url` throwing `RangeError: Maximum call stack size exceeded` on large inputs. + +- [#9245](https://github.com/Effect-TS/effect/pull/9245) [`191308b`](https://github.com/Effect-TS/effect/commit/191308bd0f5ee04557570f2f960c6e06c429bcfd) Thanks @fubhy! - Add the unstable `Schema.toCodecGoJson` codec for Go JSON responses. It normalizes optional-key `null` values to missing keys and required array or record `null` values to empty collections, while preserving schemas that already accept `null`. Encoding is unchanged from `Schema.toCodecJson`. + +- [#9170](https://github.com/Effect-TS/effect/pull/9170) [`4b96570`](https://github.com/Effect-TS/effect/commit/4b96570ed28d93b14e958742933d227dc5e74b15) Thanks @effect-bot! - Preserve literal member inference in Schema.Enum for inline object literals without requiring `as const`. + +- [#9409](https://github.com/Effect-TS/effect/pull/9409) [`d84ec3f`](https://github.com/Effect-TS/effect/commit/d84ec3fab0346c01bc59f6624d1c046ca4140087) Thanks @effect-bot! - Preserve filter-reported error paths when message annotations override schema check messages in Standard Schema and default error formatting. Filter message annotations now also override directly returned messages, including when the annotation is an empty string. Avoid applying struct messages to multiple filter failures. + +- [#9720](https://github.com/Effect-TS/effect/pull/9720) [`392aa1c`](https://github.com/Effect-TS/effect/commit/392aa1cb275ac9fdb9b4b1831d8c06196bba54b9) Thanks @Marve10s! - Fix `Schema.toFormatter` throwing on `ReadonlySet`, `ReadonlyMap`, `HashSet`, `HashMap` and `Chunk` values that contain symbols or null-prototype objects, and keep `Chunk` elements in iteration order. + + Map and set entries are now sorted by their formatted output instead of by their raw values, so they can print in a different order than before, including for primitive keys and values. + +- [#9556](https://github.com/Effect-TS/effect/pull/9556) [`99e9bd9`](https://github.com/Effect-TS/effect/commit/99e9bd98a354d83202e2522f3ade9ee4bf4a862c) Thanks @gcanti! - Remove the type-level uniqueness check from `Schema.toTaggedUnion` so generic member lists compile again. Duplicate discriminants still throw at runtime. + +- [`6f6f45f`](https://github.com/Effect-TS/effect/commit/6f6f45f480a3f191aed6a82ca5c301f60ef2cf84) Thanks @gcanti! - Fix `Schema.toJsonSchemaDocument` to support numeric record keys and BigInt template literals, and approximate tuples with post-rest elements. + +- [#9318](https://github.com/Effect-TS/effect/pull/9318) [`13fc07e`](https://github.com/Effect-TS/effect/commit/13fc07edeb939ca6229b858c9d9a8c5243ac21b5) Thanks @effect-bot! - Preserve synchronous throws from schema transformations, filters, middleware, and custom declaration parsers as defects when decoding or encoding, including at the root schema. Exit, Effect, and Promise parsers now report these defects through their result instead of throwing before returning. + +- [#9596](https://github.com/Effect-TS/effect/pull/9596) [`a4b38dd`](https://github.com/Effect-TS/effect/commit/a4b38dde7f3fdc1b54a49bffdaca912b4c47c76e) Thanks @magoz! - Match finite number strings in linear time. `Schema.isStringFinite`, `Number` record keys, `Number` template literal parts and `Schema.toCodecStringTree` number fields no longer take quadratic time to reject long digit runs such as `"111…x"`. The JSON Schema `pattern` for these schemas changes to the same linear regular expression and accepts exactly the same strings. + +- [#9724](https://github.com/Effect-TS/effect/pull/9724) [`8b18580`](https://github.com/Effect-TS/effect/commit/8b185807bb99f6de3d9564447e2f4e08d2f8863d) Thanks @Marve10s! - Fix `Schema.Tuple` encoding leaving an empty slot when a trailing optional element such as `Schema.OptionFromOptionalKey` is omitted. + +- [#9391](https://github.com/Effect-TS/effect/pull/9391) [`0c477fe`](https://github.com/Effect-TS/effect/commit/0c477fe417d3ecab86bb8d9afd0c392e9cb8a3da) Thanks @effect-bot! - Propagate all release defects from `ScopedCache.invalidateAll` and cache owner-scope shutdown after attempting every entry release. Successful bulk invalidation now returns `undefined` as declared. + +- [#9243](https://github.com/Effect-TS/effect/pull/9243) [`9a5331d`](https://github.com/Effect-TS/effect/commit/9a5331db94bfdca4f48317d062fbeb96054a9049) Thanks @fubhy! - Add `effect/SemVer` for strict Semantic Versioning 2.0.0 parsing, formatting, comparison, and version bumps, and `effect/SemVerRange` for npm-style range matching and version selection. Add schemas and string codecs for both types. + +- [#9272](https://github.com/Effect-TS/effect/pull/9272) [`a706d73`](https://github.com/Effect-TS/effect/commit/a706d73db8a8d460877c5d7e8a793a52d45be307) Thanks @Marve10s! - Fix `Sink.collect` and `Sink.take` throwing a `RangeError` on large input chunks. + +- [#9320](https://github.com/Effect-TS/effect/pull/9320) [`c9a1e72`](https://github.com/Effect-TS/effect/commit/c9a1e72c5965aa9fe01a669ebfd523f803f5a189) Thanks @effect-bot! - Fall back to client computation when atom hydration data fails schema decoding or a streamed initial result cannot be encoded, instead of throwing or leaving the atom waiting indefinitely. + +- [#9487](https://github.com/Effect-TS/effect/pull/9487) [`89eaaf8`](https://github.com/Effect-TS/effect/commit/89eaaf8581908e248c0b1e47baa7906e0c541635) Thanks @jayhemnani9910! - Fix `Socket.readerString` and `Socket.toChannelString` corrupting characters split across binary frames. Incomplete characters are emitted as U+FFFD before string frames or on close code 1000. + +- [#9292](https://github.com/Effect-TS/effect/pull/9292) [`8215811`](https://github.com/Effect-TS/effect/commit/8215811d51d28d37b4732a6ab03de4047ada62ea) Thanks @tim-smart! - `SqlEventLogServerUnencrypted` publication and `Reactivity` invalidations inside a SQL transaction now wait for the outermost commit and are dropped on rollback. + +- [#9289](https://github.com/Effect-TS/effect/pull/9289) [`ad69018`](https://github.com/Effect-TS/effect/commit/ad6901857e05e7b436a0149b5ece7f3a36d9ff33) Thanks @tim-smart! - Add `SqlClient.afterCommit`, which runs an effect once the outermost transaction commits, or immediately outside a transaction. + +- [#9576](https://github.com/Effect-TS/effect/pull/9576) [`a733eba`](https://github.com/Effect-TS/effect/commit/a733eba9f699737a80a9e7276521980bd929075a) Thanks @effect-bot! - Add `SqlClient.BeginTransaction`, a reference that replaces the statement starting a top-level transaction, such as `BEGIN ISOLATION LEVEL SERIALIZABLE`, saving the round trip of a separate `SET TRANSACTION`. Nested transactions keep their savepoints. `SqlClient.makeWithTransaction` passes the statement to `begin` as a second argument. + +- [#9472](https://github.com/Effect-TS/effect/pull/9472) [`7282d5f`](https://github.com/Effect-TS/effect/commit/7282d5f1bcc06d7065429ef81a3d17c64043fde4) Thanks @tim-smart! - Improve `SqlEventJournal` uploads and query performance. + + - Checkpoint successful uploads per remote without waiting for an echo, while forwarding older entries imported later. + - Index conflict, entry-ordering and remote-sequence queries. + - Migrate existing databases on first open using a writable connection. The first upload after migration may resend unconfirmed entries. + - Run `write` and `writeMany` callbacks in the journal transaction, rolling back their SQL on the same client if the write fails or is interrupted. + +- [#9233](https://github.com/Effect-TS/effect/pull/9233) [`da33572`](https://github.com/Effect-TS/effect/commit/da33572776763e45802ce8c2fe14ec101f9a5c34) Thanks @effect-bot! - Fix SQL event journals reporting incoming remote entries as conflicts. Remote writes now report only previously stored newer entries and atomically persist entries and remote sequences after all callbacks succeed, allowing failed writes to be retried. + +- [#9606](https://github.com/Effect-TS/effect/pull/9606) [`d9e2d12`](https://github.com/Effect-TS/effect/commit/d9e2d1221768e9a3e5e00165cb95b86e0c126cc0) Thanks @effect-bot! - Fence SQL `PersistedQueue` settlements and lock refreshes by acquisition, so a handler whose lock lapsed can no longer complete, fail, retry or release a row after the same store re-claims it. + +- [#9517](https://github.com/Effect-TS/effect/pull/9517) [`dc923b8`](https://github.com/Effect-TS/effect/commit/dc923b847ac7d9673b4f0aa7c136557287b2e5e3) Thanks @effect-bot! - Allow queued SQL transactions to be interrupted while waiting for a connection, respecting the caller's interruptibility. Interrupted or failed acquisition releases the connection scope and its permits, and a libSQL transaction interrupted during `BEGIN` is rolled back. + + `SqlClient.makeWithTransaction` (unstable) now owns the connection scope: `acquireConnection` receives the mask's `restore` function and returns a scoped connection effect instead of a `[scope, connection]` tuple. + +- [#9599](https://github.com/Effect-TS/effect/pull/9599) [`1b74086`](https://github.com/Effect-TS/effect/commit/1b740862e3d25ec5d4b6f86b9fe792b6dabbf84c) Thanks @effect-bot! - Preserve typed SQL errors without cleanup defects after SQLite automatically rolls back a transaction in sqlite-node, sqlite-bun and in-memory sqlite-wasm. OPFS and React Native clients are unchanged. + +- [#9485](https://github.com/Effect-TS/effect/pull/9485) [`8a5507b`](https://github.com/Effect-TS/effect/commit/8a5507ba00f641edf71227d79b9f90efdb634fe2) Thanks @itisrmk! - Fix `SqlMessageStorage` ID precision loss on SQLite drivers that return numbers, including Cloudflare Durable Objects. + +- [#9398](https://github.com/Effect-TS/effect/pull/9398) [`23460fa`](https://github.com/Effect-TS/effect/commit/23460fad704c730926bffe3cc6874268ef6680ec) Thanks @effect-bot! - Treat persisted cache entries that no longer decode as cache misses, recomputing and replacing them after success or error schema changes. + +- [#9264](https://github.com/Effect-TS/effect/pull/9264) [`0dbbea6`](https://github.com/Effect-TS/effect/commit/0dbbea65a93c59147a4bc407cfcda38ea26341b2) Thanks @effect-bot! - Redirect static directory index URLs without a trailing slash to their canonical form with a 301 response, preserving router prefixes and query strings so relative links resolve correctly. Clients that do not follow redirects must request the trailing-slash URL directly. + +- [#9622](https://github.com/Effect-TS/effect/pull/9622) [`70aad3f`](https://github.com/Effect-TS/effect/commit/70aad3ffe617984f5b599aeed1bc7f04899aca55) Thanks @Marve10s! - Keep recursive `Stream.concat` chains through `Stream.suspend` linear in their length. + +- [#9274](https://github.com/Effect-TS/effect/pull/9274) [`0909236`](https://github.com/Effect-TS/effect/commit/090923611b209a19f3281a7b1d0c15a9b653a0c7) Thanks @Marve10s! - Fix `Stream.mapAccum` throwing a `RangeError` when its callback returns a large array. + +- [#9623](https://github.com/Effect-TS/effect/pull/9623) [`a9e66d8`](https://github.com/Effect-TS/effect/commit/a9e66d889017b302355ba5b8bc2b2050ca1ea5aa) Thanks @Marve10s! - Keep `Stream.retry` and `Channel.retry` work constant per attempt instead of growing with every retry. + +- [#9624](https://github.com/Effect-TS/effect/pull/9624) [`ee5c0d5`](https://github.com/Effect-TS/effect/commit/ee5c0d5f43348747b04b5f7bf1b39355614c2f2c) Thanks @Marve10s! - Fix `Stream.zip` and related zip operators taking quadratic time when the two streams emit chunks of different sizes. + +- [#9492](https://github.com/Effect-TS/effect/pull/9492) [`631c5d6`](https://github.com/Effect-TS/effect/commit/631c5d6d9a81e6389964b878764fb9aea1538247) Thanks @Marve10s! - Fix `Stream.zipWithArray` discarding right-side leftovers when the callback returns leftovers for both streams. + +- [#9750](https://github.com/Effect-TS/effect/pull/9750) [`7e60c1a`](https://github.com/Effect-TS/effect/commit/7e60c1ae62a96c768f9c67d706f494213947f22f) Thanks @armancharan! - Fix `String.capitalize` and `String.uncapitalize` for strings that start with an astral letter such as `"𐐨"`. + +- [#9494](https://github.com/Effect-TS/effect/pull/9494) [`4cddd3c`](https://github.com/Effect-TS/effect/commit/4cddd3c41e71fc4990d37c42b1bcf2ce3780b41d) Thanks @Marve10s! - Fix suspended `Schema.Struct` parsers overwriting earlier results when the same parsing Effect is rerun. + +- [#9722](https://github.com/Effect-TS/effect/pull/9722) [`8727e5d`](https://github.com/Effect-TS/effect/commit/8727e5d2bec8e2d729b7aeafb725a4ab21d533d2) Thanks @Marve10s! - Fix `Stream.switchMap` and `Channel.switchMap` not finalizing inner streams that were replaced before they started. + +- [#9428](https://github.com/Effect-TS/effect/pull/9428) [`2bc97fa`](https://github.com/Effect-TS/effect/commit/2bc97faa0e06b3973780dae26029e2c59f5dba38) Thanks @jpowersdev! - Fix `Tar.decode` dropping a leading byte order mark from entry names, link targets and PAX keys, so a name such as `"\uFEFF/etc/passwd"` no longer decodes as `/etc/passwd`. + +- [#9392](https://github.com/Effect-TS/effect/pull/9392) [`f9a7f05`](https://github.com/Effect-TS/effect/commit/f9a7f056ba31aef435933acf082ec8c01ac7decc) Thanks @effect-bot! - Keep TestClock diagnostics visible when TestConsole captures test output, including warnings about time not advancing and busy fibers. Ordinary test console output remains captured. + +- [#9434](https://github.com/Effect-TS/effect/pull/9434) [`d978cca`](https://github.com/Effect-TS/effect/commit/d978cca02914bec0ddf66bb2d68d90b220096fb6) Thanks @jpowersdev! - Add the experimental `TextDiff` module for array, line and Unicode code point edit scripts, replay, inversion and unified diff formatting. Use `maxEdits` to bound search work on large inputs, trading minimality for a valid script when the cap is exceeded. + +- [#6428](https://github.com/Effect-TS/effect/pull/6428) [`b334ccf`](https://github.com/Effect-TS/effect/commit/b334ccf361a0f11362e7aabdbf70b35f53a39d61) Thanks @coyaSONG! - Add `FileSystem.lstat`, which returns the `File.Info` of a path without following symbolic links. Custom `FileSystem` implementations, including those passed to `FileSystem.make`, must now provide `lstat`. + +- [#9538](https://github.com/Effect-TS/effect/pull/9538) [`0b50cfc`](https://github.com/Effect-TS/effect/commit/0b50cfc644313afe4c318073893231ec1946eb5f) Thanks @jayhemnani9910! - Allow empty quoted keys such as `"" = 1` in `Toml.parse`, as permitted by TOML 1.0. + +- [#9741](https://github.com/Effect-TS/effect/pull/9741) [`5b7d2b5`](https://github.com/Effect-TS/effect/commit/5b7d2b5c90bc77eb9f8be1b91166c53f89aa1d0c) Thanks @danieljvdm! - Fix `Toolkit.handle` dropping additional failures and their annotations when a tool handler fails in `"error"` failure mode. + +- [#8065](https://github.com/Effect-TS/effect/pull/8065) [`6ee26b1`](https://github.com/Effect-TS/effect/commit/6ee26b16b51b380c41dbfa84d612aa525b420fce) Thanks @danieljvdm! - Interrupt Toolkit handlers when their result stream closes, including early termination. Add `Toolkit.PreliminaryBuffer` so callers can opt in to bounding buffered preliminary results. + +- [#9271](https://github.com/Effect-TS/effect/pull/9271) [`eab7e6a`](https://github.com/Effect-TS/effect/commit/eab7e6acfc7e422857565b98b9eddd3a98de1b92) Thanks @Marve10s! - Fix Trie iterators restarting from the first entry when iterated again. They now continue from their current position. + +- [#9701](https://github.com/Effect-TS/effect/pull/9701) [`e618f10`](https://github.com/Effect-TS/effect/commit/e618f1050f41424439eb82478fcf1c5da5a3244d) Thanks @effect-bot! - Require an explicit trusted proxy policy before applying forwarded headers. + + `HttpMiddleware.xForwardedHeaders` now takes `{ trustedProxies }` and only applies `X-Forwarded-*` headers when the connecting peer is trusted. `X-Forwarded-For` is read from the right, skipping trusted proxies, and `X-Forwarded-Host` / `X-Forwarded-Proto` are only used when they line up with it. + + `HttpServerRequest.toURL` no longer reads `x-forwarded-proto` directly. It uses the new `request.protocol`, set by the trusted proxy middleware, before the transport scheme. The connecting peer stays available as `request.peerAddress`. + + ```ts + import { HttpMiddleware } from "effect/http" + import { IpNetwork } from "effect/net" + + // before + HttpMiddleware.xForwardedHeaders + + // after + HttpMiddleware.xForwardedHeaders({ + trustedProxies: [IpNetwork.fromStringUnsafe("10.0.0.0/8")] + }) + ``` + +- [#9560](https://github.com/Effect-TS/effect/pull/9560) [`da14b1f`](https://github.com/Effect-TS/effect/commit/da14b1f18d71a18fc431b56d82cffe09aeb92708) Thanks @effect-bot! - Add opt-in typed reactivity keys. Augment `Reactivity.Keys` to map key names to ID types, and the public reactivity APIs (`Reactivity`, `Atom`, `AtomHttpApi`, `AtomRpc`, `SqlClient.reactive` and `EventLog.groupReactivity`) reject undeclared key names and mismatched IDs. With no keys declared, the accepted types are unchanged. + +- [#9516](https://github.com/Effect-TS/effect/pull/9516) [`3314fc3`](https://github.com/Effect-TS/effect/commit/3314fc30c22d5c9d737f3df7201f4a41424f701b) Thanks @tim-smart! - Preserve defects in mixed failure causes across Effect, Stream and Channel. + + - Typed recovery skips causes containing a defect and converts every typed error to a defect: `Fail(e) + Die(d)` becomes `Die(e) + Die(d)`, preserving reason order and interrupts. Use `catchCause` or `catchDefect` to recover from these causes. + - An interrupt alongside a typed error does not block recovery, including Effect retry and repeat error paths. Retrying or re-raising only the typed error can discard that interrupt. + - The same rule applies to failing `repeat` / `repeatOrElse` policies. Retry-policy failures still propagate unchanged. + - Error mapping in Effect (including eager variants), Exit, Stream and Channel now maps every typed error in place, preserving defects, interrupts and reason order instead of keeping only the first mapped failure. Mapping does not convert typed errors to defects. + +- [#8351](https://github.com/Effect-TS/effect/pull/8351) [`66db624`](https://github.com/Effect-TS/effect/commit/66db624670032a02144d42b6e64b83b324c813cd) Thanks @ArjunCodess! - Add `UserDirectories` to `effect/persistence`, a service that resolves the home, config, data, cache and state directories for an application following XDG and platform conventions. + +- [#9531](https://github.com/Effect-TS/effect/pull/9531) [`4b8a50d`](https://github.com/Effect-TS/effect/commit/4b8a50db9649603f382f33640698020ca79547d0) Thanks @jayhemnani9910! - Report an initial UTF-16 BOM in `Xml.decode` as `Unsupported` instead of `Syntax`. + +- [#9581](https://github.com/Effect-TS/effect/pull/9581) [`8d52668`](https://github.com/Effect-TS/effect/commit/8d52668533e9d4a6d6b3be9fe58bd5e69fa80b65) Thanks @jayhemnani9910! - Fix `Yaml.parse` block scalars including less indented comments in their value. + +- [#9608](https://github.com/Effect-TS/effect/pull/9608) [`1de1055`](https://github.com/Effect-TS/effect/commit/1de10559e7263747751bfb032fd4b0fd8aa28724) Thanks @jayhemnani9910! - Fix `Yaml.parse` rejecting tabs in block scalar content and discarding the whitespace of whitespace-only block scalar lines. Leading empty lines more indented than the block scalar content are now rejected. + +- [#9652](https://github.com/Effect-TS/effect/pull/9652) [`db130e6`](https://github.com/Effect-TS/effect/commit/db130e64557fafe370819d19f78e3b19a21275d6) Thanks @okarin1511! - Fix `Yaml.parse` rejecting anchors defined inside flow collections, such as `{a: &ref 1, b: *ref}`. + +- [#9700](https://github.com/Effect-TS/effect/pull/9700) [`5fa5fa5`](https://github.com/Effect-TS/effect/commit/5fa5fa5b9bff77352fb9ff2d424bf46bd6309986) Thanks @fubhy! - Fix `Yaml.parse` rejecting flow collections that span multiple lines, such as a JSON-style mapping with its entries and closing bracket on separate lines. + ## 4.0.5 ### Patch Changes diff --git a/repos/effect/packages/effect/HTTPAPI.md b/repos/effect/packages/effect/HTTPAPI.md index 2864513dbe..9205f770fd 100644 --- a/repos/effect/packages/effect/HTTPAPI.md +++ b/repos/effect/packages/effect/HTTPAPI.md @@ -1362,11 +1362,334 @@ Layer.launch(ApiLayer).pipe(NodeRuntime.runMain) ## Streaming Requests -To receive large or continuous data from the client, register the endpoint with `.handleRaw`, which opts out of automatic payload decoding and exposes the raw `HttpServerRequest`. The request body is then available as a `Stream` of `Uint8Array` chunks through `request.stream`, so the handler can consume it incrementally instead of buffering it in memory. +To receive large or continuous uploads, declare the payload with a streaming schema. These are the same constructors used for [streaming responses](#streaming-responses), and the slot decides the direction: -The payload schema still describes the endpoint in the generated documentation, but with `.handleRaw` the handler decides how the body is consumed. Note that the declared content type is no longer enforced at runtime: requests with a different `Content-Type` header are not rejected with `415`, so any such validation is up to the handler. +| Declaration | Handler receives | Default content type | +| ----------------------------------------------------------------------------------- | --------------------------------------------------------- | -------------------------- | +| `HttpApiSchema.StreamUint8Array({ contentType?, maxBytes? })` | `Stream` | `application/octet-stream` | +| `HttpApiSchema.StreamNdjson({ schema, contentType?, maxBytes?, maxElementBytes? })` | `Stream` | `application/x-ndjson` | -**Example** (Handling Streaming Requests) +The handler gets a lazy `Stream`. The server reads the socket only as the handler pulls, with no read-ahead, so not pulling is the backpressure. `HttpApiSchema.StreamSse` is response-only and is rejected in `payload`. + +### Streaming Raw Bytes + +**Example** (Receiving a Binary Upload) + +```ts +import { NodeHttpServer, NodeRuntime } from "@effect/platform-node" +import { Effect, Layer, Schema, Stream } from "effect" +import { HttpRouter } from "effect/http" +import { HttpApi, HttpApiBuilder, HttpApiEndpoint, HttpApiGroup, HttpApiSchema } from "effect/http-api" +import { createServer } from "node:http" + +const Api = HttpApi.make("myApi").add( + HttpApiGroup.make("group").add( + HttpApiEndpoint.post("upload", "/upload", { + // Stream the body as raw bytes and stop after 100 MiB + // (default content type: application/octet-stream) + payload: HttpApiSchema.StreamUint8Array({ maxBytes: "100 MiB" }), + success: Schema.String + }) + ) +) + +const GroupLayer = HttpApiBuilder.group( + Api, + "group", + (handlers) => + handlers.handle("upload", ({ payload }) => + // payload: Stream + payload.pipe( + // The body is read as the fold pulls, never buffered as a whole + Stream.runFold(() => 0, (total, chunk) => total + chunk.length), + Effect.map((total) => `received ${total} bytes`) + )) +) + +const ApiLayer = HttpApiBuilder.layer(Api).pipe( + Layer.provide(GroupLayer), + HttpRouter.serve, + Layer.provide(NodeHttpServer.layer(createServer, { port: 3000 })) +) + +Layer.launch(ApiLayer).pipe(NodeRuntime.runMain) + +// printf 'abc' | curl -X POST 'http://localhost:3000/upload' --data-binary @- -H "Content-Type: application/octet-stream" +// "received 3 bytes" +``` + +### Streaming NDJSON + +`HttpApiSchema.StreamNdjson({ schema })` declares a body of newline-delimited JSON, one encoded element per line. The handler receives decoded elements, and the element schema's decoding services are added to the handler's requirements. + +**Example** (Importing Records Line by Line) + +```ts +import { NodeHttpServer, NodeRuntime } from "@effect/platform-node" +import { Effect, Layer, Schema, Stream } from "effect" +import { HttpRouter } from "effect/http" +import { HttpApi, HttpApiBuilder, HttpApiEndpoint, HttpApiGroup, HttpApiSchema } from "effect/http-api" +import { createServer } from "node:http" + +const Event = Schema.Struct({ + id: Schema.Number, + name: Schema.String +}) + +const Api = HttpApi.make("myApi").add( + HttpApiGroup.make("group").add( + HttpApiEndpoint.post("importEvents", "/events", { + // One JSON-encoded Event per line + // (default content type: application/x-ndjson) + payload: HttpApiSchema.StreamNdjson({ schema: Event }), + success: Schema.String + }) + ) +) + +const GroupLayer = HttpApiBuilder.group( + Api, + "group", + (handlers) => + handlers.handle("importEvents", ({ payload }) => { + // payload: Stream + let accepted = 0 + return payload.pipe( + Stream.runForEach((_event) => + Effect.sync(() => { + // store the event... + accepted++ + }) + ), + Effect.map(() => `accepted ${accepted} records`), + // A bad record ends the stream. Every record before it has already + // been handled, so report how far the import got + Effect.catchTag("HttpApiSchemaError", () => + Effect.succeed(`accepted ${accepted} records, record ${accepted + 1} is invalid`)) + // HttpApiPayloadTooLarge is not caught, so it becomes an empty 413 + ) + }) +) + +const ApiLayer = HttpApiBuilder.layer(Api).pipe( + Layer.provide(GroupLayer), + HttpRouter.serve, + Layer.provide(NodeHttpServer.layer(createServer, { port: 3000 })) +) + +Layer.launch(ApiLayer).pipe(NodeRuntime.runMain) + +// printf '{"id":1,"name":"a"}\n{"id":2,"name":"b"}\nnope\n' | curl -X POST 'http://localhost:3000/events' --data-binary @- -H "Content-Type: application/x-ndjson" +// "accepted 2 records, record 3 is invalid" +``` + +The wire format: + +- The server accepts only the declared content type, compared after normalization, and answers `415` otherwise. There are no aliases: for JSON Lines, pass `contentType: "application/jsonl"`. +- `\n`, `\r\n` and a bare `\r` each end a line. Empty lines are skipped, so a client can send `\n` as a keep-alive. A line containing only whitespace is malformed. +- A missing final newline is fine if the last line parses. If it doesn't, the issue message says "unterminated final line", so a truncated body is recognizable. +- A leading UTF-8 BOM is stripped. Invalid UTF-8 is an error, never replaced with U+FFFD. +- The first malformed line, or the first element that fails to decode, fails the stream with an `HttpApiSchemaError` (`kind: "Payload"`), after every element before it has been delivered. A failure ends the stream, so a handler can't skip a bad record and keep reading. To skip bad records, declare a lenient element schema and filter. +- NDJSON can't detect a body that was cut cleanly at a line boundary. It looks like a shorter, complete body. + +### Limits and Errors + +Limits are set on the declaration only: + +- `maxBytes` caps the whole body. It has no default, and it does not fall back to `MaxBodySize`, which only applies to buffered reads. +- `maxElementBytes` (NDJSON only) caps the UTF-8 bytes of one line, without its terminator. It defaults to 1 MiB, and a longer line fails before the rest of it is buffered. +- Both count the bytes the handler sees, after the `HttpMiddleware.decompression` middleware. +- Both count only bytes that are actually read. After `Stream.take(1)`, `maxBytes` never fires. + +If the request's `content-length` exceeds `maxBytes`, the server answers an empty `413` with `Connection: close` as soon as params, headers and query have decoded. The handler doesn't run and no body byte is read. + +The payload stream's errors are added to the handler's error channel automatically. A handler can catch them after a partial read, as in the example above, or let them through. The server maps the ones it lets through before responding: + +| Error | Response | +| ---------------------------- | ------------------------------------ | +| `HttpApiSchemaError` | `400`, the same as buffered payloads | +| `HttpApiPayloadTooLarge` | empty `413` | +| Body read failure (a defect) | `500`, reported | + +`HttpApiPayloadTooLarge` carries `limit` (`"maxBytes"` or `"maxElementBytes"`) and its configured `size`. + +`HttpMiddleware.decompression` also caps the decompressed body at `maxDecompressedSize`, which defaults to `MaxBodySize` and then to 10 MiB. That cap applies to streaming payloads too, and breaching it answers `413`. Raise `maxDecompressedSize` on servers that accept large compressed uploads. + +### Body Lifetime + +- Middleware runs first, then params, headers and query, then the payload. A params, headers or query failure answers `400` without reading a body byte, and the handler always gets the body unread. +- Read the payload stream once. The body is never buffered for a replay, so a repeated read keeps the adapter's native behavior: an empty body, a typed `RequestParseError`, or a replay of the caller's stream under `HttpApiTest`. Don't rely on any of them. +- A handler can respond without reading the whole body, for example to reject an upload early. Over HTTP/1.1 the Node server then drains up to 256 KiB of what is left. Past that it sends `Connection: close` and closes the socket. + +### Mixing Buffered and Streaming Payloads + +One streaming payload can share the `payload` array with buffered payloads that use other content types. The server picks one by the request's content type, and the handler tells them apart with `Stream.isStream`. + +**Example** (Accepting a JSON Array or an NDJSON Stream) + +```ts +import { Effect, Schema, Stream } from "effect" +import { HttpApi, HttpApiBuilder, HttpApiEndpoint, HttpApiGroup, HttpApiSchema } from "effect/http-api" + +const Event = Schema.Struct({ + id: Schema.Number, + name: Schema.String +}) + +const Api = HttpApi.make("myApi").add( + HttpApiGroup.make("group").add( + HttpApiEndpoint.post("importEvents", "/events", { + payload: [ + // application/json: the whole batch in one body + Schema.Array(Event), + // application/x-ndjson: one event per line, streamed + HttpApiSchema.StreamNdjson({ schema: Event }) + ], + success: Schema.Number + }) + ) +) + +const GroupLayer = HttpApiBuilder.group( + Api, + "group", + (handlers) => + handlers.handle("importEvents", ({ payload }) => + Stream.isStream(payload) + ? Stream.runCount(payload) + : Effect.succeed(payload.length)) +) +``` + +Endpoint construction throws when: + +- two streaming payloads are declared, because the client could not choose an encoding for a `Stream` +- a streaming and a buffered payload share a media type, compared after normalization, so `; charset=utf-8` does not make them distinct +- the method is `GET`, `HEAD`, `OPTIONS` or `TRACE`, which is also a type error. Every other method, including `DELETE` and `QUERY`, accepts a streaming payload + +### Sending Streams with the Derived Client + +The derived client takes a `Stream` as the payload: `Stream` for `StreamUint8Array`, and `Stream` for `StreamNdjson`. It sends the stream as it is pulled, without buffering, with the declared content type and no `content-length`. NDJSON elements are encoded lazily, each as one line of JSON followed by `\n`. + +The method infers only `R` from the stream, plus the services the element schema needs to encode. The stream's error isn't one of the endpoint's declared errors, so it's not in the method's type. A failing source aborts the upload, and the method fails with the `HttpClientError` the adapter reports. If the source fails while a streaming success response is being read, the response stream fails with that `HttpClientError` instead. An element that fails to encode fails the upload the same way. + +**Example** (Uploading a Stream of Records) + +```ts +import { Effect, Schema, Stream } from "effect" +import { FetchHttpClient } from "effect/http" +import { HttpApi, HttpApiClient, HttpApiEndpoint, HttpApiGroup, HttpApiSchema } from "effect/http-api" + +const Event = Schema.Struct({ + id: Schema.Number, + name: Schema.String +}) + +const Api = HttpApi.make("myApi").add( + HttpApiGroup.make("group").add( + HttpApiEndpoint.post("importEvents", "/events", { + payload: HttpApiSchema.StreamNdjson({ schema: Event }), + success: Schema.String + }) + ) +) + +const program = Effect.gen(function*() { + const client = yield* HttpApiClient.make(Api, { + baseUrl: "http://localhost:3000" + }) + + // Any Stream of Events works + const events = Stream.range(1, 1000).pipe( + Stream.map((id) => ({ id, name: `event ${id}` })) + ) + + const result = yield* client.group.importEvents({ payload: events }) + console.log(result) +}) +// program: Effect + +Effect.runFork(program.pipe(Effect.provide(FetchHttpClient.layer))) +``` + +Things to know: + +- A `400` or `413` from the server is an `HttpClientError` with a `DecodeError` reason, as a buffered `400` is today. There's no client-side `HttpApiPayloadTooLarge`, because the `413` body is empty. +- The client doesn't check `maxBytes` or `maxElementBytes` before sending. A `HttpClient.mapRequest` that sets `content-length` lets the server answer an early `413` on Node clients. `FetchHttpClient` strips `content-length`. +- With a streaming success response, the upload runs until either side ends. Otherwise it's interrupted once the method has decoded the response, whether that's an error status or a buffered success. +- Retries and redirects replay the stream. `HttpClient.retry` and 307/308 redirects run the payload stream again. That's right for a source that can be re-read, like a file, and wrong for a one-shot source like `Stream.fromQueue`: a retry would send only what is left in the queue. Don't retry uploads from such sources, or buffer them first. +- On a mixed payload, a `Stream` value is always sent with the streaming member, even when a buffered member's `Type` would also accept it (`Schema.Unknown`, `Schema.Any`). A `Uint8Array` or an array is never wrapped into a stream. +- `FetchHttpClient` needs a runtime that can send a `ReadableStream` request body. Node, Bun and Deno can. In browsers, Chrome sends streaming uploads only over HTTP/2 or later, and over HTTP/1.1 the request fails with a `TransportError`. + +### Streaming Both Ways + +A streaming payload can be combined with a streaming success response. Neither direction is buffered, and response headers go out as soon as the handler returns its stream. + +**Example** (Transforming an Upload as It Arrives) + +```ts +import { Effect, Schema, Stream } from "effect" +import { HttpApi, HttpApiBuilder, HttpApiEndpoint, HttpApiGroup, HttpApiSchema } from "effect/http-api" + +const Event = Schema.Struct({ + id: Schema.Number, + name: Schema.String +}) + +const Api = HttpApi.make("myApi").add( + HttpApiGroup.make("group").add( + HttpApiEndpoint.post("enrich", "/enrich", { + payload: HttpApiSchema.StreamNdjson({ schema: Event }), + success: HttpApiSchema.StreamNdjson({ schema: Event }) + }), + HttpApiEndpoint.post("enrichBatch", "/enrich-batch", { + payload: HttpApiSchema.StreamNdjson({ schema: Event }), + success: HttpApiSchema.StreamNdjson({ schema: Event }) + }) + ) +) + +const enrich = (event: typeof Event.Type) => ({ ...event, name: event.name.toUpperCase() }) + +const GroupLayer = HttpApiBuilder.group( + Api, + "group", + (handlers) => + handlers + // Interleaved: each output line is written as its input line arrives. + // Needs a full-duplex client + .handle("enrich", ({ payload }) => Effect.succeed(Stream.map(payload, enrich))) + // Portable: read the whole upload, then respond + .handle("enrichBatch", ({ payload }) => + Stream.runCollect(payload).pipe( + Effect.map((events) => Stream.fromIterable(events.map(enrich))) + )) +) +``` + +Interleaving is best effort, and both ends must be full duplex. Node's `node:http` and undici, and Bun's and Deno's `fetch`, are. Browsers send uploads with `duplex: "half"` and don't read the response until the upload has ended. A handler that pipes its input to its output, like `enrich` above, then deadlocks once the data outgrows the flow-control windows. The portable pattern is the one `enrichBatch` uses: consume the upload, then respond. + +Once the response headers are sent, the payload stream is pulled inside the response stream. A payload failure is then a response failure: the connection is aborted and the client sees a body read error. For a `StreamSse` response, map the payload error into the declared SSE `error` schema to send it as a typed failure. + +On Deno, an HTTP/1.1 request body read can stall while the handler returns a streamed response (denoland/deno#36966), which affects this pattern until the fix ships. + +### Proxying a Streaming Upload + +`HttpServerRequest.toClientRequest` turns the incoming request into an outbound `HttpClientRequest` that streams the same body. The body is not buffered, so if the outbound client retries the proxied request or follows a 307/308 redirect, the retry resends whatever the adapter gives a second read, usually an empty body. A proxy that retries has to buffer the body itself, or turn retries off. + +### Testing with HttpApiTest + +`HttpApiTest` passes a `Stream` payload to the handler unbuffered, pulling each element from the caller's stream as the handler reads it. Limits, the `400` and both `413` paths work in memory. The early `413` needs a `content-length`, which the client doesn't send for streams, so set one through `HttpClient.mapRequest`. When the caller's stream fails, the handler's read of the payload fails as a defect and the test call dies with it, because `HttpApiTest` doesn't turn defects into responses. Inside a streaming response, the response body fails instead. + +Socket behavior and client aborts are not emulated: no draining of unread bodies, no `Connection: close`, no interrupted handler, and no `499`. + +### Reading the Raw Request Body + +To take over body handling completely, register the endpoint with `.handleRaw`. It opts out of automatic payload decoding and exposes the raw `HttpServerRequest`, whose body is available as a `Stream` of `Uint8Array` chunks through `request.stream`. + +The payload schema still describes the endpoint in the generated documentation, but with `.handleRaw` the handler decides how the body is consumed. The declared content type is no longer enforced at runtime: requests with a different `Content-Type` header are not rejected with `415`, so any such validation is up to the handler. + +**Example** (Reading the Raw Request Body) ```ts import { NodeHttpServer, NodeRuntime } from "@effect/platform-node" @@ -1845,6 +2168,7 @@ Layer.launch(ApiLayer).pipe(NodeRuntime.runMain) To stream data to the client over time, declare the success schema with one of the streaming constructors: - `HttpApiSchema.StreamUint8Array()` streams raw bytes (default content type: `application/octet-stream`) +- `HttpApiSchema.StreamNdjson({ schema })` streams typed values as newline-delimited JSON (default content type: `application/x-ndjson`) - `HttpApiSchema.StreamSse(options)` streams typed Server-Sent Events (default content type: `text/event-stream`) With a streaming success schema, the handler returns a `Stream` directly, and the derived client resolves to a typed `Stream` on the consuming side. The stream declaration is also rendered in the OpenAPI documentation. @@ -1963,9 +2287,64 @@ Layer.launch(ApiLayer).pipe(NodeRuntime.runMain) To control the full event shape instead, pass an `events` schema whose Encoded side matches the SSE wire format: a required `data` string plus optional `id` and `event` fields. The handler then emits whole events. +### Streaming NDJSON + +`HttpApiSchema.StreamNdjson({ schema })` declares a response of newline-delimited JSON. Each stream element is encoded with `schema` and written as one line of JSON followed by `\n`, including the last. The wire format is the same as for [NDJSON request payloads](#streaming-ndjson). + +**Example** (Implementing an NDJSON Endpoint) + +```ts +import { NodeHttpServer, NodeRuntime } from "@effect/platform-node" +import { Effect, Layer, Schedule, Schema, Stream } from "effect" +import { HttpRouter } from "effect/http" +import { HttpApi, HttpApiBuilder, HttpApiEndpoint, HttpApiGroup, HttpApiSchema } from "effect/http-api" +import { createServer } from "node:http" + +const Message = Schema.Struct({ + text: Schema.String +}) + +const Api = HttpApi.make("myApi").add( + HttpApiGroup.make("group").add( + HttpApiEndpoint.get("getMessages", "/messages", { + // Each stream element is written as one line of JSON + // (default content type: application/x-ndjson) + success: HttpApiSchema.StreamNdjson({ schema: Message }) + }) + ) +) + +const GroupLayer = HttpApiBuilder.group( + Api, + "group", + (handlers) => + handlers.handle("getMessages", () => + Effect.succeed( + Stream.make({ text: "one" }, { text: "two" }, { text: "three" }).pipe( + Stream.schedule(Schedule.spaced("500 millis")) + ) + )) +) + +const ApiLayer = HttpApiBuilder.layer(Api).pipe( + Layer.provide(GroupLayer), + HttpRouter.serve, + Layer.provide(NodeHttpServer.layer(createServer, { port: 3000 })) +) + +Layer.launch(ApiLayer).pipe(NodeRuntime.runMain) + +// curl 'http://localhost:3000/messages' --no-buffer +// {"text":"one"} +// {"text":"two"} +// {"text":"three"} +``` + +`StreamNdjson` has no `error` schema. If the handler's stream fails after the response has started, the connection is aborted, so the client sees a body read error and never a clean end. Typed failures mid-stream need `StreamSse` (see [Typed Stream Failures](#typed-stream-failures)). + ### Consuming Streams with the Derived Client -For both constructors, calling the endpoint on the derived client resolves to a typed `Stream`: +For all three constructors, calling the endpoint on the derived client resolves to a typed `Stream`: ```ts const events = yield * client.group.getEvents() @@ -1974,6 +2353,18 @@ const events = yield * client.group.getEvents() yield * Stream.runForEach(events, (message) => Console.log(message.text)) ``` +The element types and errors are: + +| Success schema | Client stream | +| -------------------------- | ------------------------------------------------------------------------------------------------------ | +| `StreamUint8Array()` | `Stream` | +| `StreamNdjson({ schema })` | `Stream` | +| `StreamSse(options)` | `Stream`, where `E` is the `error` schema | + +An NDJSON response is decoded with the same rules as an NDJSON request payload. A malformed line or an element that fails to decode is a `SchemaError`. The client also enforces the declared `maxElementBytes` (1 MiB by default), so a server that never sends `\n` can't make it buffer forever. A breach is an `HttpClientError` with a `DecodeError` reason. A server failure mid-stream is the `HttpClientError` from reading the body. + +If the endpoint also has a [streaming payload](#streaming-requests), the upload can still be running while the response is read, and a late failure of the payload stream fails the response stream with its error too. + ### Typed Stream Failures `HttpApiSchema.StreamSse` accepts an `error` schema describing typed stream failures. Since the HTTP status is already sent when the stream fails, the failure cannot travel as a normal error response. Instead, the server encodes the full `Cause` into a reserved `effect/http-api/stream/failure` event, and the derived client decodes that event and fails the stream with the original cause. @@ -2003,6 +2394,65 @@ Things to know: - An endpoint may declare at most one streaming success schema, `HEAD` endpoints cannot declare one, and streaming schemas are not supported in error responses. - The `effect/http-api/stream/failure` event name is reserved: `events` schemas may not declare it, and the client treats such an event as a stream failure only when its `data` decodes to a `Cause`. +## Upgrading the Connection + +`HttpApiSchema.Upgrade({ protocol })` declares an endpoint that switches the connection to another protocol with an HTTP/1.1 upgrade, like Docker's `Upgrade: tcp` for container attach. After `101 Switching Protocols` the connection carries raw bytes in both directions as a `Socket`. + +The handler receives the connection as `socket` and returns when it is done with it. The `101` is sent when the handler acquires the socket's reader, so the handler can still fail with a declared error before then. Socket operations fail with `SocketError`, which the handler handles itself, since the response has already switched protocols. + +**Example** (Implementing an Attach Endpoint) + +```ts +import { Effect, Schema } from "effect" +import { HttpApi, HttpApiBuilder, HttpApiEndpoint, HttpApiGroup, HttpApiSchema } from "effect/http-api" + +class NotFound extends Schema.TaggedError()("NotFound", {}, { httpApiStatus: 404 }) {} + +const Api = HttpApi.make("myApi").add( + HttpApiGroup.make("containers").add( + HttpApiEndpoint.post("attach", "/containers/:id/attach", { + params: { id: Schema.String }, + success: HttpApiSchema.Upgrade({ protocol: "tcp" }), + error: NotFound + }) + ) +) + +const GroupLayer = HttpApiBuilder.group( + Api, + "containers", + (handlers) => + handlers.handle("attach", ({ params, socket }) => + Effect.gen(function*() { + if (params.id !== "abc") return yield* new NotFound() + const reader = yield* socket.reader // sends the 101 + const writer = yield* socket.writer + yield* writer.write("attached\n") + const chunks = yield* reader.pull + yield* writer.writeAll(chunks) + }).pipe( + // the connection is already upgraded, so socket failures end the handler + Effect.catchTag("SocketError", () => Effect.void), + Effect.scoped + )) +) +``` + +The derived client sends `Connection: Upgrade` and `Upgrade: tcp`, and resolves to the `101` response headers and the connection. Any other response is decoded as one of the endpoint's errors. + +```ts +const { headers, socket } = yield * client.containers.attach({ params: { id: "abc" } }) +const reader = yield * socket.reader +``` + +Things to know: + +- The client needs a backend that can hand over the connection: `NodeHttpClient.layerNodeHttp` (Node, Bun 1.4+, Deno) or `NodeHttpClient.layerUndici` (Node, Deno). Fetch and `XMLHttpRequest` cannot send upgrade requests, so the call fails with a `TransportError` before anything is sent. +- Serve upgrade endpoints with `NodeHttpServer`, which also works on Bun 1.4+ and Deno. `Bun.serve` and `Deno.serve` only upgrade to WebSocket, so `BunHttpServer` and `DenoHttpServer` fail these upgrades. +- Requests that do not ask for the declared protocol get `426 Upgrade Required`. +- An upgrade endpoint cannot declare a payload or other success schemas, and upgrades are not supported in error responses. For WebSocket, use `HttpServerRequest.upgrade` and `Socket.makeWebSocket` instead. +- Releasing the writer's scope half-closes the connection, and releasing the reader's scope closes it. + # Error Handling ## Adding Custom Error Responses diff --git a/repos/effect/packages/effect/MCP.md b/repos/effect/packages/effect/MCP.md index 5460022953..49559c6f31 100644 --- a/repos/effect/packages/effect/MCP.md +++ b/repos/effect/packages/effect/MCP.md @@ -80,13 +80,16 @@ The part layers are merged into one layer that has a MCP server implementation a `McpServer.layerStdio` is used to create a standard I/O–based MCP server identified by its name and version. Its ordered, non-empty `protocols` declaration names implemented protocol adapters rather than arbitrary version strings. This release supports `McpProtocol.v2024_11_05`, -`McpProtocol.v2025_03_26`, and `McpProtocol.v2025_06_18`. The `v2024_11_05` adapter implements that -revision's RPC schemas and stdio framing, including its batch policy. It does not implement the -historical two-endpoint HTTP+SSE transport. `McpServer.layerHttp` instead offers the 2024 RPC schema -through the same single-endpoint HTTP compatibility transport used by the 2025 adapters. Because of +`McpProtocol.v2025_03_26`, `McpProtocol.v2025_06_18`, `McpProtocol.v2025_11_25`, and +`McpProtocol.v2026_07_28`. Both `layerStdio` and `layerHttp` support all five adapters; +`layerHttpRequestScoped` supports only the stateless `v2026_07_28` adapter. +The `v2024_11_05` adapter implements that revision's RPC schemas and stdio framing, including its +batch policy. It does not implement the historical two-endpoint HTTP+SSE transport. +`McpServer.layerHttp` instead offers the 2024 RPC schema through the same single-endpoint HTTP +compatibility transport used by the newer adapters. Because of the layer architecture the server implementation can be easily exchanged with this HTTP-based implementation. Finally, a logging layer is added with -`Logger.layer([Logger.consolePretty({ stderr: true })])`, ensuring logs are written to `stderr`. +`Layer.succeed(Logger.LogToStderr, true)`, ensuring logs are written to `stderr`. This is essential when using stdio, as any output to `stdout` would interfere with the protocol communication. @@ -370,3 +373,199 @@ const ServerLayer = Layer.mergeAll( // Run the server Layer.launch(ServerLayer).pipe(NodeRuntime.runMain) ``` + +## Connecting to an MCP server + +`McpClient` connects to an MCP server, discovers its tools, prompts and resources, and calls them. +The client and its `McpClientError` module are experimental, so their API may change in minor +releases. It supports the `2025-11-25` and `2026-07-28` protocol revisions over stdio and Streamable HTTP. +The transport selects the revision explicitly; the client does not negotiate one. + +The client covers discovery, tool calls, prompt retrieval, resource reads and Toolkit adaptation. +Resource templates, completion, logging, subscriptions, roots, sampling, elicitation, OAuth and +automatic retries are not implemented. Server requests other than `ping` receive a method-not-found +response, and modern `input_required` results fail with reason `UnsupportedError`. + +### Connecting over HTTP + +`McpClient.layerHttp` provides a connected client from an `HttpClient`. The layer scope owns the +connection: legacy sessions are closed with a `DELETE` when the scope ends. + +```typescript +import { Effect, Layer } from "effect" +import { McpClient, McpProtocol } from "effect/ai" +import { FetchHttpClient } from "effect/http" + +const ClientLayer = McpClient.layerHttp({ + name: "report-workflow", + version: "1.0.0", + protocol: McpProtocol.v2026_07_28, + url: "http://localhost:3000/mcp" +}).pipe(Layer.provide(FetchHttpClient.layer)) + +const program = Effect.gen(function*() { + const client = yield* McpClient.McpClient + const tools = yield* client.listTools() + yield* Effect.log(tools.map((tool) => tool.name)) +}) + +await Effect.runPromise(program.pipe(Effect.provide(ClientLayer))) +``` + +Transport `headers` can carry static credentials. For rotating tokens, wrap the `HttpClient` with +`HttpClient.mapRequestEffect` and read the current token there. + +### Connecting to a subprocess + +`McpClient.layerStdio` spawns the server with the `ChildProcessSpawner` service and exchanges +newline-delimited JSON-RPC over its stdin and stdout. Server logs belong on stderr. + +```typescript +import { NodeServices } from "@effect/platform-node" +import { Effect, Layer } from "effect" +import { McpClient, McpProtocol } from "effect/ai" +import { ChildProcess } from "effect/process" + +const ClientLayer = McpClient.layerStdio({ + name: "report-workflow", + version: "1.0.0", + protocol: McpProtocol.v2025_11_25, + command: ChildProcess.make("node", ["./mcp-server.mjs"]) +}).pipe(Layer.provide(NodeServices.layer)) + +const program = Effect.gen(function*() { + const client = yield* McpClient.McpClient + const tools = yield* client.listTools() + yield* Effect.log(tools.map((tool) => tool.name)) +}) + +await Effect.runPromise(program.pipe(Effect.provide(ClientLayer))) +``` + +When the scope closes, the client closes stdin and waits five seconds for the process to exit, then +sends `SIGTERM`, and `SIGKILL` five seconds later. + +To build a client without a layer, provide a `McpClient.Transport` from `McpClient.http` or +`McpClient.stdio` to `McpClient.make` inside an `Effect.scoped` region, and keep every use of the +client inside that scope. + +### Calling tools + +`callTool` takes the descriptor returned by `listTools`, not just a name: on modern HTTP the +descriptor's `x-mcp-header` annotations decide which arguments are mirrored into request headers. +Without a `schema` the full `McpSchema.CallToolResult` is returned, including `isError`, so the +caller decides how to treat tool-reported failures. With a `schema`, `structuredContent` is decoded +and a tool-reported failure fails with reason `ToolError` carrying the original result. + +```typescript +import { Effect, Schema } from "effect" +import { McpClient, McpClientError } from "effect/ai" + +const Report = Schema.Struct({ count: Schema.Number }) + +export const readReport = Effect.fn("readReport")(function*(reportId: string) { + const client = yield* McpClient.McpClient + const tools = yield* client.listTools() + const tool = tools.find((tool) => tool.name === "read_report") + if (tool === undefined) { + return yield* new McpClientError.McpClientError({ + reason: new McpClientError.UnsupportedError({ description: "Server does not expose read_report" }) + }) + } + return yield* client.callTool({ tool, arguments: { reportId }, schema: Report }, { timeout: "2 seconds" }) +}) +``` + +Remote input schemas are returned as JSON Schema metadata only. The client does not validate +arguments against them; define the argument contract in your application or in a `Toolkit`. + +### Handling failures + +Every failure is an `McpClientError.McpClientError` whose `reason` is one of the reason classes +exported by `McpClientError`: `TransportError`, `ProtocolError` (with the JSON-RPC `code` and +`data`), `HttpError` (with `status` and the `www-authenticate` and `retry-after` headers), +`ClosedError` (with `sessionExpired`), `TimeoutError` (with `operation`), `LimitError`, +`UnsupportedError`, `ValidationError`, `ConfigurationError` and `ToolError` (with the tool +`result`). Each reason derives its `message` from these fields and an optional `description`. +`McpClientError.isMcpClientError` and `McpClientError.isMcpClientErrorReason` identify them. + +```typescript +import { Effect, Schema } from "effect" +import type { McpClient, McpSchema } from "effect/ai" + +const Report = Schema.Struct({ count: Schema.Number }) + +export const readWithDeadline = (client: McpClient.McpClient, tool: McpSchema.Tool, reportId: string) => + client.callTool({ tool, arguments: { reportId }, schema: Report }, { timeout: "2 seconds" }).pipe( + Effect.map((report) => ({ status: "ready" as const, report })), + Effect.catchReason("McpClientError", "TimeoutError", () => Effect.succeed({ status: "deferred" as const })) + ) +``` + +`Effect.catchReason` and `Effect.catchReasons` recover from selected reasons; +`Effect.catchTag("McpClientError", ...)` handles them all. The client never retries, so a timeout +says nothing about whether the server completed the call. + +### Limits and deadlines + +Initialization and each complete discovery run under a single deadline, 60 seconds by default and +configurable per client with `timeout` or per call with the `timeout` option. Tool calls, prompts and +resource reads have no deadline unless you pass their `timeout` option or compose `Effect.timeout`. +Discovery follows pagination until the server stops returning a cursor and fails with `LimitError` +after 100 pages, 10,000 items or a repeated cursor. A missing server capability makes the list +operations return an empty array and the retrieval operations fail with `UnsupportedError`. + +Discovery omits tools that require task-augmented execution. Modern HTTP discovery also omits tools +whose `x-mcp-header` annotations are invalid. Messages larger than `maxMessageBytes` (16 MiB by +default) are rejected in both directions. + +The client creates no spans; wrap calls with `Effect.withSpan` or `Effect.fn` as needed. + +### Exposing discovered tools to a language model + +`McpClient.toolkit` discovers every tool and returns a `Toolkit.WithHandler` that `LanguageModel` +accepts. An optional prefix qualifies the model-facing names. Results become text by joining text +blocks, embedded text resources and resource links; when there is no content, `structuredContent` +is serialized as JSON. Binary content and tool-reported errors fail the handler. + +```typescript +import { Effect } from "effect" +import { LanguageModel, McpClient } from "effect/ai" + +export const readRemoteReport = (client: McpClient.McpClient) => + Effect.gen(function*() { + const toolkit = yield* McpClient.toolkit(client, "reports__") + return yield* LanguageModel.generateText({ prompt: "Read the quarterly report", toolkit }) + }) +``` + +The toolkit is a snapshot: build a new one to pick up changes on the server. For custom naming, +validation, approval or result conversion, define tools with `Tool.make` or `Tool.dynamic` and +call `client.callTool` from their handlers. + +### Prompts and resources + +`listPrompts` and `listResources` return complete snapshots. `getPrompt` takes a discovered +`McpSchema.Prompt` with optional string arguments; `readResource` takes any URI. Results reuse the +`McpSchema.GetPromptResult` and `McpSchema.ReadResourceResult` types shared with the server. + +```ts +import { Effect } from "effect" +import type { McpClient } from "effect/ai" + +export const loadContext = (client: McpClient.McpClient) => + Effect.gen(function*() { + const prompts = yield* client.listPrompts() + const prompt = prompts.find((prompt) => prompt.name === "review") + if (prompt !== undefined) { + const result = yield* client.getPrompt({ prompt, arguments: { code: "const answer = 42" } }) + yield* Effect.log(result.messages) + } + + const resources = yield* client.listResources() + for (const resource of resources) { + const result = yield* client.readResource({ uri: resource.uri }) + yield* Effect.log(result.contents) + } + }) +``` diff --git a/repos/effect/packages/effect/SCHEMA.md b/repos/effect/packages/effect/SCHEMA.md index ac4a29956a..67cac3f095 100644 --- a/repos/effect/packages/effect/SCHEMA.md +++ b/repos/effect/packages/effect/SCHEMA.md @@ -2174,7 +2174,7 @@ This helper has some advantages over a dedicated constructor: - You can choose among multiple possible tag fields if present. - It supports unions that include nested unions. -Each member must have a unique discriminant property key. `toTaggedUnion` rejects overlapping discriminant types in known member tuples at compile time, including nested unions. It also throws when it encounters a duplicate at runtime. Numeric and string values that resolve to the same property key, such as `1` and `"1"`, are considered duplicates. +Each member must have a unique discriminant property key. `toTaggedUnion` throws when it encounters a duplicate at runtime, including in nested unions. Numeric and string values that resolve to the same property key, such as `1` and `"1"`, are considered duplicates. **Note**. If the tag is the standard `_tag` field, you can use `Schema.TaggedUnion` instead. @@ -2586,7 +2586,9 @@ The default formatter chooses the error label from the level that failed: An `identifier` does not name a failed filter. Use `expected` to name the filter in the default formatter, or `message` to replace the filter failure -message completely. +message completely. A filter's `message` annotation also overrides messages +returned by the filter, whether they are returned directly or attached to a +nested path. Without a `message` annotation, returned messages are preserved. **Example** (Schema identifier versus filter expected message) @@ -4254,7 +4256,7 @@ Effect.runPromiseExit(program).then((exit) => console.log(JSON.stringify(exit, n "_tag": "Failure", "cause": { "_id": "Cause", - "failures": [ + "reasons": [ { "_tag": "Fail", "error": { @@ -5136,6 +5138,34 @@ console.log(serialized) // date1 uses your custom number format, date2 uses the default ISO string format ``` +#### Decoding JSON Written by Go + +Go's `encoding/json` writes a nil pointer, slice, or map as `null`, and API documents rarely say which fields can be `null`. `Schema.toCodecGoJson` is `Schema.toCodecJson` with these rules added at every level of the schema: + +- an optional key that is `null` decodes as a missing key +- a required array that is `null` decodes as `[]` +- a required record that is `null` decodes as `{}` + +Encoding is unchanged from `Schema.toCodecJson`. A key whose schema already accepts `null` (such as `Schema.NullOr(X)`) keeps that behavior. Array and record rules also apply to top-level and nested collections, but not tuples, non-empty arrays, or structs; collection checks still apply. The codec also accepts `encoding/json/v2` output, which uses `[]` and `{}` for nil slices and maps by default. + +This unstable codec deliberately normalizes nil collections to empty ones. Go distinguishes nil from non-nil empty slices and maps, and unmarshalling missing keys, `null`, and empty collections can affect existing values differently. Use `Schema.toCodecJson` with explicit nullable schemas if those distinctions matter. + +```ts +import { Schema } from "effect" + +const Container = Schema.Struct({ + Id: Schema.String, + Names: Schema.Array(Schema.String), + Labels: Schema.Record(Schema.String, Schema.String), + Status: Schema.optionalKey(Schema.String) +}) + +const codec = Schema.toCodecGoJson(Container) + +console.log(Schema.decodeUnknownSync(codec)({ Id: "a", Names: null, Labels: null, Status: null })) +// Output: { Id: 'a', Names: [], Labels: {} } +``` + ### StringTree Canonical Codec The `StringTree` codec converts all values to strings, keeping the structure but not the original types. @@ -6404,11 +6434,38 @@ would strip. Use the imported schema's actual checks and parse options when reas Import translates a Draft 2020-12 subset. `$dynamicRef`, `contains`, `dependentRequired`, `dependentSchemas`, active `if` / `then` / `else`, `unevaluatedItems`, and `unevaluatedProperties` are rejected with an error identifying the unsupported keyword. Inactive conditional keywords and `minContains` / `maxContains` without -`contains` have no validation effect and are ignored. Unknown extension keywords are ignored and their semantics are not -enforced. Objects and arrays used as `const` values or `enum` members are rejected. Only strings, numbers, booleans, and +`contains` have no validation effect and are ignored. Unknown extension keywords are ignored by default and their +semantics are not enforced. Objects and arrays used as `const` values or `enum` members are rejected. Only strings, numbers, booleans, and null are supported. The optional `onEnter` callback can normalize each JSON Schema node before it is translated. +Both import functions accept `includeAnnotationKey` to preserve selected custom keys as Effect schema annotations. +It pairs with the exporter's option of the same name: + +```ts +import { JsonSchema, Schema, SchemaRepresentation } from "effect" + +const document = JsonSchema.fromSchemaDraft2020_12({ + type: "string", + "x-sensitive": true +}) +const options = { includeAnnotationKey: (key: string) => key.startsWith("x-") } +const schema = SchemaRepresentation.fromJsonSchemaDocument(document, options) +const exported = Schema.toJsonSchemaDocument(schema, options) +// exported.schema: { type: "string", "x-sensitive": true } +``` + +Standard annotations such as `title` and `description` are imported independently of the predicate. Selection runs +after `onEnter` and only considers additional own enumerable keys. JSON Schema structural and validation keywords, +Effect's reserved annotation keys, and keys starting with `~` are excluded before calling the predicate. Selected +values must be JSON-compatible. Objects and arrays are retained as opaque metadata, without interpreting nested +schema keywords. Exceptions raised by the predicate pass through unchanged. + +Selected metadata stays on the value's schema when intersections merge checks or simplify to literals. When annotations +collide, later values overwrite earlier ones according to the existing merge order. Metadata belonging only to eliminated +alternatives or unreachable children is discarded. The resulting schema can have a different structure from the input; +retaining annotations does not provide a lossless round trip or enforce custom validation rules. + Intersections of overlapping unions are limited to disjoint root-type partitions and finite primitive `anyOf` literal sets. Other union intersections, including cases that would duplicate a nested choice, are rejected. @@ -6767,9 +6824,9 @@ const schema = Schema.Struct({ const r = SchemaParser.decodeUnknownExit(schema)({ a: "", c: [] }, { errors: "all" }) if (r._tag === "Failure") { - const failures = r.cause.failures - if (failures[0]?._tag === "Fail") { - const failureResult = SchemaIssue.makeFormatterStandardSchemaV1()(failures[0].error) + const reasons = r.cause.reasons + if (reasons[0]?._tag === "Fail") { + const failureResult = SchemaIssue.makeFormatterStandardSchemaV1()(reasons[0].error) const serializer = Schema.toCodecJson(Schema.StandardSchemaV1FailureResult) console.dir(Schema.encodeSync(serializer)(failureResult), { depth: null }) } diff --git a/repos/effect/packages/effect/benchmark/encoding/Xml.ts b/repos/effect/packages/effect/benchmark/encoding/Xml.ts new file mode 100644 index 0000000000..ddb592463f --- /dev/null +++ b/repos/effect/packages/effect/benchmark/encoding/Xml.ts @@ -0,0 +1,115 @@ +import { Effect, Stream } from "effect" +import { Xml } from "effect/encoding" +import { Bench } from "tinybench" + +const bench = new Bench() + +// A feed-style document: text, references and a few attributes per item +const feed = (items: number): string => { + const parts = ["\n\n\nFeed\n"] + for (let i = 0; i < items; i++) { + parts.push( + `\n Item ${i} & friends\n` + + ` https://example.com/items/${i}?a=1&b=2\n` + + ` urn:uuid:${i}-0000-4000-8000-000000000000\n` + + ` Mon, 01 Jan 2024 00:00:00 GMT\n` + + ` Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor ` + + `incididunt ut labore et dolore magna aliqua <b>bold</b> quis nostrud exercitation.\n` + + ` newseffect\n\n` + ) + } + parts.push("\n\n") + return parts.join("") +} + +// An attribute-heavy document with little text +const table = (rows: number): string => { + const parts = [""] + for (let i = 0; i < rows; i++) { + parts.push( + `` + ) + } + parts.push("
") + return parts.join("") +} + +const split = (text: string, size: number): Array => { + const chunks: Array = [] + for (let i = 0; i < text.length; i += size) chunks.push(text.slice(i, i + size)) + return chunks +} + +const splitBytes = (bytes: Uint8Array, size: number): Array => { + const chunks: Array = [] + for (let i = 0; i < bytes.length; i += size) chunks.push(bytes.subarray(i, i + size)) + return chunks +} + +const decodeString = (chunks: Array) => + Effect.runPromise( + Stream.fromIterable(chunks).pipe(Stream.pipeThroughChannel(Xml.decodeString()), Stream.runCount) + ) + +const decode = (chunks: Array) => + Effect.runPromise(Stream.fromIterable(chunks).pipe(Stream.pipeThroughChannel(Xml.decode()), Stream.runCount)) + +const elements = (chunks: Array, path: Array) => + Effect.runPromise( + Stream.fromIterable(chunks).pipe( + Stream.pipeThroughChannel(Xml.decodeString()), + Stream.pipeThroughChannel(Xml.elements(path)), + Stream.runCount + ) + ) + +const events = (text: string) => + Effect.runPromise( + Stream.make(text).pipe(Stream.pipeThroughChannel(Xml.decodeString()), Stream.runCollect) + ) + +const encodeString = (events: ReadonlyArray) => + Effect.runPromise( + Stream.fromIterable(events).pipe(Stream.pipeThroughChannel(Xml.encodeString()), Stream.runCount) + ) + +const encode = (events: ReadonlyArray) => + Effect.runPromise(Stream.fromIterable(events).pipe(Stream.pipeThroughChannel(Xml.encode()), Stream.runCount)) + +const encoder = new TextEncoder() +const documents = [ + { name: "feed", text: feed(8000), path: ["rss", "channel", "item"] }, + { name: "table", text: table(40000), path: ["table", "row"] } +] + +for (const { name, path, text } of documents) { + const bytes = encoder.encode(text) + const decoded = await events(text) + bench + .add(`${name} decodeString (1 chunk, ${(text.length / 1024 / 1024).toFixed(1)} MiB)`, async function() { + await decodeString([text]) + }) + .add(`${name} decodeString (64 KiB chunks)`, async function() { + await decodeString(split(text, 64 * 1024)) + }) + .add(`${name} decodeString (1 KiB chunks)`, async function() { + await decodeString(split(text, 1024)) + }) + .add(`${name} decode (64 KiB chunks)`, async function() { + await decode(splitBytes(bytes, 64 * 1024)) + }) + .add(`${name} elements (64 KiB chunks)`, async function() { + await elements(split(text, 64 * 1024), path) + }) + .add(`${name} encodeString`, async function() { + await encodeString(decoded) + }) + .add(`${name} encode`, async function() { + await encode(decoded) + }) +} + +await bench.run() + +console.table(bench.table()) diff --git a/repos/effect/packages/effect/benchmark/eventlog/Bandwidth.md b/repos/effect/packages/effect/benchmark/eventlog/Bandwidth.md new file mode 100644 index 0000000000..d312b539fb --- /dev/null +++ b/repos/effect/packages/effect/benchmark/eventlog/Bandwidth.md @@ -0,0 +1,18 @@ +# EventLog bandwidth + +Run from the repository root, optionally passing the number of entries to write (default 1000): + +```sh +nix develop -c pnpm --dir packages/effect exec node benchmark/eventlog/Bandwidth.ts 1000 +``` + +Two devices with the same identity connect to one in-memory server through the socket RPC protocols with `RpcSerialization.layerSchemaBinary`. The sockets are in-memory, but every frame is serialized as it would be on a real connection, and the byte counts are the serialized frames. One device writes the entries one `EventLog.write` at a time; the other receives them. + +Columns: + +- `writer up` / `writer down`: bytes the writing device sent and received while writing, after the handshake. +- `reader down`: bytes the other device received. +- `resume sequence`: the writer journal's `nextRemoteSequence` afterwards. +- `resume down`: bytes a fresh connection receives when it resumes changes from that sequence. + +The file only uses APIs that predate write acknowledgements, so it can be copied into an older checkout to compare. diff --git a/repos/effect/packages/effect/benchmark/eventlog/Bandwidth.ts b/repos/effect/packages/effect/benchmark/eventlog/Bandwidth.ts new file mode 100644 index 0000000000..5f91791019 --- /dev/null +++ b/repos/effect/packages/effect/benchmark/eventlog/Bandwidth.ts @@ -0,0 +1,223 @@ +import { Effect, Layer, Queue, Schema } from "effect" +import { + EventGroup, + EventJournal, + EventLog, + EventLogEncryption, + EventLogMessage, + EventLogRemote, + EventLogServerEncrypted, + EventLogServerUnencrypted +} from "effect/eventlog" +import { NetAddress } from "effect/net" +import { RpcClient, RpcSerialization, RpcServer } from "effect/rpc" +import { Socket, SocketServer } from "effect/socket" + +// Counts the bytes an event log client sends and receives while it writes, +// using the socket RPC protocols over an in-memory socket so that every frame +// is serialized exactly as it would be on a real connection. + +const entries = Number(process.argv[2] ?? 1000) +const storeId = EventLogMessage.StoreId.make("default") + +const UserGroup = EventGroup.empty.add({ + tag: "UserCreated", + primaryKey: (payload) => payload.id, + payload: Schema.Struct({ id: Schema.String, name: Schema.String }) +}) +const schema = EventLog.schema(UserGroup) + +const servers = [ + { + name: "encrypted", + handlers: EventLogServerEncrypted.layerRpcHandlers.pipe( + Layer.provide(EventLogServerEncrypted.layerStorageMemory) + ), + remote: EventLogRemote.layerEncrypted + }, + { + name: "unencrypted", + handlers: EventLogServerUnencrypted.layerRpcHandlers.pipe( + Layer.provide([ + EventLogServerUnencrypted.layerStorageMemory, + EventLog.layerRegistry, + EventLogServerUnencrypted.layerStoreMappingStatic({ storeId }), + Layer.succeed(EventLogServerUnencrypted.EventLogServerAuthorization, { + authorizeWrite: () => Effect.void, + authorizeRead: () => Effect.void, + authorizeIdentity: () => Effect.void + }) + ]) + ), + remote: EventLogRemote.layerUnencrypted + } +] as const + +// Event log messages carry binary fields, which JSON serialization rejects. +const serialization = RpcSerialization.layerSchemaBinary() + +const encoder = new TextEncoder() + +// A connected pair of in-memory sockets. `down` counts the bytes the server +// writes, `up` the bytes the client writes. +const makeSocketPair = Effect.gen(function*() { + const toServer = yield* Queue.unbounded() + const toClient = yield* Queue.unbounded() + const bytes = { up: 0, down: 0 } + const side = ( + incoming: Queue.Queue, + outgoing: Queue.Queue, + count: (bytes: number) => void + ) => { + const write = (chunk: Uint8Array | string | Socket.CloseEvent) => { + if (Socket.isCloseEvent(chunk)) return Effect.void + count(typeof chunk === "string" ? encoder.encode(chunk).byteLength : chunk.byteLength) + return Queue.offer(outgoing, chunk).pipe(Effect.asVoid) + } + return Socket.make({ + reader: Effect.succeed({ + pull: Queue.takeAll(incoming), + upgrade: () => Effect.void + }), + writer: Effect.succeed({ + write, + writeAll: (chunks) => Effect.forEach(chunks, write, { discard: true }) + }) + }) + } + return { + bytes, + client: side(toClient, toServer, (n) => bytes.up += n), + server: side(toServer, toClient, (n) => bytes.down += n) + } +}) + +const makeServer = Effect.fnUntraced(function*(server: (typeof servers)[number]) { + const sockets = yield* Queue.unbounded() + const socketServer = SocketServer.SocketServer.of({ + address: NetAddress.inetAddressUnsafe(NetAddress.ipv4Loopback, 0), + run: (handler) => + Queue.take(sockets).pipe( + Effect.flatMap((socket) => Effect.forkChild(handler(socket))), + Effect.forever + ) + }) + yield* Layer.build( + RpcServer.layer(EventLogMessage.EventLogRemoteRpcs).pipe( + Layer.provide(server.handlers), + Layer.provide(Layer.effect(RpcServer.Protocol, RpcServer.makeProtocolSocketServer)), + Layer.provide(Layer.succeed(SocketServer.SocketServer, socketServer)), + Layer.provide(serialization) + ) + ) + + // Connects a remote registered with the given registry layer. + const connectWith = Effect.fnUntraced(function*( + identity: EventLog.Identity["Service"], + registry: Layer.Layer + ) { + const pair = yield* makeSocketPair + yield* Queue.offer(sockets, pair.server) + const context = yield* Layer.build( + server.remote.pipe( + Layer.provide(RpcClient.layerProtocolSocket()), + Layer.provide(Layer.succeed(Socket.Socket, pair.client)), + Layer.provide(serialization), + Layer.provideMerge(registry) + ) + ) + const remote = yield* EventLogRemote.EventLogRemote.pipe(Effect.provide(context)) + yield* remote.whenAuthenticated(Effect.void).pipe(Effect.provideService(EventLog.Identity, identity)) + return { bytes: pair.bytes, context, remote } + }) + + // Connects a device whose event log replicates through the remote. + const connectDevice = (identity: EventLog.Identity["Service"], journal: EventJournal.EventJournal["Service"]) => + connectWith( + identity, + EventLog.layer( + schema, + EventLog.group(UserGroup, (handlers) => handlers.handle("UserCreated", () => Effect.void)).pipe( + Layer.provide(EventLog.layerRegistry) + ) + ).pipe( + Layer.provide(Layer.succeed(EventJournal.EventJournal, journal)), + Layer.provide(Layer.succeed(EventLog.Identity, identity)) + ) + ) + + // Connects only a remote, without an event log consuming its changes. + const connectRemote = (identity: EventLog.Identity["Service"]) => connectWith(identity, EventLog.layerRegistry) + + return { connectDevice, connectRemote } +}) + +const waitUntil = Effect.fnUntraced(function*(check: Effect.Effect) { + for (let i = 0; i < 6000; i++) { + if (yield* check) return + yield* Effect.sleep("5 millis") + } + return yield* Effect.die("timed out") +}) + +// Waits until a byte counter has not moved for a while. +const settle = Effect.fnUntraced(function*(read: () => number) { + let last = read() + let still = 0 + while (still < 4) { + yield* Effect.sleep("50 millis") + const current = read() + still = current === last ? still + 1 : 0 + last = current + } +}) + +const run = Effect.fnUntraced(function*(server: (typeof servers)[number]) { + const { connectDevice, connectRemote } = yield* makeServer(server) + const identity = yield* EventLog.makeIdentity + const writerJournal = yield* EventJournal.makeMemory + const readerJournal = yield* EventJournal.makeMemory + const writer = yield* connectDevice(identity, writerJournal) + // Another device with the same identity, which receives every write. + const reader = yield* connectDevice(identity, readerJournal) + yield* settle(() => writer.bytes.down + reader.bytes.down) + const writerStart = { ...writer.bytes } + const readerStart = { ...reader.bytes } + + const log = yield* EventLog.EventLog.pipe(Effect.provide(writer.context)) + for (let i = 0; i < entries; i++) { + yield* log.write({ schema, event: "UserCreated", payload: { id: `user-${i}`, name: `User ${i}` } }) + } + yield* waitUntil(Effect.map(readerJournal.entries, (stored) => stored.length === entries)) + yield* settle(() => writer.bytes.down + writer.bytes.up) + + // A fresh connection resumes from the writer's journal. + const startSequence = yield* writerJournal.nextRemoteSequence(writer.remote.id) + const resumed = yield* connectRemote(identity) + const resumedStart = resumed.bytes.down + yield* resumed.remote.changes({ identity, storeId, startSequence }) + yield* settle(() => resumed.bytes.down) + + return { + server: server.name, + entries, + "writer up": writer.bytes.up - writerStart.up, + "writer down": writer.bytes.down - writerStart.down, + "reader down": reader.bytes.down - readerStart.down, + "resume sequence": startSequence, + "resume down": resumed.bytes.down - resumedStart + } +}, Effect.scoped) + +const program = Effect.gen(function*() { + const rows = [] + for (const server of servers) { + rows.push(yield* run(server)) + } + console.table(rows) +}).pipe(Effect.provide(EventLogEncryption.layerSubtle)) + +Effect.runPromise(program).catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/repos/effect/packages/effect/package.json b/repos/effect/packages/effect/package.json index da512a3781..9a155ee4a0 100644 --- a/repos/effect/packages/effect/package.json +++ b/repos/effect/packages/effect/package.json @@ -1,7 +1,7 @@ { "name": "effect", "type": "module", - "version": "4.0.5", + "version": "4.0.6", "license": "MIT", "description": "The missing standard library for TypeScript, for writing production-grade software.", "homepage": "https://effect.website", @@ -54,8 +54,17 @@ "./workflow": "./src/workflow/index.ts", "./workers": "./src/workers/index.ts", "./*": "./src/*.ts", + "./ai/internal/*": null, "./cli/internal/*": null, "./cluster/internal/*": null, + "./eventlog/internal/*": null, + "./http/internal/*": null, + "./http/FindMyWay/internal/*": null, + "./http/MultipartParser/internal/*": null, + "./http-api/internal/*": null, + "./observability/internal/*": null, + "./rpc/internal/*": null, + "./workflow/internal/*": null, "./internal/*": null, "./index": null, "./*/index": null @@ -101,6 +110,15 @@ "./internal/*": null, "./cli/internal/*": null, "./cluster/internal/*": null, + "./ai/internal/*": null, + "./eventlog/internal/*": null, + "./http/internal/*": null, + "./http/FindMyWay/internal/*": null, + "./http/MultipartParser/internal/*": null, + "./http-api/internal/*": null, + "./observability/internal/*": null, + "./rpc/internal/*": null, + "./workflow/internal/*": null, "./index": null, "./*/index": null } @@ -117,10 +135,11 @@ "ajv-draft-04": "^1.0.0", "ast-types": "^0.16.3", "esbuild": "^0.28.2", + "fake-indexeddb": "^6.2.5", "immer": "^11.1.21", "miniflare": "5.20260925.0-alpha", - "protobufjs": "^8.8.0", - "tinybench": "^6.2.1", + "protobufjs": "^8.8.1", + "tinybench": "^6.2.2", "valibot": "^1.5.0" } } diff --git a/repos/effect/packages/effect/src/Array.ts b/repos/effect/packages/effect/src/Array.ts index 31f0b052d8..c360656595 100644 --- a/repos/effect/packages/effect/src/Array.ts +++ b/repos/effect/packages/effect/src/Array.ts @@ -2620,7 +2620,7 @@ export const rotate: { * **When to use** * * Use when checking membership with caller-provided equality instead of - * `Equal.equivalence()`. + * `Equal.asEquivalence()`. * * **Example** (Checking with custom equality) * @@ -2632,7 +2632,7 @@ export const rotate: { * pipe([1, 2, 3, 4], containsNumber(3)) // => true * ``` * - * @see {@link contains} for the `Equal.equivalence()` variant + * @see {@link contains} for the `Equal.asEquivalence()` variant * * @stability stable * @category predicates @@ -2652,7 +2652,7 @@ export const containsWith = (isEquivalent: (self: A, that: A) => boolean): { }) /** - * Checks whether an array contains a value, using `Equal.equivalence()` for + * Checks whether an array contains a value, using `Equal.asEquivalence()` for * comparison. * * **When to use** @@ -3086,7 +3086,7 @@ export const window: { * ) // => [["a", "a"], ["b", "b", "b"], ["c"], ["a"]] * ``` * - * @see {@link group} for grouping adjacent elements with `Equal.equivalence()` + * @see {@link group} for grouping adjacent elements with `Equal.asEquivalence()` * @see {@link groupBy} for grouping all elements into a record by key, regardless of adjacency * * @stability stable @@ -3116,7 +3116,7 @@ export const groupWith: { ) /** - * Groups consecutive equal elements using `Equal.equivalence()`. + * Groups consecutive equal elements using `Equal.asEquivalence()`. * * **When to use** * @@ -3252,13 +3252,22 @@ const hashBucketsHas = (buckets: HashBuckets, value: unknown): boolean => { } /** - * Computes the union of two arrays using a custom equivalence, removing - * duplicates. + * Computes the union of two iterables using a custom equivalence. * * **When to use** * * Use when you need the union of two arrays but duplicate detection must use a - * custom equivalence instead of the default `Equal.equivalence()`. + * custom equivalence instead of the default `Equal.asEquivalence()`. + * + * **Details** + * + * Keeps the elements of `self`, then appends elements of `that` that have no + * equivalent in `self`. Calls `isEquivalent` with an element from `self` first + * and an element from `that` second. + * + * **Gotchas** + * + * Each input must already be deduplicated. * * **Example** (Computing unions with custom equality) * @@ -3268,7 +3277,7 @@ const hashBucketsHas = (buckets: HashBuckets, value: unknown): boolean => { * Array.unionWith([1, 2], [2, 3], (a, b) => a === b) // => [1, 2, 3] * ``` * - * @see {@link union} for the `Equal.equivalence()` variant + * @see {@link union} for the `Equal.asEquivalence()` variant * @see {@link intersectionWith} for keeping elements present in both arrays * @see {@link differenceWith} for keeping elements present only in the first array * @@ -3292,13 +3301,21 @@ export const unionWith: { isEquivalent: (self: A, that: B) => boolean ): NonEmptyArray (self: Iterable, that: Iterable, isEquivalent: (self: A, that: B) => boolean): Array -} = dual(3, (self: Iterable, that: Iterable, isEquivalent: (self: A, that: A) => boolean): Array => { +} = dual(3, (self: Iterable, that: Iterable, isEquivalent: (self: A, that: B) => boolean): Array => { const a = fromIterable(self) const b = fromIterable(that) if (isReadonlyArrayNonEmpty(a)) { if (isReadonlyArrayNonEmpty(b)) { - const dedupe = dedupeWith(isEquivalent) - return dedupe(appendAll(a, b)) + const out: Array = [...a] + next: for (const value of b) { + for (const candidate of a) { + if (isEquivalent(candidate, value)) { + continue next + } + } + out.push(value) + } + return out } return a } @@ -3306,8 +3323,12 @@ export const unionWith: { }) /** - * Computes the union of two arrays, removing duplicates using - * `Equal.equivalence()`. + * Computes the union of two iterables using `Equal.asEquivalence()`. + * + * **Gotchas** + * + * Each input must already be deduplicated. Use {@link dedupe} to remove + * duplicates from an input first. * * **Example** (Computing array unions) * @@ -3367,7 +3388,7 @@ export const union: { * Array.intersectionWith(isEquivalent)(array2)(array1) // => [{ id: 1 }, { id: 3 }] * ``` * - * @see {@link intersection} for the `Equal.equivalence()` variant + * @see {@link intersection} for the `Equal.asEquivalence()` variant * @see {@link unionWith} for keeping values from either array with custom equality * @see {@link differenceWith} for keeping values only from the first array with custom equality * @@ -3390,7 +3411,7 @@ export const intersectionWith = (isEquivalent: (self: A, that: A) => boolean) } /** - * Computes the intersection of two arrays using `Equal.equivalence()`. Order is + * Computes the intersection of two arrays using `Equal.asEquivalence()`. Order is * determined by the first array. * * **When to use** @@ -3444,7 +3465,7 @@ export const intersection: { * Array.differenceWith((a, b) => a === b)([1, 2, 3], [2, 3, 4]) // => [1] * ``` * - * @see {@link difference} for the `Equal.equivalence()` variant + * @see {@link difference} for the `Equal.asEquivalence()` variant * @see {@link unionWith} for keeping values from either array with custom equality * @see {@link intersectionWith} for keeping values present in both arrays with custom equality * @@ -3468,12 +3489,12 @@ export const differenceWith = (isEquivalent: (self: A, that: A) => boolean): /** * Computes elements in the first array that are not in the second, using - * `Equal.equivalence()`. + * `Equal.asEquivalence()`. * * **When to use** * * Use when you need to keep values from the first array that are absent from - * the second and the default `Equal.equivalence()` comparison is appropriate. + * the second and the default `Equal.asEquivalence()` comparison is appropriate. * * **Example** (Computing array differences) * @@ -4642,7 +4663,7 @@ export const dedupeWith: { ) /** - * Removes duplicates using `Equal.equivalence()`, preserving the order of the + * Removes duplicates using `Equal.asEquivalence()`, preserving the order of the * first occurrence. * * **When to use** @@ -4726,7 +4747,7 @@ export const dedupeAdjacentWith: { }) /** - * Removes consecutive duplicate elements using `Equal.equivalence()`. + * Removes consecutive duplicate elements using `Equal.asEquivalence()`. * * **When to use** * diff --git a/repos/effect/packages/effect/src/Bytes.ts b/repos/effect/packages/effect/src/Bytes.ts new file mode 100644 index 0000000000..106f295065 --- /dev/null +++ b/repos/effect/packages/effect/src/Bytes.ts @@ -0,0 +1,111 @@ +/** + * Joins and compares byte sequences stored in `Uint8Array` values. + * + * Byte-to-text conversions live in `TextEncoder` / `TextDecoder`, and hex or + * base64 conversions live in the `Encoding` modules. + * + * @stability unstable + * @since 4.0.6 + */ +import * as Equ from "./Equivalence.ts" +import * as order from "./Order.ts" + +/** + * Joins byte chunks into one new `Uint8Array`. + * + * **When to use** + * + * Use to combine frames, headers, or streamed chunks into a single buffer. + * + * **Gotchas** + * + * The result is always a new array, even when there is zero or one chunk, so + * it never aliases an input. + * + * **Example** (Joining byte chunks) + * + * ```ts import.meta.vitest + * import { Bytes } from "effect" + * + * const bytes = Bytes.concat([new Uint8Array([1, 2]), new Uint8Array([3])]) + * Array.from(bytes) // => [1, 2, 3] + * ``` + * + * @stability unstable + * @category combinators + * @since 4.0.6 + */ +export const concat = (chunks: ReadonlyArray): Uint8Array => { + let length = 0 + for (let i = 0; i < chunks.length; i++) { + length += chunks[i].length + } + const out = new Uint8Array(length) + let offset = 0 + for (let i = 0; i < chunks.length; i++) { + out.set(chunks[i], offset) + offset += chunks[i].length + } + return out +} + +/** + * Compares byte sequences for equality, byte by byte. + * + * **Gotchas** + * + * This returns as soon as a byte differs, so it is not constant-time. Do not + * use it to compare secrets. + * + * **Example** (Comparing byte sequences) + * + * ```ts import.meta.vitest + * import { Bytes } from "effect" + * + * Bytes.Equivalence(new Uint8Array([1, 2]), new Uint8Array([1, 2])) // => true + * Bytes.Equivalence(new Uint8Array([1, 2]), new Uint8Array([1])) // => false + * ``` + * + * @stability unstable + * @category instances + * @since 4.0.6 + */ +export const Equivalence: Equ.Equivalence = Equ.make((self, that) => { + if (self.length !== that.length) { + return false + } + for (let i = 0; i < self.length; i++) { + if (self[i] !== that[i]) { + return false + } + } + return true +}) + +/** + * Orders byte sequences lexicographically by unsigned byte value, placing a + * prefix before any longer sequence that starts with it. + * + * **Example** (Sorting byte sequences) + * + * ```ts import.meta.vitest + * import { Bytes } from "effect" + * + * Bytes.Order(new Uint8Array([1, 2]), new Uint8Array([1, 3])) // => -1 + * Bytes.Order(new Uint8Array([2]), new Uint8Array([1, 9])) // => 1 + * Bytes.Order(new Uint8Array([1]), new Uint8Array([1, 0])) // => -1 + * ``` + * + * @stability unstable + * @category instances + * @since 4.0.6 + */ +export const Order: order.Order = order.make((self, that) => { + const length = Math.min(self.length, that.length) + for (let i = 0; i < length; i++) { + if (self[i] !== that[i]) { + return self[i] < that[i] ? -1 : 1 + } + } + return self.length < that.length ? -1 : self.length > that.length ? 1 : 0 +}) diff --git a/repos/effect/packages/effect/src/Channel.ts b/repos/effect/packages/effect/src/Channel.ts index 35f3d6955f..ba5951ad09 100644 --- a/repos/effect/packages/effect/src/Channel.ts +++ b/repos/effect/packages/effect/src/Channel.ts @@ -22,7 +22,7 @@ import type * as Filter from "./Filter.ts" import type { LazyArg } from "./Function.ts" import { constant, constTrue, constVoid, dual, identity as identity_ } from "./Function.ts" import * as Count from "./internal/count.ts" -import { ClockRef, endSpan } from "./internal/effect.ts" +import { ClockRef, endSpan, findRecoverableError } from "./internal/effect.ts" import { addSpanStackTrace } from "./internal/tracer.ts" import * as Iterable from "./Iterable.ts" import * as Latch from "./Latch.ts" @@ -488,7 +488,7 @@ const asyncQueue = ( Effect.tap((queue) => Scope.provide(f(queue), scope).pipe( Effect.catchCause((cause) => Queue.failCause(queue, cause)), - Effect.forkIn(scope) + Effect.forkIn(scope, { startImmediately: true }) ) ) ) @@ -1701,6 +1701,12 @@ export const fromSchedule = ( /** * Creates a channel from a lazily supplied Web `ReadableStream`. * + * **Details** + * + * Read failures are mapped with `onError`. Finalization cancels the reader and + * releases its lock, or only releases the lock when `releaseLockOnEnd` is set. + * A rejected cancel hook becomes a defect. + * * **Example** (Reading from a Web stream) * * ```ts import.meta.vitest @@ -1743,6 +1749,7 @@ export const pullIntoWritableStream = (options: { readonly writable: WritableStream readonly onError: (error: unknown) => E readonly closeOnDone?: boolean | undefined + readonly onAbort?: ((cause: Cause.Cause) => void) | undefined }): Pull.Pull => Effect.acquireUseRelease( Effect.sync(() => options.writable.getWriter()), @@ -1773,10 +1780,11 @@ export const pullIntoWritableStream = (options: { : loop return Effect.onError( withClose, - (cause) => - Pull.isDoneCause(cause) - ? Effect.void - : Effect.promise(() => writer.abort(cause).catch(constVoid)) + (cause) => { + if (Pull.isDoneCause(cause)) return Effect.void + options.onAbort?.(cause as Cause.Cause) + return Effect.promise(() => writer.abort(cause).catch(constVoid)) + } ) }, (writer) => Effect.sync(() => writer.releaseLock()) @@ -1867,7 +1875,12 @@ export const fromTransformStream = (options: { pull: upstream, writable: transform.writable, onError: options.onError, - closeOnDone: options.closeOnDone + closeOnDone: options.closeOnDone, + // Aborting errors the readable, so record the failure first. Otherwise a + // pending read reports the abort reason instead. + onAbort: (cause) => { + exit.current = Exit.failCause(cause) + } }).pipe( Effect.catchCause((cause) => { if (!Pull.isDoneCause(cause)) { @@ -1918,7 +1931,13 @@ const readableStreamToPullUnsafe = (options: { options.scope, options.releaseLockOnEnd ? Effect.sync(() => reader.releaseLock()) - : Effect.promise(() => reader.cancel().catch(constVoid)) + : Effect.promise(() => + reader.cancel().catch((error) => + // `closed` resolves only when the cancel hook failed. An errored + // stream rejects it with an error that was already reported. + reader.closed.then(() => Promise.reject(error), constVoid) + ) + ).pipe(Effect.ensuring(Effect.sync(() => reader.releaseLock()))) ), pull ) @@ -2235,7 +2254,7 @@ const mapEffectConcurrent = < const pull = yield* toTransform(self)(upstream, scope) const concurrencyN = options.concurrency === "unbounded" ? Number.MAX_SAFE_INTEGER - : options.concurrency + : Math.min(options.concurrency, Number.MAX_SAFE_INTEGER) const queue = yield* Queue.bounded>(0) yield* Scope.addFinalizer(forkedScope, Queue.shutdown(queue)) @@ -3323,12 +3342,12 @@ export const repeat: { let meta = Schedule.CurrentMetadata.defaultValue() return repeatLoop( provideServiceEffect(self, Schedule.CurrentMetadata, Effect.sync(() => meta)), - (done) => - step(done).pipe( + (exit): Pull.Pull => + Exit.isFailure(exit) ? Effect.failCause(exit.cause) : step(exit.value).pipe( Effect.map((meta_) => { meta = meta_ }), - Pull.catchDone(() => Cause.done(done)) + Pull.catchDone(() => Cause.done(exit.value)) ) ) }), @@ -3337,19 +3356,21 @@ export const repeat: { const repeatLoop = ( self: Channel, - onDone: (done: OutDone) => Pull.Pull -): Channel => + onExit: (exit: Exit.Exit, scope: Scope.Scope) => Pull.Pull +): Channel => fromTransform((upstream, scope) => Effect.sync(() => { - let currentPull: Pull.Pull | undefined - const makePull = (): Pull.Pull => { + let currentPull: Pull.Pull | undefined + const makePull = (): Pull.Pull => { const runScope = Scope.forkUnsafe(scope) return Effect.flatMap(toTransform(self)(upstream, runScope), (pull) => { - currentPull = Pull.catchDone(pull, (done) => - Scope.close(runScope, Exit.void).pipe( - Effect.flatMap(() => onDone(done as OutDone)), + currentPull = Effect.catchCause(pull, (cause) => { + const exit = Pull.doneExitFromCause(cause) as Exit.Exit + return Scope.close(runScope, exit).pipe( + Effect.flatMap(() => onExit(exit, scope)), Effect.flatMap(makePull) - )) + ) + }) return currentPull }) } @@ -3366,7 +3387,8 @@ const repeatLoop = ( self: Channel -): Channel => repeatLoop(self, () => Effect.void) +): Channel => + repeatLoop(self, (exit) => Exit.isFailure(exit) ? Effect.failCause(exit.cause) : Effect.void) /** * Runs a schedule step for each output element while preserving the emitted @@ -4277,6 +4299,12 @@ export const catchCause: { * * Typed failures and interruptions are not caught. * + * **Gotchas** + * + * `catchDefect` recovers any cause that contains a defect, including causes + * that also contain typed errors. Typed error handlers such as `catch` skip + * those causes, so defects always take precedence. + * * **Example** (Recovering from a defect) * * ```ts import.meta.vitest @@ -4788,12 +4816,19 @@ const catch_: { InErr & InErr1, InDone & InDone1, Env | Env1 -> => catchCauseFilter(self, Cause.findError, (e) => f(e))) +> => catchCauseFilter(self, findRecoverableError, (e) => f(e))) export { /** * Recovers from typed channel errors by running a fallback channel. * + * **Gotchas** + * + * Defects take precedence over typed errors. If the cause also contains a + * defect, the handler does not run and the failure is propagated with every + * typed error converted to a defect. An interruption alongside a typed error + * does not prevent recovery. + * * @stability stable * @category error handling * @since 4.0.0 @@ -4902,6 +4937,12 @@ export const tapError: { * are handled by `orElse` when provided. Without `orElse`, non-matching errors * are re-failed. * + * **Gotchas** + * + * If the cause contains a defect, the handler is skipped and every typed error + * is converted to a defect, preserving reason order and interrupts. An + * interruption alongside a typed error does not prevent recovery. + * * @see {@link catch_ catch} for recovering from every typed channel error * @see {@link catchFilter} for selecting typed errors with a `Filter` * @see {@link catchTag} for selecting tagged typed errors @@ -4930,7 +4971,7 @@ export const catchIf: { InDone2 = unknown, Env2 = never >( - refinement: Predicate.Refinement, + refinement: Predicate.Refinement, EB>, f: (failure: EB) => Channel, orElse?: | (( @@ -5018,7 +5059,7 @@ export const catchIf: { Env2 = never >( self: Channel, - refinement: Predicate.Refinement, + refinement: Predicate.Refinement, EB>, f: (failure: EB) => Channel, orElse?: | (( @@ -5147,6 +5188,12 @@ export const catchIf: { * filter results are handled by `orElse` when provided. Without `orElse`, * failed filter results are re-failed. * + * **Gotchas** + * + * If the cause contains a defect, the handler is skipped and every typed error + * is converted to a defect, preserving reason order and interrupts. An + * interruption alongside a typed error does not prevent recovery. + * * @see {@link catchIf} for selecting typed errors with a predicate * @see {@link catchTag} for selecting tagged typed errors * @see {@link catchCauseFilter} for selecting full causes with a `Filter` @@ -5311,6 +5358,12 @@ export const catchFilter: { * errors are handled by `orElse` when provided. Without `orElse`, * non-matching errors are re-failed. * + * **Gotchas** + * + * If the cause contains a defect, the handler is skipped and every typed error + * is converted to a defect, preserving reason order and interrupts. An + * interruption alongside a typed error does not prevent recovery. + * * @stability stable * @category error handling * @since 4.0.0 @@ -5462,6 +5515,12 @@ export const catchTag: { /** * Catches a specific reason within a tagged error. * + * **Gotchas** + * + * If the cause contains a defect, the handler is skipped and every typed error + * is converted to a defect, preserving reason order and interrupts. An + * interruption alongside a typed error does not prevent recovery. + * * **Example** (Recovering from nested reasons) * * ```ts import.meta.vitest @@ -5672,6 +5731,12 @@ export const catchReason: { /** * Catches multiple reasons within a tagged error using an object of handlers. * + * **Gotchas** + * + * If the cause contains a defect, the handler is skipped and every typed error + * is converted to a defect, preserving reason order and interrupts. An + * interruption alongside a typed error does not prevent recovery. + * * @stability stable * @category error handling * @since 4.0.0 @@ -5944,9 +6009,12 @@ export const unwrapReason: { ) as any) /** - * Returns a new channel, which is the same as this one, except the failure - * value of the returned channel is created by applying the specified function - * to the failure value of this channel. + * Transforms the typed errors of a channel using the specified function. + * + * **Details** + * + * Maps every typed error in the cause, preserving defects, interrupts and + * reason order. Unlike typed recovery, mapping does not convert errors to defects. * * @stability stable * @category error handling @@ -5965,7 +6033,11 @@ export const mapError: { } = dual(2, ( self: Channel, f: (err: OutErr) => OutErr2 -): Channel => catch_(self, (err) => fail(f(err)))) +): Channel => + transformPull( + self, + (pull) => Effect.succeed(Effect.mapError(pull, (err) => Cause.isDone(err) ? err : f(err))) + )) /** * Converts all errors in the channel to defects (unrecoverable failures). @@ -6013,6 +6085,13 @@ export const orDie = < * * Use the `log` option to emit the full {@link Cause} when the channel fails. * + * **Gotchas** + * + * Defects take precedence over typed errors. If the cause also contains a + * defect, the failure is not ignored and is propagated with every typed error + * converted to a defect. An interruption alongside a typed error does not + * prevent the error from being ignored. + * * @stability stable * @category error handling * @since 4.0.0 @@ -6112,6 +6191,12 @@ export const ignoreCause: < * Returns a new channel that retries this channel according to the specified * schedule whenever it fails. * + * **Gotchas** + * + * Defects take precedence over typed errors. A failure whose cause contains a + * defect is not retried and is propagated with every typed error converted to a + * defect. A typed error alongside an interruption is retried. + * * @stability stable * @category error handling * @since 4.0.0 @@ -6155,29 +6240,20 @@ export const retry: { return Effect.void }) const resolvedSchedule = typeof schedule === "function" ? schedule(identity_) : schedule - const loop: Channel< - OutElem, - OutErr | SE, - OutDone, - InElem, - InErr, - InDone, - Env | SR - > = catch_( - withReset, - Effect.fnUntraced( - function*(error) { - if (!step) { - step = yield* Schedule.toStepWithMetadata(resolvedSchedule) - } - meta = yield* step(error) - return loop - }, - (effect, error) => Pull.catchDone(effect, () => Effect.succeed(fail(error))), - unwrap + return repeatLoop(withReset, (exit, scope) => { + if (Exit.isSuccess(exit)) return Cause.done(exit.value) + const error = findRecoverableError(exit.cause) + if (Result.isFailure(error)) return Effect.failCause(error.failure) + return Effect.gen(function*() { + if (!step) { + step = yield* Schedule.toStepWithMetadata(resolvedSchedule) + } + meta = yield* step(error.success) + }).pipe( + Pull.catchDone(() => Effect.fail(error.success)), + Scope.provide(scope) ) - ) - return loop + }) })) /** @@ -6425,7 +6501,7 @@ export const mergeAll: { : Math.max(1, concurrency) const semaphore = switch_ ? undefined : Semaphore.makeUnsafe(concurrencyN) const doneLatch = yield* Latch.make(true) - const fibers = new Set>() + const fibers = new Map, Scope.Closeable>() const queue = yield* Queue.bounded>( bufferSize @@ -6455,10 +6531,14 @@ export const mergeAll: { const childPull = yield* toTransform(channel)(upstream, childScope) while (fibers.size >= concurrencyN) { - const fiber = Iterable.headUnsafe(fibers) + const [fiber, fiberScope] = Iterable.headUnsafe(fibers) fibers.delete(fiber) if (fibers.size === 0) yield* doneLatch.open yield* Fiber.interrupt(fiber) + // A worker interrupted before it starts never closes its own + // scope. Finalizer failures of replaced workers are discarded + // either way. + yield* Effect.ignoreCause(Scope.close(fiberScope, Exit.interrupt())) } const fiber = yield* childPull.pipe( @@ -6482,7 +6562,7 @@ export const mergeAll: { ) doneLatch.closeUnsafe() - fibers.add(fiber) + fibers.set(fiber, childScope) } }).pipe( Effect.catchCause((cause) => { @@ -6850,8 +6930,9 @@ export const splitLines = (): Channel< * **Details** * * Input chunks are decoded with streaming enabled so multi-byte characters may - * span `Uint8Array` boundaries. The optional `encoding` and `options` are - * passed to `TextDecoder`. + * span `Uint8Array` boundaries. On normal completion, incomplete trailing + * characters become U+FFFD or, with `fatal: true`, a defect. The optional + * `encoding` and `options` are passed to `TextDecoder`. * * @stability stable * @category decoding @@ -6872,7 +6953,15 @@ export const decodeText = ( Effect.sync(() => { const decoder = new TextDecoder(encoding, options) const streamOptions = { stream: true } - return Effect.map(upstream, Arr.map((line) => decoder.decode(line, streamOptions))) + let pull: Pull.Pull, Err, Done> = upstream.pipe( + Effect.map(Arr.map((bytes) => decoder.decode(bytes, streamOptions))), + Pull.catchDone((leftover) => { + pull = Cause.done(leftover as Done) + const last = decoder.decode() + return last.length > 0 ? Effect.succeed(Arr.of(last)) : pull + }) + ) + return Effect.suspend(() => pull) }) ) diff --git a/repos/effect/packages/effect/src/Chunk.ts b/repos/effect/packages/effect/src/Chunk.ts index 364d98e478..dcca7c24b9 100644 --- a/repos/effect/packages/effect/src/Chunk.ts +++ b/repos/effect/packages/effect/src/Chunk.ts @@ -521,6 +521,11 @@ export const reverse: >(self: S) => Chunk.With(self: Chunk, index: number): Option } = dual( 2, - (self: Chunk, index: number): Option => - index < 0 || index >= self.length ? O.none() : O.some(getUnsafe(self, index)) + (self: Chunk, index: number): Option => { + const i = Math.floor(index) + return i >= 0 && i < self.length ? O.some(getUnchecked(self, i)) : O.none() + } ) /** @@ -624,7 +631,8 @@ export const fromNonEmptyArrayUnsafe = (self: NonEmptyReadonlyArray): NonE * * **Gotchas** * - * Throws if the index is out of bounds. + * Throws if the floored index is out of bounds, including for `NaN` and + * infinite indexes. * * **Example** (Accessing elements unsafely) * @@ -649,32 +657,33 @@ export const getUnsafe: { (self: Chunk, index: number): A } = dual(2, (self: Chunk, index: number): A => { const i = Math.floor(index) + if (!(i >= 0 && i < self.length)) { + throw new Error(`Index out of bounds: ${i}`) + } + return getUnchecked(self, i) +}) + +const getUnchecked = (self: Chunk, i: number): A => { switch (self.backing._tag) { case "IEmpty": { throw new Error(`Index out of bounds: ${i}`) } case "ISingleton": { - if (index !== 0) { - throw new Error(`Index out of bounds: ${i}`) - } return self.backing.a } case "IArray": { - if (i >= self.length || i < 0) { - throw new Error(`Index out of bounds: ${i}`) - } return self.backing.array[i]! } case "IConcat": { return i < self.left.length - ? getUnsafe(self.left, i) - : getUnsafe(self.right, i - self.left.length) + ? getUnchecked(self.left, i) + : getUnchecked(self.right, i - self.left.length) } case "ISlice": { - return getUnsafe(self.backing.chunk, i + self.backing.offset) + return getUnchecked(self.backing.chunk, i + self.backing.offset) } } -}) +} /** * Appends the specified element to the end of the `Chunk`. @@ -2224,7 +2233,12 @@ export const takeWhile: { }) /** - * Creates a Chunks of unique values, in order, from all given Chunks. + * Computes the union of two chunks using Effect equality. + * + * **Gotchas** + * + * Each input must already be deduplicated. Use {@link dedupe} to remove + * duplicates from an input first. * * **Example** (Unioning chunks) * @@ -2234,11 +2248,6 @@ export const takeWhile: { * const chunk1 = Chunk.make(1, 2, 3) * const chunk2 = Chunk.make(3, 4, 5) * Chunk.toArray(Chunk.union(chunk1, chunk2)) // => [1, 2, 3, 4, 5] - * - * // Handles duplicates within the same chunk - * const withDupes1 = Chunk.make(1, 1, 2) - * const withDupes2 = Chunk.make(2, 3, 3) - * Chunk.toArray(Chunk.union(withDupes1, withDupes2)) // => [1, 2, 3] * ``` * * @stability stable diff --git a/repos/effect/packages/effect/src/Config.ts b/repos/effect/packages/effect/src/Config.ts index ba18193729..610c2ebbb0 100644 --- a/repos/effect/packages/effect/src/Config.ts +++ b/repos/effect/packages/effect/src/Config.ts @@ -706,7 +706,6 @@ const toConfigCursorAST = memoize((root: SchemaAST.AST): SchemaAST.AST => { seen.add(ast) switch (ast._tag) { case "Objects": { - const matchesIndex = ast.indexSignatures.map((is) => SchemaParser._is(is.parameter)) const materialize = Effect.fnUntraced(function*(cursor: ConfigCursor) { if (cursor.node?._tag !== "Record") { return undefined @@ -716,9 +715,15 @@ const toConfigCursorAST = memoize((root: SchemaAST.AST): SchemaAST.AST => { for (const property of ast.propertySignatures) { if (typeof property.name === "string") keys.add(property.name) } - if (matchesIndex.length > 0) { + if (ast.indexSignatures.length > 0) { + const input: Record = {} for (const key of node.keys) { - if (matchesIndex.some((matches) => matches(key))) keys.add(key) + InternalRecord.assignProperty(input, key, undefined) + } + for (const is of ast.indexSignatures) { + for (const key of SchemaAST.getIndexSignatureKeys(input, is.parameter)) { + if (typeof key === "string") keys.add(key) + } } } const out: Record = {} diff --git a/repos/effect/packages/effect/src/ConfigProvider.ts b/repos/effect/packages/effect/src/ConfigProvider.ts index b07eef48f8..6d728b454c 100644 --- a/repos/effect/packages/effect/src/ConfigProvider.ts +++ b/repos/effect/packages/effect/src/ConfigProvider.ts @@ -1127,11 +1127,12 @@ function interpolate( parsed: Record, seen: ReadonlySet ): string { - const reference = /(? 0 && envValue[match.index - 1] === "\\") continue const [, name] = match let end = reference.lastIndex let defaultValue = "" diff --git a/repos/effect/packages/effect/src/Cron.ts b/repos/effect/packages/effect/src/Cron.ts index 477667a2a2..2bee2f2af9 100644 --- a/repos/effect/packages/effect/src/Cron.ts +++ b/repos/effect/packages/effect/src/Cron.ts @@ -815,8 +815,8 @@ export const next = (cron: Cron, now?: DateTime.DateTime.Input): Date => { * * **Gotchas** * - * The search is strict: if the supplied date/time already matches the schedule, - * the result is the earlier occurrence. + * The search is strict: if the supplied date/time is exactly an occurrence of + * the schedule, the result is the earlier occurrence. * * @see {@link next} for finding the next scheduled occurrence * @@ -866,7 +866,10 @@ const stepCron = (cron: Cron, now: DateTime.DateTime.Input | undefined, directio }).epochMilliseconds <= zoned.epochMilliseconds const result = dateTime.mutate(zoned, (current) => { - current.setUTCSeconds(current.getUTCSeconds() + tick, 0) + // Start from the nearest whole second strictly past the input in the search + // direction. Going backward, a fractional input already has one. + const offset = reverse && current.getUTCMilliseconds() > 0 ? 0 : tick + current.setUTCSeconds(current.getUTCSeconds() + offset, 0) for (let i = 0; i < 10_000; i++) { if (cron.seconds.size !== 0) { diff --git a/repos/effect/packages/effect/src/Duration.ts b/repos/effect/packages/effect/src/Duration.ts index 5d4a3a4026..d02081a090 100644 --- a/repos/effect/packages/effect/src/Duration.ts +++ b/repos/effect/packages/effect/src/Duration.ts @@ -35,6 +35,10 @@ const bigint60 = BigInt(60) const bigint1e3 = BigInt(1_000) const bigint1e6 = BigInt(1_000_000) const bigint1e9 = BigInt(1_000_000_000) +const bigintNanosPerMinute = BigInt(60_000_000_000) +const bigintNanosPerHour = BigInt(3_600_000_000_000) +const bigintNanosPerDay = BigInt(86_400_000_000_000) +const bigintNanosPerWeek = BigInt(604_800_000_000_000) const roundTiesAwayFromZero = (input: number): bigint => BigInt(Math.sign(input) * Math.round(Math.abs(input))) @@ -43,6 +47,14 @@ const roundMillisToNanos = (millis: number): bigint => const fromNanosNumber = (nanos: number): Duration => make(Number.isFinite(nanos) ? roundTiesAwayFromZero(nanos) : nanos) +const nanosToUnit = (nanos: bigint, divisor: number): number => { + const value = Number(nanos) + // Preserve ordinary-input rounding; scale first only when nanoseconds overflow. + if (Number.isFinite(value)) return value / divisor + const scale = BigInt(divisor) + return Number(nanos / scale) + Number(nanos % scale) / divisor +} + const parseNanos = (input: string, scale: bigint): bigint => { const decimalIndex = input.indexOf(".") if (decimalIndex === -1) return BigInt(input) * scale @@ -62,6 +74,14 @@ const divideTiesAwayFromZero = (numerator: bigint, denominator: bigint): bigint return numerator < bigint0 ? -rounded : rounded } +const fromMillisString = (input: string, scale: number): Duration => { + const millis = Number(input) * scale + if (!Number.isFinite(millis)) return make(millis) + const nanos = parseNanos(input, BigInt(scale) * bigint1e6) + // Keep the millisecond representation only when it preserves the exact value. + return make(Number.isInteger(millis) && BigInt(millis) * bigint1e6 === nanos ? millis : nanos) +} + const nanosToHrTime = (nanos: bigint): [seconds: number, nanos: number] => { const sign = nanos < bigint0 ? -bigint1 : bigint1 const absolute = nanos < bigint0 ? -nanos : nanos @@ -274,26 +294,25 @@ export const fromInputUnsafe = (input: Input): Duration => { if (unit === "micro" || unit === "micros") { return nanos(parseNanos(valueStr, bigint1e3)) } - const value = Number(valueStr) switch (unit) { case "milli": case "millis": - return millis(value) + return fromMillisString(valueStr, 1) case "second": case "seconds": - return seconds(value) + return fromMillisString(valueStr, 1_000) case "minute": case "minutes": - return minutes(value) + return fromMillisString(valueStr, 60_000) case "hour": case "hours": - return hours(value) + return fromMillisString(valueStr, 3_600_000) case "day": case "days": - return days(value) + return fromMillisString(valueStr, 86_400_000) case "week": case "weeks": - return weeks(value) + return fromMillisString(valueStr, 604_800_000) } break } @@ -313,7 +332,10 @@ export const fromInputUnsafe = (input: Input): Duration => { if (input[0] === Infinity || input[1] === Infinity) { return infinity } - return fromNanosNumber(input[0] * 1_000_000_000 + input[1]) + const nanos = input[0] * 1_000_000_000 + input[1] + return Number.isFinite(nanos) && Number.isInteger(input[0]) && Number.isInteger(input[1]) + ? make(BigInt(input[0]) * bigint1e9 + BigInt(input[1])) + : fromNanosNumber(nanos) } const obj = input as DurationObject let millis = 0 @@ -324,13 +346,41 @@ export const fromInputUnsafe = (input: Input): Duration => { if (obj.minutes) millis += obj.minutes * 60_000 if (obj.seconds) millis += obj.seconds * 1_000 if (obj.milliseconds) millis += obj.milliseconds - if (!obj.microseconds && !obj.nanoseconds) return make(millis) - return fromNanosNumber(millis * 1_000_000 + (obj.microseconds ?? 0) * 1_000 + (obj.nanoseconds ?? 0)) + if (!obj.microseconds && !obj.nanoseconds) { + if (!Number.isFinite(millis)) return make(millis) + const exact = sumObjectNanos(obj) + // keep the Millis representation when the numeric total is already exact + return exact === undefined || BigInt(millis) * bigint1e6 === exact ? make(millis) : make(exact) + } + const nanos = millis * 1_000_000 + (obj.microseconds ?? 0) * 1_000 + (obj.nanoseconds ?? 0) + const exact = Number.isFinite(nanos) ? sumObjectNanos(obj) : undefined + return exact === undefined ? fromNanosNumber(nanos) : make(exact) } } return invalid(input) } +const isIntegerOrUnset = (n: number | undefined): boolean => !n || Number.isInteger(n) + +// sums the fields exactly, or returns undefined if any field is fractional +const sumObjectNanos = (obj: DurationObject): bigint | undefined => { + if ( + !isIntegerOrUnset(obj.weeks) || !isIntegerOrUnset(obj.days) || !isIntegerOrUnset(obj.hours) || + !isIntegerOrUnset(obj.minutes) || !isIntegerOrUnset(obj.seconds) || !isIntegerOrUnset(obj.milliseconds) || + !isIntegerOrUnset(obj.microseconds) || !isIntegerOrUnset(obj.nanoseconds) + ) return undefined + let nanos = bigint0 + if (obj.weeks) nanos += BigInt(obj.weeks) * bigintNanosPerWeek + if (obj.days) nanos += BigInt(obj.days) * bigintNanosPerDay + if (obj.hours) nanos += BigInt(obj.hours) * bigintNanosPerHour + if (obj.minutes) nanos += BigInt(obj.minutes) * bigintNanosPerMinute + if (obj.seconds) nanos += BigInt(obj.seconds) * bigint1e9 + if (obj.milliseconds) nanos += BigInt(obj.milliseconds) * bigint1e6 + if (obj.microseconds) nanos += BigInt(obj.microseconds) * bigint1e3 + if (obj.nanoseconds) nanos += BigInt(obj.nanoseconds) + return nanos +} + const invalid = (input: unknown): never => { throw new Error(`Invalid Input: ${input}`) } @@ -817,7 +867,7 @@ export const weeks = (weeks: number): Duration => make(weeks * 604_800_000) export const toMillis = (self: Input): number => match(fromInputUnsafe(self), { onMillis: identity, - onNanos: (nanos) => Number(nanos) / 1_000_000, + onNanos: (nanos) => nanosToUnit(nanos, 1_000_000), onInfinity: () => Infinity, onNegativeInfinity: () => -Infinity }) @@ -841,7 +891,7 @@ export const toMillis = (self: Input): number => export const toSeconds = (self: Input): number => match(fromInputUnsafe(self), { onMillis: (millis) => millis / 1_000, - onNanos: (nanos) => Number(nanos) / 1_000_000_000, + onNanos: (nanos) => nanosToUnit(nanos, 1_000_000_000), onInfinity: () => Infinity, onNegativeInfinity: () => -Infinity }) @@ -865,7 +915,7 @@ export const toSeconds = (self: Input): number => export const toMinutes = (self: Input): number => match(fromInputUnsafe(self), { onMillis: (millis) => millis / 60_000, - onNanos: (nanos) => Number(nanos) / 60_000_000_000, + onNanos: (nanos) => nanosToUnit(nanos, 60_000_000_000), onInfinity: () => Infinity, onNegativeInfinity: () => -Infinity }) @@ -889,7 +939,7 @@ export const toMinutes = (self: Input): number => export const toHours = (self: Input): number => match(fromInputUnsafe(self), { onMillis: (millis) => millis / 3_600_000, - onNanos: (nanos) => Number(nanos) / 3_600_000_000_000, + onNanos: (nanos) => nanosToUnit(nanos, 3_600_000_000_000), onInfinity: () => Infinity, onNegativeInfinity: () => -Infinity }) @@ -913,7 +963,7 @@ export const toHours = (self: Input): number => export const toDays = (self: Input): number => match(fromInputUnsafe(self), { onMillis: (millis) => millis / 86_400_000, - onNanos: (nanos) => Number(nanos) / 86_400_000_000_000, + onNanos: (nanos) => nanosToUnit(nanos, 86_400_000_000_000), onInfinity: () => Infinity, onNegativeInfinity: () => -Infinity }) @@ -937,7 +987,7 @@ export const toDays = (self: Input): number => export const toWeeks = (self: Input): number => match(fromInputUnsafe(self), { onMillis: (millis) => millis / 604_800_000, - onNanos: (nanos) => Number(nanos) / 604_800_000_000_000, + onNanos: (nanos) => nanosToUnit(nanos, 604_800_000_000_000), onInfinity: () => Infinity, onNegativeInfinity: () => -Infinity }) diff --git a/repos/effect/packages/effect/src/Effect.ts b/repos/effect/packages/effect/src/Effect.ts index 94a25e42f1..6410d6568d 100644 --- a/repos/effect/packages/effect/src/Effect.ts +++ b/repos/effect/packages/effect/src/Effect.ts @@ -1,3 +1,5 @@ +/// + /** * Describes workflows that run only when executed by the Effect runtime. * @@ -882,8 +884,9 @@ export const whileLoop: (options: { * * **Details** * - * An optional `AbortSignal` can be provided to allow for interruption of the - * wrapped `Promise` API. + * If `evaluate` declares a parameter, it receives an `AbortSignal` that is + * aborted if the effect is interrupted. Zero-argument thunks do not allocate + * an `AbortController`. * * **Gotchas** * @@ -893,6 +896,13 @@ export const whileLoop: (options: { * Interruption aborts the provided `AbortSignal`, but the underlying * asynchronous operation only stops if it observes that signal. * + * The parameter check reads the `length` property of `evaluate`, which does + * not count a rest parameter or any parameter from the first default value + * onward. `(...args) => ...` and `(signal = fallback) => ...` therefore + * receive `undefined` instead of a signal, as do wrappers that forward + * arguments through rest parameters. Declare the signal as an ordinary + * parameter: `(signal) => ...`. + * * **Example** (Wrapping a non-rejecting Promise) * * ```ts import.meta.vitest @@ -937,9 +947,10 @@ export const promise: ( * Passing `{ try, catch }` uses `catch` to map failures to an error of type * `E`. * - * The thunk receives an `AbortSignal` that is aborted if the effect is - * interrupted. The underlying asynchronous operation only stops if it observes - * that signal. + * If the thunk declares a parameter, it receives an `AbortSignal` that is + * aborted if the effect is interrupted. The underlying asynchronous operation + * only stops if it observes that signal. Zero-argument thunks do not allocate + * an `AbortController`. * * **Gotchas** * @@ -947,6 +958,13 @@ export const promise: ( * defect. Return the error value you want in the error channel instead of * throwing it. * + * The parameter check reads the thunk's `length` property, which does not + * count a rest parameter or any parameter from the first default value + * onward. `(...args) => ...` and `(signal = fallback) => ...` therefore + * receive `undefined` instead of a signal, as do wrappers that forward + * arguments through rest parameters. Declare the signal as an ordinary + * parameter: `(signal) => ...`. + * * **Example** (Wrapping a fetch request that may fail) * * ```ts import.meta.vitest @@ -1230,10 +1248,20 @@ export { * * **Details** * - * The registration function receives a `resume` callback and, when requested, - * an `AbortSignal`. Call `resume` at most once with the effect that should - * complete the fiber; later calls are ignored. Return an optional cleanup - * effect from the registration function to run if the fiber is interrupted. + * The registration function receives a `resume` callback and, if it declares + * a second parameter, an `AbortSignal` that is aborted if the fiber is + * interrupted. Call `resume` at most once with the effect that should complete + * the fiber; later calls are ignored. Return an optional cleanup effect from + * the registration function to run if the fiber is interrupted. + * + * **Gotchas** + * + * The parameter check reads the registration function's `length` property, + * which must be at least `2` and does not count a rest parameter or any + * parameter from the first default value onward. `(...args) => ...` and + * `(resume, signal = fallback) => ...` therefore receive `undefined` instead + * of a signal, as do wrappers that forward arguments through rest parameters. + * Declare both as ordinary parameters: `(resume, signal) => ...`. * * **Example** (Integrating callback APIs) * @@ -1889,7 +1917,9 @@ export const fromResult: (result: Result.Result) => Effect = i * `Option.some` becomes a successful effect with the contained value, while * `Option.none` becomes a failed effect. By default the failure is a * `NoSuchElementError`, but you can provide an `onNone` callback to customize - * the error value. + * the error value. For `Option.none`, `onNone` is evaluated on each execution, + * not when the effect is constructed. It is never called for `Option.some`. + * If `onNone` throws, the effect dies with the thrown value as a defect. * * **Example** (Converting an Option into an Effect) * @@ -2302,9 +2332,12 @@ export const tap: { * * **Gotchas** * - * `result` only captures typed, recoverable failures. Defects and - * interruptions are not captured inside the `Result` and still fail the - * effect. + * `result` only captures typed, recoverable failures. If the cause contains a + * defect, nothing is captured and every typed error is converted to a defect, + * preserving reason order and interrupts. A cause with only interruptions + * still fails the effect, but an interruption alongside a typed error does not + * prevent capture. The first typed error becomes a `Result.Failure` and the + * interruption is discarded. * * **Example** (Capturing success or failure as Result) * @@ -2346,9 +2379,12 @@ export const result: (self: Effect) => Effect Effect, E2 | E3 | (A3 extends unassigned ? E : never), R | R2 | R3> ( self: Effect, - refinement: Predicate.Refinement, + refinement: Predicate.Refinement, EB>, f: (e: EB) => Effect, orElse?: ((e: Exclude) => Effect) | undefined ): Effect, E2 | E3 | (A3 extends unassigned ? Exclude : never), R | R2 | R3> @@ -3484,6 +3561,12 @@ export const catchIf: { * `orElse` when provided. Without `orElse`, the original failure cause is * preserved. * + * **Gotchas** + * + * If the cause contains a defect, the handler is skipped and every typed error + * is converted to a defect, preserving reason order and interrupts. An + * interruption alongside a typed error does not prevent recovery. + * * @see {@link catchIf} for predicate-based recovery from typed errors * @see {@link catchTag} for recovering from a single tagged error * @see {@link catchTags} for recovering from several tagged errors @@ -3652,6 +3735,9 @@ export const catchCauseFilter: { * Only the failure channel is transformed. The success channel and requirements * are preserved. * + * Maps every typed error in the cause, preserving defects, interrupts and + * reason order. Unlike typed recovery, mapping does not convert errors to defects. + * * **Example** (Transforming the error channel) * * ```ts import.meta.vitest @@ -3699,6 +3785,9 @@ export const mapError: { * the error and the success values without altering the overall success or * failure status of the effect. * + * Maps every typed error in the cause, preserving defects, interrupts and + * reason order. Unlike typed recovery, mapping does not convert errors to defects. + * * **Example** (Transforming success and failure channels) * * ```ts import.meta.vitest @@ -4148,7 +4237,9 @@ export declare namespace Retry { * applied. For example, `Schedule.recurs(3)` allows up to three retries after * the initial attempt. * - * Defects and interruptions are not retried. + * Defects take precedence over typed errors. A failure whose cause contains a + * defect is not retried and is propagated with every typed error converted to a + * defect. A typed error alongside an interruption is retried. * * **Example** (Retrying with a schedule) * @@ -4218,6 +4309,14 @@ export const retry: { * If the retries are exhausted and the effect still fails, it runs a fallback * effect instead. * + * **Gotchas** + * + * Defects take precedence over typed errors. A failure whose cause contains a + * defect is not retried and is propagated with every typed error converted to a + * defect. A typed error alongside an interruption is retried. When the cause + * contains a defect, `orElse` does not run. + * This applies to source failures only; policy failures propagate unchanged. + * * **Example** (Falling back after retries are exhausted) * * ```ts import.meta.vitest @@ -4274,8 +4373,8 @@ export const retryOrElse: { * **Details** * * Use `sandbox` when downstream error handling needs to distinguish typed - * failures, defects, and interruptions. Use `unsandbox` to restore the original - * typed error channel after cause-level handling. + * failures, defects, and interruptions. Use `Effect.catch(Effect.failCause)` to + * restore the original typed error channel after cause-level handling. * * **Example** (Exposing failures as causes) * @@ -4314,6 +4413,13 @@ export const sandbox: ( * Use the `log` option to emit the full {@link Cause} when the effect fails, * and `message` to prepend a custom log message. * + * **Gotchas** + * + * Defects take precedence over typed errors. If the cause also contains a + * defect, the failure is not ignored and is propagated with every typed error + * converted to a defect. An interruption alongside a typed error does not + * prevent the error from being ignored. + * * **Example** (Discarding success and failure values) * * ```ts import.meta.vitest @@ -4538,6 +4644,12 @@ export const withErrorReporting: < * * Defects and interruptions are not recovered by this operator. * + * **Gotchas** + * + * If the cause contains a defect, the handler is skipped and every typed error + * is converted to a defect, preserving reason order and interrupts. An + * interruption alongside a typed error does not prevent recovery. + * * **Example** (Replacing failures with a value) * * ```ts import.meta.vitest @@ -5421,6 +5533,12 @@ export const when: { * to process the result if the effect succeeds, or handle the error if the * effect fails. * + * **Gotchas** + * + * If the cause contains a defect, the handler is skipped and every typed error + * is converted to a defect, preserving reason order and interrupts. An + * interruption alongside a typed error does not prevent recovery. + * * **Example** (Matching success and failure values) * * ```ts import.meta.vitest @@ -5486,6 +5604,12 @@ export const match: { * the handlers immediately without fiber scheduling. For unresolved effects, * it falls back to the regular `match` behavior. * + * **Gotchas** + * + * If the cause contains a defect, the handler is skipped and every typed error + * is converted to a defect, preserving reason order and interrupts. An + * interruption alongside a typed error does not prevent recovery. + * * **Example** (Pattern matching eagerly when possible) * * ```ts import.meta.vitest @@ -5741,6 +5865,12 @@ export const matchCauseEffect: { * performing logging, recovery, notification, or other effectful work. The * returned effect succeeds or fails according to the handler that is run. * + * **Gotchas** + * + * If the cause contains a defect, the handler is skipped and every typed error + * is converted to a defect, preserving reason order and interrupts. An + * interruption alongside a typed error does not prevent recovery. + * * **Example** (Matching success and failure with effectful handlers) * * ```ts import.meta.vitest @@ -5976,9 +6106,9 @@ export const contextWith: ( ) => Effect = internal.contextWith /** - * Provides dependencies to an effect using layers or a context. Use `options.local` - * to build the layer every time; by default, layers are shared between provide - * calls. + * Provides dependencies to an effect using layers or a context. A layer already + * built by an enclosing `provide` or runtime is reused. Sibling `provide` calls + * build their own copies. Use `options.local` to build a fresh layer subtree. * * **Example** (Providing dependencies with a layer) * @@ -7788,6 +7918,10 @@ export const forever: < * The source effect is always evaluated once before the schedule is stepped. * The schedule controls additional repetitions, not the initial execution. * + * If the effect or policy fails with a cause that contains a defect, the + * failure is propagated with every typed error converted to a defect. + * An interruption alongside a typed error does not block the typed error path. + * * **Example** (Repeating successful effects with a schedule) * * ```ts import.meta.vitest @@ -7877,6 +8011,13 @@ export const repeat: { * otherwise it receives `None`. If the schedule completes normally, the * returned effect succeeds with the schedule's output. * + * **Gotchas** + * + * Defects take precedence over typed errors from either the effect or policy. + * If the cause contains a defect, `orElse` does not run and every typed error + * is converted to a defect. An interruption alongside a typed error does not + * prevent `orElse` from running. + * * **Example** (Recovering after repetition stops) * * ```ts import.meta.vitest @@ -8752,8 +8893,8 @@ export const requestUnsafe: ( * You can use the `forkChild` method whenever you want to execute an effect in a * new fiber, concurrently and without "blocking" the fiber executing other * effects. Using fibers can be tricky, so instead of using this method - * directly, consider other higher-level methods, such as `raceWith`, - * `zipPar`, and so forth. + * directly, consider higher-level methods such as `Effect.race`, + * `Effect.raceFirst`, or `Effect.all` with a concurrency option. * * The fiber returned by this method has methods to interrupt the fiber and to * wait for it to finish executing the effect. See `Fiber` for more @@ -8891,9 +9032,14 @@ export const forkScoped: < : (self: Effect) => Effect, never, R | Scope> = internal.forkScoped /** - * Forks the effect into a new fiber attached to the global scope. Because the - * new fiber is attached to the global scope, when the fiber executing the - * returned effect terminates, the forked fiber will continue running. + * Forks the effect into a new fiber with no parent and no scope, so its lifetime + * is independent of the forking fiber. + * + * **Details** + * + * The detached fiber inherits the forking fiber's context. It continues running + * until it completes or is explicitly interrupted through the returned handle, + * even after the forking fiber terminates. * * **Example** (Forking a detached fiber) * @@ -9051,11 +9197,11 @@ export interface RunOptions { * import { Effect, Fiber } from "effect" * const output: Array = [] * - * // ┌─── Effect + * // ┌─── Effect * // ▼ * const program = Effect.sync(() => { output.push("running...") }).pipe(Effect.as("done")) * - * // ┌─── RuntimeFiber + * // ┌─── Fiber * // ▼ * const fiber = Effect.runFork(program) * @@ -9390,9 +9536,10 @@ export const runPromiseExitWith: ( * * **Details** * - * If the effect fails, dies, is interrupted, or performs asynchronous work, - * `runSync` throws a `FiberFailure` instead of returning a value. Use - * `runSyncExit` when you want the failure captured as an `Exit`. + * If the effect fails, dies, or is interrupted, `runSync` throws the squashed + * cause instead of returning a value. Typed errors can be thrown directly, + * including non-Error values. Asynchronous work throws `Cause.AsyncFiberError`. + * Use `runSyncExit` when you want the failure captured as an `Exit`. * * **Example** (Running a synchronous effect) * @@ -15354,7 +15501,7 @@ type EffectifyArgs) => any> = Parameters exten * **Details** * * The original function is called with the supplied arguments plus a final - * callback. A non-null callback error fails the returned effect, while a + * callback. A truthy callback error fails the returned effect, while a * successful callback value becomes the effect success. Use `onError` to map * callback errors and `onSyncError` to turn synchronous throws into typed * failures; otherwise synchronous throws become defects. @@ -15415,16 +15562,26 @@ export const effectify: { ((fn: Function, onError?: (e: any, args: any) => any, onSyncError?: (e: any, args: any) => any) => (...args: Array) => callback((resume) => { + let synchronous = true + let pending: Effect | undefined try { fn(...args, (err: globalThis.Error | null, result: A) => { - if (err) { - resume(onError ? suspend(() => fail(onError(err, args))) : fail(err)) + const effect = err + ? onError ? suspend(() => fail(onError(err, args))) : fail(err) + : succeed(result) + if (synchronous) { + // Keep the first callback result until registration succeeds. + pending ??= effect } else { - resume(succeed(result)) + resume(effect) } }) } catch (err) { - resume(onSyncError ? fail(onSyncError(err, args)) : die(err)) + pending = onSyncError ? fail(onSyncError(err, args)) : die(err) + } + synchronous = false + if (pending) { + resume(pending) } })) as any @@ -15582,6 +15739,9 @@ export const mapEager: { * transform. Failure effects apply the mapping function immediately, and * pending effects fall back to regular `mapError` behavior. * + * Maps every typed error in the cause, preserving defects, interrupts and + * reason order. Unlike typed recovery, mapping does not convert errors to defects. + * * **Example** (Mapping errors eagerly when possible) * * ```ts import.meta.vitest @@ -15629,6 +15789,9 @@ export const mapErrorEager: { * `onFailure` immediately. Pending effects fall back to regular `mapBoth` * behavior. * + * Maps every typed error in the cause, preserving defects, interrupts and + * reason order. Unlike typed recovery, mapping does not convert errors to defects. + * * **Example** (Mapping both channels eagerly when possible) * * ```ts import.meta.vitest @@ -15723,6 +15886,12 @@ export const flatMapEager: { * Failure effects apply the catch function immediately, and pending effects * fall back to regular `catch` behavior. * + * **Gotchas** + * + * If the cause contains a defect, the handler is skipped and every typed error + * is converted to a defect, preserving reason order and interrupts. An + * interruption alongside a typed error does not prevent recovery. + * * **Example** (Catching failures eagerly when possible) * * ```ts import.meta.vitest diff --git a/repos/effect/packages/effect/src/Equivalence.ts b/repos/effect/packages/effect/src/Equivalence.ts index ecfd537fb6..9120258ebc 100644 --- a/repos/effect/packages/effect/src/Equivalence.ts +++ b/repos/effect/packages/effect/src/Equivalence.ts @@ -362,7 +362,8 @@ export const combine: { * comparison stops at the first equivalence that returns `false`. Empty * collections return an equivalence that always returns `true`. The result is * also an equivalence that satisfies reflexive, symmetric, and transitive - * properties. + * properties. The collection is materialized when the equivalence is created, + * so it must be finite. * * **Example** (Combining multiple field equivalences) * @@ -414,15 +415,17 @@ export const combine: { * @category combining * @since 2.0.0 */ -export const combineAll = (collection: Iterable>): Equivalence => - make((x, y) => { - for (const equivalence of collection) { +export const combineAll = (collection: Iterable>): Equivalence => { + const equivalences = Array.from(collection) + return make((x, y) => { + for (const equivalence of equivalences) { if (!equivalence(x, y)) { return false } } return true }) +} /** * Transforms an equivalence relation by mapping the input values before comparison. diff --git a/repos/effect/packages/effect/src/Exit.ts b/repos/effect/packages/effect/src/Exit.ts index 00f9f46c9b..1874d53088 100644 --- a/repos/effect/packages/effect/src/Exit.ts +++ b/repos/effect/packages/effect/src/Exit.ts @@ -835,8 +835,10 @@ export const map: { * * **Gotchas** * - * Only transforms typed errors (Fail reasons). If the Cause contains only - * defects or interruptions, the failure passes through unchanged. + * Maps every typed error in the cause, preserving defects, interrupts and + * reason order. Unlike typed recovery, mapping does not convert errors to defects. + * + * If the Cause contains only defects or interruptions, no mapper is called. * * **Example** (Mapping over an error) * @@ -868,11 +870,14 @@ export const mapError: { * **Details** * * `onSuccess` transforms the value if the Exit is a Success. `onFailure` - * transforms the typed error if the Exit is a Failure with a Fail reason. + * transforms each typed error if the Exit is a Failure with Fail reasons. * Allocates a new Exit. * * **Gotchas** * + * Maps every typed error in the cause, preserving defects, interrupts and + * reason order. Unlike typed recovery, mapping does not convert errors to defects. + * * If the Cause contains only defects or interruptions, the failure passes * through unchanged. * diff --git a/repos/effect/packages/effect/src/FiberHandle.ts b/repos/effect/packages/effect/src/FiberHandle.ts index 015862edf3..d5103ba0f9 100644 --- a/repos/effect/packages/effect/src/FiberHandle.ts +++ b/repos/effect/packages/effect/src/FiberHandle.ts @@ -62,6 +62,8 @@ export interface FiberHandle extends Pipeable, state: { readonly _tag: "Open" fiber: Fiber.Fiber | undefined + /** Every fiber the handle still owns: the current one plus replaced fibers that are still interrupting. */ + readonly fibers: Set> } | { readonly _tag: "Closed" } @@ -105,7 +107,7 @@ const Proto = { const makeUnsafe = (): FiberHandle => { const self = Object.create(Proto) - self.state = { _tag: "Open", fiber: undefined } + self.state = { _tag: "Open", fiber: undefined, fibers: new Set() } self.deferred = Deferred.makeUnsafe() return self } @@ -154,12 +156,7 @@ export const make = (): Effect.Effect { - if (self.state._tag === "Open" && fiber === self.state.fiber) { - self.state.fiber = undefined + if (self.state._tag === "Open") { + self.state.fibers.delete(fiber) + if (fiber === self.state.fiber) { + self.state.fiber = undefined + } } if ( Exit.isFailure(exit) && diff --git a/repos/effect/packages/effect/src/FiberMap.ts b/repos/effect/packages/effect/src/FiberMap.ts index 2790e50a3e..671f50afb1 100644 --- a/repos/effect/packages/effect/src/FiberMap.ts +++ b/repos/effect/packages/effect/src/FiberMap.ts @@ -67,6 +67,8 @@ export interface FiberMap state: { readonly _tag: "Open" readonly backing: MutableHashMap.MutableHashMap> + /** Every fiber the map still owns: the keyed ones plus replaced fibers that are still interrupting. */ + readonly fibers: Set> } | { readonly _tag: "Closed" } @@ -122,7 +124,7 @@ const makeUnsafe = ( deferred: Deferred.Deferred ): FiberMap => { const self = Object.create(Proto) - self.state = { _tag: "Open", backing } + self.state = { _tag: "Open", backing, fibers: new Set() } self.deferred = deferred return self } @@ -176,7 +178,7 @@ export const make = (): Effect.Effect Effect.Effect + /** + * Get information about a file at `path` without following symbolic links. + * See `File.Info` for metadata limits. + */ + readonly lstat: ( + path: string + ) => Effect.Effect /** * Create a directory at `path`. You can optionally specify the mode and * whether to recursively create nested directories. @@ -671,6 +678,9 @@ export const makeNoop = (fileSystem: Partial): FileSystem => link(path) { return Effect.fail(notFound("link", path)) }, + lstat(path) { + return Effect.fail(notFound("lstat", path)) + }, makeDirectory() { return Effect.die("not implemented") }, diff --git a/repos/effect/packages/effect/src/Iterable.ts b/repos/effect/packages/effect/src/Iterable.ts index 3806375859..5777f6a5ed 100644 --- a/repos/effect/packages/effect/src/Iterable.ts +++ b/repos/effect/packages/effect/src/Iterable.ts @@ -2282,21 +2282,22 @@ export const dedupeAdjacentWith: { let first = true let last: A function next(): IteratorResult { - const result = iterator.next() - if (result.done) { - return { done: true, value: undefined } - } - if (first) { - first = false - last = result.value - return result - } - const current = result.value - if (isEquivalent(last, current)) { - return next() + while (true) { + const result = iterator.next() + if (result.done) { + return { done: true, value: undefined } + } + if (first) { + first = false + last = result.value + return result + } + const current = result.value + if (!isEquivalent(last, current)) { + last = current + return result + } } - last = current - return result } return { next } } diff --git a/repos/effect/packages/effect/src/JsonSchema.ts b/repos/effect/packages/effect/src/JsonSchema.ts index 125b00f61d..4b60929deb 100644 --- a/repos/effect/packages/effect/src/JsonSchema.ts +++ b/repos/effect/packages/effect/src/JsonSchema.ts @@ -416,9 +416,15 @@ export function fromSchemaOpenApi3_1(js: JsonSchema): Document<"draft-2020-12"> * * **Gotchas** * - * OpenAPI 3.0 `nullable` is applied only when the same Schema Object has an - * explicit string `type`; other constraints such as `enum` are left - * unchanged. Unknown keywords, vendor extensions, and annotation values are + * OpenAPI 3.0 specifies that `nullable` applies only when the same Schema + * Object has an explicit string `type`. For compatibility with tooling that + * uses a single-member `allOf` for nullable references, this converter also + * rewrites `{ nullable: true, allOf: [schema] }` to + * `{ anyOf: [schema, { type: "null" }] }` when there is no string `type` or + * existing `anyOf`. Sibling constraints such as `enum` remain unchanged and + * may still reject null. + * + * Unknown keywords, vendor extensions, and annotation values are * copied opaquely unless their name would become active in Draft 2020-12 and * change meaning, in which case conversion throws. Siblings of a valid * OpenAPI 3.0 `$ref` are ignored. @@ -1331,8 +1337,17 @@ function convertOpenApi30(root: JsonSchema): JsonSchema { } rejectKeywordCollisions(source, OPEN_API_30_TO_2020_COLLISIONS, "Draft 2020-12", "OpenAPI 3.0") + const nullableAllOf = isOpenApi30NullableAllOf(source) + for (const key of Object.keys(source)) { const value = source[key] + if (nullableAllOf && key === "allOf") { + InternalRecord.assignProperty(out, "anyOf", [ + context.schemaAt(source.allOf[0], ["allOf", "0"], ["anyOf", "0"]), + { type: "null" } + ]) + continue + } if ( convertSubschemaKeyword( out, @@ -1369,6 +1384,16 @@ function convertOpenApi30(root: JsonSchema): JsonSchema { }) } +function isOpenApi30NullableAllOf( + source: JsonSchema +): source is JsonSchema & { readonly allOf: readonly [unknown] } { + return source.nullable === true && + typeof source.type !== "string" && + source.anyOf === undefined && + Array.isArray(source.allOf) && + source.allOf.length === 1 +} + function convertOpenApiExclusiveBound( source: JsonSchema, out: JsonSchema, diff --git a/repos/effect/packages/effect/src/Layer.ts b/repos/effect/packages/effect/src/Layer.ts index 982f864e80..2c7c6948fe 100644 --- a/repos/effect/packages/effect/src/Layer.ts +++ b/repos/effect/packages/effect/src/Layer.ts @@ -1278,7 +1278,7 @@ export const mergeAll = , ...Array, Error, Services -> => fromBuild((memoMap, scope) => mergeAllEffect(layers, memoMap, scope)) +> => fromBuildMemo((memoMap, scope) => mergeAllEffect(layers, memoMap, scope)) /** * Merges this layer with another layer concurrently, producing a new layer with @@ -1366,7 +1366,7 @@ const provideWith = ( thatContext: Context.Context ) => Context.Context ) => - fromBuild((memoMap, scope) => + fromBuildMemo((memoMap, scope) => internalEffect.flatMap( Array.isArray(that) ? mergeAllEffect(that as NonEmptyArray>, memoMap, scope) diff --git a/repos/effect/packages/effect/src/ManagedRuntime.ts b/repos/effect/packages/effect/src/ManagedRuntime.ts index 9f11d44122..f5ba8669dc 100644 --- a/repos/effect/packages/effect/src/ManagedRuntime.ts +++ b/repos/effect/packages/effect/src/ManagedRuntime.ts @@ -1,3 +1,5 @@ +/// + /** * Runs many effects against services built once from a `Layer`. * @@ -10,11 +12,12 @@ * @stability stable * @since 2.0.0 */ -import type * as Context from "./Context.ts" +import * as Context from "./Context.ts" +import * as Deferred from "./Deferred.ts" import * as Effect from "./Effect.ts" import * as Exit from "./Exit.ts" import * as Fiber from "./Fiber.ts" -import { type FiberImpl, scopeRemoveFinalizerUnsafe } from "./internal/effect.ts" +import { fiberEnterUninterruptibleUnsafe, type FiberImpl, scopeRemoveFinalizerUnsafe } from "./internal/effect.ts" import * as Layer from "./Layer.ts" import { hasProperty } from "./Predicate.ts" import * as Scope from "./Scope.ts" @@ -209,6 +212,10 @@ export interface ManagedRuntime { * **When to use** * * Use to release this runtime's layer resources from Promise-based code. + * + * **Details** + * + * Calls made while a disposal is in progress wait for it to finish. */ readonly dispose: () => Promise @@ -228,10 +235,32 @@ export interface ManagedRuntime { * **When to use** * * Use to release this runtime's layer resources from an `Effect` workflow. + * + * **Details** + * + * Disposal runs uninterruptibly. Calls made while a disposal is in progress + * wait for it to finish, and interrupting a waiting call only stops the wait. + * Fibers started by this runtime, their child fibers, and the runtime's + * finalizers do not wait: the disposal interrupts or runs them, so waiting + * would deadlock. */ readonly disposeEffect: Effect.Effect } +// The runtimes whose disposal the current fiber is running. A disposal adds its +// runtime to the set it inherits, so a finalizer that disposes another runtime +// still belongs to the outer disposals. Finalizers and their parallel fibers +// inherit the set from the disposing fiber. It is not provided to the layer +// build, because finalizers of layers shared through a MemoMap run with the +// build context of the runtime that built them, which may not be the runtime +// disposing them. +const DisposingRuntimes = Context.Reference>>( + "effect/ManagedRuntime/DisposingRuntimes", + { + defaultValue: () => new Set() + } +) + /** * Creates a `ManagedRuntime` from a layer. * @@ -252,6 +281,11 @@ export interface ManagedRuntime { * Dispose the runtime when it is no longer needed. A runtime cannot be reused * after disposal. * + * A failed layer build is also cached: subsequent runs return the same build + * failure without rebuilding the layer. For transient failures, retry inside + * the layer before passing it to `make`, or dispose the runtime and create a + * new one to attempt a fresh build. + * * **Example** (Creating a managed runtime) * * ```ts import.meta.vitest @@ -298,8 +332,17 @@ export const make = ( const scope = Scope.makeUnsafe("sequential") const layerScope = Scope.forkUnsafe(scope, "sequential") const fiberScope = Scope.forkUnsafe(scope, "parallel") + // Fibers started by the runtime, including the layer build, are interrupted + // and awaited by its disposal, so they and their child fibers must not wait + // for it. + const managedFibers = new WeakSet>() + const disposingFibers = new WeakSet>() const defaultRunOptions: Effect.RunOptions = { - onFiberStart: Fiber.runIn(fiberScope) + onFiberStart(fiber) { + managedFibers.add(fiber) + if (disposingFibers.has(fiber)) return + Fiber.runIn(fiber, fiberScope) + } } const mergeRunOptions = (options?: O): O => options @@ -314,6 +357,7 @@ export const make = ( } : defaultRunOptions as O let buildFiber: Fiber.Fiber, ER> | undefined + let disposed: Deferred.Deferred | undefined const contextEffect = Effect.withFiber, ER>((fiber) => { if (!buildFiber) { buildFiber = Effect.runFork( @@ -324,7 +368,15 @@ export const make = ( self.cachedContext = context }) ), - { ...defaultRunOptions, scheduler: fiber.cache.scheduler } + { + scheduler: fiber.cache.scheduler, + // Unlike requests, the build joins fiberScope even if it disposed the + // runtime, so disposal still interrupts it. + onFiberStart(fiber) { + managedFibers.add(fiber) + Fiber.runIn(fiber, fiberScope) + } + } ) } return Effect.flatten(Fiber.await(buildFiber)) @@ -347,14 +399,28 @@ export const make = ( return self.dispose() }, disposeEffect: Effect.withFiber((fiber) => { - // Closing fiberScope interrupts and awaits every managed fiber. Skip the + if (disposed !== undefined) { + if (fiber.getRef(DisposingRuntimes).has(self)) return Effect.void + for (let current = fiber as FiberImpl | undefined; current; current = current._parent) { + if (managedFibers.has(current)) return Effect.void + } + return Deferred.await(disposed) + } + // Once `disposed` is set the close must run, or later calls would wait forever. + fiberEnterUninterruptibleUnsafe(fiber) + const done = disposed = Deferred.makeUnsafe() + // Closing fiberScope interrupts and awaits every managed fiber. Exempt the // disposing fiber and its ancestors, which would otherwise await themselves. for (let current = fiber as FiberImpl | undefined; current; current = current._parent) { + disposingFibers.add(current) scopeRemoveFinalizerUnsafe(fiberScope, current) } ;(self as Mutable>).contextEffect = Effect.die("ManagedRuntime disposed") self.cachedContext = undefined - return Scope.close(self.scope, Exit.void) + return Scope.close(self.scope, Exit.void).pipe( + Effect.provideService(DisposingRuntimes, new Set(fiber.getRef(DisposingRuntimes)).add(self)), + Effect.ensuring(Deferred.done(done, Exit.void)) + ) }), runFork(effect: Effect.Effect, options?: Effect.RunOptions): Fiber.Fiber { return self.cachedContext === undefined ? diff --git a/repos/effect/packages/effect/src/Metric.ts b/repos/effect/packages/effect/src/Metric.ts index 9b8e316357..f0b05ab3ec 100644 --- a/repos/effect/packages/effect/src/Metric.ts +++ b/repos/effect/packages/effect/src/Metric.ts @@ -17,7 +17,7 @@ import * as Context from "./Context.ts" import * as Duration from "./Duration.ts" import type { Effect } from "./Effect.ts" import type { Exit } from "./Exit.ts" -import { constUndefined, dual } from "./Function.ts" +import { constUndefined, constVoid, dual, identity } from "./Function.ts" import * as InternalEffect from "./internal/effect.ts" import * as InternalMetric from "./internal/metric.ts" import * as InternalRecord from "./internal/record.ts" @@ -1678,6 +1678,7 @@ abstract class Metric$ implements Metric { declare readonly State: Covariant #extraAttributes: Metric.AttributeSet | undefined + #mergedAttributes: Metric.AttributeSet | undefined #key = "" readonly id: string @@ -1709,29 +1710,61 @@ abstract class Metric$ implements Metric { abstract createHooks(): Metric.Hooks hook(context: Context.Context): Metric.Hooks { - const extraAttributes = Context.get(context, CurrentMetricAttributes) - if (extraAttributes !== this.#extraAttributes) { - this.#extraAttributes = extraAttributes - this.#key = makeKey(this, mergeAttributes(this.attributes, extraAttributes)) - } + const key = this.key(context) // The registry owns the series, so it is looked up on every update. const registry = Context.get(context, MetricRegistry) - let metadata = registry.get(this.#key) + let metadata = registry.get(key) if (Predicate.isUndefined(metadata)) { - metadata = { - id: this.id, - type: this.type, - description: this.description, - attributes: Object.keys(extraAttributes).length > 0 - ? mergeAttributes(this.attributes, extraAttributes) - : this.attributes, - hooks: this.createHooks() - } - registry.set(this.#key, metadata) + metadata = this.metadata(this.createHooks()) + registry.set(key, metadata) } return metadata.hooks } + removeUnsafe(context: Context.Context): void { + Context.get(context, MetricRegistry).delete(this.key(context)) + } + + registerUnsafe(context: Context.Context, hooks: Metric.Hooks): Registration { + const key = this.key(context) + // Keep the registry from registration time, so removal does not depend on + // the context it is called from. + const registry = Context.get(context, MetricRegistry) + const metadata = this.metadata({ get: hooks.get, update: constVoid, modify: constVoid }) + registry.set(key, metadata) + return { + remove() { + if (registry.get(key) === metadata) { + registry.delete(key) + } + } + } + } + + // Merges the attributes once per contextual attribute set, so series key and + // metadata share the result. + private key(context: Context.Context): string { + const extraAttributes = Context.get(context, CurrentMetricAttributes) + if (extraAttributes !== this.#extraAttributes) { + this.#extraAttributes = extraAttributes + const merged = mergeAttributes(this.attributes, extraAttributes) + this.#mergedAttributes = Object.keys(extraAttributes).length > 0 ? merged : this.attributes + this.#key = makeKey(this, merged) + } + return this.#key + } + + // Must follow `key(context)`, which computes the merged attributes. + private metadata(hooks: Metric.Hooks): Metric.Metadata { + return { + id: this.id, + type: this.type, + description: this.description, + attributes: this.#mergedAttributes, + hooks + } + } + pipe() { return pipeArguments(this, arguments) } @@ -1992,29 +2025,50 @@ class SummaryMetric extends Metric$ extends Metric$ { +// Every operation maps the context first and the input second, so wrappers +// such as `withAttributes` address the same series for update, remove and +// register. +class MetricTransform extends Metric$ { type: Metric.Type readonly metric: Metric - override readonly valueUnsafe: (context: Context.Context) => State - override readonly updateUnsafe: (input: Input2, context: Context.Context) => void - override readonly modifyUnsafe: (input: Input2, context: Context.Context) => void + readonly mapInput: (input: Input2, context: Context.Context) => Input + readonly mapContext: (context: Context.Context) => Context.Context constructor( metric: Metric, - valueUnsafe: (context: Context.Context) => State, - updateUnsafe: (input: Input2, context: Context.Context) => void, - modifyUnsafe: (input: Input2, context: Context.Context) => void + mapInput: (input: Input2, context: Context.Context) => Input, + mapContext: (context: Context.Context) => Context.Context ) { super(metric.id, metric.description, metric.attributes) this.metric = metric - this.valueUnsafe = valueUnsafe - this.updateUnsafe = updateUnsafe - this.modifyUnsafe = modifyUnsafe + this.mapInput = mapInput + this.mapContext = mapContext this.type = metric.type } createHooks(): Metric.Hooks { return (this.metric as any).createHooks() } + override valueUnsafe(context: Context.Context): State { + return this.metric.valueUnsafe(this.mapContext(context)) + } + override updateUnsafe(input: Input2, context: Context.Context): void { + const mapped = this.mapContext(context) + this.metric.updateUnsafe(this.mapInput(input, mapped), mapped) + } + override modifyUnsafe(input: Input2, context: Context.Context): void { + const mapped = this.mapContext(context) + this.metric.modifyUnsafe(this.mapInput(input, mapped), mapped) + } + override removeUnsafe(context: Context.Context): void { + removeUnsafe(this.metric, this.mapContext(context)) + } + override registerUnsafe(context: Context.Context, hooks: Metric.Hooks): Registration { + return registerUnsafe(this.metric, this.mapContext(context), { + get: hooks.get, + update: constVoid, + modify: constVoid + }) + } } /** @@ -2732,6 +2786,148 @@ export const update: { InternalEffect.contextWith((services) => InternalEffect.sync(() => self.updateUnsafe(input, services))) ) +/** + * Handle for a series registered with `Metric.registerUnsafe`. + * + * **Details** + * + * `remove` deletes the series from the registry it was registered into, but + * only while the registry still holds this registration. Calling it again, or + * after another registration has replaced the series, does nothing. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export interface Registration { + readonly remove: () => void +} + +/** + * Removes the series of a metric for the current attributes from the metric + * registry. + * + * **When to use** + * + * Use when a series describes something that has gone away, such as a + * connection or an entity, so the registry and exporters stop tracking it. + * + * **Details** + * + * The series is identified the same way `update` identifies it: by the metric + * and its attributes merged with `CurrentMetricAttributes`. Other series of the + * same metric are left alone. A later update creates the series again from its + * initial state. Removing a series that does not exist does nothing. + * + * **Example** (Removing a series) + * + * ```ts import.meta.vitest + * import { Effect, Metric } from "effect" + * + * const connections = Metric.gauge("connections") + * + * const program = Effect.gen(function*() { + * yield* Metric.update(connections, 3).pipe( + * Effect.provideService(Metric.CurrentMetricAttributes, { pool: "a" }) + * ) + * yield* Metric.update(connections, 5).pipe( + * Effect.provideService(Metric.CurrentMetricAttributes, { pool: "b" }) + * ) + * yield* Metric.remove(connections).pipe( + * Effect.provideService(Metric.CurrentMetricAttributes, { pool: "a" }) + * ) + * const snapshots = yield* Metric.snapshot + * return snapshots.map((snapshot) => snapshot.attributes) + * }) + * + * await Effect.runPromise(Effect.provideService(program, Metric.MetricRegistry, new Map())) // => [{ pool: "b" }] + * ``` + * + * @see {@link removeUnsafe} for removing a series with an explicit `Context` + * + * @stability unstable + * @category mutations + * @since 4.0.6 + */ +export const remove = (self: Metric): Effect => + InternalEffect.contextWith((context) => InternalEffect.sync(() => removeUnsafe(self, context))) + +/** + * Removes the series of a metric for the attributes in the provided `Context` + * from the metric registry in that `Context`. + * + * **When to use** + * + * Use when a low-level integration already holds a `Context`. Use `remove` + * in application code. + * + * @see {@link remove} for the `Effect` version + * + * @stability unstable + * @category mutations + * @since 4.0.6 + */ +export const removeUnsafe = (self: Metric, context: Context.Context): void => + (self as Metric$).removeUnsafe(context) + +/** + * Registers a series backed by caller-supplied hooks, for gauges whose value is + * read when metrics are snapshotted rather than pushed with updates. + * + * **When to use** + * + * Use when the value already lives in some data structure, such as a queue + * size, and updating a metric on every change would cost more than reading the + * value at export time. + * + * **Details** + * + * The series is identified as in `remove`, using the attributes and registry + * from `context`, and replaces any series with the same identity. `hooks.get` + * is called each time the series is read, for example by `snapshot` or an + * exporter. The returned {@link Registration} removes it again. + * `update` and `modify` on this series do nothing; the supplied mutation hooks + * are ignored. + * + * **Gotchas** + * + * The registry keeps `hooks.get` reachable until the series is removed, along + * with anything it closes over. Call `remove` once the source goes away. + * + * **Example** (Reporting a queue size on snapshot) + * + * ```ts import.meta.vitest + * import { Context, Metric } from "effect" + * + * const context = Context.make(Metric.MetricRegistry, new Map()) + * const queueSize = Metric.gauge("queue_size") + * const queue: Array = ["a", "b"] + * + * const registration = Metric.registerUnsafe(queueSize, context, { + * get: () => ({ value: queue.length }), + * update: () => {}, + * modify: () => {} + * }) + * queue.push("c") + * const sizes = Metric.snapshotUnsafe(context).map((snapshot) => snapshot.state) + * registration.remove() + * + * sizes // => [{ value: 3 }] + * Metric.snapshotUnsafe(context).length // => 0 + * ``` + * + * @see {@link remove} for removing a series updated with `update` + * + * @stability unstable + * @category mutations + * @since 4.0.6 + */ +export const registerUnsafe = ( + self: Metric, + context: Context.Context, + hooks: Metric.Hooks +): Registration => (self as Metric$).registerUnsafe(context, hooks) + /** * Returns a new metric that is powered by this one, but which accepts updates * of the specified new type, which must be transformable to the input type of @@ -2785,13 +2981,7 @@ export const mapInput: { >(2, ( self: Metric, f: (input: Input2, context: Context.Context) => Input -): Metric => - new MetricTransform( - self, - (context) => self.valueUnsafe(context), - (input, context) => self.updateUnsafe(f(input, context), context), - (input, context) => self.modifyUnsafe(f(input, context), context) - )) +): Metric => new MetricTransform(self, f, identity)) /** * Returns a new metric that is powered by this one, but which accepts updates @@ -2905,13 +3095,7 @@ export const withAttributes: { self: Metric, attributes: Metric.Attributes ): Metric => { - const attributed = addAttributesToContext(attributes) - return new MetricTransform( - self, - (context) => self.valueUnsafe(attributed(context)), - (input, context) => self.updateUnsafe(input, attributed(context)), - (input, context) => self.modifyUnsafe(input, attributed(context)) - ) + return new MetricTransform(self, identity, addAttributesToContext(attributes)) }) // Metric Snapshots diff --git a/repos/effect/packages/effect/src/Number.ts b/repos/effect/packages/effect/src/Number.ts index 03fdfa973b..326bf088ca 100644 --- a/repos/effect/packages/effect/src/Number.ts +++ b/repos/effect/packages/effect/src/Number.ts @@ -708,8 +708,8 @@ function toScientificInteger(n: number): readonly [coefficient: bigint, exponent * @since 2.0.0 */ export const nextPow2 = (n: number): number => { - const nextPow = Math.ceil(Math.log(n) / Math.log(2)) - return Math.max(Math.pow(2, nextPow), 2) + const nextPow = Math.pow(2, Math.ceil(Math.log2(n))) + return Math.max(nextPow < n ? nextPow * 2 : nextPow, 2) } /** diff --git a/repos/effect/packages/effect/src/Path.ts b/repos/effect/packages/effect/src/Path.ts index 61438f46a7..0512c46fd5 100644 --- a/repos/effect/packages/effect/src/Path.ts +++ b/repos/effect/packages/effect/src/Path.ts @@ -430,7 +430,15 @@ function fromFileUrl(url: URL): Effect.Effect { } } } - return Effect.succeed(decodeURIComponent(pathname)) + return Effect.try({ + try: () => decodeURIComponent(pathname), + catch: (cause) => + new BadArgument({ + module: "Path", + method: "fromFileUrl", + cause + }) + }) } const resolve: Path["resolve"] = function resolve() { diff --git a/repos/effect/packages/effect/src/PubSub.ts b/repos/effect/packages/effect/src/PubSub.ts index 15ab11e100..30065637cc 100644 --- a/repos/effect/packages/effect/src/PubSub.ts +++ b/repos/effect/packages/effect/src/PubSub.ts @@ -1469,16 +1469,16 @@ export const takeUpTo: { } = dual(2, (self: Subscription, max: number): Effect.Effect> => Effect.suspend(() => { if (self.shutdownFlag.current) return Effect.interrupt - max = Count.normalize(max) + let limit = Count.normalize(max) let replay: Array | undefined = undefined - if (self.replayWindow.remaining >= max) { - return Effect.succeed(self.replayWindow.takeN(max)) + if (self.replayWindow.remaining >= limit) { + return Effect.succeed(self.replayWindow.takeN(limit)) } else if (self.replayWindow.remaining > 0) { replay = self.replayWindow.takeAll() - max = max - replay.length + limit = limit - replay.length } const as = self.pollers.length === 0 - ? self.subscription.pollUpTo(max) + ? self.subscription.pollUpTo(limit) : [] self.strategy.onPubSubEmptySpaceUnsafe(self.pubsub, self.subscribers) return replay ? Effect.succeed(replay.concat(as)) : Effect.succeed(as) @@ -1537,7 +1537,9 @@ const takeRemainderLoop = ( return Effect.succeed(acc) } return Effect.flatMap(takeUpTo(self, max), (bs) => { - acc.push(...bs) + for (let i = 0; i < bs.length; i++) { + acc.push(bs[i]) + } const remaining = min - bs.length if (remaining === 1) { return Effect.map(take(self), (b) => { @@ -2935,7 +2937,10 @@ class ReplayBuffer { } this.tail = this.tail.next if (this.size === this.capacity) { - this.head = this.head.next! + const dropped = this.head + this.head = dropped.next! + dropped.next = null + dropped.value = AbsentValue } else { this.size += 1 } diff --git a/repos/effect/packages/effect/src/Queue.ts b/repos/effect/packages/effect/src/Queue.ts index 6ea790b826..950b73d6a0 100644 --- a/repos/effect/packages/effect/src/Queue.ts +++ b/repos/effect/packages/effect/src/Queue.ts @@ -695,7 +695,10 @@ export const offer: { ) /** - * Adds a message to the queue synchronously. Returns `false` if the queue is done. + * Adds a message to the queue synchronously. Returns `false` without adding the + * message if the queue is closing or done, or if it is full with a non-sliding + * strategy (`"suspend"` or `"dropping"`). A queue with non-positive capacity can + * still accept a message when a taker is waiting. * * **When to use** * diff --git a/repos/effect/packages/effect/src/Random.ts b/repos/effect/packages/effect/src/Random.ts index e2fc7b61a5..75c0a38e0a 100644 --- a/repos/effect/packages/effect/src/Random.ts +++ b/repos/effect/packages/effect/src/Random.ts @@ -318,7 +318,7 @@ export const withSeed: { } = dual(2, ( self: Effect.Effect, seed: string | number -) => Effect.provideService(self, Random, ISAAC_CSPRNG(seed))) +) => Effect.suspend(() => Effect.provideService(self, Random, ISAAC_CSPRNG(seed)))) /*/////////////////////////////////////////////////////////////////////////////////////////////////// This is a derivative work copyright (c) 2025 Effectful Technologies Inc, under MIT license. diff --git a/repos/effect/packages/effect/src/RcMap.ts b/repos/effect/packages/effect/src/RcMap.ts index d3c5b6cb75..397481d774 100644 --- a/repos/effect/packages/effect/src/RcMap.ts +++ b/repos/effect/packages/effect/src/RcMap.ts @@ -4,10 +4,12 @@ * An `RcMap` runs a lookup effect the first time a key is requested, shares the * in-progress or acquired resource with other callers for the same key, and * tracks each caller through its current `Scope`. When the last scope for a key - * closes, the resource can be released, kept alive for an idle time, or removed - * by capacity limits or explicit invalidation. It is meant for resource - * lifecycles such as clients, sessions, and connections, not as a general - * mutable cache. + * closes, the resource is released immediately or kept alive until its idle + * time-to-live expires. Entries can also be explicitly invalidated. Capacity + * limits do not evict entries: idle entries count toward capacity, and a new + * key fails with `ExceededCapacityError` when the map is full. It is meant for + * resource lifecycles such as clients, sessions, and connections, not as a + * general mutable cache. * * @stability stable * @since 3.5.0 @@ -204,7 +206,8 @@ const makeUnsafe = (options: { * * Complex keys can extend `Equal` and `Hash` to allow lookups by value. * - * - `capacity`: The maximum number of resources that can be held in the map. + * - `capacity`: The maximum number of entries, including idle entries. A new key + * fails with `ExceededCapacityError` when full; existing entries are not evicted. * - `idleTimeToLive`: When the reference count reaches zero, the resource will be released after this duration. * * **Example** (Creating a reference-counted map) @@ -285,7 +288,8 @@ export const make: { Effect.sync(() => { MutableHashMap.clear(map) }) - ) + ), + Effect.flatMap(Exit.asVoidAll) ) }), self diff --git a/repos/effect/packages/effect/src/Record.ts b/repos/effect/packages/effect/src/Record.ts index 74c9af49f3..9865b45f62 100644 --- a/repos/effect/packages/effect/src/Record.ts +++ b/repos/effect/packages/effect/src/Record.ts @@ -1181,6 +1181,11 @@ export const isSubrecord: { /** * Reduces a record to a single value by combining its entries with a specified function. * + * **Details** + * + * The reducer function receives the accumulator, the current value, and the + * key of the current entry. + * * **Example** (Reducing record values) * * ```ts import.meta.vitest @@ -1189,6 +1194,18 @@ export const isSubrecord: { * Record.reduce({ a: 1, b: 2, c: 3 }, 0, (acc, value) => acc + value) // => 6 * ``` * + * **Example** (Using the key in the reducer) + * + * ```ts import.meta.vitest + * import { Record } from "effect" + * + * Record.reduce( + * { a: 1, b: 2, c: 3 }, + * "", + * (acc, value, key) => (acc === "" ? `${key}${value}` : `${acc}-${key}${value}`) + * ) // => "a1-b2-c3" + * ``` + * * @stability stable * @category folding * @since 2.0.0 diff --git a/repos/effect/packages/effect/src/References.ts b/repos/effect/packages/effect/src/References.ts index 95b479181a..1b882d53cf 100644 --- a/repos/effect/packages/effect/src/References.ts +++ b/repos/effect/packages/effect/src/References.ts @@ -563,12 +563,13 @@ export const TracerSpanLinks: Context.Reference> = refer export const TracerTimingEnabled: Context.Reference = references.TracerTimingEnabled /** - * Context reference for the log severity used when a pool finalizer reports an - * unhandled error. + * Context reference for the log severity used to report errors that have no + * caller to receive them, such as pool finalizer failures or defects in + * background callbacks. * * **When to use** * - * Use to choose whether and at which severity pool finalizer failures are + * Use to choose whether and at which severity these unhandled errors are * reported. * * **Details** @@ -577,7 +578,7 @@ export const TracerTimingEnabled: Context.Reference = references.Tracer * * **Gotchas** * - * Providing `undefined` suppresses this report; it does not fall back to + * Providing `undefined` suppresses these reports; it does not fall back to * `CurrentLogLevel`. * * @see {@link CurrentLogLevel} for the default severity used by ordinary `Effect.log` calls diff --git a/repos/effect/packages/effect/src/RequestResolver.ts b/repos/effect/packages/effect/src/RequestResolver.ts index 32d6d22ad5..dd78faa103 100644 --- a/repos/effect/packages/effect/src/RequestResolver.ts +++ b/repos/effect/packages/effect/src/RequestResolver.ts @@ -172,7 +172,9 @@ export const isRequestResolver = (u: unknown): u is RequestResolver => hasP * * `batchKey` groups request entries, `delay` schedules batch execution, * `collectWhile` can end collection early, and `runAll` receives a non-empty - * batch for one key. + * batch for one key. The batch runs with the context of the caller that created + * it, even when a later caller ends collection. Each entry retains its own + * caller context in `entry.context`. * * **Gotchas** * @@ -209,6 +211,17 @@ const defaultKey = (_request: unknown): unknown => defaultKeyObject /** * Constructs a request resolver with the specified method to run requests. * + * **Details** + * + * `runAll` runs with the context of the caller that created the batch, including + * when `collectWhile` ends collection early (for example with `batchN`). + * + * **Gotchas** + * + * Different caller contexts do not split batches. Use `entry.context` for + * per-entry services, or `makeGrouped` to separate requests by the services + * that affect the backend operation, such as tenant or authorization settings. + * * **Example** (Creating a request resolver) * * ```ts import.meta.vitest @@ -256,6 +269,14 @@ export const make = ( * **Details** * * The key can use the Equal trait to determine if two keys are equal. + * Each batch runs with the context of the caller that created that batch, + * whether collection ends after the delay or at a `collectWhile` cutoff. + * + * **Gotchas** + * + * Caller services that are not included in the key do not split batches. Include + * tenant or authorization settings in the key when they affect the backend + * operation, or read each caller's services from `entry.context`. * * **Example** (Grouping requests by key) * @@ -411,6 +432,15 @@ export const fromFunctionBatched = ( /** * Constructs a request resolver from an effectual function. * + * **Details** + * + * Each entry's effect runs with its own caller context. Services provided around + * `runAll` are not visible to it; provide them inside `f` instead. Entries with + * different caller contexts can still share a batch. + * + * With `withSpan`, spans created inside `f` are children of the caller's span, + * not the resolver span. The resolver span still links to each caller's span. + * * **Example** (Creating a resolver from an effectful function) * * ```ts import.meta.vitest @@ -451,7 +481,11 @@ export const fromEffect = ( let done = 0 for (let i = 0; i < entries.length; i++) { const entry = entries[i] - const fiber = effect.forkUnsafe(parent as any, f(entry), true) + const fiber = effect.forkUnsafe( + parent as any, + effect.setContext(f(entry), entry.context), + true + ) fiber.addObserver((exit) => { entry.completeUnsafe(exit) done++ @@ -1020,7 +1054,7 @@ export const withSpan: { links.push({ span: span.value, attributes: {} }) } return Effect.withSpan(self.runAll(entries, key), name, { - ...options, + ...opts, links, attributes: { batchSize: entries.length, @@ -1183,6 +1217,9 @@ export const withCache: { preCheck(entry) { const ocached = MutableHashMap.get(cache, entry.request) if (ocached._tag === "None") { + if (self.preCheck !== undefined && !self.preCheck(entry)) { + return false + } const cached: CacheEntry = { entry, exit: undefined, pending: [] } MutableHashMap.set(cache, entry.request, cached) const prevComplete = entry.completeUnsafe @@ -1304,6 +1341,8 @@ export const persisted: { exit === undefined || (options.staleWhileRevalidate && options.staleWhileRevalidate(exit as any, entry.request)) ) { + // Deliver stale results before tracking refresh completions for persistence. + if (exit !== undefined) entry.completeUnsafe(exit as any) const prevComplete = entry.completeUnsafe entry.completeUnsafe = function(exit) { completed.add(entry) @@ -1311,7 +1350,7 @@ export const persisted: { prevComplete(exit) } leftover.push(entry) - if (exit === undefined) continue + continue } entry.completeUnsafe(exit as any) } diff --git a/repos/effect/packages/effect/src/Resource.ts b/repos/effect/packages/effect/src/Resource.ts index ef03a2eb9e..b8e7c14206 100644 --- a/repos/effect/packages/effect/src/Resource.ts +++ b/repos/effect/packages/effect/src/Resource.ts @@ -136,7 +136,7 @@ export const auto = ( ): Effect.Effect, never, R | R2 | Scope.Scope> => Effect.tap( manual(acquire), - (self) => Effect.forkScoped(Effect.repeat(refresh(self), policy)) + (self) => Effect.forkScoped(Effect.schedule(Effect.ignoreCause(refresh(self)), policy)) ) /** diff --git a/repos/effect/packages/effect/src/Runtime.ts b/repos/effect/packages/effect/src/Runtime.ts index e9769d40c7..996908ad1c 100644 --- a/repos/effect/packages/effect/src/Runtime.ts +++ b/repos/effect/packages/effect/src/Runtime.ts @@ -126,9 +126,9 @@ export const defaultTeardown: Teardown = ( * * **Details** * - * The runner executes Effect programs as main entry points. The provided - * function receives a forked fiber and a teardown callback so it can install - * platform-specific signal handling, fiber observers, and final exit behavior. + * The setup function runs before user code and receives the main fiber and a + * teardown callback to install platform-specific signal handlers, observers, + * and exit behavior. * * Most applications should use a platform-provided runner, such as * `NodeRuntime.runMain`, rather than constructing one directly. @@ -208,23 +208,24 @@ export const makeRunMain = ( readonly disableErrorReporting?: boolean | undefined readonly teardown?: Teardown | undefined }) => { - const fiber = options?.disableErrorReporting === true - ? Effect.runFork(effect) - : Effect.runFork( - Effect.tapCause(effect, (cause) => { - if (Cause.hasInterruptsOnly(cause)) return Effect.void - const isReported = getErrorReported(Cause.squash(cause)) - return isReported ? Effect.logError(cause) : Effect.void - }) - ) - try { - const keepAlive = globalThis.setInterval(constVoid, 2_147_483_647) - fiber.addObserver(() => { - clearInterval(keepAlive) + const main = options?.disableErrorReporting === true + ? effect + : Effect.tapCause(effect, (cause) => { + if (Cause.hasInterruptsOnly(cause)) return Effect.void + const isReported = getErrorReported(Cause.squash(cause)) + return isReported ? Effect.logError(cause) : Effect.void }) - } catch {} const teardown = options?.teardown ?? defaultTeardown - return f({ fiber, teardown }) + Effect.runFork(Effect.withFiber((fiber) => { + try { + const keepAlive = globalThis.setInterval(constVoid, 2_147_483_647) + fiber.addObserver(() => { + clearInterval(keepAlive) + }) + } catch {} + f({ fiber, teardown }) + return main + })) }) declare global { diff --git a/repos/effect/packages/effect/src/Schema.ts b/repos/effect/packages/effect/src/Schema.ts index bfef2b661d..fe28508944 100644 --- a/repos/effect/packages/effect/src/Schema.ts +++ b/repos/effect/packages/effect/src/Schema.ts @@ -79,6 +79,8 @@ import * as SchemaIssue from "./SchemaIssue.ts" import * as SchemaParser from "./SchemaParser.ts" import type * as SchemaRepresentation from "./SchemaRepresentation.ts" import * as SchemaTransformation from "./SchemaTransformation.ts" +import * as SemVer_ from "./SemVer.ts" +import * as SemVerRange_ from "./SemVerRange.ts" import type { StandardJSONSchemaV1, StandardSchemaV1 } from "./StandardSchema.ts" import type { Assign, Lambda, Mutable, Simplify } from "./Struct.ts" import * as Struct_ from "./Struct.ts" @@ -123,8 +125,8 @@ export type ConstructorDefault = "no-default" | "with-default" * * **When to use** * - * Use when passing `disableChecks: true` to skip validation when you trust the - * data. Pass `parseOptions` to control parsing behavior. + * Use when passing `disableChecks: true` to skip filters and checks while still + * parsing types and structure. Pass `parseOptions` to control parsing behavior. * * @see {@link BottomWithoutNew.makeEffect} * @see {@link BottomWithoutNew.make} @@ -139,7 +141,8 @@ export interface MakeOptions { */ readonly parseOptions?: SchemaAST.ParseOptions | undefined /** - * Whether to disable validation for the schema. + * Whether to skip filters and checks. Types and structure are still parsed, + * and constructor defaults still apply. */ readonly disableChecks?: boolean | undefined @@ -2961,8 +2964,8 @@ export interface Enum readonly enums: A } /** - * Creates a schema from a TypeScript enum object. Validates that the input is one of the enum's values. - * Numeric enum values must be finite. + * Creates a schema from a TypeScript enum or an object mapping names to string or number values. + * Validates that the input is one of those values. Numeric values must be finite. * * **Gotchas** * @@ -2986,7 +2989,7 @@ export interface Enum * @category constructors * @since 4.0.0 */ -export function Enum(enums: A): Enum { +export function Enum(enums: A): Enum { return make( new SchemaAST.Enum( Object.keys(enums).filter( @@ -5632,8 +5635,8 @@ export interface compose extends * **Gotchas** * * In a custom transformation, `decode` maps `From["Type"]` to `To["Encoded"]` - * and is used on the encoding path, while `encode` maps `To["Encoded"]` to - * `From["Type"]` and is used on the decoding path. + * and is used on the decoding path, while `encode` maps `To["Encoded"]` to + * `From["Type"]` and is used on the encoding path. * * **Example** (Transforming strings to numbers with a schema transformation) * @@ -6316,12 +6319,6 @@ type Flatten = Schemas extends readonly [infer Head, ...infer Tail] ? Head extends Union ? [...Flatten, ...Flatten] : [Head, ...Flatten] : [] -type DiscriminantKey = T extends number ? `${T}` : T -type EnsureUniqueDiscriminants, Seen = never> = Tags extends - readonly [infer Head extends PropertyKey, ...infer Tail extends ReadonlyArray] - ? [DiscriminantKey & Seen] extends [never] ? EnsureUniqueDiscriminants> - : never - : unknown type MatchCasesResult = { [K in keyof Cases]-?: NonNullable extends (...args: Array) => infer R ? R : never }[keyof Cases] @@ -6406,10 +6403,9 @@ export type toTaggedUnion< * * **Gotchas** * - * Rejects overlapping discriminant types in known member tuples at compile - * time, including nested unions. Numeric and string values such as `1` and - * `"1"` count as duplicates. Also throws at runtime if multiple members use the - * same discriminant property key. + * Throws when multiple members use the same discriminant property key, + * including members of nested unions. Numeric and string values such as `1` + * and `"1"` count as duplicates. * * **Example** (Adding tagged-union utilities to an existing union) * @@ -6436,7 +6432,7 @@ export type toTaggedUnion< */ export function toTaggedUnion(tag: Tag) { return >( - self: Union & NoInfer["discriminants"]>> + self: Union ): toTaggedUnion => { const cases: Record = {} const discriminants: Array = [] @@ -7137,16 +7133,21 @@ export function isULID(annotations?: Annotations.Filter) { * @since 4.0.0 */ export function isBase64(annotations?: Annotations.Filter) { - const regExp = /^([0-9a-zA-Z+/]{4})*(([0-9a-zA-Z+/]{2}==)|([0-9a-zA-Z+/]{3}=))?$/ - return isPattern( - regExp, + const pattern = "^([0-9a-zA-Z+/]{4})*(([0-9a-zA-Z+/]{2}==)|([0-9a-zA-Z+/]{3}=))?$" + // `pattern` overflows the V8 stack on large inputs + const regExp = /^[0-9a-zA-Z+/]*={0,2}$/ + return makeFilter( + (s: string) => s.length % 4 === 0 && regExp.test(s), { expected: "a base64 encoded string", representation: { id: "effect/schema/isBase64", payload: null }, - toJsonSchema: () => ({ pattern: regExp.source }), + toJsonSchema: () => ({ pattern }), + arbitraryConstraint: { + patterns: [{ source: pattern, flags: "" }] + }, toCode: () => ({ runtime: "Schema.isBase64()" }), ...annotations } @@ -7173,16 +7174,21 @@ export function isBase64(annotations?: Annotations.Filter) { * @since 4.0.0 */ export function isBase64Url(annotations?: Annotations.Filter) { - const regExp = /^([0-9a-zA-Z-_]{4})*(([0-9a-zA-Z-_]{2}(==)?)|([0-9a-zA-Z-_]{3}(=)?))?$/ - return isPattern( - regExp, + const pattern = "^([0-9a-zA-Z-_]{4})*(([0-9a-zA-Z-_]{2}(==)?)|([0-9a-zA-Z-_]{3}(=)?))?$" + // `pattern` overflows the V8 stack on large inputs + const regExp = /^[0-9a-zA-Z-_]*={0,2}$/ + return makeFilter( + (s: string) => regExp.test(s) && (s.endsWith("=") ? s.length % 4 === 0 : s.length % 4 !== 1), { expected: "a base64url encoded string", representation: { id: "effect/schema/isBase64Url", payload: null }, - toJsonSchema: () => ({ pattern: regExp.source }), + toJsonSchema: () => ({ pattern }), + arbitraryConstraint: { + patterns: [{ source: pattern, flags: "" }] + }, toCode: () => ({ runtime: "Schema.isBase64Url()" }), ...annotations } @@ -9939,7 +9945,9 @@ export interface FormData extends instanceOf { * @category schemas * @since 4.0.0 */ -export const FormData: FormData = instanceOf(globalThis.FormData, { +// Read the global inside the guard: resolving globalThis.FormData at module +// scope loads undici on Node. +export const FormData: FormData = declare((u): u is globalThis.FormData => u instanceof globalThis.FormData, { representation: { id: "effect/schema/FormData", payload: null @@ -11277,7 +11285,7 @@ export function Cause(error: E, defe } return Effect.mapBothEager(SchemaParser.decodeUnknownEffect(failures)(input.reasons, options), { onSuccess: Cause_.fromReasons, - onFailure: (issue) => SchemaIssue.makeCompositeAtKey(ast, "failures", issue, input, options) + onFailure: (issue) => SchemaIssue.makeCompositeAtKey(ast, "reasons", issue, input, options) }) } }, @@ -11680,6 +11688,65 @@ export const DateTimeUtcFromString: DateTimeUtcFromString = String.annotate({ dateTimeUtcFromString ) ) +const dateTimeUtcFromHttpDate: SchemaTransformation.Transformation = SchemaTransformation + .transformEffect({ + decode: (s, options) => + Result_.match(Headers_.parseHttpDate(s), { + onFailure: () => + Effect.fail( + new SchemaIssue.InvalidValue( + { expected: "an HTTP date (RFC 9110 §5.6.7)" }, + s, + options + ) + ), + onSuccess: Effect.succeed + }), + encode: (utc) => Effect.succeed(Headers_.formatHttpDate(utc)) + }) +/** + * Type-level representation of {@link DateTimeUtcFromHttpDate}. + * + * @stability unstable + * @category models + * @since 4.0.0 + */ +export interface DateTimeUtcFromHttpDate extends decodeTo { + readonly "Rebuild": DateTimeUtcFromHttpDate +} +/** + * Schema for HTTP date header values, such as `Date`, `Last-Modified` and + * `Expires`, decoded as `DateTime.Utc`. + * + * **When to use** + * + * Use with HTTP API header schemas that carry dates. Decoding accepts the + * three formats of RFC 9110 §5.6.7 and reads them as UTC; encoding emits the + * preferred IMF-fixdate form (`Sun, 06 Nov 1994 08:49:37 GMT`). + * + * **Example** (Decoding an HTTP date) + * + * ```ts + * import { DateTime, Schema } from "effect" + * + * const date = Schema.decodeSync(Schema.DateTimeUtcFromHttpDate)("Sun, 06 Nov 1994 08:49:37 GMT") + * const millis = DateTime.toEpochMillis(date) // => 784111777000 + * ``` + * + * @see {@link DateTimeUtcFromString} for ISO 8601 strings + * + * @stability unstable + * @category schemas + * @since 4.0.0 + */ +export const DateTimeUtcFromHttpDate: DateTimeUtcFromHttpDate = String.annotate({ + expected: "an HTTP date string" +}).pipe( + decodeTo( + DateTimeUtc, + dateTimeUtcFromHttpDate + ) +) /** * Type-level representation of {@link DateTimeUtcFromMillis}. * @@ -15190,6 +15257,110 @@ export const Headers: Headers = declare( } ) +/** + * Type-level representation of {@link RetryAfter}. + * + * @stability unstable + * @category models + * @since 4.0.0 + */ +export interface RetryAfter extends + Union< + readonly [ + Struct<{ readonly _tag: tag<"Duration">; readonly duration: Duration }>, + Struct<{ readonly _tag: tag<"DateTime">; readonly dateTime: DateTimeUtc }> + ] + > +{ + readonly "Rebuild": RetryAfter +} + +/** + * Schema for parsed `Retry-After` header values. + * + * **Details** + * + * Matches `Headers.RetryAfter`: either `{ _tag: "Duration", duration }` or + * `{ _tag: "DateTime", dateTime }`. The duration must be finite and + * non-negative. Use {@link RetryAfterFromString} for the header string form. + * + * @stability unstable + * @category schemas + * @since 4.0.0 + */ +export const RetryAfter: RetryAfter = Union([ + Struct({ + _tag: tag("Duration"), + duration: Duration.check( + makeFilter((d) => Duration_.isFinite(d) && !Duration_.isNegative(d), { + expected: "a finite, non-negative duration" + }) + ) + }), + Struct({ _tag: tag("DateTime"), dateTime: DateTimeUtc }) +]) + +const retryAfterFromString: SchemaTransformation.Transformation = SchemaTransformation + .transformEffect({ + decode: (s, options) => + Result_.match(Headers_.parseRetryAfter(s), { + onFailure: () => + Effect.fail( + new SchemaIssue.InvalidValue( + { expected: "whole non-negative seconds or an HTTP date (RFC 9110 §10.2.3)" }, + s, + options + ) + ), + onSuccess: Effect.succeed + }), + encode: (self) => Effect.succeed(Headers_.formatRetryAfter(self)) + }) + +/** + * Type-level representation of {@link RetryAfterFromString}. + * + * @stability unstable + * @category models + * @since 4.0.0 + */ +export interface RetryAfterFromString extends decodeTo { + readonly "Rebuild": RetryAfterFromString +} + +/** + * Schema for `Retry-After` header values, decoded with + * `Headers.parseRetryAfter`. + * + * **When to use** + * + * Use with HTTP API header schemas that carry `Retry-After`. Decoding accepts + * whole non-negative seconds or an HTTP date and rejects anything else; + * encoding emits the delay as whole seconds, rounding a fractional second up, + * or the date as an IMF-fixdate. + * + * **Example** (Decoding a Retry-After header) + * + * ```ts + * import { Schema } from "effect" + * + * const value = Schema.decodeSync(Schema.RetryAfterFromString)("120") + * // => { _tag: "Duration", duration: Duration.seconds(120) } + * ``` + * + * @stability unstable + * @category schemas + * @since 4.0.0 + */ +export const RetryAfterFromString: RetryAfterFromString = String.annotate({ + expected: "a Retry-After header value" +}).pipe( + decodeTo( + RetryAfter, + retryAfterFromString + ) +) + /** * Type-level representation of {@link UrlParams}. * @@ -15743,6 +15914,198 @@ export function Result( return make(schema.ast, { success, failure }) } +// ----------------------------------------------------------------------------- +// SemVer schemas +// ----------------------------------------------------------------------------- + +const semVerStringTransformation = ( + parse: (input: string) => Result_.Result, + format: (value: A) => string +): SchemaTransformation.Transformation => + SchemaTransformation.transformEffect({ + decode: (input, options) => + Effect.mapErrorEager( + Effect.fromResult(parse(input)), + (error) => new SchemaIssue.InvalidValue({ message: error.message }, input, options) + ), + encode: (value) => Effect.succeed(format(value)) + }) +// Alphanumeric identifiers contain at least one non-digit. +const SemVerCoreArbitrary = Struct({ + major: Natural, + minor: Natural, + patch: Natural, + prerelease: ArraySchema(Union([Natural, String.check(isPattern(/^[0-9]*[A-Za-z-][0-9A-Za-z-]*$/))])) +}) +const semVerFromString = semVerStringTransformation(SemVer_.parse, SemVer_.format) +const SemVerString = String.annotate({ expected: "a string that will be decoded as a SemVer" }) +/** + * Type-level representation of {@link SemVer}. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export interface SemVer extends declare { + readonly "Rebuild": SemVer +} +/** + * Schema for `SemVer` values. + * + * **Details** + * + * The JSON and StringTree codecs encode a version as its canonical string, + * including build metadata. + * + * @see {@link SemVerFromString} for parsing version strings + * @stability unstable + * @category schemas + * @since 4.0.6 + */ +export const SemVer: SemVer = declare( + SemVer_.isSemVer, + { + representation: { + id: "effect/schema/SemVer", + payload: null + }, + toCode: () => ({ + runtime: `Schema.SemVer`, + Type: `SemVer.SemVer`, + importDeclarations: [`import * as SemVer from "effect/SemVer"`] + }), + expected: "SemVer", + toCodecArbitrary: () => + linkDecoding()( + Struct({ + ...SemVerCoreArbitrary.fields, + build: ArraySchema(String.check(isPattern(/^[0-9A-Za-z-]+$/))) + }), + SchemaGetter.transform(SemVer_.make) + ), + toCodecJson: () => + link()( + SemVerString, + semVerFromString + ), + toFormatter: () => SemVer_.format + } +) +/** + * Type-level representation of {@link SemVerFromString}. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export interface SemVerFromString extends decodeTo { + readonly "Rebuild": SemVerFromString +} +/** + * Schema that parses a Semantic Versioning 2.0.0 string into a `SemVer`. + * + * **Details** + * + * Decodes with `SemVer.parse`, rejecting a `v` prefix, surrounding whitespace, + * partial versions, and leading zeros. Encodes with `SemVer.format`, including + * build metadata. + * + * @see {@link SemVer} for validating values that are already SemVer values + * @stability unstable + * @category schemas + * @since 4.0.6 + */ +export const SemVerFromString: SemVerFromString = SemVerString.pipe( + decodeTo(SemVer, semVerFromString) +) +const semVerRangeFromString = semVerStringTransformation(SemVerRange_.parse, SemVerRange_.format) +const SemVerRangeString = String.annotate({ expected: "a string that will be decoded as a SemVer range" }) +/** + * Type-level representation of {@link SemVerRange}. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export interface SemVerRange extends declare { + readonly "Rebuild": SemVerRange +} +/** + * Schema for `SemVerRange` values. + * + * **Details** + * + * The JSON and StringTree codecs encode a range with `SemVerRange.format`. + * + * @see {@link SemVerRangeFromString} for parsing range strings + * @stability unstable + * @category schemas + * @since 4.0.6 + */ +export const SemVerRange: SemVerRange = declare( + SemVerRange_.isSemVerRange, + { + representation: { + id: "effect/schema/SemVerRange", + payload: null + }, + toCode: () => ({ + runtime: `Schema.SemVerRange`, + Type: `SemVerRange.SemVerRange`, + importDeclarations: [`import * as SemVerRange from "effect/SemVerRange"`] + }), + expected: "SemVerRange", + toCodecArbitrary: () => + linkDecoding()( + ArraySchema( + ArraySchema(Struct({ + operator: Literals(["", "=", "<", "<=", ">", ">=", "~", "^"]), + version: SemVerCoreArbitrary + })).check(isMinLength(1)) + ), + SchemaGetter.transform((sets) => + SemVerRange_.parseUnsafe( + sets.map((set) => + set.map(({ operator, version }) => `${operator}${SemVer_.format(SemVer_.make(version))}`).join(" ") + ).join(" || ") + ) + ) + ), + toCodecJson: () => + link()( + SemVerRangeString, + semVerRangeFromString + ), + toFormatter: () => SemVerRange_.format + } +) +/** + * Type-level representation of {@link SemVerRangeFromString}. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export interface SemVerRangeFromString extends decodeTo { + readonly "Rebuild": SemVerRangeFromString +} +/** + * Schema that parses an npm-style version range string into a `SemVerRange`. + * + * **Details** + * + * Decodes with `SemVerRange.parse` and encodes with `SemVerRange.format`, + * preserving parsed operators and normalizing wildcard spellings. + * + * @see {@link SemVerRange} for validating values that are already ranges + * @stability unstable + * @category schemas + * @since 4.0.6 + */ +export const SemVerRangeFromString: SemVerRangeFromString = SemVerRangeString.pipe( + decodeTo(SemVerRange, semVerRangeFromString) +) + /** * Type-level representation returned by {@link Class}. * @@ -15958,16 +16321,19 @@ function makeClass< return out } +/** + * Constructs a class instance from fields that were already validated against + * the class struct, without running the constructor's schema pass. + * + * @internal + */ +export function makeClassTrusted(self: new(input: any, options?: MakeOptions) => Self, fields: object): Self { + return new self(fields, { "~payload": { token: payloadToken, value: fields } }) +} + function getClassTransformation(self: new(...args: ReadonlyArray) => any) { return new SchemaTransformation.Transformation( - SchemaGetter.transform((input) => - new self(input, { - "~payload": { - token: payloadToken, - value: input - } - }) - ), + SchemaGetter.transform((input) => makeClassTrusted(self, input)), SchemaGetter.passthrough() ) } @@ -16051,7 +16417,9 @@ function isStruct(schema: Struct.Fields | Struct): schema is Stru * * **Gotchas** * - * Passing `disableChecks` in the options skips constructor validation. + * Passing `disableChecks: true` skips filters and checks, not parsing of types + * and structure. Wrong types and shapes are still rejected, and constructor + * defaults still apply. * * **Example** (Defining a basic class) * @@ -16769,6 +17137,108 @@ export interface toCodecArrayFromSingle extends export const toCodecArrayFromSingle: { (schema: S): toCodecArrayFromSingle } = InternalToCodec.toCodecArrayFromSingle +/** + * Type-level representation returned by {@link toCodecGoJson}. + * + * @stability unstable + * @category converting + * @since 4.0.6 + */ +export interface toCodecGoJson extends + BottomLazy< + S["ast"], + toCodecGoJson, + S["~type.parameters"], + S["~type.mutability"], + S["~type.optionality"], + S["~type.constructor.default"], + S["~encoded.mutability"], + S["~encoded.optionality"] + > +{ + readonly "Type": S["Type"] + readonly "Encoded": Json + readonly "DecodingServices": S["DecodingServices"] + readonly "EncodingServices": S["EncodingServices"] + readonly "~type.make.in": S["~type.make.in"] + readonly "~type.make": S["~type.make"] + readonly "Iso": S["Iso"] + readonly schema: S +} +/** + * Derives a JSON codec, like {@link toCodecJson}, that also accepts the + * `null` values Go's `encoding/json` writes for nil pointers, slices, and maps. + * + * **When to use** + * + * Use when decoding JSON produced by a Go service (for example the Docker + * Engine or Kubernetes APIs), where any field the API document describes as + * optional, or any slice or map, may arrive as `null`. + * + * **Details** + * + * Decoding applies these rules at every level of the schema: + * + * - An optional key that is `null` decodes as a missing key. + * - A required array (`Schema.Array`) that is `null` decodes as `[]`. + * - A required record (`Schema.Record`) that is `null` decodes as `{}`. + * + * Array and record rules also apply at the top level and to nested collection + * values. Tuples, non-empty arrays, and structs are not treated as collections; + * collection checks still apply to the empty value. + * + * Encoding is unchanged from {@link toCodecJson}: the added rules only decode + * `null` and do not emit it. A missing key stays missing. + * + * These rules also accept `encoding/json/v2` output. By default, v2 writes + * nil slices and maps as `[]` and `{}`, but it can be configured to emit + * `null` instead. + * + * **Gotchas** + * + * - Normalization deliberately discards the distinction between nil and empty + * Go slices or maps. They are distinguishable in Go, and unmarshalling a + * missing key, `null`, or an empty collection can affect existing values + * differently. Use {@link toCodecJson} with explicit nullable schemas when + * that distinction matters. + * - A key whose schema already accepts `null` (for example + * `Schema.NullOr(X)`, `Schema.Unknown`, or `Schema.Json`) keeps `null` as a + * value. + * - A required field that is not an array or a record (a nil pointer to a Go + * struct, for example) still fails on `null`. Make it optional if the API + * can omit it. + * - A nil `[]byte` is written as `null` by `encoding/json` (and as `""` by + * version 2), but its schema is a base64 string, not an array. A required + * `[]byte` field that is `null` fails; make it optional if the API can send + * `null`. + * - Only these `null` rules are covered. Field formats such as durations, + * base64 bytes, and numbers encoded as strings belong in the field schemas. + * + * **Example** (Decoding a Go response with nil fields) + * + * ```ts import.meta.vitest + * import { Schema } from "effect" + * + * const Container = Schema.Struct({ + * Id: Schema.String, + * Names: Schema.Array(Schema.String), + * Labels: Schema.Record(Schema.String, Schema.String), + * Status: Schema.optionalKey(Schema.String) + * }) + * + * const decode = Schema.decodeUnknownSync(Schema.toCodecGoJson(Container)) + * + * decode({ Id: "a", Names: null, Labels: null, Status: null }) // => { Id: "a", Names: [], Labels: {} } + * ``` + * + * @see {@link toCodecJson} for the canonical JSON codec + * @stability unstable + * @category converting + * @since 4.0.6 + */ +export const toCodecGoJson: { + (schema: S): toCodecGoJson +} = InternalToCodec.toCodecGoJson type XmlEncoderOptions = { /** Root element name for the returned XML string. Default: "root" */ readonly rootName?: string | undefined @@ -17386,7 +17856,7 @@ export declare namespace Annotations { readonly "~sentinels"?: ReadonlyArray | undefined } /** - * Annotations for filter schema nodes (created via `Schema.filter`). Extends + * Annotations for filter checks (created via `Schema.makeFilter`). Extends * {@link Augment} with an optional error message, identifier, and metadata. * Filters are intentionally non-parametric to keep them covariant. * diff --git a/repos/effect/packages/effect/src/SchemaAST.ts b/repos/effect/packages/effect/src/SchemaAST.ts index 4bf42ef434..943329d788 100644 --- a/repos/effect/packages/effect/src/SchemaAST.ts +++ b/repos/effect/packages/effect/src/SchemaAST.ts @@ -480,7 +480,7 @@ export type Encoding = readonly [Link, ...Array] * * **Details** * - * Pass to `Schema.decodeUnknown`, `Schema.encode`, and related APIs to customize + * Pass to `Schema.decodeUnknownEffect`, `Schema.encodeEffect`, and related APIs to customize * error reporting, excess property handling, check execution, and product * concurrency. Options apply throughout the parse; schema annotations do not * override them. @@ -814,7 +814,13 @@ export const Declaration: new( let run: ReturnType return (input, options) => { if (input === InternalParser.missing) return InternalParser.missingExit - return (run ??= this.run(this.typeParameters))(input, this, options) + // Keep this declaration's initialization errors outside its callback catch. + run ??= this.run(this.typeParameters) + try { + return run(input, this, options) + } catch (error) { + return Effect.die(error) + } } } private _rebuild( @@ -2435,7 +2441,7 @@ export const Arrays: new( input, len, tailThreshold: Math.max(elementLen, len - tailLen), - output: new globalThis.Array(len), + output: [], issues: undefined, options } @@ -2599,9 +2605,12 @@ const wrapPropertyKeyIssue = ( /** * floating point or integer, with optional exponent + * + * The mantissa alternation keeps rejection linear: `\d*\.?\d+` retries + * every split of a long digit run. * @internal */ -export const FINITE_PATTERN = "[+-]?\\d*\\.?\\d+(?:[Ee][+-]?\\d+)?" +export const FINITE_PATTERN = "[+-]?(?:\\d+(?:\\.\\d+)?|\\.\\d+)(?:[Ee][+-]?\\d+)?" /** * Returns the object keys that match the index signature parameter schema. @@ -3101,12 +3110,14 @@ export const Objects: new( // Resumes at the property whose parser suspended, without replaying the // properties already parsed. const resume = ( - state: ObjectParserState, + snapshot: ObjectParserState, index: number, pending: Effect.Effect ): Effect.Effect => { const property = properties![index] return Effect.flatMap(Effect.exit(pending), (exit) => { + // Copy the parsed prefix per execution so reruns cannot overwrite earlier results. + const state: ObjectParserState = { ...snapshot, out: { ...snapshot.out } } const terminal = stepProperty(state, property, exit) if (terminal) return terminal const done = () => InternalParser.succeed(state.out) @@ -4278,7 +4289,7 @@ const numberToJson = new Link( * * **Details** * - * The filter can be used with `Schema.filter` or attached directly to a + * The filter can be used with `Schema.check` or attached directly to a * `String` AST node through checks. The regular expression is cloned and its * `lastIndex` is reset before each test, so global and sticky expressions are * deterministic and the provided regular expression is not mutated. The @@ -4777,7 +4788,7 @@ function flipEncoding(ast: AST, encoding: Encoding): AST { * **Details** * * After flipping, what was decoding becomes encoding and vice versa. This is - * the core operation behind `Schema.encode` — encoding a value is decoding + * the core operation behind `Schema.encodeEffect` — encoding a value is decoding * with a flipped SchemaAST. * * - Memoized: same input reference → same output reference. @@ -5110,7 +5121,7 @@ export function getConstructorDescriptor(ast: AST): ConstructorDescriptor | unde * **Details** * * If the node has {@link Checks}, returns annotations from the last check - * (which is where user-supplied annotations end up after `.pipe(Schema.annotations(...))`). + * (which is where user-supplied annotations end up after `.pipe(Schema.annotate(...))`). * Otherwise returns the node's `annotations` directly. * * **Example** (Reading annotations) @@ -5153,7 +5164,7 @@ export const resolveAt: (key: string) => (ast: AST) => A | undefined = Intern * * **Details** * - * The identifier is typically set by `Schema.annotations({ identifier: "..." })` + * The identifier is typically set by `Schema.annotate({ identifier: "..." })` * and is used for error messages and schema identification. * * @see {@link resolve} diff --git a/repos/effect/packages/effect/src/SchemaGetter.ts b/repos/effect/packages/effect/src/SchemaGetter.ts index f99a8bd289..609b5eb2d0 100644 --- a/repos/effect/packages/effect/src/SchemaGetter.ts +++ b/repos/effect/packages/effect/src/SchemaGetter.ts @@ -992,7 +992,8 @@ export function Boolean(): Getter { * **Details** * * - Delegates to `globalThis.BigInt`. - * - Throws at runtime if the input cannot be converted (e.g. non-numeric string). + * - Fails with `SchemaIssue.InvalidValue` if the input cannot be converted + * (e.g. a non-numeric string or a non-integer number). * * **Example** (Coercing to a bigint) * @@ -1008,7 +1009,19 @@ export function Boolean(): Getter { * @since 4.0.0 */ export function BigInt(): Getter { - return transform(globalThis.BigInt) + return transformEffect((input, options) => { + try { + return Effect.succeed(globalThis.BigInt(input)) + } catch { + return Effect.fail( + new SchemaIssue.InvalidValue( + { expected: "a value that can be converted to a bigint" }, + input, + options + ) + ) + } + }) } /** @@ -1510,7 +1523,8 @@ export function encodeBase64(): Getter * * **Details** * - * The getter is pure and never fails. + * The getter is pure and never fails. The output is unpadded unless + * `padding` is `true`. * * **Example** (Encoding to Base64Url) * @@ -1530,8 +1544,11 @@ export function encodeBase64(): Getter * @category encoding * @since 4.0.0 */ -export function encodeBase64Url(): Getter { - return transform(Base64Url.encode) +export function encodeBase64Url(options?: { + /** Keep the `=` padding. Defaults to `false`. */ + readonly padding?: boolean | undefined +}): Getter { + return transform(options?.padding ? Base64Url.encodePadded : Base64Url.encode) } /** diff --git a/repos/effect/packages/effect/src/SchemaIssue.ts b/repos/effect/packages/effect/src/SchemaIssue.ts index 6e32f09112..6f60b279a5 100644 --- a/repos/effect/packages/effect/src/SchemaIssue.ts +++ b/repos/effect/packages/effect/src/SchemaIssue.ts @@ -720,8 +720,8 @@ export const InvalidType: new( * * **When to use** * - * Use when you need to detect constraint violations from `Schema.filter`, - * `Schema.minLength`, `Schema.greaterThan`, or similar checks. + * Use when you need to detect constraint violations from `Schema.check` with + * `Schema.isMinLength`, `Schema.isGreaterThan`, or similar checks. * * **Details** * @@ -1242,8 +1242,8 @@ export type CheckHook = (issue: Filter) => string | undefined * * **Details** * - * - Looks for a `message` annotation on the inner issue first, then on the - * filter itself. + * - Looks for a `message` annotation on the filter first, then on the inner + * issue. * - Returns `undefined` when no annotation is found, causing the formatter to * fall back to `"Expected "` or, when the filter reports input, * `"Expected , got "`. @@ -1255,7 +1255,7 @@ export type CheckHook = (issue: Filter) => string | undefined * @category formatting * @since 4.0.0 */ -export const defaultCheckHook: CheckHook = (issue): string | undefined => findMessage(issue.issue) ?? findMessage(issue) +export const defaultCheckHook: CheckHook = (issue): string | undefined => findMessage(issue) ?? findMessage(issue.issue) /** * Creates a {@link Formatter} that produces a `StandardSchemaV1.FailureResult`. @@ -1329,46 +1329,48 @@ function getExpectedMessage(expected: string, issue: Issue): string { return input === undefined ? `Expected ${expected}` : `Expected ${expected}, got ${input}` } +function getFilterMessage(issue: Filter, checkHook: CheckHook): string | undefined { + const message = checkHook(issue) + if (message !== undefined || issue.issue._tag !== "InvalidValue") return message + const expected = findExpected(issue.issue) + return expected === undefined + ? getExpectedMessage(formatCheck(issue.filter), issue) + : getExpectedMessage(expected, issue.issue) +} + function toDefaultIssues( issue: Issue, path: ReadonlyArray, leafHook: LeafHook, - checkHook: CheckHook + checkHook: CheckHook, + messageOverride?: string ): Array { switch (issue._tag) { - case "Filter": { - const message = checkHook(issue) - if (message !== undefined) { - return [{ path, message }] - } - if (issue.issue._tag !== "InvalidValue") { - return toDefaultIssues(issue.issue, path, leafHook, checkHook) - } - const expected = findExpected(issue.issue) - return [{ + case "Filter": + return toDefaultIssues( + issue.issue, path, - message: expected === undefined - ? getExpectedMessage(formatCheck(issue.filter), issue) - : getExpectedMessage(expected, issue.issue) - }] - } + leafHook, + checkHook, + messageOverride ?? getFilterMessage(issue, checkHook) + ) case "Encoding": - return toDefaultIssues(issue.issue, path, leafHook, checkHook) + return toDefaultIssues(issue.issue, path, leafHook, checkHook, messageOverride) case "Pointer": - return toDefaultIssues(issue.issue, [...path, ...issue.path], leafHook, checkHook) + return toDefaultIssues(issue.issue, [...path, ...issue.path], leafHook, checkHook, messageOverride) case "Composite": - return issue.issues.flatMap((issue) => toDefaultIssues(issue, path, leafHook, checkHook)) case "AnyOf": { - if (issue.issues.length === 0) { - return [{ - path, - message: findMessage(issue) ?? getExpectedMessage(InternalAnnotations.getExpected(issue.ast), issue) - }] + if (issue._tag === "Composite" || issue.issues.length > 0) { + return issue.issues.flatMap((issue) => toDefaultIssues(issue, path, leafHook, checkHook, messageOverride)) } - return issue.issues.flatMap((issue) => toDefaultIssues(issue, path, leafHook, checkHook)) + return [{ + path, + message: messageOverride ?? findMessage(issue) ?? + getExpectedMessage(InternalAnnotations.getExpected(issue.ast), issue) + }] } default: - return [{ path, message: leafHook(issue) }] + return [{ path, message: messageOverride ?? leafHook(issue) }] } } @@ -1431,45 +1433,34 @@ export function makeFormatterDefault(): Formatter { /** @internal */ export const defaultFormatter = makeFormatterDefault() -function formatIssue(issue: Issue, path: string): string { +function formatIssue(issue: Issue, path: string, messageOverride?: string): string { let message: string switch (issue._tag) { - case "Filter": { - const annotated = defaultCheckHook(issue) - if (annotated !== undefined) { - message = annotated - } else { - if (issue.issue._tag !== "InvalidValue") { - return formatIssue(issue.issue, path) - } - const expected = findExpected(issue.issue) - message = expected === undefined - ? getExpectedMessage(formatCheck(issue.filter), issue) - : getExpectedMessage(expected, issue.issue) - } - break - } + case "Filter": + return formatIssue(issue.issue, path, messageOverride ?? getFilterMessage(issue, defaultCheckHook)) case "Encoding": - return formatIssue(issue.issue, path) + return formatIssue(issue.issue, path, messageOverride) case "Pointer": - return formatIssue(issue.issue, path + formatPath(issue.path)) + return formatIssue(issue.issue, path + formatPath(issue.path), messageOverride) case "Composite": case "AnyOf": { if (issue._tag === "Composite" || issue.issues.length > 0) { - return issue.issues.map((issue) => formatIssue(issue, path)).join("\n") + return issue.issues.map((issue) => formatIssue(issue, path, messageOverride)).join("\n") } - message = findMessage(issue) ?? getExpectedMessage(InternalAnnotations.getExpected(issue.ast), issue) + message = messageOverride ?? findMessage(issue) ?? + getExpectedMessage(InternalAnnotations.getExpected(issue.ast), issue) break } default: - message = defaultLeafHook(issue) + message = messageOverride ?? defaultLeafHook(issue) break } return path ? `${message}\n at ${path}` : message } function findMessage(issue: Issue): string | undefined { - if (issue._tag === "Pointer") return + // A Composite's ast describes the whole schema, not the failures it groups. + if (issue._tag === "Pointer" || issue._tag === "Composite") return if (issue._tag === "Encoding") return findMessage(issue.issue) const annotations = issue._tag === "Filter" ? issue.filter.annotations diff --git a/repos/effect/packages/effect/src/SchemaRepresentation.ts b/repos/effect/packages/effect/src/SchemaRepresentation.ts index d6153da6ed..c591caf1e6 100644 --- a/repos/effect/packages/effect/src/SchemaRepresentation.ts +++ b/repos/effect/packages/effect/src/SchemaRepresentation.ts @@ -1944,6 +1944,32 @@ export const BigDecimalReviver: DeclarationReviver = makeFixedDeclarationR Schema.BigDecimal ) +/** + * Reviver for persisted {@link Schema.SemVer} declarations. + * + * @see {@link Schema.SemVer} for the corresponding schema + * @stability unstable + * @category schemas + * @since 4.0.6 + */ +export const SemVerReviver: DeclarationReviver = makeFixedDeclarationReviver( + "effect/schema/SemVer", + Schema.SemVer +) + +/** + * Reviver for persisted {@link Schema.SemVerRange} declarations. + * + * @see {@link Schema.SemVerRange} for the corresponding schema + * @stability unstable + * @category schemas + * @since 4.0.6 + */ +export const SemVerRangeReviver: DeclarationReviver = makeFixedDeclarationReviver( + "effect/schema/SemVerRange", + Schema.SemVerRange +) + /** * Reviver for persisted `File` declarations. * @@ -2387,7 +2413,8 @@ const jsonSchemaRevivers: ReadonlyArray = [ * * **When to use** * - * Use when you need to configure pattern handling or transform each JSON Schema node before translation. + * Use when you need to configure pattern handling, preserve custom annotations, or transform each JSON Schema node + * before translation. * * **Details** * @@ -2422,6 +2449,42 @@ export interface FromJsonSchemaOptions { * @default "error" */ readonly patterns?: "error" | "ignore" | "apply" | undefined + /** + * Selects additional JSON Schema keys to preserve as Effect schema annotations. + * + * **Details** + * + * Standard annotations such as `title` and `description` are imported independently of this predicate. + * Other keys are ignored by default. Selected values must be JSON-compatible and are kept as opaque metadata. + * Selection runs after `onEnter`. JSON Schema structural and validation keywords, Effect's reserved annotation + * keys, and keys starting with `~` are excluded before calling the predicate. Exceptions from the predicate + * pass through unchanged. + * + * **Gotchas** + * + * Import normalizes the schema. Selected annotations stay on surviving values when checks simplify, and + * annotations from eliminated alternatives or unreachable children are discarded. When schemas merge, + * later annotations overwrite earlier values for the same key, following the importer's merge order. + * Retaining an annotation does not enforce its semantics or preserve the original JSON Schema structure. + * + * **Example** (Preserving custom metadata on import and export) + * + * ```ts import.meta.vitest + * import { JsonSchema, Schema, SchemaRepresentation } from "effect" + * + * const document = JsonSchema.fromSchemaDraft2020_12({ + * type: "string", + * "x-sensitive": true + * }) + * const options = { includeAnnotationKey: (key: string) => key.startsWith("x-") } + * const schema = SchemaRepresentation.fromJsonSchemaDocument(document, options) + * + * Schema.toJsonSchemaDocument(schema, options).schema // => { type: "string", "x-sensitive": true } + * ``` + * + * @see Schema.ToJsonSchemaOptions + */ + readonly includeAnnotationKey?: ((key: string) => boolean) | undefined } /** diff --git a/repos/effect/packages/effect/src/SchemaTransformation.ts b/repos/effect/packages/effect/src/SchemaTransformation.ts index 7824aa0e3e..6a2dfdf02f 100644 --- a/repos/effect/packages/effect/src/SchemaTransformation.ts +++ b/repos/effect/packages/effect/src/SchemaTransformation.ts @@ -1818,7 +1818,7 @@ export const stringFromHexString: Transformation = new Transform * **When to use** * * Use when you need a schema transformation to store structured data in URL - * query parameters or fragments, such as composing with `Schema.parseJson` to + * query parameters or fragments, such as composing with `Schema.fromJsonString` to * round-trip JSON through a URL. * * **Details** diff --git a/repos/effect/packages/effect/src/Scope.ts b/repos/effect/packages/effect/src/Scope.ts index 2746d1a2cf..8ff3748f8d 100644 --- a/repos/effect/packages/effect/src/Scope.ts +++ b/repos/effect/packages/effect/src/Scope.ts @@ -159,8 +159,13 @@ export declare namespace State { finalizers: Map<{}, (exit: Exit) => Effect> | undefined } /** - * Represents a closed scope state where finalizers have been executed - * and the scope is no longer accepting new resources. + * Represents a closed scope state where the scope is no longer accepting + * new resources. + * + * **Details** + * + * The scope enters this state when closing starts, so its finalizers may + * still be running. * * **Example** (Inspecting a closed scope state) * @@ -481,6 +486,11 @@ export const forkUnsafe: (scope: Scope, finalizerStrategy?: "sequential" | "para * By default, finalizers run uninterruptibly, so interrupting the closing fiber * waits for them to finish; a finalizer can explicitly restore interruptibility. * + * **Gotchas** + * + * Closing a scope that is already closed returns immediately, even when the + * finalizers of an earlier close are still running. + * * **Example** (Running scope finalizers) * * ```ts import.meta.vitest diff --git a/repos/effect/packages/effect/src/ScopedCache.ts b/repos/effect/packages/effect/src/ScopedCache.ts index cdeeaa17a9..b34aa0e92e 100644 --- a/repos/effect/packages/effect/src/ScopedCache.ts +++ b/repos/effect/packages/effect/src/ScopedCache.ts @@ -7,6 +7,8 @@ * remaining entries when the cache is closed. Lookups for the same missing key * share one in-progress effect, and entries can expire, be refreshed, be * invalidated, or be evicted by capacity limits. + * Expiry cleanup is lazy: an expired entry is removed on access to that key or + * a `keys`/`values`/`entries` scan, not by a background timer. * * @stability stable * @since 4.0.0 @@ -39,14 +41,15 @@ const TypeId = "~effect/ScopedCache" * **When to use** * * Use to cache values that acquire scoped resources and must release those - * resources when entries expire, are evicted, or are invalidated. + * resources when entries are removed or the owning scope closes. * * **Details** * * Concurrent requests for the same key share the same in-flight lookup. * Entries can expire based on the lookup exit, are evicted when capacity is - * exceeded, and release their entry scopes when invalidated, evicted, expired, - * or when the cache's owning scope closes. + * exceeded, and release their entry scopes when invalidated, evicted, or when + * the cache's owning scope closes. Expired entries are cleaned up lazily on + * access to that key or a `keys`/`values`/`entries` scan, never in the background. * * @see {@link make} for creating a scoped cache with a fixed time-to-live * @see {@link makeWith} for creating a scoped cache with dynamic time-to-live @@ -127,8 +130,9 @@ export interface Entry { * **Details** * * The cache must be constructed in a `Scope`. Each lookup runs in its own entry - * scope, and that scope is closed when the entry expires, is invalidated, is - * evicted by capacity, or when the cache's owning scope closes. + * scope, and that scope is closed when the entry is invalidated, is evicted by + * capacity, or when the cache's owning scope closes. Expired entries are cleaned + * up on access to that key or a `keys`/`values`/`entries` scan, not in the background. * `requireServicesAt` controls whether lookup services are captured at * construction time or required when lookup operations run. * @@ -192,8 +196,9 @@ export const makeWith = < * * This is the constant-TTL variant of `makeWith`: values are acquired by the * lookup effect in per-entry scopes, capacity can evict older entries, and - * entry scopes are closed when entries expire, are invalidated, are evicted, or - * when the cache's owning scope closes. + * entry scopes are closed when entries are invalidated, are evicted, or when + * the cache's owning scope closes. Expiry cleanup is lazy: expired entries close + * on access to that key or a `keys`/`values`/`entries` scan, not in the background. * * @see {@link makeWith} for computing time-to-live from each lookup result and key * @@ -770,11 +775,11 @@ const invalidateAllImpl = ( // Detach the batch before finalizers can reenter the cache. const entries = Array.from(MutableHashMap.values(map)) MutableHashMap.clear(map) - const fibers = Arr.empty>() + const fibers = Arr.empty>() for (const entry of entries) { fibers.push(effect.forkUnsafe(parent as any, Scope.close(entry.scope, effect.exitVoid), true, true)) } - return effect.fiberAwaitAll(fibers) + return effect.flatMap(effect.fiberAwaitAll(fibers), Exit.asVoidAll) } /** diff --git a/repos/effect/packages/effect/src/SemVer.ts b/repos/effect/packages/effect/src/SemVer.ts new file mode 100644 index 0000000000..89798fd396 --- /dev/null +++ b/repos/effect/packages/effect/src/SemVer.ts @@ -0,0 +1,1042 @@ +/** + * Immutable [Semantic Versioning 2.0.0](https://semver.org/spec/v2.0.0.html) + * values with strict parsing, canonical formatting, spec precedence ordering, + * and version bumps. + * + * A `SemVer` has numeric major, minor, and patch components, optional + * pre-release identifiers, and optional build metadata. Structural equality + * and hashing include build metadata, while `Order` follows spec precedence + * and ignores it. Version ranges live in `SemVerRange`. + * + * @stability unstable + * @since 4.0.6 + */ +import * as Data from "./Data.ts" +import * as Equal from "./Equal.ts" +import * as Equ from "./Equivalence.ts" +import { dual } from "./Function.ts" +import * as Hash from "./Hash.ts" +import { type Inspectable, NodeInspectSymbol } from "./Inspectable.ts" +import * as Option from "./Option.ts" +import * as order from "./Order.ts" +import { type Pipeable, pipeArguments } from "./Pipeable.ts" +import { hasProperty } from "./Predicate.ts" +import * as Result from "./Result.ts" + +const TypeId = "~effect/SemVer" +const SemVerErrorTypeId = "~effect/SemVer/SemVerError" + +/** + * Represents a semantic version. + * + * **Details** + * + * Pre-release identifiers are numbers when they consist only of digits and + * strings otherwise. Build metadata identifiers are always strings because + * they may contain leading zeros. Both arrays are empty when absent. + * + * **Example** (Reading version components) + * + * ```ts import.meta.vitest + * import { SemVer } from "effect" + * + * const version = SemVer.parseUnsafe("1.2.3-rc.1+build.5") + * + * version.minor // => 2 + * version.prerelease // => ["rc", 1] + * version.build // => ["build", "5"] + * ``` + * + * @see {@link parse} for decoding version strings + * @see {@link make} for constructing versions from components + * @stability unstable + * @category models + * @since 4.0.6 + */ +export interface SemVer extends Pipeable, Equal.Equal, Inspectable { + readonly [TypeId]: typeof TypeId + readonly major: number + readonly minor: number + readonly patch: number + readonly prerelease: ReadonlyArray + readonly build: ReadonlyArray +} + +/** + * Error raised for invalid version strings, ranges, or components. + * + * @see {@link isSemVerError} for narrowing unknown values + * @stability unstable + * @category errors + * @since 4.0.6 + */ +export class SemVerError extends Data.TaggedError("SemVerError")<{ + readonly message: string + readonly input?: string +}> { + readonly [SemVerErrorTypeId]: typeof SemVerErrorTypeId = SemVerErrorTypeId +} + +/** + * Checks whether a value is a `SemVerError`. + * + * @stability unstable + * @category guards + * @since 4.0.6 + */ +export const isSemVerError = (u: unknown): u is SemVerError => hasProperty(u, SemVerErrorTypeId) + +/** @internal */ +export const invalid = (message: string): never => { + throw new SemVerError({ message }) +} + +const attempt = (evaluate: () => A): Result.Result => + Result.try({ + try: evaluate, + catch: (error) => { + if (!isSemVerError(error)) throw error + return error + } + }) + +/** @internal */ +export const tryParse = ( + label: string, + input: string, + parse: (input: string) => A +): Result.Result => + Result.mapError( + attempt(() => parse(input)), + (error) => new SemVerError({ message: `Invalid ${label} ${JSON.stringify(input)}: ${error.message}`, input }) + ) + +const numericPattern = /^(?:0|[1-9]\d*)$/ +const digitsPattern = /^\d+$/ +const identifierPattern = /^[0-9A-Za-z-]+$/ + +const checkComponent = (value: number, label: string): number => { + if (!Number.isSafeInteger(value) || value < 0) { + return invalid(`${label} must be a non-negative safe integer, received ${value}`) + } + return value +} + +/** @internal */ +export const parseComponent = (value: string, label: string): number => { + if (!numericPattern.test(value)) { + return invalid(`${label} must be a non-negative integer without leading zeros, received ${JSON.stringify(value)}`) + } + return checkComponent(Number(value), label) +} + +/** @internal */ +export const prereleaseIdentifier = (identifier: string | number): string | number => { + if (typeof identifier === "number") return checkComponent(identifier, "numeric pre-release identifier") + if (!identifierPattern.test(identifier)) { + return invalid(`pre-release identifier must be non-empty [0-9A-Za-z-], received ${JSON.stringify(identifier)}`) + } + return digitsPattern.test(identifier) ? parseComponent(identifier, "numeric pre-release identifier") : identifier +} + +const buildIdentifier = (identifier: string): string => { + if (!identifierPattern.test(identifier)) { + return invalid(`build identifier must be non-empty [0-9A-Za-z-], received ${JSON.stringify(identifier)}`) + } + return identifier +} + +/** @internal */ +export type Core = Pick + +const comparePrerelease = ( + self: ReadonlyArray, + that: ReadonlyArray +): -1 | 0 | 1 => { + if (self.length === 0) return that.length === 0 ? 0 : 1 + if (that.length === 0) return -1 + const length = Math.min(self.length, that.length) + for (let i = 0; i < length; i++) { + const a = self[i] + const b = that[i] + if (a === b) continue + if (typeof a === "number") return typeof b === "number" ? order.Number(a, b) : -1 + if (typeof b === "number") return 1 + return order.String(a, b) + } + return order.Number(self.length, that.length) +} + +/** @internal */ +export const sameCore = (self: Core, that: Core): boolean => + self.major === that.major && self.minor === that.minor && self.patch === that.patch + +/** @internal */ +export const compareCore = (self: Core, that: Core): -1 | 0 | 1 => + order.Number(self.major, that.major) || + order.Number(self.minor, that.minor) || + order.Number(self.patch, that.patch) || + comparePrerelease(self.prerelease, that.prerelease) + +const SemVerProto: Omit = { + [TypeId]: TypeId, + [Equal.symbol](this: SemVer, that: unknown): boolean { + return isSemVer(that) && Equivalence(this, that) + }, + [Hash.symbol](this: SemVer): number { + return Hash.string(format(this)) + }, + toString(this: SemVer) { + return `SemVer(${format(this)})` + }, + toJSON(this: SemVer) { + return { _id: "SemVer", value: format(this) } + }, + [NodeInspectSymbol](this: SemVer) { + return this.toJSON() + }, + pipe() { + return pipeArguments(this, arguments) + } +} + +const unchecked = ( + major: number, + minor: number, + patch: number, + prerelease: ReadonlyArray, + build: ReadonlyArray +): SemVer => { + const self = Object.create(SemVerProto) + self.major = major + self.minor = minor + self.patch = patch + self.prerelease = prerelease + self.build = build + return self +} + +const increment = (value: number, label: string): number => { + if (value >= Number.MAX_SAFE_INTEGER) return invalid(`${label} cannot be incremented past Number.MAX_SAFE_INTEGER`) + return value + 1 +} + +/** + * Checks whether a value is a `SemVer`. + * + * @stability unstable + * @category guards + * @since 4.0.6 + */ +export const isSemVer = (u: unknown): u is SemVer => hasProperty(u, TypeId) + +/** + * The numeric components and identifiers of a version. + * + * @see {@link make} for validating components into a version + * @stability unstable + * @category models + * @since 4.0.6 + */ +export interface Components { + readonly major: number + readonly minor: number + readonly patch: number + readonly prerelease?: ReadonlyArray | undefined + readonly build?: ReadonlyArray | undefined +} + +// The intersection excludes signs, leading zeros, and exponent notation. +type NumericComponent = "0" | (`${bigint}` & `${"1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9"}${string}`) + +/** + * Version string literals accepted by `fromInput`. + * + * **Details** + * + * Requires three numeric components without signs or leading zeros, followed + * by an optional `-` pre-release and `+` build suffix. Component magnitudes + * (at most `Number.MAX_SAFE_INTEGER`) and identifiers are only validated at + * runtime. + * + * @see {@link parse} for arbitrary strings + * @stability unstable + * @category models + * @since 4.0.6 + */ +export type VersionString = `${NumericComponent}.${NumericComponent}.${NumericComponent}${ + | "" + | `-${string}` + | `+${string}`}` + +/** + * Values accepted by `fromInput`: a version, a version string literal, or + * version components. + * + * @see {@link fromInput} for converting inputs + * @stability unstable + * @category models + * @since 4.0.6 + */ +export type Input = SemVer | VersionString | Components + +/** + * Creates a version from its components and throws `SemVerError` for invalid + * components. + * + * **Details** + * + * Components must be non-negative safe integers. Identifiers must be non-empty + * and contain only ASCII alphanumerics and hyphens. Digit-only pre-release + * strings are normalized to numbers and must not have leading zeros. + * + * A `SemVer` spreads into exactly these components, so `make({ ...version, + * patch: version.patch + 1 })` derives a validated version from an existing + * one. + * + * **Example** (Creating a pre-release version) + * + * ```ts import.meta.vitest + * import { SemVer } from "effect" + * + * SemVer.format(SemVer.make({ major: 1, minor: 0, patch: 0, prerelease: ["beta", "2"] })) // => "1.0.0-beta.2" + * ``` + * + * @see {@link parse} for decoding version strings + * @stability unstable + * @category constructors + * @since 4.0.6 + */ +export const make = (components: Components): SemVer => + unchecked( + checkComponent(components.major, "major"), + checkComponent(components.minor, "minor"), + checkComponent(components.patch, "patch"), + components.prerelease?.map(prereleaseIdentifier) ?? [], + components.build?.map(buildIdentifier) ?? [] + ) + +const parseOrThrow = (input: string): SemVer => { + let rest = input + let build: ReadonlyArray = [] + const plus = rest.indexOf("+") + if (plus !== -1) { + build = rest.slice(plus + 1).split(".").map(buildIdentifier) + rest = rest.slice(0, plus) + } + let prerelease: ReadonlyArray = [] + const dash = rest.indexOf("-") + if (dash !== -1) { + prerelease = rest.slice(dash + 1).split(".").map(prereleaseIdentifier) + rest = rest.slice(0, dash) + } + const core = rest.split(".") + if (core.length !== 3) return invalid("expected MAJOR.MINOR.PATCH") + return unchecked( + parseComponent(core[0], "major"), + parseComponent(core[1], "minor"), + parseComponent(core[2], "patch"), + prerelease, + build + ) +} + +/** + * Parses a strict Semantic Versioning 2.0.0 string. + * + * **Details** + * + * The input must match the spec grammar exactly. Leading zeros in numeric + * components or pre-release identifiers, empty identifiers, and numbers above + * `Number.MAX_SAFE_INTEGER` are rejected. + * + * **Gotchas** + * + * A `v` prefix, surrounding whitespace, and partial versions such as `1.2` are + * rejected because they are not semantic versions. + * + * **Example** (Parsing version strings) + * + * ```ts import.meta.vitest + * import { Result, SemVer } from "effect" + * + * Result.isSuccess(SemVer.parse("1.2.3-alpha.1+001")) // => true + * Result.isFailure(SemVer.parse("v1.2.3")) // => true + * ``` + * + * @see {@link parseUnsafe} for throwing on invalid input + * @see {@link format} for the inverse operation + * @stability unstable + * @category constructors + * @since 4.0.6 + */ +export const parse = (input: string): Result.Result => tryParse("SemVer", input, parseOrThrow) + +/** + * Parses a strict Semantic Versioning 2.0.0 string and throws `SemVerError` + * for invalid input. + * + * @see {@link parse} for non-throwing parsing + * @stability unstable + * @category constructors + * @since 4.0.6 + */ +export const parseUnsafe = (input: string): SemVer => Result.getOrThrow(parse(input)) + +/** + * Converts a version, version string, or version components into a `SemVer`. + * + * **Details** + * + * Versions are returned unchanged, strings are parsed like {@link parse}, and + * components are validated like {@link make}. Invalid input fails with + * `SemVerError`. + * + * **Example** (Accepting flexible version input) + * + * ```ts import.meta.vitest + * import { Result, SemVer } from "effect" + * + * Result.map(SemVer.fromInput("1.2.3-rc.1"), SemVer.format) // => Result.succeed("1.2.3-rc.1") + * Result.map(SemVer.fromInput({ major: 1, minor: 2, patch: 3 }), SemVer.format) // => Result.succeed("1.2.3") + * Result.isFailure(SemVer.fromInput({ major: -1, minor: 0, patch: 0 })) // => true + * ``` + * + * @see {@link fromInputUnsafe} for throwing on invalid input + * @stability unstable + * @category constructors + * @since 4.0.6 + */ +export const fromInput = (input: Input): Result.Result => { + if (isSemVer(input)) return Result.succeed(input) + if (typeof input === "string") return parse(input) + return attempt(() => make(input)) +} + +/** + * Converts a version, version string, or version components into a `SemVer` + * and throws `SemVerError` for invalid input. + * + * @see {@link fromInput} for non-throwing conversion + * @stability unstable + * @category constructors + * @since 4.0.6 + */ +export const fromInputUnsafe = (input: Input): SemVer => Result.getOrThrow(fromInput(input)) + +/** + * Formats a version as its canonical string, including pre-release identifiers + * and build metadata. + * + * @see {@link parse} for the inverse operation + * @stability unstable + * @category converting + * @since 4.0.6 + */ +export const format = (self: SemVer): string => { + let out = `${self.major}.${self.minor}.${self.patch}` + if (self.prerelease.length > 0) out += `-${self.prerelease.join(".")}` + if (self.build.length > 0) out += `+${self.build.join(".")}` + return out +} + +/** + * Checks whether a version has pre-release identifiers. + * + * @stability unstable + * @category predicates + * @since 4.0.6 + */ +export const isPrerelease = (self: SemVer): boolean => self.prerelease.length > 0 + +/** + * Checks whether a version has build metadata. + * + * **Details** + * + * Build metadata does not affect precedence, but it is part of structural + * equality. + * + * @stability unstable + * @category predicates + * @since 4.0.6 + */ +export const hasBuild = (self: SemVer): boolean => self.build.length > 0 + +/** + * Checks whether a version is in initial development, meaning its major + * version is `0`. + * + * **Details** + * + * The SemVer spec treats `0.y.z` as initial development where anything may + * change at any time and the public API should not be considered stable. + * + * @stability unstable + * @category predicates + * @since 4.0.6 + */ +export const isInitialDevelopment = (self: SemVer): boolean => self.major === 0 + +/** + * Orders versions by Semantic Versioning precedence. + * + * **Details** + * + * Major, minor, and patch are compared numerically. A pre-release has lower + * precedence than its release. Pre-release identifiers are compared left to + * right: numeric identifiers compare numerically and sort before alphanumeric + * identifiers, which compare in ASCII order. A longer list of otherwise equal + * identifiers has higher precedence. + * + * **Gotchas** + * + * Build metadata is ignored, so versions that differ only in build metadata + * compare as `0` even though they are not `Equal`. + * + * **Example** (Sorting by precedence) + * + * ```ts import.meta.vitest + * import { Array, SemVer } from "effect" + * + * const versions = ["1.0.0", "1.0.0-rc.1", "1.0.0-beta.11", "1.0.0-beta.2", "1.0.0-alpha"].map(SemVer.parseUnsafe) + * + * Array.sort(versions, SemVer.Order).map(SemVer.format) // => ["1.0.0-alpha", "1.0.0-beta.2", "1.0.0-beta.11", "1.0.0-rc.1", "1.0.0"] + * ``` + * + * @see {@link Equivalence} for structural equality including build metadata + * @stability unstable + * @category instances + * @since 4.0.6 + */ +export const Order: order.Order = order.make(compareCore) + +const identifiersEqual = Equ.Array(Equ.strictEqual()) + +/** + * Provides structural equivalence for versions, including build metadata. + * + * @see {@link Order} for precedence that ignores build metadata + * @stability unstable + * @category instances + * @since 4.0.6 + */ +export const Equivalence: Equ.Equivalence = Equ.make((self, that) => + self.major === that.major && + self.minor === that.minor && + self.patch === that.patch && + identifiersEqual(self.prerelease, that.prerelease) && + identifiersEqual(self.build, that.build) +) + +/** + * Checks whether two versions are structurally equal, including build + * metadata. + * + * @stability unstable + * @category predicates + * @since 4.0.6 + */ +export const equals: { + (that: SemVer): (self: SemVer) => boolean + (self: SemVer, that: SemVer): boolean +} = dual(2, Equivalence) + +/** + * Checks whether the first version has lower precedence than the second. + * + * @stability unstable + * @category predicates + * @since 4.0.6 + */ +export const isLessThan: { + (that: SemVer): (self: SemVer) => boolean + (self: SemVer, that: SemVer): boolean +} = order.isLessThan(Order) + +/** + * Checks whether the first version has lower or equal precedence than the + * second. + * + * @stability unstable + * @category predicates + * @since 4.0.6 + */ +export const isLessThanOrEqualTo: { + (that: SemVer): (self: SemVer) => boolean + (self: SemVer, that: SemVer): boolean +} = order.isLessThanOrEqualTo(Order) + +/** + * Checks whether the first version has higher precedence than the second. + * + * @stability unstable + * @category predicates + * @since 4.0.6 + */ +export const isGreaterThan: { + (that: SemVer): (self: SemVer) => boolean + (self: SemVer, that: SemVer): boolean +} = order.isGreaterThan(Order) + +/** + * Checks whether the first version has higher or equal precedence than the + * second. + * + * @stability unstable + * @category predicates + * @since 4.0.6 + */ +export const isGreaterThanOrEqualTo: { + (that: SemVer): (self: SemVer) => boolean + (self: SemVer, that: SemVer): boolean +} = order.isGreaterThanOrEqualTo(Order) + +/** + * Checks whether a version's precedence is within an inclusive range. + * + * @stability unstable + * @category predicates + * @since 4.0.6 + */ +export const between: { + (options: { minimum: SemVer; maximum: SemVer }): (self: SemVer) => boolean + (self: SemVer, options: { minimum: SemVer; maximum: SemVer }): boolean +} = order.isBetween(Order) + +/** + * Returns the version with lower precedence, preferring the first on ties. + * + * @stability unstable + * @category ordering + * @since 4.0.6 + */ +export const min: { + (that: SemVer): (self: SemVer) => SemVer + (self: SemVer, that: SemVer): SemVer +} = order.min(Order) + +/** + * Returns the version with higher precedence, preferring the first on ties. + * + * @stability unstable + * @category ordering + * @since 4.0.6 + */ +export const max: { + (that: SemVer): (self: SemVer) => SemVer + (self: SemVer, that: SemVer): SemVer +} = order.max(Order) + +/** + * Constrains a version's precedence to an inclusive range. + * + * @stability unstable + * @category ordering + * @since 4.0.6 + */ +export const clamp: { + (options: { minimum: SemVer; maximum: SemVer }): (self: SemVer) => SemVer + (self: SemVer, options: { minimum: SemVer; maximum: SemVer }): SemVer +} = order.clamp(Order) + +/** + * Replaces the pre-release identifiers of a version and throws `SemVerError` + * for invalid identifiers. + * + * **Details** + * + * Digit-only strings are normalized to numbers. An empty array removes the + * pre-release. Build metadata is preserved. + * + * @stability unstable + * @category transforming + * @since 4.0.6 + */ +export const withPrerelease: { + (prerelease: ReadonlyArray): (self: SemVer) => SemVer + (self: SemVer, prerelease: ReadonlyArray): SemVer +} = dual( + 2, + (self: SemVer, prerelease: ReadonlyArray): SemVer => + unchecked(self.major, self.minor, self.patch, prerelease.map(prereleaseIdentifier), self.build) +) + +/** + * Replaces the build metadata of a version and throws `SemVerError` for + * invalid identifiers. + * + * **Details** + * + * An empty array removes the build metadata. Pre-release identifiers are + * preserved. + * + * @stability unstable + * @category transforming + * @since 4.0.6 + */ +export const withBuild: { + (build: ReadonlyArray): (self: SemVer) => SemVer + (self: SemVer, build: ReadonlyArray): SemVer +} = dual( + 2, + (self: SemVer, build: ReadonlyArray): SemVer => + unchecked(self.major, self.minor, self.patch, self.prerelease, build.map(buildIdentifier)) +) + +/** + * Removes pre-release identifiers and build metadata from a version. + * + * **Example** (Finalizing a release candidate) + * + * ```ts import.meta.vitest + * import { SemVer } from "effect" + * + * SemVer.format(SemVer.release(SemVer.parseUnsafe("2.0.0-rc.3+sha.5114f85"))) // => "2.0.0" + * ``` + * + * @stability unstable + * @category transforming + * @since 4.0.6 + */ +export const release = (self: SemVer): SemVer => + self.prerelease.length === 0 && self.build.length === 0 + ? self + : unchecked(self.major, self.minor, self.patch, [], []) + +/** + * Returns the next major release. + * + * **Details** + * + * Returns the smallest major release with higher precedence, so a pre-release + * of a major version, such as `2.0.0-rc.1`, becomes `2.0.0`. Pre-release + * identifiers and build metadata are removed. + * + * **Example** (Bumping major versions) + * + * ```ts import.meta.vitest + * import { SemVer } from "effect" + * + * SemVer.format(SemVer.bumpMajor(SemVer.parseUnsafe("1.2.3"))) // => "2.0.0" + * SemVer.format(SemVer.bumpMajor(SemVer.parseUnsafe("2.0.0-rc.1"))) // => "2.0.0" + * SemVer.format(SemVer.bumpMajor(SemVer.parseUnsafe("2.1.0-rc.1"))) // => "3.0.0" + * ``` + * + * @stability unstable + * @category transforming + * @since 4.0.6 + */ +export const bumpMajor = (self: SemVer): SemVer => + unchecked( + self.minor === 0 && self.patch === 0 && self.prerelease.length > 0 ? self.major : increment(self.major, "major"), + 0, + 0, + [], + [] + ) + +/** + * Returns the next minor release. + * + * **Details** + * + * Returns the smallest minor release with higher precedence, so a pre-release + * of a minor version, such as `1.3.0-rc.1`, becomes `1.3.0`. Pre-release + * identifiers and build metadata are removed. + * + * @stability unstable + * @category transforming + * @since 4.0.6 + */ +export const bumpMinor = (self: SemVer): SemVer => + unchecked( + self.major, + self.patch === 0 && self.prerelease.length > 0 ? self.minor : increment(self.minor, "minor"), + 0, + [], + [] + ) + +/** + * Returns the next patch release. + * + * **Details** + * + * Returns the smallest patch release with higher precedence, so a pre-release + * such as `1.2.3-rc.1` becomes `1.2.3`. Pre-release identifiers and build + * metadata are removed. + * + * @stability unstable + * @category transforming + * @since 4.0.6 + */ +export const bumpPatch = (self: SemVer): SemVer => + unchecked( + self.major, + self.minor, + self.prerelease.length > 0 ? self.patch : increment(self.patch, "patch"), + [], + [] + ) + +const startPrerelease = (identifier: string | undefined): ReadonlyArray => + identifier === undefined ? [0] : [prereleaseIdentifier(identifier), 0] + +const isSemVerArgs = (args: IArguments) => isSemVer(args[0]) + +/** + * Returns the first pre-release of the next major version. + * + * **Details** + * + * Increments major, resets minor and patch, and starts the pre-release at + * `[identifier, 0]`, or `[0]` without an identifier. Build metadata is removed. + * + * @stability unstable + * @category transforming + * @since 4.0.6 + */ +export const bumpPremajor: { + (identifier?: string): (self: SemVer) => SemVer + (self: SemVer, identifier?: string): SemVer +} = dual( + isSemVerArgs, + (self: SemVer, identifier?: string): SemVer => + unchecked(increment(self.major, "major"), 0, 0, startPrerelease(identifier), []) +) + +/** + * Returns the first pre-release of the next minor version. + * + * **Details** + * + * Increments minor, resets patch, and starts the pre-release at + * `[identifier, 0]`, or `[0]` without an identifier. Build metadata is removed. + * + * @stability unstable + * @category transforming + * @since 4.0.6 + */ +export const bumpPreminor: { + (identifier?: string): (self: SemVer) => SemVer + (self: SemVer, identifier?: string): SemVer +} = dual( + isSemVerArgs, + (self: SemVer, identifier?: string): SemVer => + unchecked(self.major, increment(self.minor, "minor"), 0, startPrerelease(identifier), []) +) + +/** + * Returns the first pre-release of the next patch version. + * + * **Details** + * + * Increments patch and starts the pre-release at `[identifier, 0]`, or `[0]` + * without an identifier. Build metadata is removed. + * + * @stability unstable + * @category transforming + * @since 4.0.6 + */ +export const bumpPrepatch: { + (identifier?: string): (self: SemVer) => SemVer + (self: SemVer, identifier?: string): SemVer +} = dual( + isSemVerArgs, + (self: SemVer, identifier?: string): SemVer => + unchecked(self.major, self.minor, increment(self.patch, "patch"), startPrerelease(identifier), []) +) + +const nextPrerelease = (prerelease: ReadonlyArray): ReadonlyArray => { + for (let i = prerelease.length - 1; i >= 0; i--) { + const identifier = prerelease[i] + if (typeof identifier === "number") { + const next = prerelease.slice() + next[i] = increment(identifier, "pre-release identifier") + return next + } + } + return [...prerelease, 0] +} + +/** + * Returns the next pre-release. + * + * **Details** + * + * - A release starts the pre-release of the next patch, like + * {@link bumpPrepatch}. + * - An identifier that differs from the first pre-release identifier restarts + * the pre-release at `[identifier, 0]`. + * - Otherwise the right-most numeric identifier is incremented, or `0` is + * appended when there is none. + * + * Build metadata is removed. + * + * **Gotchas** + * + * Switching to an identifier that sorts lower, such as from `beta` to `alpha`, + * returns a version with lower precedence. + * + * **Example** (Advancing pre-releases) + * + * ```ts import.meta.vitest + * import { SemVer } from "effect" + * + * SemVer.format(SemVer.bumpPrerelease(SemVer.parseUnsafe("1.2.3"), "alpha")) // => "1.2.4-alpha.0" + * SemVer.format(SemVer.bumpPrerelease(SemVer.parseUnsafe("1.2.4-alpha.0"))) // => "1.2.4-alpha.1" + * SemVer.format(SemVer.bumpPrerelease(SemVer.parseUnsafe("1.2.4-alpha.1"), "beta")) // => "1.2.4-beta.0" + * ``` + * + * @stability unstable + * @category transforming + * @since 4.0.6 + */ +export const bumpPrerelease: { + (identifier?: string): (self: SemVer) => SemVer + (self: SemVer, identifier?: string): SemVer +} = dual(isSemVerArgs, (self: SemVer, identifier?: string): SemVer => { + if (self.prerelease.length === 0) { + return unchecked(self.major, self.minor, increment(self.patch, "patch"), startPrerelease(identifier), []) + } + if (identifier !== undefined) { + const normalized = prereleaseIdentifier(identifier) + if (self.prerelease[0] !== normalized) { + return unchecked(self.major, self.minor, self.patch, [normalized, 0], []) + } + } + return unchecked(self.major, self.minor, self.patch, nextPrerelease(self.prerelease), []) +}) + +/** + * Names of version changes, matching npm's release types. + * + * @see {@link difference} for classifying the change between two versions + * @stability unstable + * @category models + * @since 4.0.6 + */ +export type ReleaseType = "major" | "premajor" | "minor" | "preminor" | "patch" | "prepatch" | "prerelease" + +/** + * Bumps a version by release type. + * + * **When to use** + * + * Use when the release type is data, such as a CLI argument, configuration, or + * the result of {@link difference}. + * + * **Details** + * + * Dispatches to {@link bumpMajor}, {@link bumpMinor}, {@link bumpPatch}, + * {@link bumpPremajor}, {@link bumpPreminor}, {@link bumpPrepatch}, or + * {@link bumpPrerelease}, with the same semantics. + * + * **Gotchas** + * + * The pre-release `identifier` is ignored for `"major"`, `"minor"`, and + * `"patch"`. + * + * **Example** (Bumping by a release type from input) + * + * ```ts import.meta.vitest + * import { SemVer } from "effect" + * + * const version = SemVer.parseUnsafe("1.2.3") + * + * SemVer.format(SemVer.bump(version, "minor")) // => "1.3.0" + * SemVer.format(SemVer.bump(version, "premajor", "rc")) // => "2.0.0-rc.0" + * ``` + * + * **Example** (Defining custom bump rules) + * + * ```ts import.meta.vitest + * import { SemVer } from "effect" + * + * // Always increment patch, even for pre-releases, and keep build metadata + * const customBump = (version: SemVer.SemVer, releaseType: SemVer.ReleaseType): SemVer.SemVer => + * releaseType === "patch" + * ? SemVer.make({ ...version, patch: version.patch + 1, prerelease: [] }) + * : SemVer.withBuild(SemVer.bump(version, releaseType), version.build) + * + * SemVer.format(customBump(SemVer.parseUnsafe("1.2.3-rc.1+b"), "patch")) // => "1.2.4+b" + * SemVer.format(customBump(SemVer.parseUnsafe("1.2.3-rc.1+b"), "minor")) // => "1.3.0+b" + * ``` + * + * @stability unstable + * @category transforming + * @since 4.0.6 + */ +export const bump: { + (releaseType: ReleaseType, identifier?: string): (self: SemVer) => SemVer + (self: SemVer, releaseType: ReleaseType, identifier?: string): SemVer +} = dual(isSemVerArgs, (self: SemVer, releaseType: ReleaseType, identifier?: string): SemVer => { + switch (releaseType) { + case "major": + return bumpMajor(self) + case "minor": + return bumpMinor(self) + case "patch": + return bumpPatch(self) + case "premajor": + return bumpPremajor(self, identifier) + case "preminor": + return bumpPreminor(self, identifier) + case "prepatch": + return bumpPrepatch(self, identifier) + case "prerelease": + return bumpPrerelease(self, identifier) + } +}) + +/** + * Classifies the change between two versions by release type. + * + * **Details** + * + * Follows node-semver's `diff`, so the result names the bump that produces the + * higher version from the lower one. Moving to a pre-release adds the `pre` + * prefix, and finalizing a pre-release reports the level it was released at: + * `1.0.0-1` to `1.0.0` is `"major"`, `1.1.0-1` to `1.1.0` is `"minor"`, and + * `1.2.3-1` to `1.2.3` is `"patch"`. The result does not depend on argument + * order. + * + * **Gotchas** + * + * Returns `None` for versions with the same precedence, including versions + * that differ only in build metadata. + * + * **Example** (Classifying upgrades) + * + * ```ts import.meta.vitest + * import { Option, SemVer } from "effect" + * + * SemVer.difference(SemVer.parseUnsafe("1.2.3"), SemVer.parseUnsafe("1.3.0")) // => Option.some("minor") + * SemVer.difference(SemVer.parseUnsafe("1.2.3"), SemVer.parseUnsafe("2.0.0-rc.1")) // => Option.some("premajor") + * SemVer.difference(SemVer.parseUnsafe("2.0.0-rc.1"), SemVer.parseUnsafe("2.0.0")) // => Option.some("major") + * SemVer.difference(SemVer.parseUnsafe("1.2.3+a"), SemVer.parseUnsafe("1.2.3+b")) // => Option.none() + * ``` + * + * @see {@link bump} for applying a release type to a version + * @stability unstable + * @category comparisons + * @since 4.0.6 + */ +export const difference: { + (that: SemVer): (self: SemVer) => Option.Option + (self: SemVer, that: SemVer): Option.Option +} = dual(2, (self: SemVer, that: SemVer): Option.Option => { + const comparison = Order(self, that) + if (comparison === 0) return Option.none() + const [low, high] = comparison < 0 ? [self, that] : [that, self] + const highIsPrerelease = high.prerelease.length > 0 + if (low.prerelease.length > 0 && !highIsPrerelease) { + if (low.minor === 0 && low.patch === 0) return Option.some("major") + if (sameCore(low, high)) return Option.some(low.patch === 0 ? "minor" : "patch") + } + if (self.major !== that.major) return Option.some(highIsPrerelease ? "premajor" : "major") + if (self.minor !== that.minor) return Option.some(highIsPrerelease ? "preminor" : "minor") + if (self.patch !== that.patch) return Option.some(highIsPrerelease ? "prepatch" : "patch") + return Option.some("prerelease") +}) diff --git a/repos/effect/packages/effect/src/SemVerRange.ts b/repos/effect/packages/effect/src/SemVerRange.ts new file mode 100644 index 0000000000..bafa5352e3 --- /dev/null +++ b/repos/effect/packages/effect/src/SemVerRange.ts @@ -0,0 +1,703 @@ +/** + * npm-style version ranges over `SemVer` values. + * + * A `SemVerRange` keeps the syntax it was parsed from, such as `^1.2`, `~1.2.3`, + * `1.x`, or `1.2.3 - 2.3.4`. Matching follows node-semver's pre-release rules, + * except that wildcard alternatives preserve the union described in {@link matches}. + * + * @stability unstable + * @since 4.0.6 + */ +import * as Equal from "./Equal.ts" +import { dual } from "./Function.ts" +import * as Hash from "./Hash.ts" +import { type Inspectable, NodeInspectSymbol } from "./Inspectable.ts" +import * as Option from "./Option.ts" +import { type Pipeable, pipeArguments } from "./Pipeable.ts" +import { hasProperty } from "./Predicate.ts" +import * as Result from "./Result.ts" +import { + compareCore, + type Core, + fromInputUnsafe as versionFromInputUnsafe, + type Input as VersionInput, + invalid, + parseComponent, + prereleaseIdentifier, + sameCore, + type SemVer, + type SemVerError, + tryParse +} from "./SemVer.ts" + +const TypeId = "~effect/SemVerRange" + +/** + * Represents an npm-style version range. + * + * **Details** + * + * Ranges are opaque. {@link format} preserves parsed operators such as `^1.2` + * rather than expanding them into bounds. + * + * @see {@link parse} for decoding range strings + * @see {@link matches} for matching versions + * @stability unstable + * @category models + * @since 4.0.6 + */ +export interface SemVerRange extends Pipeable, Equal.Equal, Inspectable { + readonly [TypeId]: typeof TypeId +} + +interface PartialVersion { + // The components given before the first wildcard; empty for `*`. + readonly components: ReadonlyArray + // Only a full MAJOR.MINOR.PATCH may carry a pre-release. + readonly prerelease: ReadonlyArray +} + +type Operator = "=" | "<" | "<=" | ">" | ">=" | "~" | "^" + +type Comparator = + | { + readonly _tag: "Comparator" + readonly operator: Operator + readonly version: PartialVersion + } + | { + readonly _tag: "Hyphen" + readonly from: PartialVersion + readonly to: PartialVersion + } + +interface SemVerRangeImpl extends SemVerRange { + // Alternatives separated by `||`, each a set of comparators that must all match. + readonly sets: ReadonlyArray> + matcher?: (version: SemVer) => boolean + prereleaseMatcher?: (version: SemVer) => boolean +} + +/** + * Options controlling how pre-release versions match a range. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export interface MatchOptions { + /** + * Lets pre-release versions match any comparator set whose bounds contain + * them, and lowers wildcard lower bounds to include pre-releases. + * + * @default false + */ + readonly includePrerelease?: boolean | undefined +} + +const SemVerRangeProto: SemVerRange = { + [TypeId]: TypeId, + [Equal.symbol](this: SemVerRange, that: unknown): boolean { + return isSemVerRange(that) && format(this) === format(that) + }, + [Hash.symbol](this: SemVerRange): number { + return Hash.string(format(this)) + }, + toString(this: SemVerRange) { + return `SemVerRange(${format(this)})` + }, + toJSON(this: SemVerRange) { + return { _id: "SemVerRange", value: format(this) } + }, + [NodeInspectSymbol](this: SemVerRange) { + return this.toJSON() + }, + pipe() { + return pipeArguments(this, arguments) + } +} + +/** + * Checks whether a value is a `SemVerRange`. + * + * @stability unstable + * @category guards + * @since 4.0.6 + */ +export const isSemVerRange = (u: unknown): u is SemVerRange => hasProperty(u, TypeId) + +const isRangeArgs = (args: IArguments) => isSemVerRange(args[0]) + +const wildcard: PartialVersion = { components: [], prerelease: [] } +const operatorPattern = /^(<=|>=|<|>|=|~|\^)?(.*)$/ +const isWildcard = (part: string): boolean => part === "x" || part === "X" || part === "*" + +const componentLabels = ["major", "minor", "patch"] as const + +const parsePartial = (input: string): PartialVersion => { + if (input.length === 0) return invalid("expected a version after the operator") + if (input.includes("+")) { + return invalid(`build metadata is not allowed in ranges, received ${JSON.stringify(input)}`) + } + const dash = input.indexOf("-") + const parts = (dash === -1 ? input : input.slice(0, dash)).split(".") + if (parts.length > 3) return invalid(`expected at most MAJOR.MINOR.PATCH, received ${JSON.stringify(input)}`) + const components: Array = [] + for (let i = 0; i < parts.length; i++) { + if (isWildcard(parts[i])) continue + // A skipped wildcard leaves `components` shorter than the position reached. + if (components.length < i) { + return invalid(`a version component cannot follow a wildcard, received ${JSON.stringify(input)}`) + } + components.push(parseComponent(parts[i], componentLabels[i])) + } + if (dash === -1) return { components, prerelease: [] } + if (components.length < 3) { + return invalid(`a pre-release requires MAJOR.MINOR.PATCH, received ${JSON.stringify(input)}`) + } + return { components, prerelease: input.slice(dash + 1).split(".").map(prereleaseIdentifier) } +} + +const parseComparatorSet = (input: string): ReadonlyArray => { + const tokens = input.split(/\s+/).filter((token) => token.length > 0) + if (tokens.length === 0) return invalid("expected a comparator") + if (tokens.length === 3 && tokens[1] === "-") { + return [{ _tag: "Hyphen", from: parsePartial(tokens[0]), to: parsePartial(tokens[2]) }] + } + const comparators: Array = [] + for (let i = 0; i < tokens.length; i++) { + const [, operator, rest] = operatorPattern.exec(tokens[i])! + // Allow whitespace between an operator and its version, as in `>= 1.2.3`. + const version = rest.length === 0 && operator !== undefined && i + 1 < tokens.length ? tokens[++i] : rest + comparators.push({ _tag: "Comparator", operator: (operator ?? "=") as Operator, version: parsePartial(version) }) + } + return comparators +} + +const makeRange = (sets: ReadonlyArray>): SemVerRange => { + const self = Object.create(SemVerRangeProto) + self.sets = sets + return self +} + +const parseOrThrow = (input: string): SemVerRange => { + if (input.trim().length === 0) return any + return makeRange(input.split("||").map(parseComparatorSet)) +} + +/** + * Parses an npm-style version range. + * + * **Details** + * + * Supports: + * + * - comparators `=`, `<`, `<=`, `>`, `>=`, whitespace-separated as AND; + * - alternatives separated by `||`; + * - X-ranges such as `1.x`, `1.2.*`, `1`, and `*`; + * - tilde (`~1.2.3`), caret (`^1.2.3`), and hyphen (`1.2.3 - 2.3.4`) ranges. + * + * An empty string is equivalent to `*`. Invalid input fails with + * `SemVer.SemVerError`. + * + * **Gotchas** + * + * Versions must follow the strict SemVer grammar apart from wildcards, so a + * `v` prefix, leading zeros, build metadata, and empty `||` alternatives are + * rejected. + * + * **Example** (Parsing and matching a caret range) + * + * ```ts import.meta.vitest + * import { Result, SemVerRange } from "effect" + * + * const range = SemVerRange.parseUnsafe("^1.2.0 || >=3.0.0-rc.1") + * + * SemVerRange.matches(range, "1.9.0") // => true + * SemVerRange.matches(range, "2.0.0") // => false + * SemVerRange.matches(range, "3.0.0-rc.2") // => true + * Result.isFailure(SemVerRange.parse("^v1.2.0")) // => true + * ``` + * + * @see {@link parseUnsafe} for throwing on invalid input + * @see {@link format} for the inverse operation + * @stability unstable + * @category constructors + * @since 4.0.6 + */ +export const parse = (input: string): Result.Result => + tryParse("SemVer range", input, parseOrThrow) + +/** + * Parses an npm-style version range and throws `SemVer.SemVerError` for + * invalid input. + * + * @see {@link parse} for non-throwing parsing + * @stability unstable + * @category constructors + * @since 4.0.6 + */ +export const parseUnsafe = (input: string): SemVerRange => Result.getOrThrow(parse(input)) + +/** + * Values accepted by `fromInput`: a range or a range string. + * + * @see {@link fromInput} for converting inputs + * @stability unstable + * @category models + * @since 4.0.6 + */ +export type Input = SemVerRange | string + +/** + * Converts a range or range string into a `SemVerRange`. + * + * **Details** + * + * Ranges are returned unchanged and strings are parsed like {@link parse}. + * Invalid strings fail with `SemVer.SemVerError`. + * + * @see {@link fromInputUnsafe} for throwing on invalid input + * @stability unstable + * @category constructors + * @since 4.0.6 + */ +export const fromInput = (input: Input): Result.Result => + isSemVerRange(input) ? Result.succeed(input) : parse(input) + +/** + * Converts a range or range string into a `SemVerRange` and throws + * `SemVer.SemVerError` for invalid input. + * + * @see {@link fromInput} for non-throwing conversion + * @stability unstable + * @category constructors + * @since 4.0.6 + */ +export const fromInputUnsafe = (input: Input): SemVerRange => Result.getOrThrow(fromInput(input)) + +const toPartial = (version: SemVer): PartialVersion => ({ + components: [version.major, version.minor, version.patch], + prerelease: version.prerelease +}) + +const comparator = (operator: Operator, version: VersionInput): Comparator => ({ + _tag: "Comparator", + operator, + version: toPartial(versionFromInputUnsafe(version)) +}) + +const fromComparator = (operator: Operator, version: VersionInput): SemVerRange => + makeRange([[comparator(operator, version)]]) + +/** + * The range matching every release, written `*`. + * + * **Gotchas** + * + * Like every range, `*` only matches pre-releases with `includePrerelease`. + * + * @stability unstable + * @category constants + * @since 4.0.6 + */ +export const any: SemVerRange = makeRange([[{ _tag: "Comparator", operator: "=", version: wildcard }]]) + +/** + * Creates a range matching exactly one version, written `1.2.3`. + * + * **Details** + * + * Build metadata is dropped because ranges cannot express it and it does not + * affect matching, so `1.2.3+a` and `1.2.3+b` both match. Throws `SemVerError` for an invalid `SemVer.Input`. + * + * **Example** (Building ranges from versions) + * + * ```ts import.meta.vitest + * import { SemVer, SemVerRange } from "effect" + * + * const version = SemVer.parseUnsafe("1.2.3+build.5") + * + * SemVerRange.format(SemVerRange.exact(version)) // => "1.2.3" + * SemVerRange.format(SemVerRange.greaterThanOrEqualTo(version)) // => ">=1.2.3" + * SemVerRange.format(SemVerRange.caret("1.2.3")) // => "^1.2.3" + * SemVerRange.matches(SemVerRange.greaterThan(version), "1.2.4") // => true + * ``` + * + * @stability unstable + * @category constructors + * @since 4.0.6 + */ +export const exact = (version: VersionInput): SemVerRange => fromComparator("=", version) + +/** + * Creates a range matching versions with higher precedence, written `>1.2.3`. + * + * **Details** + * + * Build metadata is dropped. Throws `SemVerError` for an invalid `SemVer.Input`. + * + * @stability unstable + * @category constructors + * @since 4.0.6 + */ +export const greaterThan = (version: VersionInput): SemVerRange => fromComparator(">", version) + +/** + * Creates a range matching versions with at least the given precedence, written `>=1.2.3`. + * + * **Details** + * + * Build metadata is dropped. Throws `SemVerError` for an invalid `SemVer.Input`. + * + * @stability unstable + * @category constructors + * @since 4.0.6 + */ +export const greaterThanOrEqualTo = (version: VersionInput): SemVerRange => fromComparator(">=", version) + +/** + * Creates a range matching versions with lower precedence, written `<1.2.3`. + * + * **Details** + * + * Build metadata is dropped. Throws `SemVerError` for an invalid `SemVer.Input`. + * + * @stability unstable + * @category constructors + * @since 4.0.6 + */ +export const lessThan = (version: VersionInput): SemVerRange => fromComparator("<", version) + +/** + * Creates a range matching versions with at most the given precedence, written `<=1.2.3`. + * + * **Details** + * + * Build metadata is dropped. Throws `SemVerError` for an invalid `SemVer.Input`. + * + * @stability unstable + * @category constructors + * @since 4.0.6 + */ +export const lessThanOrEqualTo = (version: VersionInput): SemVerRange => fromComparator("<=", version) + +/** + * Creates a range matching versions compatible with a version, written `^1.2.3`. + * + * **Details** + * + * Allows changes that do not modify the left-most non-zero component: `^1.2.3` + * means `>=1.2.3 <2.0.0-0` and `^0.2.3` means `>=0.2.3 <0.3.0-0`. Build + * metadata is dropped. Throws `SemVerError` for an invalid `SemVer.Input`. + * + * @stability unstable + * @category constructors + * @since 4.0.6 + */ +export const caret = (version: VersionInput): SemVerRange => fromComparator("^", version) + +/** + * Creates a range matching patch-level changes of a version, written `~1.2.3`. + * + * **Details** + * + * `~1.2.3` means `>=1.2.3 <1.3.0-0`. Build metadata is dropped. + * Throws `SemVerError` for an invalid `SemVer.Input`. + * + * @stability unstable + * @category constructors + * @since 4.0.6 + */ +export const tilde = (version: VersionInput): SemVerRange => fromComparator("~", version) + +/** + * Creates a range matching versions within inclusive bounds, written + * `>=1.2.3 <=2.0.0`. + * + * **Details** + * + * Matches the same versions as `SemVer.between`, subject to the pre-release + * rule. Build metadata is dropped. Throws `SemVerError` for an invalid `SemVer.Input`. + * + * @stability unstable + * @category constructors + * @since 4.0.6 + */ +export const between = (options: { readonly minimum: VersionInput; readonly maximum: VersionInput }): SemVerRange => + makeRange([[comparator(">=", options.minimum), comparator("<=", options.maximum)]]) + +const formatPartial = (self: PartialVersion): string => { + if (self.components.length === 0) return "*" + const core = self.components.join(".") + return self.prerelease.length === 0 ? core : `${core}-${self.prerelease.join(".")}` +} + +const formatComparator = (self: Comparator): string => + self._tag === "Hyphen" + ? `${formatPartial(self.from)} - ${formatPartial(self.to)}` + : `${self.operator === "=" ? "" : self.operator}${formatPartial(self.version)}` + +/** + * Formats a range in canonical form. + * + * **Details** + * + * Operators and comparators are kept as parsed. Wildcard spellings are + * normalized by dropping trailing wildcard components (`1.x` and `1.*.*` + * format as `1`, and `x` as `*`), and an explicit `=` operator is omitted. + * + * @see {@link parse} for the inverse operation + * @stability unstable + * @category converting + * @since 4.0.6 + */ +export const format = (self: SemVerRange): string => + (self as SemVerRangeImpl).sets.map((set) => set.map(formatComparator).join(" ")).join(" || ") + +interface Bound extends Core { + readonly operator: "=" | "<" | "<=" | ">" | ">=" +} + +const bound = ( + operator: Bound["operator"], + [major = 0, minor = 0, patch = 0]: ReadonlyArray, + prerelease: ReadonlyArray +): Bound => ({ operator, major, minor, patch, prerelease }) + +// The expansions below follow node-semver's `Range` desugaring. `z` is the +// pre-release a wildcard lower bound carries: `[0]` when pre-releases are +// included, so `>=1.2` admits `1.2.0-alpha`, and empty otherwise. + +// Exclusive upper bound at the first pre-release of `core`, node-semver's `): Bound => bound("<", core, [0]) + +// The lowest version a partial covers. +const floor = (self: PartialVersion, z: ReadonlyArray): Bound => + bound(">=", self.components, self.components.length === 3 ? self.prerelease : z) + +// The first core above every version the first `precision` components cover. +const after = (components: ReadonlyArray, precision: number): ReadonlyArray => { + const next = components.slice(0, precision) + next[precision - 1]++ + return next +} + +const expandCaret = (self: PartialVersion, z: ReadonlyArray): ReadonlyArray => { + const n = self.components.length + if (n === 0) return [] + // The first non-zero component may not change; `^0.0.x` pins the patch. + const precision = self.components.findIndex((component) => component !== 0) + 1 || n + return [floor(self, z), below(after(self.components, precision))] +} + +const expandTilde = (self: PartialVersion, z: ReadonlyArray): ReadonlyArray => { + const n = self.components.length + if (n === 0) return [] + return [floor(self, z), below(after(self.components, Math.min(n, 2)))] +} + +const expandXRange = ( + operator: Bound["operator"], + self: PartialVersion, + z: ReadonlyArray +): ReadonlyArray => { + const n = self.components.length + if (n === 3) return [bound(operator, self.components, self.prerelease)] + if (n === 0) return operator === "<" || operator === ">" ? [below([0, 0, 0])] : [] + switch (operator) { + case "=": + return [floor(self, z), below(after(self.components, n))] + case ">": + return [bound(">=", after(self.components, n), z)] + case ">=": + return [floor(self, z)] + case "<=": + return [below(after(self.components, n))] + case "<": + return [below(self.components)] + } +} + +const expand = (self: Comparator, includePrerelease: boolean): ReadonlyArray => { + const z = includePrerelease ? [0] : [] + if (self._tag === "Hyphen") { + const { from, to } = self + // Unlike `>=`, a full hyphen lower bound without a pre-release also takes `z`. + const lower = from.components.length === 0 + ? [] + : [bound(">=", from.components, from.prerelease.length > 0 ? from.prerelease : z)] + return [...lower, ...expandXRange("<=", to, z)] + } + switch (self.operator) { + case "^": + return expandCaret(self.version, z) + case "~": + return expandTilde(self.version, z) + default: + return expandXRange(self.operator, self.version, z) + } +} + +const testBound = (self: Bound, version: SemVer): boolean => { + const result = compareCore(version, self) + switch (self.operator) { + case "=": + return result === 0 + case "<": + return result < 0 + case "<=": + return result <= 0 + case ">": + return result > 0 + case ">=": + return result >= 0 + } +} + +const compile = (self: SemVerRangeImpl, includePrerelease: boolean): (version: SemVer) => boolean => { + const sets = self.sets.map((set) => set.flatMap((comparator) => expand(comparator, includePrerelease))) + return (version) => + sets.some((bounds) => + bounds.every((bound) => testBound(bound, version)) && + (includePrerelease || version.prerelease.length === 0 || + bounds.some((bound) => bound.prerelease.length > 0 && sameCore(bound, version))) + ) +} + +const matcher = (range: SemVerRange, options: MatchOptions | undefined): (version: SemVer) => boolean => { + const self = range as SemVerRangeImpl + return options?.includePrerelease === true + ? (self.prereleaseMatcher ??= compile(self, true)) + : (self.matcher ??= compile(self, false)) +} + +/** + * Checks whether a range matches a version. + * + * **Details** + * + * Partial versions, tilde, caret, and hyphen ranges expand to bounds following + * node-semver. For example, `^1.2.3` means `>=1.2.3 <2.0.0-0`, `~1.2` means + * `>=1.2.0 <1.3.0-0`, and `<=1.2` means `<1.3.0-0`. Throws `SemVerError` when + * `version` is an invalid `SemVer.Input`. + * + * **Gotchas** + * + * A pre-release version only matches a comparator set that contains a + * comparator with a pre-release on the same major, minor, and patch. So + * `^1.2.3-beta.1` matches `1.2.3-beta.2` but not `1.3.0-rc.1`, and `*` does + * not match any pre-release. Set `includePrerelease` to disable this rule. + * + * Alternatives separated by `||` always form a union. Unlike node-semver, a `*` + * alternative does not discard pre-releases admitted by another alternative. + * + * **Example** (Matching pre-releases) + * + * ```ts import.meta.vitest + * import { SemVerRange } from "effect" + * + * const range = SemVerRange.parseUnsafe("^1.2.3-beta.1") + * + * SemVerRange.matches(range, "1.2.3-beta.2") // => true + * SemVerRange.matches(range, "1.3.0-rc.1") // => false + * SemVerRange.matches(range, "1.3.0-rc.1", { includePrerelease: true }) // => true + * ``` + * + * **Example** (Filtering versions) + * + * ```ts import.meta.vitest + * import { SemVer, SemVerRange } from "effect" + * + * const range = SemVerRange.parseUnsafe("~1.2") + * const versions = ["1.1.0", "1.2.0", "1.2.7", "1.3.0"].map(SemVer.parseUnsafe) + * + * versions.filter((version) => SemVerRange.matches(range, version)).map(SemVer.format) // => ["1.2.0", "1.2.7"] + * ``` + * + * @see {@link maxSatisfying} for selecting the highest matching version + * @stability unstable + * @category predicates + * @since 4.0.6 + */ +export const matches: { + (version: VersionInput, options?: MatchOptions): (self: SemVerRange) => boolean + (self: SemVerRange, version: VersionInput, options?: MatchOptions): boolean +} = dual( + isRangeArgs, + (self: SemVerRange, version: VersionInput, options?: MatchOptions): boolean => + matcher(self, options)(versionFromInputUnsafe(version)) +) + +const select = ( + self: SemVerRange, + versions: Iterable, + options: MatchOptions | undefined, + better: -1 | 1 +): Option.Option => { + const test = matcher(self, options) + let selected: SemVer | undefined + for (const input of versions) { + const version = versionFromInputUnsafe(input) + if (test(version) && (selected === undefined || compareCore(version, selected) === better)) { + selected = version + } + } + return Option.fromUndefinedOr(selected) +} + +/** + * Returns the version with the highest precedence that a range matches. + * + * **Details** + * + * Returns `None` when no version matches. When several versions share the + * highest precedence, the first one is returned. Throws `SemVerError` when a + * version is an invalid `SemVer.Input`. + * + * **Example** (Selecting the newest compatible version) + * + * ```ts import.meta.vitest + * import { Option, SemVer, SemVerRange } from "effect" + * + * const range = SemVerRange.parseUnsafe("^1.2.0") + * + * SemVerRange.maxSatisfying(range, ["1.2.0", "1.4.1", "2.0.0", "1.5.0-rc.1"]).pipe(Option.map(SemVer.format)) // => Option.some("1.4.1") + * ``` + * + * @see {@link minSatisfying} for the lowest matching version + * @stability unstable + * @category filtering + * @since 4.0.6 + */ +export const maxSatisfying: { + (versions: Iterable, options?: MatchOptions): (self: SemVerRange) => Option.Option + (self: SemVerRange, versions: Iterable, options?: MatchOptions): Option.Option +} = dual( + isRangeArgs, + (self: SemVerRange, versions: Iterable, options?: MatchOptions): Option.Option => + select(self, versions, options, 1) +) + +/** + * Returns the version with the lowest precedence that a range matches. + * + * **Details** + * + * Returns `None` when no version matches. When several versions share the + * lowest precedence, the first one is returned. Throws `SemVerError` when a + * version is an invalid `SemVer.Input`. + * + * @see {@link maxSatisfying} for the highest matching version + * @stability unstable + * @category filtering + * @since 4.0.6 + */ +export const minSatisfying: { + (versions: Iterable, options?: MatchOptions): (self: SemVerRange) => Option.Option + (self: SemVerRange, versions: Iterable, options?: MatchOptions): Option.Option +} = dual( + isRangeArgs, + (self: SemVerRange, versions: Iterable, options?: MatchOptions): Option.Option => + select(self, versions, options, -1) +) diff --git a/repos/effect/packages/effect/src/Sink.ts b/repos/effect/packages/effect/src/Sink.ts index 138e26d8ee..4f60592869 100644 --- a/repos/effect/packages/effect/src/Sink.ts +++ b/repos/effect/packages/effect/src/Sink.ts @@ -1215,7 +1215,9 @@ export const take = (n: number): Sink, In, In> => { return upstream.pipe( Effect.flatMap((arr) => { if (taken.length + arr.length <= count) { - taken.push(...arr) + for (let i = 0; i < arr.length; i++) { + taken.push(arr[i]) + } if (taken.length === count) { return Cause.done() } @@ -1644,7 +1646,9 @@ export const count: Sink = reduceArray(() => 0, (s, arr) => s + */ export const collect = (): Sink, In> => reduceArray(Arr.empty, (s, arr) => { - s.push(...arr) + for (let i = 0; i < arr.length; i++) { + s.push(arr[i]) + } return s }) diff --git a/repos/effect/packages/effect/src/Stream.ts b/repos/effect/packages/effect/src/Stream.ts index e102f5f57d..1050a5b2cf 100644 --- a/repos/effect/packages/effect/src/Stream.ts +++ b/repos/effect/packages/effect/src/Stream.ts @@ -823,8 +823,13 @@ export const sync = (evaluate: LazyArg): Stream => fromChannel(Channel. * @category constructors * @since 2.0.0 */ -export const suspend = (stream: LazyArg>): Stream => - fromChannel(Channel.suspend(() => stream().channel)) +export const suspend = (stream: LazyArg>): Stream => { + const suspended = fromChannel(Channel.suspend(() => stream().channel)) + suspendedStreams.set(suspended, stream) + return suspended +} + +const suspendedStreams = new WeakMap, LazyArg>>() /** * Terminates with the specified error. @@ -1259,9 +1264,10 @@ export const fromPubSubTake = (pubsub: PubSub.PubSub>): St * * **Details** * - * The stream reads from a `ReadableStreamDefaultReader`, maps read failures - * with `onError`, and closes the reader when the stream finalizes. By default - * the reader is canceled; set `releaseLockOnEnd` to release the lock instead. + * The stream reads from a `ReadableStreamDefaultReader` and maps read failures + * with `onError`. Finalization cancels the reader and releases its lock, or + * only releases the lock when `releaseLockOnEnd` is set. A rejected cancel + * hook becomes a defect. * * **Example** (Creating a stream from a ReadableStream) * @@ -1433,6 +1439,11 @@ export interface EventListener { /** * Creates a stream from an event listener. * + * **Gotchas** + * + * Event listeners cannot backpressure the source. When a bounded `bufferSize` + * is full, additional events are dropped rather than suspending the listener. + * * **Example** (Creating a stream from an event listener) * * ```ts import.meta.vitest @@ -1800,6 +1811,11 @@ export const as: { /** * Maps both the failure and success channels of a stream. * + * **Details** + * + * Maps every typed error in the cause, preserving defects, interrupts and + * reason order. Unlike typed recovery, mapping does not convert errors to defects. + * * **Example** (Mapping both the failure and success channels of a stream) * * ```ts import.meta.vitest @@ -2981,14 +2997,30 @@ export const concat: { // a single layer instead of one layer per call const concatParts = new WeakMap, readonly [Stream, Stream]>() -// Yields the channel of each non-`concat` stream in a chain, in order +// Yields the channel of each non-`concat` stream in a chain, in order. A +// `suspend` part is evaluated when it is reached and its result joins the chain function* concatChannels(self: Stream, that: Stream) { const stack = [that, self] while (stack.length > 0) { const stream = stack.pop()! const parts = concatParts.get(stream) - if (parts === undefined) yield stream.channel - else stack.push(parts[1], parts[0]) + if (parts !== undefined) { + stack.push(parts[1], parts[0]) + continue + } + const evaluate = suspendedStreams.get(stream) + if (evaluate === undefined) { + yield stream.channel + continue + } + // Evaluated here rather than in an effect, so retrying an interrupted pull + // cannot evaluate it twice. `Channel.empty` keeps a yield point + try { + stack.push(evaluate()) + } catch (defect) { + stack.push(die(defect)) + } + yield Channel.empty } } @@ -3453,24 +3485,23 @@ export const zipWith: { left: Stream, right: Stream, f: (left: AL, right: AR) => A -): Stream => zipWithArray(left, right, zipArrays(f))) - -const zipArrays = ( - f: (left: AL, right: AR) => A -) => -( - leftArr: Arr.NonEmptyReadonlyArray, - rightArr: Arr.NonEmptyReadonlyArray -) => { - const minLength = Math.min(leftArr.length, rightArr.length) - const result: Arr.NonEmptyArray = [] as any - - for (let i = 0; i < minLength; i++) { - result.push(f(leftArr[i], rightArr[i])) - } - - return [result, leftArr.slice(minLength), rightArr.slice(minLength)] as const -} +): Stream => + suspend(() => { + // Track consumed elements with cursors instead of slicing leftovers + let leftIndex = 0 + let rightIndex = 0 + return zipWithArray(left, right, (leftArr, rightArr) => { + const length = Math.min(leftArr.length - leftIndex, rightArr.length - rightIndex) + const output = Arr.makeBy(length, (i) => f(leftArr[leftIndex + i], rightArr[rightIndex + i])) + leftIndex += length + rightIndex += length + const leftDone = leftIndex === leftArr.length + const rightDone = rightIndex === rightArr.length + if (leftDone) leftIndex = 0 + if (rightDone) rightIndex = 0 + return [output, leftDone ? [] : leftArr, rightDone ? [] : rightArr] + }) + })) /** * Zips two streams by applying a function to non-empty arrays of elements. @@ -3554,31 +3585,23 @@ export const zipWithArray: { ] }) - type State = - | { _tag: "PullBoth" } - | { _tag: "PullLeft"; rightArray: Arr.NonEmptyReadonlyArray } - | { _tag: "PullRight"; leftArray: Arr.NonEmptyReadonlyArray } - let state: State = { _tag: "PullBoth" } + let leftArray: Arr.NonEmptyReadonlyArray | undefined + let rightArray: Arr.NonEmptyReadonlyArray | undefined const pull: Effect.Effect< Arr.NonEmptyReadonlyArray, EL | ER | Cause.Done, RL | RR > = Effect.gen(function*() { - const [left, right] = state._tag === "PullBoth" - ? yield* pullBoth - : state._tag === "PullLeft" - ? [yield* pullLeft, state.rightArray] - : [state.leftArray, yield* pullRight] - const result = f(left, right) - if (Arr.isReadonlyArrayNonEmpty(result[1])) { - state = { _tag: "PullRight", leftArray: result[1] } - } else if (Arr.isReadonlyArrayNonEmpty(result[2])) { - state = { _tag: "PullLeft", rightArray: result[2] } - } else { - state = { _tag: "PullBoth" } - } - return result[0] + const [left, right] = leftArray + ? [leftArray, rightArray ?? (yield* pullRight)] + : rightArray + ? [yield* pullLeft, rightArray] + : yield* pullBoth + const [output, leftoverLeft, leftoverRight] = f(left, right) + leftArray = Arr.isReadonlyArrayNonEmpty(leftoverLeft) ? leftoverLeft : undefined + rightArray = Arr.isReadonlyArrayNonEmpty(leftoverRight) ? leftoverRight : undefined + return output }) return pull @@ -3657,15 +3680,7 @@ export const zipLeft: { ( left: Stream, right: Stream - ): Stream => - zipWithArray(left, right, (leftArr, rightArr) => { - const minLength = Math.min(leftArr.length, rightArr.length) - const output = leftArr.slice(0, minLength) as Arr.NonEmptyArray - const leftoverLeft = leftArr.slice(minLength) - const leftoverRight = rightArr.slice(minLength) - - return [output, leftoverLeft, leftoverRight] as const - }) + ): Stream => zipWith(left, right, (a) => a) ) /** @@ -3699,15 +3714,7 @@ export const zipRight: { ( left: Stream, right: Stream - ): Stream => - zipWithArray(left, right, (leftArr, rightArr) => { - const minLength = Math.min(leftArr.length, rightArr.length) - const output = rightArr.slice(0, minLength) as Arr.NonEmptyArray - const leftoverLeft = leftArr.slice(minLength) - const leftoverRight = rightArr.slice(minLength) - - return [output, leftoverLeft, leftoverRight] as const - }) + ): Stream => zipWith(left, right, (_, a) => a) ) /** @@ -4905,6 +4912,12 @@ export const catchCause: { * * Typed failures and interruptions are not caught. * + * **Gotchas** + * + * `catchDefect` recovers any cause that contains a defect, including causes + * that also contain typed errors. Typed error handlers such as `catch` skip + * those causes, so defects always take precedence. + * * **Example** (Recovering from a defect) * * ```ts import.meta.vitest @@ -5002,6 +5015,13 @@ export { /** * Switches over to the stream produced by the provided function if this one fails. * + * **Gotchas** + * + * Defects take precedence over typed errors. If the cause also contains a + * defect, the handler does not run and the failure is propagated with every + * typed error converted to a defect. An interruption alongside a typed error + * does not prevent recovery. + * * **Example** (Catching stream failures) * * ```ts import.meta.vitest @@ -5218,6 +5238,12 @@ export const tapDefect: { * stream. Non-matching failures propagate downstream, so the error type is * preserved unless the filter narrows it. * + * **Gotchas** + * + * If the cause contains a defect, the handler is skipped and every typed error + * is converted to a defect, preserving reason order and interrupts. An + * interruption alongside a typed error does not prevent recovery. + * * **Example** (Catching matching failures) * * ```ts import.meta.vitest @@ -5260,7 +5286,7 @@ export const catchIf: { ) => Stream, E2 | E3 | (A3 extends unassigned ? E : never), R | R2 | R3> ( self: Stream, - refinement: Refinement, + refinement: Refinement, EB>, f: (e: EB) => Stream, orElse?: ((e: Exclude) => Stream) | undefined ): Stream, E2 | E3 | (A3 extends unassigned ? Exclude : never), R | R2 | R3> @@ -5309,6 +5335,12 @@ export const catchIf: { * Successful filter results are passed to `f`. Failed filter results go to * `orElse` when provided; otherwise the filter failure is re-failed. * + * **Gotchas** + * + * If the cause contains a defect, the handler is skipped and every typed error + * is converted to a defect, preserving reason order and interrupts. An + * interruption alongside a typed error does not prevent recovery. + * * @see {@link catchIf} for predicate or refinement based recovery * @see {@link catchTag} for `_tag` based recovery from one tagged error * @see {@link catchTags} for `_tag` based recovery from multiple tagged errors @@ -5368,6 +5400,12 @@ export const catchFilter: { * Use when you need to handle a specific error case from a stream whose error * type is a tagged union with a readonly `_tag` field. * + * **Gotchas** + * + * If the cause contains a defect, the handler is skipped and every typed error + * is converted to a defect, preserving reason order and interrupts. An + * interruption alongside a typed error does not prevent recovery. + * * **Example** (Catching tagged failures) * * ```ts import.meta.vitest @@ -5476,6 +5514,12 @@ export const catchTag: { /** * Switches to a recovery stream based on matching `_tag` handlers. * + * **Gotchas** + * + * If the cause contains a defect, the handler is skipped and every typed error + * is converted to a defect, preserving reason order and interrupts. An + * interruption alongside a typed error does not prevent recovery. + * * **Example** (Catching tagged failures with handlers) * * ```ts import.meta.vitest @@ -5598,6 +5642,12 @@ export const catchTags: { * * The handler receives the unwrapped reason. * + * **Gotchas** + * + * If the cause contains a defect, the handler is skipped and every typed error + * is converted to a defect, preserving reason order and interrupts. An + * interruption alongside a typed error does not prevent recovery. + * * **Example** (Catching a tagged error reason) * * ```ts import.meta.vitest @@ -5733,6 +5783,12 @@ export const catchReason: { /** * Catches multiple reasons within a tagged error using an object of handlers. * + * **Gotchas** + * + * If the cause contains a defect, the handler is skipped and every typed error + * is converted to a defect, preserving reason order and interrupts. An + * interruption alongside a typed error does not prevent recovery. + * * **Example** (Catching tagged error reasons) * * ```ts import.meta.vitest @@ -5961,6 +6017,11 @@ export const unwrapReason: { /** * Transforms the errors emitted by this stream using `f`. * + * **Details** + * + * Maps every typed error in the cause, preserving defects, interrupts and + * reason order. Unlike typed recovery, mapping does not convert errors to defects. + * * **Example** (Mapping stream errors) * * ```ts import.meta.vitest @@ -6135,6 +6196,12 @@ export const orElseIfEmpty: { /** * Returns a stream that emits a fallback value when this stream fails. * + * **Gotchas** + * + * If the cause contains a defect, the handler is skipped and every typed error + * is converted to a defect, preserving reason order and interrupts. An + * interruption alongside a typed error does not prevent recovery. + * * **Example** (Recovering with a fallback value) * * ```ts import.meta.vitest @@ -6208,6 +6275,13 @@ export const orDie = (self: Stream): Stream => fr * The `log` option controls whether the failure is logged before the stream * terminates. * + * **Gotchas** + * + * Defects take precedence over typed errors. If the cause also contains a + * defect, the failure is not ignored and is propagated with every typed error + * converted to a defect. An interruption alongside a typed error does not + * prevent the error from being ignored. + * * **Example** (Ignoring stream failures) * * ```ts import.meta.vitest @@ -6325,6 +6399,12 @@ export const ignoreCause: < * The schedule is reset as soon as the first element passes through the * stream again. * + * **Gotchas** + * + * Defects take precedence over typed errors. A failure whose cause contains a + * defect is not retried and is propagated with every typed error converted to a + * defect. A typed error alongside an interruption is retried. + * * **Example** (Retrying stream failures) * * ```ts import.meta.vitest @@ -7825,7 +7905,9 @@ export const mapAccum: { for (let index = 0; index < arr.length; index++) { const [newState, values] = f(state, arr[index]) state = newState - acc.push(...values) + for (let i = 0; i < values.length; i++) { + acc.push(values[i]) + } } return [state, Arr.isArrayNonEmpty(acc) ? Arr.of(acc) : emptyArr] }, @@ -9807,10 +9889,10 @@ export const decodeText: < (self: Stream, options?: { readonly encoding?: string | undefined }): Stream => - suspend(() => { - const decoder = new TextDecoder(options?.encoding) - return map(self, (chunk) => decoder.decode(chunk, { stream: true })) - }) + self.channel.pipe( + Channel.pipeTo(Channel.decodeText(options?.encoding)), + fromChannel + ) ) /** @@ -10409,7 +10491,8 @@ export const ensuring: { /** * Provides a layer or context to the stream, removing the corresponding * service requirements. Use `options.local` to build the layer every time; by - * default, layers are shared between provide calls. + * default, a layer already built by an enclosing provide or runtime is reused; + * sibling provide calls build their own copies. * * **Example** (Providing stream requirements) * @@ -11527,7 +11610,7 @@ export const mkArrayBuffer = (self: Stream): Effect.Effe * @category destructors * @since 4.0.0 */ -export const mkUint8Array = (self: Stream): Effect.Effect => +export const mkUint8Array = (self: Stream): Effect.Effect, E, R> => Channel.mkUint8Array(self.channel) /** @@ -11597,15 +11680,18 @@ export const toReadableStreamWith: { return new ReadableStream({ start(controller) { fiber = Effect.runFork(Effect.provideContext( - runForEachArray(self, (chunk) => - latch.whenOpen(Effect.sync(() => { - latch.closeUnsafe() - for (let i = 0; i < chunk.length; i++) { - controller.enqueue(chunk[i]) - } - currentResolve!() - currentResolve = undefined - }))), + Channel.runForEach( + Channel.transformPull(self.channel, (pull) => Effect.succeed(latch.whenOpen(pull))), + (chunk) => + Effect.sync(() => { + latch.closeUnsafe() + for (let i = 0; i < chunk.length; i++) { + controller.enqueue(chunk[i]) + } + currentResolve!() + currentResolve = undefined + }) + ), context )) fiber.addObserver((exit) => { diff --git a/repos/effect/packages/effect/src/String.ts b/repos/effect/packages/effect/src/String.ts index b4bb0117fd..531d23271c 100644 --- a/repos/effect/packages/effect/src/String.ts +++ b/repos/effect/packages/effect/src/String.ts @@ -200,7 +200,7 @@ export const toUpperCase = (self: S): Uppercase => self.toU export const toLowerCase = (self: T): Lowercase => self.toLowerCase() as Lowercase /** - * Capitalizes the first character of a string. + * Capitalizes the first character (Unicode code point) of a string. * * **Example** (Capitalizing a string) * @@ -209,6 +209,7 @@ export const toLowerCase = (self: T): Lowercase => self.toL * * pipe("abc", String.capitalize) // => "Abc" * String.capitalize("hello") // => "Hello" + * String.capitalize("𐐨foo") // => "𐐀foo" * ``` * * @stability stable @@ -216,13 +217,12 @@ export const toLowerCase = (self: T): Lowercase => self.toL * @since 2.0.0 */ export const capitalize = (self: T): Capitalize => { - if (self.length === 0) return self as Capitalize - - return (toUpperCase(self[0]) + self.slice(1)) as Capitalize + const [first = ""] = self + return (toUpperCase(first) + self.slice(first.length)) as Capitalize } /** - * Uncapitalizes the first character of a string. + * Uncapitalizes the first character (Unicode code point) of a string. * * **Example** (Uncapitalizing a string) * @@ -231,6 +231,7 @@ export const capitalize = (self: T): Capitalize => { * * pipe("ABC", String.uncapitalize) // => "aBC" * String.uncapitalize("Hello") // => "hello" + * String.uncapitalize("𐐀FOO") // => "𐐨FOO" * ``` * * @stability stable @@ -238,9 +239,8 @@ export const capitalize = (self: T): Capitalize => { * @since 2.0.0 */ export const uncapitalize = (self: T): Uncapitalize => { - if (self.length === 0) return self as Uncapitalize - - return (toLowerCase(self[0]) + self.slice(1)) as Uncapitalize + const [first = ""] = self + return (toLowerCase(first) + self.slice(first.length)) as Uncapitalize } /** diff --git a/repos/effect/packages/effect/src/TextDiff.ts b/repos/effect/packages/effect/src/TextDiff.ts new file mode 100644 index 0000000000..3c651c964a --- /dev/null +++ b/repos/effect/packages/effect/src/TextDiff.ts @@ -0,0 +1,766 @@ +/** + * The `TextDiff` module computes edit scripts for arrays and text, applies and + * inverts them, and formats line differences as unified diffs. + * + * Scripts are computed with Myers' O(ND) algorithm in linear space, after + * trimming common prefixes and suffixes. Scripts are minimal unless the + * optional `maxEdits` search cap is exceeded. Word diffs, semantic cleanup + * and collaborative editing protocols are out of scope. + * + * @stability experimental + * @since 4.0.6 + */ +import * as Data from "./Data.ts" +import * as Equal from "./Equal.ts" +import { hasProperty } from "./Predicate.ts" +import * as Result from "./Result.ts" + +const TextDiffErrorTypeId = "~effect/TextDiff/TextDiffError" + +/** + * One run of an edit script. + * + * **Details** + * + * `Equal` runs carry items kept from `before`, `Delete` runs carry items + * removed from `before`, and `Insert` runs carry items added from `after`. + * Scripts produced by this module never contain empty runs or two adjacent + * runs with the same tag, and within each change the `Delete` run comes before + * the `Insert` run. + * + * @see {@link diff} to compute an edit script + * @stability experimental + * @category models + * @since 4.0.6 + */ +export type Edit = EditEqual | EditInsert | EditDelete + +/** + * Run of items present in both sequences. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export interface EditEqual { + readonly _tag: "Equal" + readonly items: ReadonlyArray +} + +/** + * Run of items present only in the `after` sequence. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export interface EditInsert { + readonly _tag: "Insert" + readonly items: ReadonlyArray +} + +/** + * Run of items present only in the `before` sequence. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export interface EditDelete { + readonly _tag: "Delete" + readonly items: ReadonlyArray +} + +/** + * Error returned when an edit script does not fit the input it is applied to. + * + * **Example** (Rejecting a script made for other input) + * + * ```ts import.meta.vitest + * import { Result, TextDiff } from "effect" + * + * const edits = TextDiff.diff([1, 2], [1]) + * + * Result.mapError(TextDiff.apply([1], edits), TextDiff.isTextDiffError) // => Result.fail(true) + * ``` + * + * @see {@link apply} and {@link applyText} for the operations that return it + * @see {@link isTextDiffError} for narrowing unknown values + * @stability experimental + * @category errors + * @since 4.0.6 + */ +export class TextDiffError extends Data.TaggedError("TextDiffError")<{ + readonly message: string +}> { + readonly [TextDiffErrorTypeId]: typeof TextDiffErrorTypeId = TextDiffErrorTypeId +} + +/** + * Checks whether a value is a `TextDiffError`. + * + * @stability experimental + * @category guards + * @since 4.0.6 + */ +export const isTextDiffError = (u: unknown): u is TextDiffError => hasProperty(u, TextDiffErrorTypeId) + +/** + * Creates a function that computes a minimal edit script between two arrays + * using a custom equivalence. + * + * **When to use** + * + * Use when items should match by key or by a structural comparison other than + * `Equal.equals`. + * + * **Details** + * + * Unless the `maxEdits` cap is exceeded, the script has the + * fewest possible inserted plus deleted items. `Equal` runs carry the items + * from `before`, not necessarily the exact values from `after`. + * + * **Gotchas** + * + * Without `maxEdits`, work grows with the product of the input size and the + * number of differences, which is quadratic for unrelated inputs. Set + * `maxEdits` for large or untrusted inputs. + * + * **Example** (Matching items by id) + * + * ```ts import.meta.vitest + * import { TextDiff } from "effect" + * + * const diffById = TextDiff.diffWith<{ readonly id: number }>((a, b) => a.id === b.id) + * + * diffById([{ id: 1 }, { id: 2 }], [{ id: 2 }]) // => [{ _tag: "Delete", items: [{ id: 1 }] }, { _tag: "Equal", items: [{ id: 2 }] }] + * ``` + * + * @see {@link diff} for the `Equal.equals` variant + * @stability experimental + * @category constructors + * @since 4.0.6 + */ +export const diffWith = (isEquivalent: (self: A, that: A) => boolean) => +( + before: ReadonlyArray, + after: ReadonlyArray, + options?: { + /** + * Largest number of inserted plus deleted items searched for in each + * changed region. Common prefixes, suffixes and matching runs are free. + * When a region needs more edits, the search splits it at the furthest + * point it reached and continues on the rest, and a part that still cannot + * be matched becomes a deletion followed by an insertion. The script stays + * valid, and is minimal whenever the whole diff needs at most `maxEdits` + * edits. For a finite cap, work is O((N + M) * (maxEdits + 1)). + * + * @default Infinity + */ + readonly maxEdits?: number | undefined + } +): Array> => editScript(before, after, (i, j) => isEquivalent(before[i], after[j]), options?.maxEdits) + +/** + * Computes a minimal edit script between two arrays, comparing items with + * `Equal.equals`. + * + * **Details** + * + * Unless the `maxEdits` cap is exceeded, the script has the + * fewest possible inserted plus deleted items. The choice is deterministic, + * and each change lists its deletions before its insertions. + * + * **Gotchas** + * + * Without `maxEdits`, work is quadratic for unrelated inputs. Set `maxEdits` + * when diffing large or untrusted arrays. + * + * **Example** (Diffing two arrays) + * + * ```ts import.meta.vitest + * import { TextDiff } from "effect" + * + * TextDiff.diff([1, 2, 3, 4], [1, 3, 4, 5]) // => [{ _tag: "Equal", items: [1] }, { _tag: "Delete", items: [2] }, { _tag: "Equal", items: [3, 4] }, { _tag: "Insert", items: [5] }] + * ``` + * + * @see {@link diffWith} to compare items with a custom equivalence + * @see {@link apply} to replay a script + * @stability experimental + * @category constructors + * @since 4.0.6 + */ +export const diff = ( + before: ReadonlyArray, + after: ReadonlyArray, + options?: { + /** + * Largest number of inserted plus deleted items searched for in each + * changed region. Common prefixes, suffixes and matching runs are free. + * When a region needs more edits, the search splits it at the furthest + * point it reached and continues on the rest, and a part that still cannot + * be matched becomes a deletion followed by an insertion. The script stays + * valid, and is minimal whenever the whole diff needs at most `maxEdits` + * edits. For a finite cap, work is O((N + M) * (maxEdits + 1)). + * + * @default Infinity + */ + readonly maxEdits?: number | undefined + } +): Array> => diffWith(Equal.equals)(before, after, options) + +/** + * Computes a minimal line-level edit script between two strings. + * + * **Details** + * + * Each line keeps its `\n` terminator, so `\n` versus `\r\n` endings and a + * missing final newline are reported as changes. Joining the items of a run + * gives the exact text it covers. + * + * **Gotchas** + * + * Without `maxEdits`, work is quadratic for unrelated inputs. Set `maxEdits` + * when diffing large or untrusted text. + * + * **Example** (Diffing lines) + * + * ```ts import.meta.vitest + * import { TextDiff } from "effect" + * + * TextDiff.lines("a\nb\n", "a\nc\n") // => [{ _tag: "Equal", items: ["a\n"] }, { _tag: "Delete", items: ["b\n"] }, { _tag: "Insert", items: ["c\n"] }] + * ``` + * + * @see {@link codePoints} for a code point diff + * @see {@link unified} to format a line diff + * @stability experimental + * @category constructors + * @since 4.0.6 + */ +export const lines = (before: string, after: string, options?: { + /** + * Largest number of inserted plus deleted items searched for in each + * changed region. Common prefixes, suffixes and matching runs are free. + * When a region needs more edits, the search splits it at the furthest + * point it reached and continues on the rest, and a part that still cannot + * be matched becomes a deletion followed by an insertion. The script stays + * valid, and is minimal whenever the whole diff needs at most `maxEdits` + * edits. For a finite cap, work is O((N + M) * (maxEdits + 1)). + * + * @default Infinity + */ + readonly maxEdits?: number | undefined +}): Array> => diffStrings(splitLines(before), splitLines(after), options?.maxEdits) + +/** + * Computes a minimal code point edit script between two strings. + * + * **Details** + * + * Items are Unicode code points, so a surrogate pair is never split between + * runs. Grapheme clusters, such as an emoji with modifiers or a letter with + * combining marks, can still be split. The UTF-16 length of a run is the sum + * of its items' lengths. + * + * **Gotchas** + * + * Without `maxEdits`, work is quadratic for unrelated inputs. Set `maxEdits` + * when diffing large or untrusted text. + * + * **Example** (Diffing code points) + * + * ```ts import.meta.vitest + * import { TextDiff } from "effect" + * + * TextDiff.codePoints("cat", "cut") // => [{ _tag: "Equal", items: ["c"] }, { _tag: "Delete", items: ["a"] }, { _tag: "Insert", items: ["u"] }, { _tag: "Equal", items: ["t"] }] + * ``` + * + * @see {@link lines} for a line diff + * @see {@link applyText} to replay a script on a string + * @stability experimental + * @category constructors + * @since 4.0.6 + */ +export const codePoints = (before: string, after: string, options?: { + /** + * Largest number of inserted plus deleted items searched for in each + * changed region. Common prefixes, suffixes and matching runs are free. + * When a region needs more edits, the search splits it at the furthest + * point it reached and continues on the rest, and a part that still cannot + * be matched becomes a deletion followed by an insertion. The script stays + * valid, and is minimal whenever the whole diff needs at most `maxEdits` + * edits. For a finite cap, work is O((N + M) * (maxEdits + 1)). + * + * @default Infinity + */ + readonly maxEdits?: number | undefined +}): Array> => diffStrings(Array.from(before), Array.from(after), options?.maxEdits) + +/** + * Applies an edit script to an array. + * + * **Details** + * + * `Equal` and `Delete` runs consume items from `before` by count, `Equal` runs + * copy them to the result, and `Insert` runs add their items. Run contents are + * not compared with `before`, since under a custom equivalence `Equal` items + * can differ from the input. + * + * **Gotchas** + * + * Fails with a {@link TextDiffError} when the runs do not consume exactly + * `before.length` items. + * + * **Example** (Replaying a script) + * + * ```ts import.meta.vitest + * import { Result, TextDiff } from "effect" + * + * const edits = TextDiff.diff([1, 2, 3], [1, 3, 4]) + * + * TextDiff.apply([1, 2, 3], edits) // => Result.succeed([1, 3, 4]) + * ``` + * + * @see {@link applyText} for the string variant + * @see {@link invert} to apply a script in reverse + * @stability experimental + * @category combinators + * @since 4.0.6 + */ +export const apply = ( + before: ReadonlyArray, + edits: ReadonlyArray> +): Result.Result, TextDiffError> => { + const out: Array = [] + let index = 0 + for (const edit of edits) { + if (edit._tag === "Insert") { + for (const item of edit.items) out.push(item) + continue + } + const end = index + edit.items.length + if (end > before.length) return fail("Edit script consumes more items than the input contains") + if (edit._tag === "Equal") { + for (let i = index; i < end; i++) out.push(before[i]) + } + index = end + } + if (index !== before.length) return fail("Edit script consumes fewer items than the input contains") + return Result.succeed(out) +} + +/** + * Applies a text edit script to a string. + * + * **Details** + * + * Works with scripts from both {@link lines} and {@link codePoints}. The + * joined items of each `Equal` and `Delete` run must match `before` at the + * current position, so a script made for different text is rejected rather + * than applied. + * + * **Gotchas** + * + * Fails with a {@link TextDiffError} when a run does not match `before`, or + * when the runs do not cover all of it. + * + * **Example** (Replaying a code point diff) + * + * ```ts import.meta.vitest + * import { Result, TextDiff } from "effect" + * + * const edits = TextDiff.codePoints("cat", "cut") + * + * TextDiff.applyText("cat", edits) // => Result.succeed("cut") + * ``` + * + * @see {@link apply} for the array variant + * @stability experimental + * @category combinators + * @since 4.0.6 + */ +export const applyText = ( + before: string, + edits: ReadonlyArray> +): Result.Result => { + let out = "" + let index = 0 + for (const edit of edits) { + const text = edit.items.join("") + if (edit._tag === "Insert") { + out += text + continue + } + if (!before.startsWith(text, index)) return fail(`Edit script does not match the input at index ${index}`) + if (edit._tag === "Equal") out += text + index += text.length + } + if (index !== before.length) return fail("Edit script covers less text than the input contains") + return Result.succeed(out) +} + +/** + * Inverts an edit script, so it turns `after` back into `before`. + * + * **Details** + * + * Swaps the items of each change's `Delete` and `Insert` runs, keeping + * deletions before insertions. `Equal` runs are unchanged. + * + * **Example** (Undoing a change) + * + * ```ts import.meta.vitest + * import { Result, TextDiff } from "effect" + * + * const edits = TextDiff.diff([1, 2], [1, 3]) + * + * TextDiff.apply([1, 3], TextDiff.invert(edits)) // => Result.succeed([1, 2]) + * ``` + * + * @stability experimental + * @category combinators + * @since 4.0.6 + */ +export const invert = (edits: ReadonlyArray>): Array> => { + const out: Array> = [] + let deleted: ReadonlyArray = [] + let inserted: ReadonlyArray = [] + const flush = () => { + if (inserted.length > 0) out.push({ _tag: "Delete", items: inserted }) + if (deleted.length > 0) out.push({ _tag: "Insert", items: deleted }) + deleted = [] + inserted = [] + } + for (const edit of edits) { + if (edit._tag === "Delete") { + deleted = deleted.concat(edit.items) + } else if (edit._tag === "Insert") { + inserted = inserted.concat(edit.items) + } else { + flush() + out.push(edit) + } + } + flush() + return out +} + +/** + * Formats a line edit script as a unified diff. + * + * **Details** + * + * The script must come from {@link lines}, or otherwise hold one line per + * item. The output follows `diff -u`: hunks whose context would touch or + * overlap are merged, and a final line without a newline is followed by + * `\ No newline at end of file`. When `fromFile` or `toFile` is set, both + * header lines are emitted and an unset name falls back to the other. A script + * without changes produces an empty string. + * + * **Gotchas** + * + * Among equally small diffs, hunks can differ from those of GNU diff or Git, + * which apply extra heuristics. + * + * **Example** (Formatting a unified diff) + * + * ```ts import.meta.vitest + * import { TextDiff } from "effect" + * + * TextDiff.unified(TextDiff.lines("a\nb\n", "a\nc\n")) // => "@@ -1,2 +1,2 @@\n a\n-b\n+c\n" + * ``` + * + * @see {@link lines} to compute a line diff + * @stability experimental + * @category formatting + * @since 4.0.6 + */ +export const unified = (edits: ReadonlyArray>, options?: { + /** + * Number of unchanged lines shown around each change. + * + * @default 3 + */ + readonly context?: number | undefined + /** + * Name shown in the `---` header line. + */ + readonly fromFile?: string | undefined + /** + * Name shown in the `+++` header line. + */ + readonly toFile?: string | undefined +}): string => { + const a: Array = [] + const b: Array = [] + // Group consecutive deletions and insertions between Equal runs. + const changes: Array = [] + let change: Change | undefined + for (const edit of edits) { + if (edit._tag === "Equal") { + change = undefined + for (const line of edit.items) { + a.push(line) + b.push(line) + } + continue + } + if (edit.items.length === 0) continue + if (change === undefined) { + change = { a0: a.length, a1: a.length, b0: b.length, b1: b.length } + changes.push(change) + } + if (edit._tag === "Delete") { + for (const line of edit.items) a.push(line) + change.a1 = a.length + } else { + for (const line of edit.items) b.push(line) + change.b1 = b.length + } + } + if (changes.length === 0) return "" + + const context = Math.max(0, Math.floor(options?.context ?? 3)) + let out = "" + if (options?.fromFile !== undefined || options?.toFile !== undefined) { + out += `--- ${options.fromFile ?? options.toFile}\n+++ ${options.toFile ?? options.fromFile}\n` + } + let start = 0 + while (start < changes.length) { + let end = start + 1 + while (end < changes.length && changes[end].a0 - changes[end - 1].a1 <= 2 * context) end++ + const first = changes[start] + const last = changes[end - 1] + const a0 = Math.max(0, first.a0 - context) + const b0 = first.b0 - (first.a0 - a0) + const a1 = Math.min(a.length, last.a1 + context) + const b1 = last.b1 + (a1 - last.a1) + out += `@@ -${range(a0, a1 - a0)} +${range(b0, b1 - b0)} @@\n` + let line = a0 + for (let c = start; c < end; c++) { + const change = changes[c] + for (; line < change.a0; line++) out += formatLine(" ", a[line]) + for (let k = change.a0; k < change.a1; k++) out += formatLine("-", a[k]) + for (let k = change.b0; k < change.b1; k++) out += formatLine("+", b[k]) + line = change.a1 + } + for (; line < a1; line++) out += formatLine(" ", a[line]) + start = end + } + return out +} + +// ----------------------------------------------------------------------------- +// internal +// ----------------------------------------------------------------------------- + +const fail = (message: string) => Result.fail(new TextDiffError({ message })) + +interface Block { + readonly a: number + readonly b: number + readonly length: number +} + +interface Change { + a0: number + a1: number + b0: number + b1: number +} + +interface Region { + readonly aLo: number + readonly aHi: number + readonly bLo: number + readonly bHi: number +} + +// A diagonal run from (x, y) to (u, v) +interface Snake { + readonly x: number + readonly y: number + readonly u: number + readonly v: number +} + +const splitLines = (text: string): Array => text.match(/[^\n]*\n|[^\n]+$/g) ?? [] + +const range = (start: number, count: number): string => + count === 0 ? `${start},0` : count === 1 ? `${start + 1}` : `${start + 1},${count}` + +const formatLine = (prefix: string, line: string): string => + line.endsWith("\n") ? prefix + line : `${prefix}${line}\n\\ No newline at end of file\n` + +// Interns the tokens so the search compares integers instead of strings +const diffStrings = (a: ReadonlyArray, b: ReadonlyArray, maxEdits?: number) => { + const ids = new Map() + const intern = (token: string) => { + let id = ids.get(token) + if (id === undefined) { + id = ids.size + ids.set(token, id) + } + return id + } + const ai = Int32Array.from(a, intern) + const bi = Int32Array.from(b, intern) + return editScript(a, b, (i, j) => ai[i] === bi[j], maxEdits) +} + +const editScript = ( + a: ReadonlyArray, + b: ReadonlyArray, + equals: (i: number, j: number) => boolean, + maxEdits?: number +): Array> => { + const blocks = matchingBlocks(a.length, b.length, equals, maxEdits) + // A final empty block flushes the trailing changes + blocks.push({ a: a.length, b: b.length, length: 0 }) + const out: Array> = [] + let i = 0 + let j = 0 + for (const block of blocks) { + if (block.a > i) out.push({ _tag: "Delete", items: a.slice(i, block.a) }) + if (block.b > j) out.push({ _tag: "Insert", items: b.slice(j, block.b) }) + if (block.length > 0) out.push({ _tag: "Equal", items: a.slice(block.a, block.a + block.length) }) + i = block.a + block.length + j = block.b + block.length + } + return out +} + +/** + * Returns ordered, merged matching runs for sequences of lengths n and m, + * using the index comparator. Linear-space Myers search splits each region + * at the middle snake of a minimal path, or at the furthest point reached + * when the search cap is exceeded. + */ +const matchingBlocks = ( + n: number, + m: number, + equals: (i: number, j: number) => boolean, + maxEdits = Infinity +): Array => { + const blocks: Array = [] + const push = (a: number, b: number, length: number) => { + if (length === 0) return + const last = blocks[blocks.length - 1] + if (last !== undefined && last.a + last.length === a && last.b + last.length === b) { + blocks[blocks.length - 1] = { a: last.a, b: last.b, length: last.length + length } + } else { + blocks.push({ a, b, length }) + } + } + + // Forward and reverse furthest-reaching x per diagonal, offset by n + m + 1 + const offset = n + m + 1 + const vf = new Int32Array(2 * offset + 1) + const vb = new Int32Array(2 * offset + 1) + // The searches meet after half the edits of a region, so a region needing at + // most `maxEdits` edits is solved within `maxD` steps of each search + const maxD = Math.ceil(Math.max(0, maxEdits) / 2) + + // Returns the middle snake of a minimal path in absolute coordinates, or an + // empty snake at the furthest point reached when the region needs more than + // `maxEdits` edits, or undefined when no point inside the region was reached + const findSplit = (aLo: number, aHi: number, bLo: number, bHi: number): Snake | undefined => { + const N = aHi - aLo + const M = bHi - bLo + const delta = N - M + const odd = (delta & 1) === 1 + const limit = Math.min((N + M + 1) >> 1, maxD) + vf[offset + 1] = 0 + vb[offset + 1] = 0 + for (let d = 0; d <= limit; d++) { + for (let k = -d; k <= d; k += 2) { + let x = k === -d || (k !== d && vf[offset + k - 1] < vf[offset + k + 1]) + ? vf[offset + k + 1] + : vf[offset + k - 1] + 1 + let y = x - k + const x0 = x + const y0 = y + while (x < N && y < M && equals(aLo + x, bLo + y)) { + x++ + y++ + } + vf[offset + k] = x + if (odd && k >= delta - (d - 1) && k <= delta + (d - 1) && x + vb[offset + delta - k] >= N) { + return { x: aLo + x0, y: bLo + y0, u: aLo + x, v: bLo + y } + } + } + for (let k = -d; k <= d; k += 2) { + let x = k === -d || (k !== d && vb[offset + k - 1] < vb[offset + k + 1]) + ? vb[offset + k + 1] + : vb[offset + k - 1] + 1 + let y = x - k + const x0 = x + const y0 = y + while (x < N && y < M && equals(aHi - x - 1, bHi - y - 1)) { + x++ + y++ + } + vb[offset + k] = x + if (!odd && k >= delta - d && k <= delta + d && x + vf[offset + delta - k] >= N) { + return { x: aHi - x, y: bHi - y, u: aHi - x0, v: bHi - y0 } + } + } + } + + // Over the cap: split at the reached point that covers the most items. It + // must lie strictly inside the region so that both parts shrink + let best = 0 + let x = 0 + let y = 0 + for (let k = -limit; k <= limit; k += 2) { + const xf = vf[offset + k] + const yf = xf - k + if (xf <= N && yf >= 0 && yf <= M && xf + yf > best) { + best = xf + yf + x = aLo + xf + y = bLo + yf + } + const xb = vb[offset + k] + const yb = xb - k + if (xb <= N && yb >= 0 && yb <= M && xb + yb > best) { + best = xb + yb + x = aHi - xb + y = bHi - yb + } + } + return best > 0 && best < N + M ? { x, y, u: x, v: y } : undefined + } + + // Regions are processed in order from an explicit stack, since splits over + // the cap can nest deeper than the call stack allows + const stack: Array = [{ aLo: 0, aHi: n, bLo: 0, bHi: m }] + while (stack.length > 0) { + const task = stack.pop()! + if ("length" in task) { + push(task.a, task.b, task.length) + continue + } + let { aHi, aLo, bHi, bLo } = task + let prefix = 0 + while (aLo + prefix < aHi && bLo + prefix < bHi && equals(aLo + prefix, bLo + prefix)) prefix++ + push(aLo, bLo, prefix) + aLo += prefix + bLo += prefix + let suffix = 0 + while (aLo < aHi - suffix && bLo < bHi - suffix && equals(aHi - suffix - 1, bHi - suffix - 1)) suffix++ + aHi -= suffix + bHi -= suffix + stack.push({ a: aHi, b: bHi, length: suffix }) + // With a common prefix and suffix removed, a non-empty side on both ends + // means at least two edits, so both halves below are strictly smaller + const snake = aLo < aHi && bLo < bHi ? findSplit(aLo, aHi, bLo, bHi) : undefined + if (snake !== undefined) { + stack.push( + { aLo: snake.u, aHi, bLo: snake.v, bHi }, + { a: snake.x, b: snake.y, length: snake.u - snake.x }, + { aLo, aHi: snake.x, bLo, bHi: snake.y } + ) + } + } + return blocks +} diff --git a/repos/effect/packages/effect/src/ai/AiError.ts b/repos/effect/packages/effect/src/ai/AiError.ts index 8fbe890871..0a84f8a8d7 100644 --- a/repos/effect/packages/effect/src/ai/AiError.ts +++ b/repos/effect/packages/effect/src/ai/AiError.ts @@ -341,6 +341,15 @@ export interface ContentPolicyErrorMetadata extends ProviderMetadata {} */ export interface InvalidRequestErrorMetadata extends ProviderMetadata {} +/** + * Provider-specific metadata attached to `InvalidResponseError`. + * + * @stability unstable + * @category configuration + * @since 4.1.0 + */ +export interface InvalidResponseErrorMetadata extends ProviderMetadata {} + /** * Provider-specific metadata attached to `InternalProviderError`. * @@ -835,7 +844,59 @@ export class InternalProviderError extends Schema.Error( } /** - * Error indicating failure to parse or validate LLM output. + * Error indicating the provider response does not conform to the expected format. + * + * **Details** + * + * Invalid response errors are not retryable because retrying does not resolve an + * incompatible provider response format. Optional HTTP context can retain request + * and response details for diagnosis. + * + * @see {@link InvalidOutputError} for retryable model-generated output failures + * @stability unstable + * @category errors + * @since 4.1.0 + */ +export class InvalidResponseError extends Schema.Error( + "effect/ai/AiError/InvalidResponseError" +)({ + _tag: Schema.tag("InvalidResponseError"), + description: Schema.String, + metadata: providerMetadataWithDefaults(), + http: Schema.optional(HttpContext) +}) { + /** + * Marks `InvalidResponseError` as a semantic AI error reason for runtime guards. + * + * @since 4.1.0 + */ + readonly [ReasonTypeId] = ReasonTypeId + + /** + * Invalid response errors are not retryable because the response format is incompatible. + * + * @since 4.1.0 + */ + get isRetryable(): boolean { + return false + } + + /** + * Creates an InvalidResponseError from a provider response schema error. + * + * @since 4.1.0 + */ + static fromSchemaError(error: Schema.SchemaError): InvalidResponseError { + return new InvalidResponseError({ description: error.message }) + } + + override get message(): string { + return `Invalid response: ${this.description}` + } +} + +/** + * Error indicating failure to parse or validate model-generated output. * * **Details** * @@ -903,9 +964,7 @@ export class InvalidOutputError extends Schema.Error( * @since 4.0.0 */ static fromSchemaError(error: Schema.SchemaError): InvalidOutputError { - return new InvalidOutputError({ - description: error.message - }) + return new InvalidOutputError({ description: error.message }) } override get message(): string { @@ -1520,6 +1579,7 @@ export type AiErrorReason = | AuthenticationError | ContentPolicyError | InvalidRequestError + | InvalidResponseError | InternalProviderError | NetworkError | InvalidOutputError @@ -1557,6 +1617,7 @@ export const AiErrorReason: Schema.Union<[ typeof AuthenticationError, typeof ContentPolicyError, typeof InvalidRequestError, + typeof InvalidResponseError, typeof InternalProviderError, typeof NetworkError, typeof InvalidOutputError, @@ -1576,6 +1637,7 @@ export const AiErrorReason: Schema.Union<[ AuthenticationError, ContentPolicyError, InvalidRequestError, + InvalidResponseError, InternalProviderError, NetworkError, InvalidOutputError, diff --git a/repos/effect/packages/effect/src/ai/Chat.ts b/repos/effect/packages/effect/src/ai/Chat.ts index a68b1e37f4..ce0d2cc209 100644 --- a/repos/effect/packages/effect/src/ai/Chat.ts +++ b/repos/effect/packages/effect/src/ai/Chat.ts @@ -18,8 +18,7 @@ import * as Duration from "../Duration.ts" import * as Effect from "../Effect.ts" import * as Layer from "../Layer.ts" import * as Option from "../Option.ts" -import type { PersistenceError } from "../persistence/Persistence.ts" -import { BackingPersistence } from "../persistence/Persistence.ts" +import { BackingPersistence, PersistenceError } from "../persistence/Persistence.ts" import * as Predicate from "../Predicate.ts" import * as Ref from "../Ref.ts" import * as Schema from "../Schema.ts" @@ -926,13 +925,19 @@ export const makePersisted = Effect.fnUntraced(function*(options: { return yield* new ChatNotFoundError({ chatId }) } // Decode the encoded previous history - const history = yield* decodeHistory(previousHistory) + const history = yield* decodeHistory(previousHistory).pipe( + Effect.mapError((cause) => + new PersistenceError({ + message: `Failed to decode persisted history for chat "${chatId}"`, + cause + }) + ) + ) // Hydrate the chat history yield* Ref.set(chat.history, history) // Convert the chat to a persisted chat return yield* toPersisted(chatId, chat, ttl) - }, - Effect.catchTag("SchemaError", Effect.die) + } ) const get = Effect.fnUntraced( diff --git a/repos/effect/packages/effect/src/ai/Decision.ts b/repos/effect/packages/effect/src/ai/Decision.ts index 1775bd80bf..ba1032bfd4 100644 --- a/repos/effect/packages/effect/src/ai/Decision.ts +++ b/repos/effect/packages/effect/src/ai/Decision.ts @@ -1,13 +1,15 @@ /** * Defines decisions that a `DecisionModel` answers over a single input. * Pair an input schema with named classification, rating, or probability - * decisions using `make`, then answer them with `DecisionModel.decide`. + * decisions using `make`, then answer them with `DecisionModel.decide`, or + * with `DecisionModel.decideSettled` to keep valid answers when others fail. * * @see {@link make} for building a definition from an input schema and decisions * * @stability unstable * @since 4.0.0 */ +import type * as Result from "../Result.ts" import type * as Schema from "../Schema.ts" /** @@ -175,6 +177,52 @@ export type Answers> = { readonly [K in keyof Decisions]: Answer } +/** + * Reason and description of the first failed answer validation check. + * + * **Details** + * + * - `"Missing"`: the provider returned no answer for the decision. + * - `"Refused"`: the provider declined to answer the decision. + * - `"Invalid"`: the answer has an invalid kind, value, confidence, or distribution. + * - `"NoDistribution"`: a classify or rate answer is otherwise valid but has + * no probabilities. + * + * Except for refusals, `description` is the message `DecisionModel.decide` + * fails with for the same answer. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export interface Unanswered { + readonly _tag: "Unanswered" + readonly reason: "Missing" | "Refused" | "NoDistribution" | "Invalid" + readonly description: string +} + +/** + * Validated answer or the reason the decision is unanswered. + * + * @see {@link Unanswered} for the failure reasons + * + * @stability unstable + * @category utility types + * @since 4.0.6 + */ +export type Outcome = Result.Result, Unanswered> + +/** + * Settled results keyed by decision name. + * + * @stability unstable + * @category utility types + * @since 4.0.6 + */ +export type Outcomes> = { + readonly [K in keyof Decisions]: Outcome +} + /** * Input schema and named decisions that run together in one provider call. * diff --git a/repos/effect/packages/effect/src/ai/DecisionModel.ts b/repos/effect/packages/effect/src/ai/DecisionModel.ts index 954dc7b934..22e78a100d 100644 --- a/repos/effect/packages/effect/src/ai/DecisionModel.ts +++ b/repos/effect/packages/effect/src/ai/DecisionModel.ts @@ -2,15 +2,19 @@ * Defines the provider-neutral service for structured decisions. * `decide` encodes one input as JSON, sends its named decisions in one provider * call, and validates the answers. Failures are reported as `AiError` values. + * `decideSettled` makes the same call but reports each decision separately, so + * valid answers survive when others are missing, refused, or invalid. * @stability unstable * @since 4.0.0 */ import * as Context from "../Context.ts" import * as Effect from "../Effect.ts" import * as Predicate from "../Predicate.ts" +import * as Result from "../Result.ts" import * as Schema from "../Schema.ts" import * as AiError from "./AiError.ts" import type * as Decision from "./Decision.ts" +import type * as Response from "./Response.ts" /** * Service key for answering decisions about an input. Create implementations @@ -87,7 +91,17 @@ export interface Image { } /** - * Answers for every decision in a definition together with usage metadata. + * Provider-specific response metadata keyed by provider name. Provider + * packages augment this interface. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export interface DecideResponseMetadata extends Response.ProviderMetadata {} + +/** + * Answers for every decision in a definition with usage and response metadata. * * @see {@link DecisionUsage} for token usage metadata * @@ -98,6 +112,43 @@ export interface Image { export interface DecideResponse> { readonly answers: Decision.Answers readonly usage: DecisionUsage + /** + * Response id reported by the provider. + * + * @since 4.0.6 + */ + readonly id?: string | undefined + /** + * Model that answered. Can differ from the requested model when the + * provider resolves an alias. + * + * @since 4.0.6 + */ + readonly modelId?: string | undefined + /** + * Provider-specific metadata, such as reported cost. + * + * @since 4.0.6 + */ + readonly metadata: DecideResponseMetadata +} + +/** + * Per-decision outcomes, token usage, and response metadata for the whole + * request. + * + * @see {@link decideSettled} for producing this response + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export interface DecideSettledResponse> { + readonly outcomes: Decision.Outcomes + readonly usage: DecisionUsage + readonly id?: string | undefined + readonly modelId?: string | undefined + readonly metadata: DecideResponseMetadata } /** @@ -117,7 +168,7 @@ export interface ProviderOptions { } /** - * Provider answer for a classify decision. + * Provider classify answer. Omit `probabilities` when no distribution is available. * * @stability unstable * @category models @@ -126,14 +177,13 @@ export interface ProviderOptions { export interface ProviderClassifyAnswer { readonly _tag: "Classify" readonly label: string - readonly probabilities: Readonly> + readonly probabilities?: Readonly> | undefined readonly confidence?: number | undefined } /** - * Provider answer for a rate decision. - * The core derives the label from the highest probability, choosing the - * first criteria entry on ties. + * Provider rate answer. The highest probability determines the label, with + * ties going to the first criterion. Omit `probabilities` when unavailable. * * @stability unstable * @category models @@ -142,7 +192,7 @@ export interface ProviderClassifyAnswer { export interface ProviderRateAnswer { readonly _tag: "Rate" readonly rating: number - readonly probabilities: Readonly> + readonly probabilities?: Readonly> | undefined readonly confidence?: number | undefined } @@ -158,6 +208,17 @@ export interface ProviderProbabilityAnswer { readonly probability: number } +/** + * Provider refusal to answer a decision. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export interface ProviderRefusalAnswer { + readonly _tag: "Refusal" +} + /** * Provider answer for any decision kind. * @@ -165,12 +226,18 @@ export interface ProviderProbabilityAnswer { * @category models * @since 4.0.0 */ -export type ProviderAnswer = ProviderClassifyAnswer | ProviderRateAnswer | ProviderProbabilityAnswer +export type ProviderAnswer = + | ProviderClassifyAnswer + | ProviderRateAnswer + | ProviderProbabilityAnswer + | ProviderRefusalAnswer /** * Provider response for a decision request. * `answers` is keyed like the requested decisions. Each answer is validated - * against its decision before it is returned to the caller. + * against its decision before it is returned to the caller. Omit the key of a + * decision the provider did not answer. `id`, `modelId` and `metadata` are + * passed through; `metadata` defaults to `{}`. * * @stability unstable * @category models @@ -182,6 +249,9 @@ export interface ProviderResponse { readonly inputTokens: number | undefined readonly outputTokens: number | undefined } + readonly id?: string | undefined + readonly modelId?: string | undefined + readonly metadata?: DecideResponseMetadata | undefined } /** @@ -197,14 +267,21 @@ export interface DecisionModel { definition: Decision.Definition, options: DecideOptions ) => Effect.Effect, AiError.AiError, Input["EncodingServices"]> + readonly decideSettled: >( + definition: Decision.Definition, + options: DecideOptions + ) => Effect.Effect, AiError.AiError, Input["EncodingServices"]> } -const invalidOutput = (description: string): AiError.AiError => - AiError.make({ - module: "DecisionModel", - method: "decide", - reason: new AiError.InvalidOutputError({ description }) - }) +const unanswered = ( + reason: Decision.Unanswered["reason"], + description: string +): Result.Result => Result.fail({ _tag: "Unanswered", reason, description }) + +const invalid = (description: string) => unanswered("Invalid", description) + +const aiError = (method: "decide" | "decideSettled", reason: AiError.AiErrorReason): AiError.AiError => + AiError.make({ module: "DecisionModel", method, reason }) const isFiniteNumber = (value: unknown): value is number => Predicate.isNumber(value) && Number.isFinite(value) @@ -222,41 +299,41 @@ const validateDistribution = ( labels: ReadonlyArray, answer: Record, roundingError: number -): Record | AiError.AiError => { +): Result.Result, Decision.Unanswered> => { const raw = answer.probabilities if (!Predicate.isObject(raw)) { - return invalidOutput(`Provider returned no probabilities for decision "${key}"`) + return unanswered("NoDistribution", `Provider returned no probabilities for decision "${key}"`) } const probabilities: Record = Object.create(null) let total = 0 for (const label of labels) { const value = raw[label] if (!isUnitInterval(value)) { - return invalidOutput(`Provider returned no probability for label "${label}" of decision "${key}"`) + return invalid(`Provider returned no probability for label "${label}" of decision "${key}"`) } probabilities[label] = value total += value } const drift = Math.abs(total - 1) if (total === 0 || drift > labels.length * roundingError + sumTolerance) { - return invalidOutput(`Provider returned probabilities that do not sum to 1 for decision "${key}"`) + return invalid(`Provider returned probabilities that do not sum to 1 for decision "${key}"`) } if (drift > sumTolerance) { for (const label of labels) probabilities[label] /= total } - return probabilities + return Result.succeed(probabilities) } const validateConfidence = ( key: string, answer: Record -): number | undefined | AiError.AiError => { +): Result.Result => { if (answer.confidence === undefined) { - return undefined + return Result.succeed(undefined) } return isUnitInterval(answer.confidence) - ? answer.confidence - : invalidOutput(`Provider returned confidence outside [0, 1] for decision "${key}"`) + ? Result.succeed(answer.confidence) + : invalid(`Provider returned confidence outside [0, 1] for decision "${key}"`) } const validateAnswer = ( @@ -264,83 +341,105 @@ const validateAnswer = ( decision: Decision.Any, answer: unknown, roundingError: number -): Decision.Answer | AiError.AiError => { +): Decision.Outcome => { if (!Predicate.isObject(answer)) { - return invalidOutput(`Provider returned no answer for decision "${key}"`) + return unanswered("Missing", `Provider returned no answer for decision "${key}"`) + } + if (answer._tag === "Refusal") { + return unanswered("Refused", `Provider refused to answer decision "${key}"`) } if (answer._tag !== decision._tag) { - return invalidOutput( + return invalid( `Provider returned a "${String(answer._tag)}" answer for decision "${key}", expected "${decision._tag}"` ) } switch (decision._tag) { case "Classify": { const labels = Object.keys(decision.criteria) - if (!Predicate.isString(answer.label) || !labels.includes(answer.label)) { - return invalidOutput(`Provider returned an unknown label for decision "${key}"`) - } - const confidence = validateConfidence(key, answer) - if (AiError.isAiError(confidence)) { - return confidence - } - const probabilities = validateDistribution(key, labels, answer, roundingError) - if (AiError.isAiError(probabilities)) { - return probabilities + const label = answer.label + if (!Predicate.isString(label) || !labels.includes(label)) { + return invalid(`Provider returned an unknown label for decision "${key}"`) } - return { label: answer.label, probabilities, ...(confidence === undefined ? undefined : { confidence }) } + return Result.gen(function*() { + const confidence = yield* validateConfidence(key, answer) + const probabilities = yield* validateDistribution(key, labels, answer, roundingError) + return { label, probabilities, ...(confidence === undefined ? undefined : { confidence }) } + }) } case "Rate": { const levels = decision.criteria.map((level) => level.label) - if (!isFiniteNumber(answer.rating) || answer.rating < 0 || answer.rating > levels.length - 1) { - return invalidOutput(`Provider returned a rating outside [0, ${levels.length - 1}] for decision "${key}"`) + const rating = answer.rating + if (!isFiniteNumber(rating) || rating < 0 || rating > levels.length - 1) { + return invalid(`Provider returned a rating outside [0, ${levels.length - 1}] for decision "${key}"`) } - const confidence = validateConfidence(key, answer) - if (AiError.isAiError(confidence)) { - return confidence - } - const probabilities = validateDistribution(key, levels, answer, roundingError) - if (AiError.isAiError(probabilities)) { - return probabilities - } - let label = levels[0] - for (let i = 1; i < levels.length; i++) { - if (probabilities[levels[i]] > probabilities[label]) { - label = levels[i] + return Result.gen(function*() { + const confidence = yield* validateConfidence(key, answer) + const probabilities = yield* validateDistribution(key, levels, answer, roundingError) + let label = levels[0] + for (let i = 1; i < levels.length; i++) { + if (probabilities[levels[i]] > probabilities[label]) { + label = levels[i] + } } - } - return { rating: answer.rating, label, probabilities, ...(confidence === undefined ? undefined : { confidence }) } + return { rating, label, probabilities, ...(confidence === undefined ? undefined : { confidence }) } + }) } case "Probability": { if (!isUnitInterval(answer.probability)) { - return invalidOutput(`Provider returned a probability outside [0, 1] for decision "${key}"`) + return invalid(`Provider returned a probability outside [0, 1] for decision "${key}"`) } - return { probability: answer.probability } + return Result.succeed({ probability: answer.probability }) } } } -const validateAnswers = >( +const settleAnswers = >( decisions: Decisions, answers: Readonly>, roundingError: number -): Effect.Effect, AiError.AiError> => { - const validated: Record> = Object.create(null) +): Decision.Outcomes => { + const outcomes: Record> = Object.create(null) for (const key of Object.keys(decisions)) { - const answer = validateAnswer(key, decisions[key], answers[key], roundingError) - if (AiError.isAiError(answer)) { - return Effect.fail(answer) + outcomes[key] = validateAnswer(key, decisions[key], answers[key], roundingError) + } + return outcomes as Decision.Outcomes +} + +const requireAnswers = >( + outcomes: Decision.Outcomes +): Effect.Effect, AiError.AiError> => { + const answers: Record> = Object.create(null) + const refused: Array = [] + let firstUnanswered: Decision.Unanswered | undefined + for (const [key, outcome] of Object.entries(outcomes) as Array<[string, Decision.Outcome]>) { + if (Result.isSuccess(outcome)) { + answers[key] = outcome.success + } else if (outcome.failure.reason === "Refused") { + refused.push(`"${key}"`) + } else { + firstUnanswered ??= outcome.failure } - validated[key] = answer } - return Effect.succeed(validated as Decision.Answers) + if (refused.length > 0) { + return Effect.fail(aiError( + "decide", + new AiError.ContentPolicyError({ description: `Provider refused to answer decisions ${refused.join(", ")}` }) + )) + } + if (firstUnanswered !== undefined) { + return Effect.fail(aiError("decide", new AiError.InvalidOutputError({ description: firstUnanswered.description }))) + } + return Effect.succeed(answers as Decision.Answers) } /** * Creates a `DecisionModel` that encodes inputs as JSON and validates provider answers. * Answers must cover every decision and use its labels. Distributions must * sum to 1 within `1e-6`, optional confidence must be in `[0, 1]`, and ratings must be - * in `[0, criteria.length - 1]`. Invalid answers fail with - * `AiError.InvalidOutputError`; encoding failures use `AiError.InvalidUserInputError`. + * in `[0, criteria.length - 1]`. With `decide`, refused decisions fail with + * `AiError.ContentPolicyError` and other unanswered decisions fail with + * `AiError.InvalidOutputError`; `decideSettled` reports them per decision + * instead. Encoding failures use `AiError.InvalidUserInputError`. * * **Details** * @@ -367,52 +466,52 @@ export const make = (params: { }): Effect.Effect => Effect.sync(() => { const roundingError = params.probabilityPrecision === undefined ? 0 : 0.5 * 10 ** -params.probabilityPrecision - return DecisionModel.of({ - [TypeId]: TypeId, - decide: >( - definition: Decision.Definition, - options: DecideOptions - ): Effect.Effect, AiError.AiError, Input["EncodingServices"]> => - Schema.encodeEffect(Schema.toCodecJson(definition.input))(options.input).pipe( - Effect.mapError((error) => - AiError.make({ - module: "DecisionModel", - method: "decide", - reason: new AiError.InvalidUserInputError({ description: error.message }) - }) - ), - Effect.flatMap((state) => { - const images = options.images?.length ? options.images : undefined - if (images !== undefined && params.supportsImages !== true) { - return Effect.fail( - AiError.make({ - module: "DecisionModel", - method: "decide", - reason: new AiError.InvalidUserInputError({ - description: "This decision model does not accept images" - }) - }) + const settle = >( + method: "decide" | "decideSettled", + definition: Decision.Definition, + options: DecideOptions + ): Effect.Effect, AiError.AiError, Input["EncodingServices"]> => + Schema.encodeEffect(Schema.toCodecJson(definition.input))(options.input).pipe( + Effect.mapError((error) => aiError(method, new AiError.InvalidUserInputError({ description: error.message }))), + Effect.flatMap((state) => { + const images = options.images?.length ? options.images : undefined + if (images !== undefined && params.supportsImages !== true) { + return Effect.fail( + aiError( + method, + new AiError.InvalidUserInputError({ description: "This decision model does not accept images" }) ) - } - return params.decide({ - state, - decisions: definition.decisions, - ...(images === undefined ? undefined : { images }) - }) - }), - Effect.flatMap((response) => - Effect.map( - validateAnswers(definition.decisions, response.answers, roundingError), - (answers): DecideResponse => ({ - answers, - usage: new DecisionUsage({ - inputTokens: response.usage.inputTokens, - outputTokens: response.usage.outputTokens - }) - }) ) + } + return params.decide({ + state, + decisions: definition.decisions, + ...(images === undefined ? undefined : { images }) + }) + }), + Effect.map((response): DecideSettledResponse => ({ + outcomes: settleAnswers(definition.decisions, response.answers, roundingError), + usage: new DecisionUsage({ + inputTokens: response.usage.inputTokens, + outputTokens: response.usage.outputTokens + }), + ...(response.id === undefined ? undefined : { id: response.id }), + ...(response.modelId === undefined ? undefined : { modelId: response.modelId }), + metadata: response.metadata ?? {} + })) + ) + return DecisionModel.of({ + [TypeId]: TypeId, + decide: (definition, options) => + settle("decide", definition, options).pipe( + Effect.flatMap(({ outcomes, ...response }) => + Effect.map(requireAnswers(outcomes), (answers) => ({ answers, ...response })) ), (effect) => Effect.withSpan(effect, "DecisionModel.decide", { captureStackTrace: false }) + ), + decideSettled: (definition, options) => + settle("decideSettled", definition, options).pipe( + (effect) => Effect.withSpan(effect, "DecisionModel.decideSettled", { captureStackTrace: false }) ) }) }) @@ -423,6 +522,9 @@ export const make = (params: { * Explicit `undefined` fields become `null`; absent fields stay absent. * Custom declarations need a JSON codec annotation or encoding fails. * Returned answers and probability dictionaries have null prototypes. + * Fails with `AiError.ContentPolicyError` if the provider refuses any decision, + * and otherwise with `AiError.InvalidOutputError` for the first unanswered + * decision in definition order. * * **Example** (Triaging a ticket) * @@ -449,6 +551,7 @@ export const make = (params: { * ``` * * @see {@link DecisionModel} for the service this function requires + * @see {@link decideSettled} for keeping valid answers when others fail * * @stability unstable * @category decisions @@ -459,3 +562,56 @@ export const decide = ): Effect.Effect, AiError.AiError, DecisionModel | Input["EncodingServices"]> => DecisionModel.use((model) => model.decide(definition, options)) + +/** + * Answers each decision with a `Result` containing a validated answer or a + * `Decision.Unanswered` reason. Input encoding and provider failures still + * fail the effect. Token usage covers the whole call. + * + * **Details** + * + * Provider response decoding errors fail the whole call. Per-decision reasons + * reflect the first failed validation check; `"NoDistribution"` requires an + * otherwise valid answer. + * + * **Example** (Falling back for unanswered decisions) + * + * ```ts + * import { Effect, Result, Schema } from "effect" + * import { Decision, DecisionModel } from "effect/ai" + * + * const TicketTriage = Decision.make({ + * input: Schema.String, + * decisions: { + * department: Decision.classify({ + * instructions: "Which team should handle this", + * criteria: { billing: "payments", technical: "bugs" } + * }), + * urgent: Decision.probability({ instructions: "The message is time-sensitive" }) + * } + * }) + * + * const program = Effect.gen(function*() { + * const { outcomes } = yield* DecisionModel.decideSettled(TicketTriage, { + * input: "My card was charged twice, please fix this today" + * }) + * const department = Result.isSuccess(outcomes.department) && + * outcomes.department.success.probabilities.billing >= 0.8 + * ? "billing" + * : "needs review" + * const urgent = Result.isSuccess(outcomes.urgent) ? outcomes.urgent.success.probability : undefined + * return { department, urgent } + * }) + * ``` + * + * @see {@link decide} for failing unless every decision is answered + * + * @stability unstable + * @category decisions + * @since 4.0.6 + */ +export const decideSettled = >( + definition: Decision.Definition, + options: DecideOptions +): Effect.Effect, AiError.AiError, DecisionModel | Input["EncodingServices"]> => + DecisionModel.use((model) => model.decideSettled(definition, options)) diff --git a/repos/effect/packages/effect/src/ai/EmbeddingModel.ts b/repos/effect/packages/effect/src/ai/EmbeddingModel.ts index 4d3ac0c6f2..b9a8d9865d 100644 --- a/repos/effect/packages/effect/src/ai/EmbeddingModel.ts +++ b/repos/effect/packages/effect/src/ai/EmbeddingModel.ts @@ -209,8 +209,17 @@ const invalidProviderResponse = (description: string): AiError.AiError => * provider, while `embedMany([])` returns an empty response without calling the * provider. * + * Concurrent `embed` calls share one `embedMany` call when `batchKey` returns + * equal keys under `Equal.equals`. The batch runs with the first caller's + * context, so return every value `embedMany` reads from that context. Without + * `batchKey`, all callers share one batch. + * * **Gotchas** * + * Context that `batchKey` does not return is not isolated. Per-call + * `FetchHttpClient.RequestInit`, or any service read by an injected HTTP + * client, takes the first caller's value for the whole batch. + * * Provider responses are interpreted positionally and must contain exactly one * result for each requested input. If the provider returns a different number * of results, `embed` and `embedMany` fail with `AiError.InvalidOutputError`. @@ -225,20 +234,23 @@ const invalidProviderResponse = (description: string): AiError.AiError => */ export const make: (params: { readonly embedMany: (options: ProviderOptions) => Effect.Effect + readonly batchKey?: ((context: Context.Context) => unknown) | undefined }) => Effect.Effect = Effect.fnUntraced(function*(params) { - const resolver = RequestResolver.make((entries) => - Effect.flatMap( - params.embedMany({ - inputs: entries.map((entry) => entry.request.input) - }), - (response) => - Effect.map(mapProviderResults(entries.length, response.results), (embeddings) => { - for (let i = 0; i < entries.length; i++) { - entries[i].completeUnsafe(Exit.succeed(embeddings[i])) - } - }) - ) - ).pipe( + const resolver = RequestResolver.makeGrouped({ + key: ({ context }) => params.batchKey?.(context), + resolver: (entries) => + Effect.flatMap( + params.embedMany({ + inputs: entries.map((entry) => entry.request.input) + }), + (response) => + Effect.map(mapProviderResults(entries.length, response.results), (embeddings) => { + for (let i = 0; i < entries.length; i++) { + entries[i].completeUnsafe(Exit.succeed(embeddings[i])) + } + }) + ) + }).pipe( RequestResolver.withSpan("EmbeddingModel.resolver") ) diff --git a/repos/effect/packages/effect/src/ai/McpClient.ts b/repos/effect/packages/effect/src/ai/McpClient.ts new file mode 100644 index 0000000000..ec7c2ad8c9 --- /dev/null +++ b/repos/effect/packages/effect/src/ai/McpClient.ts @@ -0,0 +1,1211 @@ +/** + * Connects to MCP servers with scoped Effect clients. + * + * Clients discover tools, prompts and resources using the protocol revision selected by + * their transport. `callTool` returns tool-reported errors as results; supplying a schema + * turns them into a `ToolError` reason that retains the original result. Transport and + * protocol failures fail the Effect. Interactive server requests, subscriptions, OAuth + * and retries are not included. + * + * @stability experimental + * @since 4.0.6 + */ +import * as ByteSize from "../ByteSize.ts" +import type * as Cause from "../Cause.ts" +import * as Context from "../Context.ts" +import * as Deferred from "../Deferred.ts" +import * as Duration from "../Duration.ts" +import * as Effect from "../Effect.ts" +import * as Base64 from "../encoding/Base64.ts" +import * as Sse from "../encoding/Sse.ts" +import * as HttpClient from "../http/HttpClient.ts" +import * as HttpClientRequest from "../http/HttpClientRequest.ts" +import type * as HttpClientResponse from "../http/HttpClientResponse.ts" +import * as Layer from "../Layer.ts" +import * as Option from "../Option.ts" +import * as Predicate from "../Predicate.ts" +import type * as ChildProcess from "../process/ChildProcess.ts" +import { ChildProcessSpawner } from "../process/ChildProcessSpawner.ts" +import * as Queue from "../Queue.ts" +import type * as Rpc from "../rpc/Rpc.ts" +import * as Schema from "../Schema.ts" +import type * as Scope from "../Scope.ts" +import * as Stream from "../Stream.ts" +import { BASE64_SENTINEL_PREFIX, BASE64_SENTINEL_SUFFIX } from "./internal/mcpProtocol.ts" +import { + ClosedError, + ConfigurationError, + HttpError, + isMcpClientError, + LimitError, + McpClientError, + type McpClientErrorReason, + ProtocolError, + TimeoutError, + ToolError, + TransportError, + UnsupportedError, + ValidationError +} from "./McpClientError.ts" +import type * as McpProtocol from "./McpProtocol.ts" +import * as McpSchema from "./McpSchema.ts" +import * as Tool from "./Tool.ts" +import * as Toolkit from "./Toolkit.ts" + +// ============================================================================= +// Models +// ============================================================================= + +/** + * Supported MCP wire protocol revisions. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export type ProtocolVersion = "2025-11-25" | "2026-07-28" + +/** + * Service tag for applications that provide one default MCP connection. + * + * @stability experimental + * @category services + * @since 4.0.6 + */ +export const McpClient: Context.Service = Context.Service("effect/ai/McpClient") + +/** + * A scoped connection to one MCP server. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export interface McpClient { + readonly [TypeId]: typeof TypeId + readonly protocolVersion: ProtocolVersion + readonly serverInfo: McpSchema.Implementation | undefined + readonly instructions?: string | undefined + /** + * Returns every tool page as one snapshot, bounded by the deadline, 100 pages and 10,000 tools. + * `timeout` overrides the client's discovery deadline. + */ + readonly listTools: ( + options?: { readonly timeout?: Duration.Input | undefined } + ) => Effect.Effect, McpClientError> + /** Returns every prompt page as one snapshot with the same bounds as `listTools`. */ + readonly listPrompts: ( + options?: { readonly timeout?: Duration.Input | undefined } + ) => Effect.Effect, McpClientError> + /** Gets the messages for a discovered prompt. Unbounded unless `timeout` is supplied. */ + readonly getPrompt: ( + params: { + readonly prompt: McpSchema.Prompt + readonly arguments?: typeof McpSchema.GetPrompt.payloadSchema.Type["arguments"] + }, + options?: { readonly timeout?: Duration.Input | undefined } + ) => Effect.Effect + /** Returns every resource page as one snapshot with the same bounds as `listTools`. */ + readonly listResources: ( + options?: { readonly timeout?: Duration.Input | undefined } + ) => Effect.Effect, McpClientError> + /** Reads the complete contents of a resource URI. Unbounded unless `timeout` is supplied. */ + readonly readResource: ( + params: { readonly uri: string }, + options?: { readonly timeout?: Duration.Input | undefined } + ) => Effect.Effect + /** + * Calls a discovered tool, optionally decoding with a schema. Errors retain the full result. + * Unbounded unless `timeout` is supplied. + */ + readonly callTool: { + ( + params: { + readonly tool: McpSchema.Tool + readonly schema?: undefined + readonly arguments?: typeof McpSchema.CallTool.payloadSchema.Type["arguments"] + }, + options?: { readonly timeout?: Duration.Input | undefined } + ): Effect.Effect + ( + params: { + readonly tool: McpSchema.Tool + readonly schema: S + readonly arguments?: typeof McpSchema.CallTool.payloadSchema.Type["arguments"] + }, + options?: { readonly timeout?: Duration.Input | undefined } + ): Effect.Effect + ( + params: { + readonly tool: McpSchema.Tool + readonly schema?: S | undefined + readonly arguments?: typeof McpSchema.CallTool.payloadSchema.Type["arguments"] + }, + options?: { readonly timeout?: Duration.Input | undefined } + ): Effect.Effect + } +} + +/** + * Returns whether a value is an MCP client. + * + * @stability experimental + * @category guards + * @since 4.0.6 + */ +export const isMcpClient = (value: unknown): value is McpClient => Predicate.hasProperty(value, TypeId) + +// ============================================================================= +// Constructors +// ============================================================================= + +/** + * Constructs and connects a client using the supplied MCP transport service and its protocol adapter. + * + * **Details** + * + * `name`, `version` and the optional `title`, `description`, `websiteUrl` and + * `icons` identify the client to the server. `timeout` defaults to 60 seconds + * and bounds initialization and each complete `listTools`, `listPrompts` or + * `listResources` scan. `callTool`, `getPrompt` and `readResource` have no + * deadline unless their own `timeout` option is supplied. + * Schema decoding requires the schema's services from the caller. + * + * @stability experimental + * @category constructors + * @since 4.0.6 + */ +export const make: (options: { + readonly name: string + readonly version: string + readonly title?: string | undefined + readonly description?: string | undefined + readonly websiteUrl?: string | undefined + readonly icons?: ReadonlyArray | undefined + readonly timeout?: Duration.Input | undefined +}) => Effect.Effect< + McpClient, + McpClientError, + Scope.Scope | Transport +> = Effect.fnUntraced(function*(options: Options): Effect.fn.Return< + McpClient, + McpClientError, + Scope.Scope | Transport +> { + const transport = yield* Transport + const protocol = transport.protocol + const version = protocol.protocolVersion + const modern = version === "2026-07-28" + const timeout = options.timeout ?? DEFAULT_TIMEOUT + const closed = yield* Deferred.make() + yield* Effect.addFinalizer(() => + Deferred.fail(closed, fail(new ClosedError({ description: "MCP client scope closed" }))) + ) + let nextId = 1 + + const clientInfo = encodeImplementation({ + name: options.name, + title: options.title, + version: options.version, + description: options.description, + websiteUrl: options.websiteUrl, + icons: options.icons + }) + + // Modern requests carry the client identity in every request. + // https://modelcontextprotocol.io/specification/2026-07-28/basic/index + const meta = { + "io.modelcontextprotocol/protocolVersion": version, + "io.modelcontextprotocol/clientInfo": clientInfo, + "io.modelcontextprotocol/clientCapabilities": {} + } + + const request = Effect.fnUntraced(function*( + method: string, + params: Record, + tool?: McpSchema.Tool + ): Effect.fn.Return { + const rpc = protocol.clientRpcs.requests.get(method) as Rpc.AnyWithProps | undefined + if (rpc === undefined) { + return yield* fail(new UnsupportedError({ description: `Unsupported MCP method: ${method}` })) + } + + const codecs = protocol.payloadCodecs(rpc) + const payload = yield* codecs.decode(modern ? { ...params, _meta: meta } : params).pipe( + Effect.flatMap(codecs.encode), + Effect.mapError(validationError) + ) + + const response = yield* transport.request( + { jsonrpc: "2.0", id: nextId++, method, params: payload as Readonly> }, + tool + ) + if ("error" in response) { + return yield* fail( + new ProtocolError({ + code: response.error.code, + data: response.error.data, + description: response.error.message + }) + ) + } + + let result: unknown = response.result + // Older modern peers may omit the complete discriminator. + if (modern && isRecord(result) && result.resultType === undefined) result = { ...result, resultType: "complete" } + if (method === "tools/list" && isRecord(result) && Array.isArray(result.tools)) { + result = { ...result, tools: result.tools.filter((tool) => !requiresTasks(tool)) } + } + + const validated = yield* decodeUnknown(rpc.successSchema, result) + if (isRecord(result) && result.resultType === "input_required") { + return yield* fail(new UnsupportedError({ description: "MCP input continuation is not supported" })) + } + const publicRpc = publicRpcs.get(method) + return (publicRpc === undefined ? validated : yield* decodeUnknown(publicRpc.successSchema, result)) as A + }) + + // Initialization and discovery default to `timeout`; other operations are bounded only on request. + const withDeadline = ( + effect: Effect.Effect, + operation: string, + duration: Duration.Input | undefined + ) => { + const open = Effect.raceFirst(effect, Deferred.await(closed)) + return duration === undefined ? open : open.pipe( + Effect.timeoutOrElse({ + duration, + orElse: () => Effect.fail(fail(new TimeoutError({ operation }))) + }) + ) + } + + const initialize = Effect.gen(function*() { + if (!modern) { + return yield* request("initialize", { + protocolVersion: version, + capabilities: {}, + clientInfo + }) + } + const discover = request("server/discover", {}) + // Retry once when the server rejects a version that it reports as supported. + return yield* discover.pipe(Effect.catchIf((error) => { + const reason = error.reason + if (reason._tag !== "ProtocolError" && reason._tag !== "HttpError") return false + const rejection = Option.getOrUndefined(decodeVersionRejection(reason.data)) + return reason.code === McpSchema.UNSUPPORTED_PROTOCOL_VERSION_ERROR_CODE && rejection?.requested === version && + rejection.supported.includes(version) + }, () => discover)) + }) + + const { capabilities, instructions, serverInfo } = yield* withDeadline( + Effect.gen(function*() { + const result = yield* initialize + if (modern ? !result.supportedVersions?.includes(version) : result.protocolVersion !== version) { + return yield* fail( + new UnsupportedError({ description: "Server does not support the explicitly selected protocol version" }) + ) + } + const identity = modern ? result._meta?.["io.modelcontextprotocol/serverInfo"] : result.serverInfo + const capabilities = yield* decodeServerCapabilities(result.capabilities) + const serverInfo = identity === undefined ? undefined : yield* decodeImplementation(identity) + if (!modern) yield* transport.notify({ jsonrpc: "2.0", method: "notifications/initialized" }) + return { capabilities, instructions: result.instructions, serverInfo } + }).pipe( + Effect.mapError((cause) => + Schema.isSchemaError(cause) + ? fail(new ProtocolError({ description: "Invalid MCP initialization result", cause })) + : cause + ) + ), + "initialization", + timeout + ) + + // Retrieval requires the capability; discovery treats a missing one as empty. + const call = ( + kind: Capability, + method: string, + params: Record, + tool?: McpSchema.Tool + ): Effect.Effect => + capabilities[kind] === undefined + ? Effect.fail(fail(new UnsupportedError({ description: `Server does not advertise ${kind}` }))) + : request(method, params, tool) + + const list = (kind: K) => + Effect.fnUntraced( + function*(_options?: CallOptions) { + if (capabilities[kind] === undefined) return [] + const items: Array = [] + const cursors = new Set() + let cursor: string | undefined + for (let page = 0; page < MAX_DISCOVERY_PAGES; page++) { + const result = yield* request>(`${kind}/list`, cursor === undefined ? {} : { cursor }) + if (items.length + result[kind].length > MAX_DISCOVERY_ITEMS) { + return yield* limitError(`MCP ${kind} discovery exceeded its item limit`) + } + items.push(...result[kind]) + if (result.nextCursor === undefined) return items + if (cursors.has(result.nextCursor)) { + return yield* limitError(`MCP ${kind} discovery repeated a cursor`) + } + cursors.add(cursor = result.nextCursor) + } + return yield* limitError(`MCP ${kind} discovery exceeded its page limit`) + }, + (effect, callOptions) => withDeadline(effect, `${kind} discovery`, callOptions?.timeout ?? timeout) + ) + + const listTools = list<"tools", McpSchema.Tool>("tools") + + const callTool = Effect.fnUntraced( + function*( + params: { + readonly tool: McpSchema.Tool + readonly schema?: S | undefined + readonly arguments?: typeof McpSchema.CallTool.payloadSchema.Type["arguments"] + }, + _options?: CallOptions + ) { + const result = yield* call( + "tools", + "tools/call", + { name: params.tool.name, ...(params.arguments === undefined ? {} : { arguments: params.arguments }) }, + params.tool + ) + if (params.schema === undefined) return result + if (result.isError === true) { + return yield* fail(new ToolError({ result })) + } + return yield* Schema.decodeUnknownEffect(params.schema)(result.structuredContent).pipe( + Effect.mapError((cause) => + fail(new ValidationError({ description: "MCP structured result validation failed", cause })) + ) + ) + }, + (effect, _params, callOptions) => withDeadline(effect, "tools/call", callOptions?.timeout) + ) + + return { + [TypeId]: TypeId, + protocolVersion: version, + serverInfo, + instructions, + listTools: (callOptions) => + listTools(callOptions).pipe(Effect.map((tools) => tools.filter((tool) => transport.isCallable(tool)))), + listPrompts: list("prompts"), + listResources: list("resources"), + getPrompt: (params, callOptions) => + withDeadline( + call("prompts", "prompts/get", { + name: params.prompt.name, + ...(params.arguments === undefined ? {} : { arguments: params.arguments }) + }), + "prompts/get", + callOptions?.timeout + ), + readResource: (params, callOptions) => + withDeadline( + call("resources", "resources/read", { uri: params.uri }), + "resources/read", + callOptions?.timeout + ), + callTool: callTool as McpClient["callTool"] + } +}) + +// ============================================================================= +// Toolkit +// ============================================================================= + +/** + * Discovers remote tools and binds their handlers using the standard Toolkit implementation. + * + * **Details** + * + * Pass an optional prefix such as `"reports__"` to qualify model tool names. + * Remote JSON Schemas are advertised without local schema import. Results become + * text; tool-reported and operational failures fail the Effect. Use `Tool.make` + * or `Tool.dynamic` with `Toolkit.make` for custom validation, approval or results. + * Build a new toolkit to discover changes between model turns. + * + * @stability experimental + * @category constructors + * @since 4.0.6 + */ +export const toolkit: (client: McpClient, prefix?: string) => Effect.Effect< + Toolkit.WithHandler< + Record< + string, + Tool.Tool + > + >, + McpClientError +> = Effect.fnUntraced(function*(client: McpClient, prefix = "") { + const descriptors = yield* client.listTools() + const tools: Array< + Tool.Tool + > = [] + const handlers: Record Effect.Effect> = Object.create(null) + + for (const descriptor of descriptors) { + const name = `${prefix}${descriptor.name}` + if (!/^[a-zA-Z0-9_-]{1,64}$/.test(name) || Object.hasOwn(handlers, name)) { + return yield* fail(new ConfigurationError({ description: `Invalid or duplicate MCP model tool name: ${name}` })) + } + tools.push( + Tool.dynamic(name, { + parameters: descriptor.inputSchema, + description: descriptor.description, + success: Schema.String, + failure: McpClientError + }).annotate(Tool.Strict, false) + ) + handlers[name] = Effect.fnUntraced(function*(params: unknown) { + const args = yield* decodeToolArguments(params).pipe( + Effect.mapError((cause) => + fail(new ValidationError({ description: "MCP tool arguments must be a JSON object", cause })) + ) + ) + const result = yield* client.callTool({ tool: descriptor, arguments: args }) + if (result.isError === true) { + return yield* fail(new ToolError({ result })) + } + return yield* toText(result) + }) + } + + const toolkit = Toolkit.make(...tools) + return yield* toolkit.pipe(Effect.provideContext(yield* toolkit.toHandlers(handlers))) +}) + +// ============================================================================= +// Transports +// ============================================================================= + +/** + * Service supplied by the built-in MCP transport constructors and layers. + * + * **Details** + * + * Transport construction owns cleanup through its Effect scope. + * + * @stability experimental + * @category services + * @since 4.0.6 + */ +export const Transport: Context.Service = Context.Service( + "effect/ai/McpClient/Transport" +) + +/** + * Exchanges JSON-RPC messages with one MCP server using its protocol adapter. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export interface Transport { + readonly [TransportTypeId]: typeof TransportTypeId + readonly protocol: McpProtocol.ProtocolAdapter + /** Sends a request and waits for its response. `tool` supplies header routing for tool calls. */ + readonly request: ( + message: McpSchema.JsonRpcRequest, + tool?: McpSchema.Tool + ) => Effect.Effect + /** Sends a notification without waiting for a reply. */ + readonly notify: (message: McpSchema.JsonRpcNotification) => Effect.Effect + /** Returns whether the transport can route a call to the tool. */ + readonly isCallable: (tool: McpSchema.Tool) => boolean +} + +/** + * Starts a scoped subprocess using newline-delimited JSON-RPC. + * + * @stability experimental + * @category constructors + * @since 4.0.6 + */ +export const stdio: (options: { + readonly command: ChildProcess.Command + readonly protocol: McpProtocol.ProtocolAdapter + readonly maxMessageBytes?: ByteSize.Input | undefined +}) => Effect.Effect = Effect.fnUntraced( + function*(options: { + readonly command: ChildProcess.Command + readonly protocol: Protocol + readonly maxMessageBytes?: ByteSize.Input | undefined + }): Effect.fn.Return { + const maxBytes = yield* messageLimit(options.maxMessageBytes) + const spawner = yield* ChildProcessSpawner + const process = yield* spawner.spawn(options.command).pipe( + Effect.mapError((cause) => fail(new TransportError({ description: "Cannot spawn MCP server", cause }))) + ) + + const pending = new Map>() + const failure = yield* Deferred.make() + let failed: McpClientError | undefined + + const close = (error: McpClientError) => + Effect.suspend(() => { + failed ??= error + return Deferred.fail(failure, failed) + }) + + const input = yield* Queue.unbounded() + + const write = Effect.fnUntraced(function*(message: McpSchema.JsonRpcMessage) { + if (failed !== undefined) return yield* failed + const json = yield* encodeMessage(message, maxBytes) + yield* Queue.offer(input, json + "\n") + }) + + const receive = Effect.fnUntraced(function*(message: McpSchema.JsonRpcMessage) { + if (isRequest(message)) return yield* write(replyToServer(message)) + if (!isResponse(message)) return + // Without an ID the error cannot be correlated, so no pending request can be trusted. + if (message.id === undefined || message.id === null) { + return yield* fail(new ProtocolError({ description: "Server returned an uncorrelated JSON-RPC error" })) + } + const response = pending.get(message.id) + if (response !== undefined) yield* Deferred.succeed(response, message) + }) + + yield* Stream.fromQueue(input).pipe( + Stream.encodeText, + Stream.run(process.stdin), + Effect.catch((cause) => close(fail(new TransportError({ description: "MCP stdin failed", cause })))), + Effect.forkScoped + ) + yield* Stream.runDrain(process.stderr).pipe(Effect.ignore, Effect.forkScoped) + const drained = yield* Deferred.make() + let buffer = "" + yield* process.stdout.pipe( + Stream.decodeText(), + Stream.runForEach(Effect.fnUntraced(function*(chunk) { + buffer += chunk + let start = 0 + let index: number + while ((index = buffer.indexOf("\n", start)) >= 0) { + const line = buffer.slice(start, index) + start = index + 1 + if (line.trim() !== "") yield* receive(yield* parseMessage(line, maxBytes)) + } + buffer = buffer.slice(start) + if (buffer.length > maxBytes) { + return yield* limitError("MCP message exceeds maxMessageBytes") + } + })), + Effect.mapError((cause) => + isMcpClientError(cause) + ? cause + : fail(new TransportError({ description: "MCP stdout failed", cause })) + ), + Effect.andThen(Effect.fail(fail(new TransportError({ description: "MCP server closed stdout" })))), + Effect.catch(close), + Effect.ensuring(Deferred.succeed(drained, undefined)), + Effect.forkScoped + ) + // Output written just before exit is delivered before pending requests fail. + yield* process.exitCode.pipe( + Effect.map((code) => `MCP process exited (${code})`), + Effect.orElseSucceed(() => "MCP process exited"), + Effect.tap(() => Deferred.await(drained).pipe(Effect.timeoutOption(CLOSE_TIMEOUT))), + Effect.flatMap((description) => close(fail(new TransportError({ description })))), + Effect.forkScoped + ) + + // Close stdin, then escalate to SIGTERM and SIGKILL if the server does not exit. + // https://modelcontextprotocol.io/specification/2025-11-25/basic/lifecycle#stdio + yield* Effect.addFinalizer(Effect.fnUntraced(function*() { + yield* close(fail(new ClosedError({ description: "MCP transport closed" }))) + yield* Queue.end(input) + const exited = yield* process.exitCode.pipe(Effect.timeoutOption(CLOSE_TIMEOUT), Effect.option) + if (Option.isNone(exited) || Option.isNone(exited.value)) { + yield* process.kill({ killSignal: "SIGTERM", forceKillAfter: Duration.toMillis(CLOSE_TIMEOUT) }).pipe( + Effect.ignore + ) + } + })) + + return { + [TransportTypeId]: TransportTypeId, + protocol: options.protocol, + isCallable: () => true, + notify: write, + request: Effect.fnUntraced(function*(message) { + const response = yield* Deferred.make() + pending.set(message.id, response) + + return yield* write(message).pipe( + Effect.andThen(Deferred.await(response)), + Effect.raceFirst(Deferred.await(failure)), + Effect.onInterrupt(() => + message.method === "initialize" + ? Effect.void + : Deferred.isDone(response).pipe( + Effect.flatMap((done) => done ? Effect.void : write(cancelled(message.id)).pipe(Effect.ignore)) + ) + ), + Effect.ensuring(Effect.sync(() => pending.delete(message.id))) + ) + }) + } + } +) + +/** + * Provides a connected MCP client over stdio using the child process spawner. + * + * **Details** + * + * The layer scope owns the client and subprocess. The options are those of + * {@link make} and {@link stdio}. + * + * @see {@link layerHttp} + * @stability experimental + * @category layers + * @since 4.0.6 + */ +export const layerStdio: (options: { + readonly name: string + readonly version: string + readonly title?: string | undefined + readonly description?: string | undefined + readonly websiteUrl?: string | undefined + readonly icons?: ReadonlyArray | undefined + readonly timeout?: Duration.Input | undefined + readonly command: ChildProcess.Command + readonly protocol: McpProtocol.ProtocolAdapter + readonly maxMessageBytes?: ByteSize.Input | undefined +}) => Layer.Layer< + McpClient, + McpClientError, + ChildProcessSpawner +> = (options) => Layer.effect(McpClient)(make(options)).pipe(Layer.provide(Layer.effect(Transport)(stdio(options)))) + +/** + * Creates a Streamable HTTP transport with request-owned JSON or SSE responses. + * + * @stability experimental + * @category constructors + * @since 4.0.6 + */ +export const http: (options: { + readonly url: string | URL + readonly protocol: McpProtocol.ProtocolAdapter + readonly headers?: Readonly> | undefined + readonly maxMessageBytes?: ByteSize.Input | undefined +}) => Effect.Effect = Effect.fnUntraced( + function*(options: { + readonly url: string | URL + readonly protocol: Protocol + readonly headers?: Readonly> | undefined + readonly maxMessageBytes?: ByteSize.Input | undefined + }): Effect.fn.Return { + const maxBytes = yield* messageLimit(options.maxMessageBytes) + const client = HttpClient.withScope(yield* HttpClient.HttpClient) + const modern = options.protocol.protocolVersion === "2026-07-28" + const closed = yield* Deferred.make() + let failed: McpClientError | undefined + // Legacy servers may assign a session during initialization. + let session: string | undefined + + const baseHeaders = { + ...options.headers, + accept: "application/json, text/event-stream", + "mcp-protocol-version": options.protocol.protocolVersion + } + const headers = (): Record => + session === undefined ? baseHeaders : { ...baseHeaders, "mcp-session-id": session } + + const httpError = Effect.fnUntraced(function*(response: HttpClientResponse.HttpClientResponse) { + // Error bodies may carry a JSON-RPC error without an ID, e.g. an unsupported protocol version. + const remote = mediaType(response) === "application/json" + ? yield* readBody(response, maxBytes).pipe( + Effect.flatMap((text) => parseMessage(text, maxBytes)), + Effect.map((message) => "error" in message ? message.error : undefined), + Effect.orElseSucceed(() => undefined) + ) + : undefined + return yield* fail( + new HttpError({ + status: response.status, + code: remote?.code, + data: remote?.data, + description: remote?.message, + wwwAuthenticate: response.headers["www-authenticate"], + retryAfter: response.headers["retry-after"] + }) + ) + }) + + // Posts one message and returns the successful response, tracking legacy sessions. + const send = Effect.fnUntraced(function*(message: McpSchema.JsonRpcMessage, tool?: McpSchema.Tool) { + if (failed !== undefined) return yield* failed + const body = yield* encodeMessage(message, maxBytes) + const request = HttpClientRequest.post(options.url).pipe( + HttpClientRequest.setHeaders({ ...headers(), ...(modern ? yield* routingHeaders(message, tool) : {}) }), + HttpClientRequest.bodyText(body, "application/json") + ) + const response = yield* client.execute(request).pipe( + // Clients using `filterStatusOk` surface error responses as failures. + Effect.catch((cause) => + cause.response === undefined + ? Effect.fail(fail(new TransportError({ description: "MCP HTTP request failed", cause }))) + : Effect.succeed(cause.response) + ) + ) + + // A legacy session ends when the server no longer recognizes its ID. + if (session !== undefined && response.status === 404) { + failed ??= fail(new ClosedError({ sessionExpired: true })) + return yield* failed + } + if (response.status < 200 || response.status >= 300) return yield* httpError(response) + if (!modern) session = response.headers["mcp-session-id"] ?? session + return response + }) + + const exchange = (effect: Effect.Effect) => + effect.pipe( + Effect.mapError((cause) => + isMcpClientError(cause) + ? cause + : fail(new TransportError({ description: "Cannot read MCP response", cause })) + ), + Effect.raceFirst(Deferred.await(closed)), + Effect.scoped + ) + + const notify = (message: McpSchema.JsonRpcNotification | McpSchema.JsonRpcResponse) => + exchange(Effect.asVoid(send(message))) + + const readResponse = Effect.fnUntraced(function*( + response: HttpClientResponse.HttpClientResponse, + request: McpSchema.JsonRpcRequest + ) { + const match = Effect.fnUntraced(function*(message: McpSchema.JsonRpcMessage) { + if (isRequest(message)) { + yield* notify(replyToServer(message)) + return undefined + } + if (!isResponse(message)) return undefined + // A response on this exchange belongs to its request; an error may omit the ID. + if (message.id !== request.id && !("error" in message && message.id == null)) { + return yield* fail(new ProtocolError({ description: "HTTP MCP response id does not match its request" })) + } + return message + }) + + switch (mediaType(response)) { + case "application/json": { + const message = yield* match(yield* parseMessage(yield* readBody(response, maxBytes), maxBytes)) + if (message === undefined) break + return message + } + case "text/event-stream": { + const events: Array = [] + const parser = Sse.makeParser((event) => { + // Resumption is unsupported, so retry directives and event IDs are ignored. + if (event._tag === "Event" && event.data !== "") events.push(event.data) + }, { maxEventSize: maxBytes }) + const message = yield* response.stream.pipe( + Stream.decodeText(), + Stream.flatMap((chunk) => { + const error = parser.feed(chunk) + const parsed = Stream.fromIterable(events.splice(0)) + // Events before an oversized one can still complete the request. + return error === undefined ? parsed : parsed.pipe( + Stream.concat(Stream.fail(limitError("MCP SSE event exceeds maxMessageBytes"))) + ) + }), + Stream.mapEffect((data) => parseMessage(data, maxBytes).pipe(Effect.flatMap(match))), + Stream.filter(Predicate.isNotUndefined), + Stream.runHead + ) + if (Option.isNone(message)) break + return message.value + } + default: { + return yield* fail(new ProtocolError({ description: "Unsupported MCP response Content-Type" })) + } + } + return yield* fail(new TransportError({ description: "MCP response ended before the terminal response" })) + }) + + yield* Effect.addFinalizer(Effect.fnUntraced(function*() { + failed ??= fail(new ClosedError({ description: "MCP HTTP transport closed" })) + yield* Deferred.fail(closed, failed) + if (session === undefined || failed.reason._tag !== "ClosedError" || failed.reason.sessionExpired) return + // Explicitly terminate a legacy session. + // https://modelcontextprotocol.io/specification/2025-11-25/basic/transports#session-management + yield* client.execute(HttpClientRequest.delete(options.url).pipe(HttpClientRequest.setHeaders(headers()))).pipe( + Effect.scoped, + Effect.timeoutOption(CLOSE_TIMEOUT), + Effect.ignore + ) + })) + + return { + [TransportTypeId]: TransportTypeId, + protocol: options.protocol, + isCallable: (tool) => !modern || headerParameters(tool) !== undefined, + notify, + request: (message, tool) => + exchange( + send(message, tool).pipe( + Effect.flatMap((response) => + response.status === 202 || response.status === 204 + ? fail(new ProtocolError({ description: "MCP request received no response" })) + : readResponse(response, message) + ) + ) + ).pipe( + // Modern servers treat a closed response stream as cancellation. + Effect.onInterrupt(() => + modern || message.method === "initialize" + ? Effect.void + : notify(cancelled(message.id)).pipe(Effect.timeoutOption(CLOSE_TIMEOUT), Effect.ignore) + ) + ) + } + } +) + +/** + * Provides a connected MCP client over Streamable HTTP using the HTTP client. + * + * **Details** + * + * The layer scope owns the client and connection. The options are those of + * {@link make} and {@link http}. + * + * **Example** (Providing an HTTP client connection) + * + * ```ts import.meta.vitest + * import { Layer } from "effect" + * import { McpClient, McpProtocol } from "effect/ai" + * import { FetchHttpClient } from "effect/http" + * + * const ClientLayer = McpClient.layerHttp({ + * name: "report-workflow", + * version: "1.0.0", + * protocol: McpProtocol.v2026_07_28, + * url: "http://localhost:3000/mcp" + * }).pipe(Layer.provide(FetchHttpClient.layer)) + * ``` + * + * @see {@link layerStdio} + * @stability experimental + * @category layers + * @since 4.0.6 + */ +export const layerHttp: (options: { + readonly name: string + readonly version: string + readonly title?: string | undefined + readonly description?: string | undefined + readonly websiteUrl?: string | undefined + readonly icons?: ReadonlyArray | undefined + readonly timeout?: Duration.Input | undefined + readonly url: string | URL + readonly protocol: McpProtocol.ProtocolAdapter + readonly headers?: Readonly> | undefined + readonly maxMessageBytes?: ByteSize.Input | undefined +}) => Layer.Layer< + McpClient, + McpClientError, + HttpClient.HttpClient +> = (options) => Layer.effect(McpClient)(make(options)).pipe(Layer.provide(Layer.effect(Transport)(http(options)))) + +// ============================================================================= +// Internal +// ============================================================================= + +const TypeId = "~effect/ai/McpClient" as const +const TransportTypeId = "~effect/ai/McpClient/Transport" as const +const DEFAULT_MAX_MESSAGE_BYTES = ByteSize.mebibytes(16) +const DEFAULT_TIMEOUT = Duration.seconds(60) +const CLOSE_TIMEOUT = Duration.seconds(5) +const MAX_DISCOVERY_PAGES = 100 +const MAX_DISCOVERY_ITEMS = 10_000 +const HEADER_TOKEN = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/ +const encoder = new TextEncoder() +const decoder = new TextDecoder() + +type Protocol = McpProtocol.ProtocolAdapter + +type Options = Parameters[0] + +type Capability = "tools" | "prompts" | "resources" + +interface CallOptions { + readonly timeout?: Duration.Input | undefined +} + +interface InitializeResult { + readonly protocolVersion?: string + readonly supportedVersions?: ReadonlyArray + readonly capabilities: unknown + readonly serverInfo?: unknown + readonly instructions?: string + readonly _meta?: Record +} + +type ListResult = Record> & { readonly nextCursor?: string } +const fail = (reason: McpClientErrorReason) => new McpClientError({ reason }) +const limitError = (description: string) => fail(new LimitError({ description })) + +const validationError = (cause: Schema.SchemaError) => + fail(new ValidationError({ description: "Invalid MCP payload", cause })) + +const isRecord = Predicate.isReadonlyObject +const decodeMessage = Schema.decodeUnknownEffect(McpSchema.JsonRpcMessage, { onExcessProperty: "error" }) + +const decodeVersionRejection = Schema.decodeUnknownOption(Schema.Struct({ + requested: Schema.String, + supported: Schema.Array(Schema.String) +})) + +const decodeServerCapabilities = Schema.decodeUnknownEffect(McpSchema.ServerCapabilities) +const decodeImplementation = Schema.decodeUnknownEffect(McpSchema.Implementation) +const encodeImplementation = Schema.encodeSync(McpSchema.Implementation) + +// The protocol payload codec validates argument values; the toolkit only checks for an object. +const decodeToolArguments = Schema.decodeUnknownEffect(Schema.Record(Schema.String, Schema.Unknown)) + +const decodeUnknown = (schema: Schema.Constraint, value: unknown) => + Schema.decodeUnknownEffect(schema as Schema.ConstraintCodec)(value).pipe( + Effect.mapError(validationError) + ) + +const publicRpcs: ReadonlyMap = + McpSchema.ClientRequestRpcs.requests + +// Tools that require task-augmented execution cannot be called without the tasks extension. +const requiresTasks = (tool: unknown) => + isRecord(tool) && isRecord(tool.execution) && tool.execution.taskSupport === "required" + +const messageLimit = (input: ByteSize.Input = DEFAULT_MAX_MESSAGE_BYTES) => { + const bytes = Option.flatMap(ByteSize.fromInput(input), ByteSize.toNumber) + return Option.isSome(bytes) && bytes.value > 0 + ? Effect.succeed(bytes.value) + : Effect.fail(fail(new ConfigurationError({ description: "maxMessageBytes must be a positive safe integer" }))) +} + +const encodeMessage = Effect.fnUntraced(function*(message: McpSchema.JsonRpcMessage, maxBytes: number) { + const json = yield* Effect.try({ + try: () => JSON.stringify(message), + catch: (cause) => fail(new TransportError({ description: "Cannot encode MCP message", cause })) + }) + if (encoder.encode(json).length > maxBytes) { + return yield* limitError("MCP message exceeds maxMessageBytes") + } + return json +}) + +const parseMessage = Effect.fnUntraced(function*(text: string, maxBytes: number) { + if (encoder.encode(text).length > maxBytes) { + return yield* limitError("MCP message exceeds maxMessageBytes") + } + const json = yield* Effect.try({ + try: () => JSON.parse(text) as unknown, + catch: (cause) => fail(new ProtocolError({ description: "Invalid MCP JSON", cause })) + }) + return yield* decodeMessage(json).pipe( + Effect.mapError((cause) => fail(new ProtocolError({ description: "Invalid JSON-RPC message", cause }))) + ) +}) + +const isRequest = (message: McpSchema.JsonRpcMessage): message is McpSchema.JsonRpcRequest => + "method" in message && "id" in message + +const isResponse = (message: McpSchema.JsonRpcMessage): message is McpSchema.JsonRpcResponse => !("method" in message) + +// The client advertises no capabilities, so it answers pings and declines every other server request. +const replyToServer = (request: McpSchema.JsonRpcRequest): McpSchema.JsonRpcResponse => + request.method === "ping" + ? { jsonrpc: "2.0", id: request.id, result: {} } + : { + jsonrpc: "2.0", + id: request.id, + error: { code: McpSchema.METHOD_NOT_FOUND_ERROR_CODE, message: `Unsupported MCP method: ${request.method}` } + } + +const cancelled = (id: McpSchema.RequestId): McpSchema.JsonRpcNotification => ({ + jsonrpc: "2.0", + method: "notifications/cancelled", + params: { requestId: id } +}) + +// ============================================================================= +// Internal: Streamable HTTP +// ============================================================================= + +const mediaType = (response: HttpClientResponse.HttpClientResponse) => + response.headers["content-type"]?.split(";", 1)[0].trim().toLowerCase() + +const readBody = (response: HttpClientResponse.HttpClientResponse, maxBytes: number) => + response.stream.pipe( + Stream.mapError((cause) => fail(new TransportError({ description: "Cannot read MCP response", cause }))), + Stream.limitBytes(maxBytes, () => Stream.fail(limitError("MCP response exceeds maxMessageBytes"))), + Stream.mkUint8Array, + Effect.map((bytes) => decoder.decode(bytes)) + ) + +// Values that are not plain visible ASCII are sent Base64 encoded. +// https://modelcontextprotocol.io/specification/2026-07-28/basic/transports#custom-headers-from-tool-parameters +const headerValue = (value: string) => + /^[\x20-\x7e]*$/.test(value) && value.trim() === value && + !(value.startsWith(BASE64_SENTINEL_PREFIX) && value.endsWith(BASE64_SENTINEL_SUFFIX)) + ? value + : `${BASE64_SENTINEL_PREFIX}${Base64.encode(value)}${BASE64_SENTINEL_SUFFIX}` + +interface HeaderParameter { + readonly path: ReadonlyArray + readonly header: string + readonly type: string +} + +// JSON Schema keywords whose values are schemas, maps of schemas or arrays of schemas. +const SCHEMA_KEYWORDS = new Set([ + "additionalProperties", + "unevaluatedProperties", + "propertyNames", + "items", + "additionalItems", + "unevaluatedItems", + "contains", + "not", + "if", + "then", + "else", + "contentSchema" +]) + +const SCHEMA_MAP_KEYWORDS = new Set([ + "properties", + "patternProperties", + "$defs", + "definitions", + "dependentSchemas", + "dependencies" +]) + +const SCHEMA_ARRAY_KEYWORDS = new Set(["allOf", "anyOf", "oneOf", "prefixItems", "items"]) + +// Collects `x-mcp-header` annotations along chains of `properties` keys, or +// returns undefined when an annotation is invalid or found anywhere else. +// https://modelcontextprotocol.io/specification/2026-07-28/basic/transports/streamable-http#schema-extension +const headerParameters = (tool: McpSchema.Tool): ReadonlyArray | undefined => { + const parameters: Array = [] + const names = new Set() + + // `path` is undefined once the traversal leaves a properties-only chain. + const visit = (schema: unknown, path: ReadonlyArray | undefined): boolean => { + if (!isRecord(schema)) return true + const name = schema["x-mcp-header"] + if (name !== undefined) { + if ( + path === undefined || path.length === 0 || typeof name !== "string" || !HEADER_TOKEN.test(name) || + names.has(name.toLowerCase()) || !["string", "integer", "boolean"].includes(schema.type as string) + ) return false + names.add(name.toLowerCase()) + parameters.push({ path, header: `mcp-param-${name.toLowerCase()}`, type: schema.type as string }) + } + return Object.entries(schema).every(([keyword, value]) => { + if (SCHEMA_MAP_KEYWORDS.has(keyword) && isRecord(value)) { + return Object.entries(value).every(([key, child]) => + visit(child, keyword === "properties" && path !== undefined ? [...path, key] : undefined) + ) + } + if (Array.isArray(value)) { + return !SCHEMA_ARRAY_KEYWORDS.has(keyword) || value.every((child) => visit(child, undefined)) + } + return !SCHEMA_KEYWORDS.has(keyword) || visit(value, undefined) + }) + } + + return visit(tool.inputSchema, []) ? parameters : undefined +} + +const routingHeaders = Effect.fnUntraced(function*(message: McpSchema.JsonRpcMessage, tool?: McpSchema.Tool) { + if (!("method" in message)) return {} + const headers: Record = { "mcp-method": message.method } + const params = message.params ?? {} + const name = message.method === "resources/read" ? params.uri : params.name + if ( + typeof name === "string" && + (message.method === "tools/call" || message.method === "prompts/get" || message.method === "resources/read") + ) { + headers["mcp-name"] = headerValue(name) + } + if (message.method !== "tools/call") return headers + + const parameters = tool === undefined ? undefined : headerParameters(tool) + if (parameters === undefined) { + return yield* fail(new UnsupportedError({ description: "Tool has invalid x-mcp-header annotations" })) + } + for (const { header, path, type } of parameters) { + let value: unknown = params.arguments + for (const key of path) value = isRecord(value) && Object.hasOwn(value, key) ? value[key] : undefined + if (value === undefined || value === null) continue + if (type === "integer" ? !Number.isSafeInteger(value) : typeof value !== type) { + return yield* fail(new ValidationError({ description: `Invalid value for MCP header parameter ${header}` })) + } + headers[header] = headerValue(String(value)) + } + return headers +}) + +// ============================================================================= +// Internal: Toolkit +// ============================================================================= + +const toText = (result: McpSchema.CallToolResult): Effect.Effect => { + const unsupported = (content: string) => + Effect.fail( + fail(new UnsupportedError({ description: `MCP ${content} requires an application text converter`, result })) + ) + const parts: Array = [] + for (const block of result.content) { + switch (block.type) { + case "text": { + parts.push(block.text) + break + } + case "resource_link": { + parts.push( + `${block.title ?? block.name}: ${block.uri}${block.description === undefined ? "" : `\n${block.description}`}` + ) + break + } + case "resource": { + if (!("text" in block.resource)) return unsupported("binary resources") + parts.push(block.resource.text) + break + } + default: { + return unsupported(`${block.type} content`) + } + } + } + return Effect.succeed( + parts.length === 0 && result.structuredContent !== undefined + ? JSON.stringify(result.structuredContent) + : parts.join("\n") + ) +} diff --git a/repos/effect/packages/effect/src/ai/McpClientError.ts b/repos/effect/packages/effect/src/ai/McpClientError.ts new file mode 100644 index 0000000000..a9a702b380 --- /dev/null +++ b/repos/effect/packages/effect/src/ai/McpClientError.ts @@ -0,0 +1,329 @@ +/** + * Errors produced by `McpClient` operations and transports. + * + * Every failure is an `McpClientError` whose `reason` is one of the reason + * classes in this module: transport, protocol, HTTP, unsupported feature, + * closed connection, timeout, limit, configuration, validation or + * tool-reported errors. + * + * @stability experimental + * @since 4.0.6 + */ +import * as Predicate from "../Predicate.ts" +import * as Schema from "../Schema.ts" +import * as McpSchema from "./McpSchema.ts" + +const TypeId = "~effect/ai/McpClientError" as const +const ReasonTypeId = "~effect/ai/McpClientError/Reason" as const + +/** + * The connection to the server failed: a process or HTTP request could not be + * started, written or read, or ended before the response. + * + * @stability experimental + * @category reasons + * @since 4.0.6 + */ +export class TransportError extends Schema.Error("effect/ai/McpClientError/TransportError")({ + _tag: Schema.tag("TransportError"), + description: Schema.optional(Schema.String), + cause: Schema.optional(Schema.Defect()) +}) { + readonly [ReasonTypeId] = ReasonTypeId + + override get message(): string { + return this.description ?? "MCP transport failed" + } +} + +/** + * The server returned a JSON-RPC error, or sent a message that is not valid + * JSON-RPC for the selected protocol. + * + * @stability experimental + * @category reasons + * @since 4.0.6 + */ +export class ProtocolError extends Schema.Error("effect/ai/McpClientError/ProtocolError")({ + _tag: Schema.tag("ProtocolError"), + code: Schema.optional(Schema.Int), + data: Schema.optional(Schema.Unknown), + description: Schema.optional(Schema.String), + cause: Schema.optional(Schema.Defect()) +}) { + readonly [ReasonTypeId] = ReasonTypeId + + override get message(): string { + const message = this.description ?? "MCP protocol error" + return this.code === undefined ? message : `${message} (code ${this.code})` + } +} + +/** + * The server answered an HTTP request with a non-success status. A JSON-RPC + * error in the response body supplies `code`, `data` and `description`. + * + * @stability experimental + * @category reasons + * @since 4.0.6 + */ +export class HttpError extends Schema.Error("effect/ai/McpClientError/HttpError")({ + _tag: Schema.tag("HttpError"), + status: Schema.Int, + code: Schema.optional(Schema.Int), + data: Schema.optional(Schema.Unknown), + description: Schema.optional(Schema.String), + wwwAuthenticate: Schema.optional(Schema.String), + retryAfter: Schema.optional(Schema.String) +}) { + readonly [ReasonTypeId] = ReasonTypeId + + override get message(): string { + const message = `MCP HTTP status ${this.status}` + return this.description === undefined ? message : `${message}: ${this.description}` + } +} + +/** + * The server or the selected protocol does not support the requested + * operation, or a tool result cannot be converted for a language model. + * + * @stability experimental + * @category reasons + * @since 4.0.6 + */ +export class UnsupportedError extends Schema.Error("effect/ai/McpClientError/UnsupportedError")({ + _tag: Schema.tag("UnsupportedError"), + description: Schema.String, + result: Schema.optional(McpSchema.CallToolResult) +}) { + readonly [ReasonTypeId] = ReasonTypeId + + override get message(): string { + return this.description + } +} + +/** + * The client, its transport or its legacy HTTP session is closed. + * + * @stability experimental + * @category reasons + * @since 4.0.6 + */ +export class ClosedError extends Schema.Error("effect/ai/McpClientError/ClosedError")({ + _tag: Schema.tag("ClosedError"), + sessionExpired: Schema.optional(Schema.Boolean), + description: Schema.optional(Schema.String) +}) { + readonly [ReasonTypeId] = ReasonTypeId + + override get message(): string { + return this.sessionExpired === true ? "MCP session expired" : this.description ?? "MCP connection closed" + } +} + +/** + * An operation did not complete within its deadline. + * + * @stability experimental + * @category reasons + * @since 4.0.6 + */ +export class TimeoutError extends Schema.Error("effect/ai/McpClientError/TimeoutError")({ + _tag: Schema.tag("TimeoutError"), + operation: Schema.String +}) { + readonly [ReasonTypeId] = ReasonTypeId + + override get message(): string { + return `MCP ${this.operation} deadline exceeded` + } +} + +/** + * A message or a discovery scan exceeded a client limit. + * + * @stability experimental + * @category reasons + * @since 4.0.6 + */ +export class LimitError extends Schema.Error("effect/ai/McpClientError/LimitError")({ + _tag: Schema.tag("LimitError"), + description: Schema.String +}) { + readonly [ReasonTypeId] = ReasonTypeId + + override get message(): string { + return this.description + } +} + +/** + * Client or toolkit options are invalid. + * + * @stability experimental + * @category reasons + * @since 4.0.6 + */ +export class ConfigurationError + extends Schema.Error("effect/ai/McpClientError/ConfigurationError")({ + _tag: Schema.tag("ConfigurationError"), + description: Schema.String, + cause: Schema.optional(Schema.Defect()) + }) +{ + readonly [ReasonTypeId] = ReasonTypeId + + override get message(): string { + return this.description + } +} + +/** + * A request payload, tool argument or result failed schema validation. + * + * @stability experimental + * @category reasons + * @since 4.0.6 + */ +export class ValidationError extends Schema.Error("effect/ai/McpClientError/ValidationError")({ + _tag: Schema.tag("ValidationError"), + description: Schema.String, + cause: Schema.optional(Schema.Defect()) +}) { + readonly [ReasonTypeId] = ReasonTypeId + + override get message(): string { + return this.description + } +} + +/** + * A tool reported an execution error with `isError`. The complete result is + * retained. + * + * @stability experimental + * @category reasons + * @since 4.0.6 + */ +export class ToolError extends Schema.Error("effect/ai/McpClientError/ToolError")({ + _tag: Schema.tag("ToolError"), + result: McpSchema.CallToolResult +}) { + readonly [ReasonTypeId] = ReasonTypeId + + override get message(): string { + const text = this.result.content.find((block) => block.type === "text") + return text === undefined ? "MCP tool reported an error" : `MCP tool reported an error: ${text.text}` + } +} + +/** + * Union of MCP client and transport failure reasons. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export type McpClientErrorReason = + | TransportError + | ProtocolError + | HttpError + | UnsupportedError + | ClosedError + | TimeoutError + | LimitError + | ConfigurationError + | ValidationError + | ToolError + +/** + * Schema for the reasons carried by an `McpClientError`. + * + * @stability experimental + * @category schemas + * @since 4.0.6 + */ +export const McpClientErrorReason: Schema.Union<[ + typeof TransportError, + typeof ProtocolError, + typeof HttpError, + typeof UnsupportedError, + typeof ClosedError, + typeof TimeoutError, + typeof LimitError, + typeof ConfigurationError, + typeof ValidationError, + typeof ToolError +]> = Schema.Union([ + TransportError, + ProtocolError, + HttpError, + UnsupportedError, + ClosedError, + TimeoutError, + LimitError, + ConfigurationError, + ValidationError, + ToolError +]) + +/** + * Error wrapping an MCP client or transport failure reason. + * + * **Details** + * + * Use `Effect.catchReason` or `Effect.catchReasons` to recover from selected + * reasons. The error's `message` and `cause` come from its reason. + * + * **Example** (Recovering from a deadline failure) + * + * ```ts import.meta.vitest + * import { Effect } from "effect" + * import { McpClientError } from "effect/ai" + * + * const operation = Effect.fail(new McpClientError.McpClientError({ + * reason: new McpClientError.TimeoutError({ operation: "tools/call" }) + * })) + * const recovered = operation.pipe( + * Effect.catchReason("McpClientError", "TimeoutError", () => Effect.succeed("deferred")) + * ) + * + * await Effect.runPromise(recovered) // => "deferred" + * ``` + * + * @stability experimental + * @category errors + * @since 4.0.6 + */ +export class McpClientError extends Schema.Error("effect/ai/McpClientError/McpClientError")({ + _tag: Schema.tag("McpClientError"), + reason: McpClientErrorReason +}) { + readonly [TypeId] = TypeId + + override readonly cause = this.reason + + override get message(): string { + return this.reason.message + } +} + +/** + * Returns whether a value is an `McpClientError`. + * + * @stability experimental + * @category guards + * @since 4.0.6 + */ +export const isMcpClientError = (u: unknown): u is McpClientError => Predicate.hasProperty(u, TypeId) + +/** + * Returns whether a value is one of the `McpClientError` reasons. + * + * @stability experimental + * @category guards + * @since 4.0.6 + */ +export const isMcpClientErrorReason = (u: unknown): u is McpClientErrorReason => Predicate.hasProperty(u, ReasonTypeId) diff --git a/repos/effect/packages/effect/src/ai/McpProtocol.ts b/repos/effect/packages/effect/src/ai/McpProtocol.ts index e06a892a51..b46b02bac1 100644 --- a/repos/effect/packages/effect/src/ai/McpProtocol.ts +++ b/repos/effect/packages/effect/src/ai/McpProtocol.ts @@ -106,9 +106,9 @@ export interface TransportPolicy { */ export interface StatelessRuntimeProfile { readonly protocolVersion: string - readonly clientCapabilities: Schema.JsonObject + readonly clientCapabilities: Readonly> readonly clientInfo?: McpSchema.Implementation | undefined - readonly requestMetadata: Schema.JsonObject + readonly requestMetadata: Readonly> } /** diff --git a/repos/effect/packages/effect/src/ai/McpSchema.ts b/repos/effect/packages/effect/src/ai/McpSchema.ts index 02fa724498..f15d16d1e8 100644 --- a/repos/effect/packages/effect/src/ai/McpSchema.ts +++ b/repos/effect/packages/effect/src/ai/McpSchema.ts @@ -94,6 +94,10 @@ export const optional = ( }) ) +// Objects are checked one level deep. Incoming payloads come from JSON.parse and +// outgoing ones are serialized through Schema.toCodecJson. +const UnknownObject = Schema.Record(Schema.String, Schema.Unknown) + // ============================================================================= // Common // ============================================================================= @@ -119,6 +123,109 @@ export const RequestId: Schema.Union<[ */ export type RequestId = typeof RequestId.Type +/** + * A JSON-RPC request. + * + * @stability unstable + * @category schemas + * @since 4.0.6 + */ +export const JsonRpcRequest = Schema.Struct({ + jsonrpc: Schema.Literal("2.0"), + id: RequestId, + method: Schema.String, + params: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)) +}) + +/** + * A JSON-RPC request. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export type JsonRpcRequest = typeof JsonRpcRequest.Type + +/** + * A JSON-RPC notification without a request identifier. + * + * @stability unstable + * @category schemas + * @since 4.0.6 + */ +export const JsonRpcNotification = Schema.Struct({ + jsonrpc: Schema.Literal("2.0"), + method: Schema.String, + params: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)) +}) + +/** + * A JSON-RPC notification without a request identifier. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export type JsonRpcNotification = typeof JsonRpcNotification.Type & { readonly id?: never } + +/** + * A JSON-RPC response carrying a result or remote error. + * + * @stability unstable + * @category schemas + * @since 4.0.6 + */ +export const JsonRpcResponse = Schema.Union([ + Schema.Struct({ + jsonrpc: Schema.Literal("2.0"), + id: RequestId, + result: Schema.Unknown + }), + Schema.Struct({ + jsonrpc: Schema.Literal("2.0"), + id: Schema.optional(Schema.Union([RequestId, Schema.Null])), + error: Schema.Struct({ + code: Schema.Int, + message: Schema.String, + data: Schema.optional(Schema.Unknown) + }) + }) +]) + +/** + * A JSON-RPC response carrying exactly one result or remote error. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export type JsonRpcResponse = + | (typeof JsonRpcResponse.members[0]["Type"] & { readonly error?: never }) + | (typeof JsonRpcResponse.members[1]["Type"] & { readonly result?: never }) + +/** + * Schema for JSON-RPC messages exchanged by MCP peers. + * + * **Details** + * + * Only the envelope is validated. `params`, `result` and error `data` are left + * for the method's own schema to decode. + * + * @stability unstable + * @category schemas + * @since 4.0.6 + */ +export const JsonRpcMessage = Schema.Union([JsonRpcRequest, JsonRpcNotification, JsonRpcResponse]) + +/** + * JSON-RPC messages exchanged by MCP peers. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export type JsonRpcMessage = JsonRpcRequest | JsonRpcNotification | JsonRpcResponse + /** * Schema for MCP progress tokens that associate progress notifications with the * original request. @@ -165,7 +272,7 @@ export class RequestMeta extends Schema.Opaque()(Schema.Struct({ */ progressToken: optional(ProgressToken) }), - [Schema.JsonObject] + [UnknownObject] )) })) {} @@ -186,7 +293,7 @@ export class ResultMeta extends Schema.Opaque()(Schema.Struct({ * This result property is reserved by the protocol to allow clients and * servers to attach additional metadata to their responses. */ - _meta: optional(Schema.JsonObject) + _meta: optional(UnknownObject) })) {} /** @@ -206,7 +313,7 @@ export class NotificationMeta extends Schema.Opaque()(Schema.S * This parameter name is reserved by MCP to allow clients and servers to * attach additional metadata to their notifications. */ - _meta: optional(Schema.JsonObject) + _meta: optional(UnknownObject) })) {} /** @@ -400,7 +507,7 @@ export class ClientCapabilities extends Schema.Class( * Optional extensions capabilities advertised by the client. * Keys are extension identifiers following / (e.g. "io.modelcontextprotocol/ui"). */ - extensions: optional(Schema.Record(Schema.TemplateLiteral([Schema.String, "/", Schema.String]), Schema.Json)), + extensions: optional(Schema.Record(Schema.TemplateLiteral([Schema.String, "/", Schema.String]), Schema.Unknown)), /** * Present if the client supports listing roots. */ @@ -464,7 +571,7 @@ export class ServerCapabilities extends Schema.Opaque()(Sche * Optional extensions capabilities advertised by the server. * Keys are extension identifiers following / (e.g. "io.modelcontextprotocol/ui"). */ - extensions: optional(Schema.Record(Schema.TemplateLiteral([Schema.String, "/", Schema.String]), Schema.Json)), + extensions: optional(Schema.Record(Schema.TemplateLiteral([Schema.String, "/", Schema.String]), Schema.Unknown)), /** * Present if the server supports sending log messages to the client. */ @@ -623,6 +730,16 @@ export const PARSE_ERROR_CODE = -32700 as const */ export const HEADER_MISMATCH_ERROR_CODE = -32020 as const +/** + * Represents the MCP error code returned when a server does not support the + * protocol version requested by a client. + * + * @stability unstable + * @category constants + * @since 4.0.6 + */ +export const UNSUPPORTED_PROTOCOL_VERSION_ERROR_CODE = -32022 as const + /** * Represents an MCP/JSON-RPC error for invalid JSON that could not be parsed. * @@ -987,7 +1104,7 @@ export class Resource extends Schema.Class( * This parameter name is reserved by MCP to allow clients and servers to * attach additional metadata to resources. */ - _meta: optional(Schema.JsonObject) + _meta: optional(UnknownObject) }) {} /** @@ -1038,7 +1155,7 @@ export class ResourceTemplate extends Schema.Class( /** * Optional additional metadata for the client. */ - _meta: optional(Schema.JsonObject) + _meta: optional(UnknownObject) }) {} /** @@ -1060,7 +1177,7 @@ export class ResourceContents extends Schema.Opaque()(Schema.S /** * Optional additional metadata for the client. */ - _meta: optional(Schema.JsonObject) + _meta: optional(UnknownObject) })) {} /** @@ -1319,7 +1436,7 @@ export class Prompt extends Schema.Class( * Icons that clients can display for this prompt. */ icons: optional(Schema.Array(Icon)), - _meta: optional(Schema.JsonObject) + _meta: optional(UnknownObject) }) {} /** @@ -1339,7 +1456,7 @@ export class TextContent extends Schema.Opaque()(Schema.Struct({ * Optional annotations for the client. */ annotations: optional(Annotations), - _meta: optional(Schema.JsonObject) + _meta: optional(UnknownObject) })) {} /** @@ -1364,7 +1481,7 @@ export class ImageContent extends Schema.Opaque()(Schema.Struct({ * Optional annotations for the client. */ annotations: optional(Annotations), - _meta: optional(Schema.JsonObject) + _meta: optional(UnknownObject) })) {} /** @@ -1389,7 +1506,7 @@ export class AudioContent extends Schema.Opaque()(Schema.Struct({ * Optional annotations for the client. */ annotations: optional(Annotations), - _meta: optional(Schema.JsonObject) + _meta: optional(UnknownObject) })) {} /** @@ -1411,7 +1528,7 @@ export class EmbeddedResource extends Schema.Opaque()(Schema.S * Optional annotations for the client. */ annotations: optional(Annotations), - _meta: optional(Schema.JsonObject) + _meta: optional(UnknownObject) })) {} /** @@ -1629,16 +1746,17 @@ export class ToolAnnotations extends Schema.Opaque()(Schema.Str * * **Details** * - * Property definitions and additional root keywords are constrained to JSON - * values. The open root supports generated keywords such as `$defs`. + * Property schemas may be objects or booleans. The open root supports + * generated keywords such as `$defs`. Nested values are not validated when + * decoding. * * @stability unstable * @category tools * @since 4.0.0 */ -export type ToolJson = Schema.JsonObject & { +export type ToolJson = { readonly [x: string]: unknown } & { readonly type: "object" - readonly properties?: Readonly> | undefined + readonly properties?: Readonly> | undefined readonly required?: ReadonlyArray | undefined } @@ -1652,10 +1770,10 @@ export type ToolJson = Schema.JsonObject & { export const ToolJson: Schema.Codec = Schema.StructWithRest( Schema.Struct({ type: Schema.Literal("object"), - properties: optional(Schema.Record(Schema.String, Schema.JsonObject)), + properties: optional(Schema.Record(Schema.String, Schema.Union([UnknownObject, Schema.Boolean]))), required: optional(Schema.Array(Schema.String)) }), - [Schema.JsonObject] + [UnknownObject] ) /** @@ -1669,7 +1787,7 @@ export const ToolJson: Schema.Codec = Schema.StructWithRest( * @category tools * @since 4.0.0 */ -export type ToolOutputJson = Schema.JsonObject +export type ToolOutputJson = { readonly [x: string]: unknown } /** * Schema for {@link ToolOutputJson}. @@ -1678,7 +1796,7 @@ export type ToolOutputJson = Schema.JsonObject * @category tools * @since 4.0.0 */ -export const ToolOutputJson: Schema.Codec = Schema.JsonObject +export const ToolOutputJson: Schema.Codec = UnknownObject /** * Schema for the definition of a tool the client can call. @@ -1723,7 +1841,7 @@ export class Tool extends Schema.Class( * This parameter name is reserved by MCP to allow clients and servers to * attach additional metadata to resources. */ - _meta: optional(Schema.JsonObject) + _meta: optional(UnknownObject) }) {} /** @@ -1775,7 +1893,7 @@ export class CallToolResult extends Schema.Class("@effect/ai/Mcp /** * An optional JSON value containing the structured result of the tool call. */ - structuredContent: optional(Schema.Json), + structuredContent: optional(Schema.Unknown), /** * Whether the tool call ended in an error. * @@ -1955,7 +2073,7 @@ export class ToolUseContent extends Schema.Class("@effect/ai/Mcp * Arguments supplied to the tool. */ input: Schema.Record(Schema.String, Schema.Unknown), - _meta: optional(Schema.JsonObject) + _meta: optional(UnknownObject) }) {} /** @@ -1978,12 +2096,12 @@ export class ToolResultContent extends Schema.Class("@effect/ /** * Optional structured result returned by the tool. */ - structuredContent: optional(Schema.Json), + structuredContent: optional(Schema.Unknown), /** * Whether tool execution ended in an error. */ isError: optional(Schema.Boolean), - _meta: optional(Schema.JsonObject) + _meta: optional(UnknownObject) }) {} /** @@ -2020,7 +2138,7 @@ export type SamplingMessageContentBlock = typeof SamplingMessageContentBlock.Typ export class SamplingMessage extends Schema.Opaque()(Schema.Struct({ role: Role, content: Schema.Union([SamplingMessageContentBlock, Schema.Array(SamplingMessageContentBlock)]), - _meta: optional(Schema.JsonObject) + _meta: optional(UnknownObject) })) {} /** @@ -2141,7 +2259,7 @@ export class CreateMessageResult extends Schema.Class( )({ role: Role, content: Schema.Union([SamplingMessageContentBlock, Schema.Array(SamplingMessageContentBlock)]), - _meta: optional(Schema.JsonObject), + _meta: optional(UnknownObject), /** * The name of the model that generated the message. */ @@ -2360,7 +2478,7 @@ export class Root extends Schema.Class( /** * Optional additional metadata associated with the root. */ - _meta: optional(Schema.JsonObject) + _meta: optional(UnknownObject) }) {} /** @@ -2799,9 +2917,9 @@ export type ElicitResult = typeof ElicitResult.Type * @since 4.0.0 */ export type McpInputRequest = - | { readonly method: "roots/list"; readonly params?: Schema.JsonObject | undefined } - | { readonly method: "sampling/createMessage"; readonly params: Schema.JsonObject } - | { readonly method: "elicitation/create"; readonly params: Schema.JsonObject } + | { readonly method: "roots/list"; readonly params?: Readonly> | undefined } + | { readonly method: "sampling/createMessage"; readonly params: Readonly> } + | { readonly method: "elicitation/create"; readonly params: Readonly> } /** * Version-neutral response supplied by an MCP client for a prior input request. @@ -2810,7 +2928,7 @@ export type McpInputRequest = * @category models * @since 4.0.0 */ -export type McpInputResponse = Schema.JsonObject +export type McpInputResponse = Readonly> /** * Indicates that an MCP operation requires additional keyed input from the client. @@ -2970,7 +3088,7 @@ export class McpRequestContext extends Context.Service - | Schema.JsonObject + | Readonly> | undefined readonly inputResponses?: Readonly> | undefined readonly requestState?: string | undefined @@ -2994,7 +3112,7 @@ export class McpServerClient extends Context.Service> | undefined readonly getClient: Effect.Effect< McpReverseClient, never, diff --git a/repos/effect/packages/effect/src/ai/McpServer.ts b/repos/effect/packages/effect/src/ai/McpServer.ts index a44d32e6f5..7cf92985a0 100644 --- a/repos/effect/packages/effect/src/ai/McpServer.ts +++ b/repos/effect/packages/effect/src/ai/McpServer.ts @@ -424,214 +424,210 @@ export class McpServer extends Context.Service - Effect.gen(function*() { - const existingIndex = tools.findIndex(({ tool }) => tool.name === options.tool.name) - if (existingIndex === -1) { - tools.push(options) - } else { - tools[existingIndex] = options - } - const enabledWhen = Context.getOrUndefined(options.annotations, EnabledWhen) - yield* internalCore.tools.register({ - descriptor: new McpTool({ - ...options.tool, - title: options.tool.title ?? options.tool.annotations?.title - }), - isVisible: (profile) => - enabledWhen === undefined || enabledWhen( - { - protocolVersion: profile.protocolVersion, - capabilities: profile.clientCapabilities, - clientInfo: profile.clientInfo - } - ), - handle: (call, invocation) => - provideInvocationContext(options.handle(call.arguments), invocation).pipe( - Effect.catchTags({ - InternalError: (error) => - Effect.fail( - new McpCore.ToolExecutionError({ - name: options.tool.name, - message: error.message - }) - ), - InvalidParams: (error) => - Effect.fail( - new (invocation.requestContext.requestState === undefined && - invocation.requestContext.inputResponses === undefined - ? McpCore.InvalidToolInput - : McpCore.InvalidToolContinuation)({ - name: options.tool.name, - message: error.message - }) - ) - }), - Effect.map((result) => - Predicate.isTagged(result, "InputRequired") - ? McpCore.OperationOutcome.InputRequired(result) - : McpCore.OperationOutcome.Complete(result) - ) + addTool: Effect.fnUntraced(function*(options) { + const existingIndex = tools.findIndex(({ tool }) => tool.name === options.tool.name) + if (existingIndex === -1) { + tools.push(options) + } else { + tools[existingIndex] = options + } + const enabledWhen = Context.getOrUndefined(options.annotations, EnabledWhen) + yield* internalCore.tools.register({ + descriptor: new McpTool({ + ...options.tool, + title: options.tool.title ?? options.tool.annotations?.title + }), + isVisible: (profile) => + enabledWhen === undefined || enabledWhen( + { + protocolVersion: profile.protocolVersion, + capabilities: profile.clientCapabilities, + clientInfo: profile.clientInfo + } + ), + handle: (call, invocation) => + provideInvocationContext(options.handle(call.arguments), invocation).pipe( + Effect.catchTags({ + InternalError: (error) => + Effect.fail( + new McpCore.ToolExecutionError({ + name: options.tool.name, + message: error.message + }) + ), + InvalidParams: (error) => + Effect.fail( + new (invocation.requestContext.requestState === undefined && + invocation.requestContext.inputResponses === undefined + ? McpCore.InvalidToolInput + : McpCore.InvalidToolContinuation)({ + name: options.tool.name, + message: error.message + }) + ) + }), + Effect.map((result) => + Predicate.isTagged(result, "InputRequired") + ? McpCore.OperationOutcome.InputRequired(result) + : McpCore.OperationOutcome.Complete(result) ) - }) - yield* notifications.client["notifications/tools/list_changed"]({}) - }), - callTool: (request) => - Effect.gen(function*() { - const client = yield* McpServerClient - const result = yield* internalCore.tools.call( - request, - McpProtocolInternal.invocationFromClient(client) - ).pipe( - Effect.mapError((error) => - new InvalidParams({ - message: error._tag === "ToolNotFound" - ? `Tool '${error.name}' not found` - : error.message - }) ) + }) + yield* notifications.client["notifications/tools/list_changed"]({}) + }), + callTool: Effect.fnUntraced(function*(request) { + const client = yield* McpServerClient + const result = yield* internalCore.tools.call( + request, + McpProtocolInternal.invocationFromClient(client) + ).pipe( + Effect.mapError((error) => + new InvalidParams({ + message: error._tag === "ToolNotFound" + ? `Tool '${error.name}' not found` + : error.message + }) ) - if (result._tag === "InputRequired") { - return yield* new InvalidParams({ message: "Client input is not supported by this MCP protocol" }) - } - return result.value - }), + ) + if (result._tag === "InputRequired") { + return yield* new InvalidParams({ message: "Client input is not supported by this MCP protocol" }) + } + return result.value + }), get resources() { return resources }, get resourceTemplates() { return resourceTemplates }, - addResource: (options) => - Effect.gen(function*() { - const existingIndex = resources.findIndex(({ resource }) => resource.uri === options.resource.uri) - if (existingIndex === -1) { - resources.push(options) - } else { - resources[existingIndex] = options - } - yield* internalCore.resources.register({ - descriptor: options.resource, - isVisible: (profile) => { - const enabledWhen = Context.getOrUndefined(options.annotations, EnabledWhen) - return enabledWhen === undefined || enabledWhen({ - protocolVersion: profile.protocolVersion, - capabilities: profile.clientCapabilities, - clientInfo: profile.clientInfo - }) - }, - read: (invocation) => provideInvocationContext(options.handle, invocation) - }) - yield* notifications.client["notifications/resources/list_changed"]({}) - }), - addResourceTemplate: ({ annotations, completions, handle, routerPath, template }) => - Effect.gen(function*() { - const existingIndex = resourceTemplates.findIndex(({ template: current }) => - current.uriTemplate === template.uriTemplate + addResource: Effect.fnUntraced(function*(options) { + const existingIndex = resources.findIndex(({ resource }) => resource.uri === options.resource.uri) + if (existingIndex === -1) { + resources.push(options) + } else { + resources[existingIndex] = options + } + yield* internalCore.resources.register({ + descriptor: options.resource, + isVisible: (profile) => { + const enabledWhen = Context.getOrUndefined(options.annotations, EnabledWhen) + return enabledWhen === undefined || enabledWhen({ + protocolVersion: profile.protocolVersion, + capabilities: profile.clientCapabilities, + clientInfo: profile.clientInfo + }) + }, + read: (invocation) => provideInvocationContext(options.handle, invocation) + }) + yield* notifications.client["notifications/resources/list_changed"]({}) + }), + addResourceTemplate: Effect.fnUntraced(function*({ annotations, completions, handle, routerPath, template }) { + const existingIndex = resourceTemplates.findIndex(({ template: current }) => + current.uriTemplate === template.uriTemplate + ) + if (existingIndex === -1) { + resourceTemplates.push({ template, annotations }) + } else { + resourceTemplates[existingIndex] = { template, annotations } + } + const templateMatcher = makeUriMatcher() + templateMatcher.add(routerPath, true) + yield* internalCore.resources.registerTemplate({ + descriptor: template, + isVisible: (profile) => { + const enabledWhen = Context.getOrUndefined(annotations, EnabledWhen) + return enabledWhen === undefined || enabledWhen({ + protocolVersion: profile.protocolVersion, + capabilities: profile.clientCapabilities, + clientInfo: profile.clientInfo + }) + }, + match: (uri) => { + const match = templateMatcher.find(uri) + if (match === undefined) { + return undefined + } + const params: Array = [] + for (const key of Object.keys(match.params)) { + params[Number(key)] = match.params[key]! + } + return params + }, + read: (uri, params, invocation) => provideInvocationContext(handle(uri, Array.from(params)), invocation) + }) + for (const [param, handle] of Object.entries(completions)) { + yield* internalCore.completions.register( + `resource/${template.uriTemplate}/${param}`, + (request, invocation) => + provideInvocationContext(handle(request.argument.value, request.context), invocation).pipe( + Effect.map((result) => ({ + values: result.completion.values, + total: result.completion.total, + hasMore: result.completion.hasMore, + metadata: result._meta + })) + ) ) - if (existingIndex === -1) { - resourceTemplates.push({ template, annotations }) - } else { - resourceTemplates[existingIndex] = { template, annotations } - } - const templateMatcher = makeUriMatcher() - templateMatcher.add(routerPath, true) - yield* internalCore.resources.registerTemplate({ - descriptor: template, - isVisible: (profile) => { - const enabledWhen = Context.getOrUndefined(annotations, EnabledWhen) - return enabledWhen === undefined || enabledWhen({ - protocolVersion: profile.protocolVersion, - capabilities: profile.clientCapabilities, - clientInfo: profile.clientInfo - }) - }, - match: (uri) => { - const match = templateMatcher.find(uri) - if (match === undefined) { - return undefined - } - const params: Array = [] - for (const key of Object.keys(match.params)) { - params[Number(key)] = match.params[key]! - } - return params - }, - read: (uri, params, invocation) => provideInvocationContext(handle(uri, Array.from(params)), invocation) - }) - for (const [param, handle] of Object.entries(completions)) { - yield* internalCore.completions.register( - `resource/${template.uriTemplate}/${param}`, - (request, invocation) => - provideInvocationContext(handle(request.argument.value, request.context), invocation).pipe( - Effect.map((result) => ({ - values: result.completion.values, - total: result.completion.total, - hasMore: result.completion.hasMore, - metadata: result._meta - })) - ) - ) - } - yield* notifications.client["notifications/resources/list_changed"]({}) - }), - findResource: (uri) => - Effect.gen(function*() { - const client = yield* McpServerClient - return yield* internalCore.resources.read( - uri, - McpProtocolInternal.invocationFromClient(client) - ).pipe( - Effect.catchTag("ResourceNotFound", (error) => - Effect.fail(new InvalidParams({ message: `Resource '${error.uri}' not found` }))) + } + yield* notifications.client["notifications/resources/list_changed"]({}) + }), + findResource: Effect.fnUntraced(function*(uri) { + const client = yield* McpServerClient + return yield* internalCore.resources.read( + uri, + McpProtocolInternal.invocationFromClient(client) + ).pipe( + Effect.catchTag( + "ResourceNotFound", + (error) => Effect.fail(new InvalidParams({ message: `Resource '${error.uri}' not found` })) ) - }), + ) + }), get prompts() { return prompts }, - addPrompt: (options) => - Effect.gen(function*() { - const existingIndex = prompts.findIndex(({ prompt }) => prompt.name === options.prompt.name) - if (existingIndex === -1) { - prompts.push(options) - } else { - prompts[existingIndex] = options - } - yield* internalCore.prompts.register({ - descriptor: options.prompt, - isVisible: (profile) => { - const enabledWhen = Context.getOrUndefined(options.annotations, EnabledWhen) - return enabledWhen === undefined || enabledWhen({ - protocolVersion: profile.protocolVersion, - capabilities: profile.clientCapabilities, - clientInfo: profile.clientInfo - }) - }, - get: (params, invocation) => - provideInvocationContext(options.handle(params), invocation).pipe( - Effect.map((result) => - Predicate.isTagged(result, "InputRequired") - ? McpCore.OperationOutcome.InputRequired(result) - : McpCore.OperationOutcome.Complete(result) - ) + addPrompt: Effect.fnUntraced(function*(options) { + const existingIndex = prompts.findIndex(({ prompt }) => prompt.name === options.prompt.name) + if (existingIndex === -1) { + prompts.push(options) + } else { + prompts[existingIndex] = options + } + yield* internalCore.prompts.register({ + descriptor: options.prompt, + isVisible: (profile) => { + const enabledWhen = Context.getOrUndefined(options.annotations, EnabledWhen) + return enabledWhen === undefined || enabledWhen({ + protocolVersion: profile.protocolVersion, + capabilities: profile.clientCapabilities, + clientInfo: profile.clientInfo + }) + }, + get: (params, invocation) => + provideInvocationContext(options.handle(params), invocation).pipe( + Effect.map((result) => + Predicate.isTagged(result, "InputRequired") + ? McpCore.OperationOutcome.InputRequired(result) + : McpCore.OperationOutcome.Complete(result) ) - }) - for (const [param, handle] of Object.entries(options.completions)) { - yield* internalCore.completions.register( - `prompt/${options.prompt.name}/${param}`, - (request, invocation) => - provideInvocationContext(handle(request.argument.value, request.context), invocation).pipe( - Effect.map((result) => ({ - values: result.completion.values, - total: result.completion.total, - hasMore: result.completion.hasMore, - metadata: result._meta - })) - ) ) - } - yield* notifications.client["notifications/prompts/list_changed"]({}) - }), + }) + for (const [param, handle] of Object.entries(options.completions)) { + yield* internalCore.completions.register( + `prompt/${options.prompt.name}/${param}`, + (request, invocation) => + provideInvocationContext(handle(request.argument.value, request.context), invocation).pipe( + Effect.map((result) => ({ + values: result.completion.values, + total: result.completion.total, + hasMore: result.completion.hasMore, + metadata: result._meta + })) + ) + ) + } + yield* notifications.client["notifications/prompts/list_changed"]({}) + }), getPromptResult: Effect.fnUntraced(function*({ arguments: params, name }) { const client = yield* McpServerClient const outcome = yield* internalCore.prompts.get( @@ -916,18 +912,15 @@ const runWithRuntime = Effect.fnUntraced(function*( const isInitialize = rpc._tag.endsWith("/initialize") if (!isInitialize && !session) { const fiber = Fiber.getCurrent()! - const httpRequest = Context.getOrUndefined(fiber.context, HttpServerRequest.HttpServerRequest) - if (httpRequest) { - appendPreResponseHandlerUnsafe( - httpRequest, - () => - Effect.succeed( - HttpServerResponse.empty({ - status: headers[MCP_SESSION_ID_HEADER] === undefined ? 400 : 404 - }) - ) - ) - } + appendPreResponseHandlerUnsafe( + fiber.context, + () => + Effect.succeed( + HttpServerResponse.empty({ + status: headers[MCP_SESSION_ID_HEADER] === undefined ? 400 : 404 + }) + ) + ) return Effect.die(new Error(`Mcp-Session-Id does not exist`)) } // RPC middleware erases the correlation between the initialize tag @@ -966,7 +959,7 @@ const runWithRuntime = Effect.fnUntraced(function*( clientInfo: profile.clientInfo, initializePayload, requestMetadata: Predicate.isReadonlyObject(payload) && Predicate.isReadonlyObject(payload._meta) - ? payload._meta as Schema.JsonObject + ? payload._meta as Readonly> : undefined, getClient: RcMap.get( clients, @@ -1033,7 +1026,7 @@ const runWithRuntime = Effect.fnUntraced(function*( ? Context.getOrUndefined(fiber.context, HttpServerRequest.HttpServerRequest) : undefined if (httpRequest !== undefined && request._tag !== "Eof") { - appendPreResponseHandlerUnsafe(httpRequest, (_, response) => + appendPreResponseHandlerUnsafe(fiber.context, (_, response) => Effect.succeed( response.status === 200 && response.body._tag === "Uint8Array" && @@ -1095,7 +1088,7 @@ const runWithRuntime = Effect.fnUntraced(function*( ) } if (httpRequest !== undefined && session !== undefined) { - appendPreResponseHandlerUnsafe(httpRequest, (_, res) => + appendPreResponseHandlerUnsafe(fiber.context, (_, res) => Effect.succeed( runtime.resolveRequest(clientId, headers) === undefined ? HttpServerResponse.empty({ status: 404 }) @@ -1111,17 +1104,15 @@ const runWithRuntime = Effect.fnUntraced(function*( const isClientNotification = selectedProtocol.clientNotificationRpcs.requests.has(request.tag) if (rpc && isClientNotification && request.isNotification) { if (!session && selectedProtocol.runtime._tag === "Stateful") { - if (httpRequest) { - appendPreResponseHandlerUnsafe( - httpRequest, - () => - Effect.succeed( - HttpServerResponse.empty({ - status: headers[MCP_SESSION_ID_HEADER] === undefined ? 400 : 404 - }) - ) - ) - } + appendPreResponseHandlerUnsafe( + fiber.context, + () => + Effect.succeed( + HttpServerResponse.empty({ + status: headers[MCP_SESSION_ID_HEADER] === undefined ? 400 : 404 + }) + ) + ) return } const decode = selectedProtocol.payloadCodecs(rpc).decode(request.payload) @@ -2192,10 +2183,16 @@ const isParameterValidationError = ( ): error is AiError.AiError & { readonly reason: AiError.ToolParameterValidationError } => AiError.isAiError(error) && error.reason._tag === "ToolParameterValidationError" -// MCP requires an object root, so a top-level `$ref` is inlined. +// MCP requires an object root. A non-recursive root is compiled inline rather +// than as a definition; a recursive root keeps its definition and the top-level +// `$ref` is inlined. const toolJsonSchema = (schema: Schema.Constraint, strict: boolean): JsonSchema.JsonSchema => { + const root = SchemaAST.getContextOwner(SchemaAST.getLastEncoding(Schema.toCodecJson(schema).ast)) const document = InternalStructuredOutput.resolveTopLevelReference( - Schema.toJsonSchemaDocument(schema, { onExcessProperty: strict ? "error" : "ignore" }) + Schema.toJsonSchemaDocument(schema, { + onExcessProperty: strict ? "error" : "ignore", + referencePolicy: ({ ast, identifier, occurrences }) => ast === root && occurrences === 1 ? undefined : identifier + }) ) return Object.keys(document.definitions).length === 0 ? document.schema diff --git a/repos/effect/packages/effect/src/ai/Prompt.ts b/repos/effect/packages/effect/src/ai/Prompt.ts index 45f30c9f9d..150075810c 100644 --- a/repos/effect/packages/effect/src/ai/Prompt.ts +++ b/repos/effect/packages/effect/src/ai/Prompt.ts @@ -465,9 +465,15 @@ export interface FilePartEncoded extends BasePartEncoded<"file", FilePartOptions */ readonly fileName?: string | undefined /** - * File data as base64 string of data, a byte array, or a URL. + * File data as a base64 string, a byte array, a URL, or the tagged JSON + * form that encoding produces for byte arrays and URLs. */ - readonly data: string | Uint8Array | URL + readonly data: + | string + | Uint8Array + | URL + | { readonly _tag: "Uint8Array"; readonly base64: string } + | { readonly _tag: "URL"; readonly href: string } } /** @@ -491,7 +497,18 @@ export const FilePart: Schema.Struct<{ readonly type: Schema.Literal<"file"> readonly mediaType: Schema.String readonly fileName: Schema.optional - readonly data: Schema.Union + readonly data: Schema.Union< + readonly [ + Schema.String, + Schema.decodeTo< + Schema.Uint8Array, + Schema.TaggedStruct<"Uint8Array", { readonly base64: Schema.Uint8ArrayFromBase64 }> + >, + Schema.decodeTo>, + Schema.Uint8Array, + Schema.URL + ] + > readonly "~effect/ai/Prompt/Part": Schema.withDecodingDefaultKey> readonly options: Schema.withDecodingDefault< Schema.$Record< @@ -504,7 +521,27 @@ export const FilePart: Schema.Struct<{ type: Schema.Literal("file"), mediaType: Schema.String, fileName: Schema.optional(Schema.String), - data: Schema.Union([Schema.String, Schema.Uint8Array, Schema.URL]) + // The tagged members come first so that encoding produces JSON-safe data, + // while the raw members keep decoding byte arrays and URLs directly + data: Schema.Union([ + Schema.String, + Schema.TaggedStruct("Uint8Array", { base64: Schema.Uint8ArrayFromBase64 }).pipe(Schema.decodeTo( + Schema.Uint8Array, + SchemaTransformation.transform({ + decode: ({ base64 }) => base64, + encode: (base64) => ({ _tag: "Uint8Array", base64 }) as const + }) + )), + Schema.TaggedStruct("URL", { href: Schema.URLFromString }).pipe(Schema.decodeTo( + Schema.URL, + SchemaTransformation.transform({ + decode: ({ href }) => href, + encode: (href) => ({ _tag: "URL", href }) as const + }) + )), + Schema.Uint8Array, + Schema.URL + ]) }).annotate({ identifier: "FilePart" }) /** @@ -1407,36 +1444,7 @@ export const UserMessage: Schema.Struct<{ never, never >, - Schema.$Array< - Schema.Union< - readonly [ - Schema.Struct<{ - readonly type: Schema.Literal<"text"> - readonly text: Schema.String - readonly "~effect/ai/Prompt/Part": Schema.withDecodingDefaultKey> - readonly options: Schema.withDecodingDefault< - Schema.$Record< - Schema.String, - Schema.NullOr> - > - > - }>, - Schema.Struct<{ - readonly type: Schema.Literal<"file"> - readonly mediaType: Schema.String - readonly fileName: Schema.optional - readonly data: Schema.Union - readonly "~effect/ai/Prompt/Part": Schema.withDecodingDefaultKey> - readonly options: Schema.withDecodingDefault< - Schema.$Record< - Schema.String, - Schema.NullOr> - > - > - }> - ] - > - > + Schema.$Array> ] > readonly "~effect/ai/Prompt/Message": Schema.withDecodingDefaultKey> diff --git a/repos/effect/packages/effect/src/ai/Response.ts b/repos/effect/packages/effect/src/ai/Response.ts index ed3e15077e..deb29613b9 100644 --- a/repos/effect/packages/effect/src/ai/Response.ts +++ b/repos/effect/packages/effect/src/ai/Response.ts @@ -170,6 +170,21 @@ const toolPartSchemas = (tool: Tool.Any) => { return parts } +type AnyToolParts = ToolCallPart | ToolResultPart + +const toolPartSchemasFor = ( + toolkit: Toolkit.Any | Toolkit.WithHandler | undefined +): ReadonlyArray => { + if (toolkit === undefined) { + return [ + makeToolCallPart(Schema.String, Schema.Unknown), + makeToolResultPart(Schema.String, Schema.Unknown, Schema.Unknown) + ] + } + const parts = Object.values(toolkit.tools as Record).map(toolPartSchemas) + return [...parts.map((part) => part.toolCall), ...parts.map((part) => part.toolResult)] +} + /** * Creates a Schema for all response parts based on a toolkit. * @@ -178,6 +193,9 @@ const toolPartSchemas = (tool: Tool.Any) => { * Generates a schema that includes all possible response parts, with tool call * and tool result parts dynamically created based on the provided toolkit. * + * When no toolkit is provided, tool call and tool result parts accept any tool + * name, with parameters and results decoded as `unknown`. + * * **Example** (Building a response parts schema) * * ```ts import.meta.vitest @@ -193,28 +211,25 @@ const toolPartSchemas = (tool: Tool.Any) => { * * const allPartsSchema = Response.AllParts(myToolkit) * Schema.isSchema(allPartsSchema) // => true + * + * const anyToolPartsSchema = Response.AllParts() + * Schema.isSchema(anyToolPartsSchema) // => true * ``` * * @stability unstable * @category schemas * @since 4.0.0 */ -export const AllParts = >( - toolkit: T -): Schema.Codec< - AllParts : Toolkit.WithHandlerTools>, - AllPartsEncoded, - Tool.ResultDecodingServices[keyof Toolkit.Tools]>, - Tool.ResultEncodingServices[keyof Toolkit.Tools]> -> => { - const toolCalls: Array = [] - const toolResults: Array = [] - for (const tool of Object.values(toolkit.tools as Record)) { - const parts = toolPartSchemas(tool) - toolCalls.push(parts.toolCall) - toolResults.push(parts.toolResult) - } - return Schema.Union([ +export const AllParts: { + >(toolkit: T): Schema.Codec< + AllParts : Toolkit.WithHandlerTools>, + AllPartsEncoded, + Tool.ResultDecodingServices[keyof Toolkit.Tools]>, + Tool.ResultEncodingServices[keyof Toolkit.Tools]> + > + (): Schema.Codec | AnyToolParts, AllPartsEncoded> +} = (toolkit?: Toolkit.Any | Toolkit.WithHandler) => + Schema.Union([ TextPart, TextStartPart, TextDeltaPart, @@ -233,10 +248,8 @@ export const AllParts = >( ResponseMetadataPart, FinishPart, ErrorPart, - ...toolCalls, - ...toolResults + ...toolPartSchemasFor(toolkit) ]) as any -} // ============================================================================= // Parts @@ -289,26 +302,25 @@ export type PartEncoded = /** * Creates a Schema for non-streaming response parts based on a toolkit. * + * **Details** + * + * When no toolkit is provided, tool call and tool result parts accept any tool + * name, with parameters and results decoded as `unknown`. + * * @stability unstable * @category schemas * @since 4.0.0 */ -export const Part = >( - toolkit: T -): Schema.Codec< - Part : Toolkit.WithHandlerTools>, - PartEncoded, - Tool.ResultDecodingServices[keyof Toolkit.Tools]>, - Tool.ResultEncodingServices[keyof Toolkit.Tools]> -> => { - const toolCalls: Array = [] - const toolResults: Array = [] - for (const tool of Object.values(toolkit.tools as Record)) { - const parts = toolPartSchemas(tool) - toolCalls.push(parts.toolCall) - toolResults.push(parts.toolResult) - } - return Schema.Union([ +export const Part: { + >(toolkit: T): Schema.Codec< + Part : Toolkit.WithHandlerTools>, + PartEncoded, + Tool.ResultDecodingServices[keyof Toolkit.Tools]>, + Tool.ResultEncodingServices[keyof Toolkit.Tools]> + > + (): Schema.Codec | AnyToolParts, PartEncoded> +} = (toolkit?: Toolkit.Any | Toolkit.WithHandler) => + Schema.Union([ TextPart, ReasoningPart, ToolApprovalRequestPart, @@ -317,10 +329,8 @@ export const Part = >( UrlSourcePart, ResponseMetadataPart, FinishPart, - ...toolCalls, - ...toolResults + ...toolPartSchemasFor(toolkit) ]) as any -} // ============================================================================= // Stream Parts @@ -386,26 +396,25 @@ export type StreamPartEncoded = /** * Creates a Schema for streaming response parts based on a toolkit. * + * **Details** + * + * When no toolkit is provided, tool call and tool result parts accept any tool + * name, with parameters and results decoded as `unknown`. + * * @stability unstable * @category schemas * @since 4.0.0 */ -export const StreamPart = >( - toolkit: T -): Schema.Codec< - StreamPart : Toolkit.WithHandlerTools>, - StreamPartEncoded, - Tool.ResultDecodingServices[keyof Toolkit.Tools]>, - Tool.ResultEncodingServices[keyof Toolkit.Tools]> -> => { - const toolCalls: Array = [] - const toolResults: Array = [] - for (const tool of Object.values(toolkit.tools as Record)) { - const parts = toolPartSchemas(tool) - toolCalls.push(parts.toolCall) - toolResults.push(parts.toolResult) - } - return Schema.Union([ +export const StreamPart: { + >(toolkit: T): Schema.Codec< + StreamPart : Toolkit.WithHandlerTools>, + StreamPartEncoded, + Tool.ResultDecodingServices[keyof Toolkit.Tools]>, + Tool.ResultEncodingServices[keyof Toolkit.Tools]> + > + (): Schema.Codec | AnyToolParts, StreamPartEncoded> +} = (toolkit?: Toolkit.Any | Toolkit.WithHandler) => + Schema.Union([ TextStartPart, TextDeltaPart, TextEndPart, @@ -422,10 +431,8 @@ export const StreamPart = >( ResponseMetadataPart, FinishPart, ErrorPart, - ...toolCalls, - ...toolResults + ...toolPartSchemasFor(toolkit) ]) as any -} // ============================================================================= // utility types @@ -1522,15 +1529,17 @@ export const ToolCallPart: = ( name: Name, params: Params -) => +) => makeToolCallPart(Schema.Literal(name), params) as any + +const makeToolCallPart = (name: Schema.Top, params: Schema.Constraint) => Schema.Struct({ ...BasePart.fields, type: Schema.Literal("tool-call"), id: Schema.String, - name: Schema.Literal(name), + name, params, providerExecuted: Schema.Boolean.pipe(Schema.withDecodingDefaultKey(Effect.succeed(false))) - }).annotate({ identifier: "ToolCallPart" }) as any + }).annotate({ identifier: "ToolCallPart" }) /** * Constructs a new tool call part. @@ -1739,14 +1748,19 @@ export const ToolResultPart = < ToolResultPartEncoded, Success["DecodingServices"] | Failure["DecodingServices"], Success["EncodingServices"] | Failure["EncodingServices"] -> => { +> => + makeToolResultPart(Schema.Literal(name), success, failure).annotate({ + identifier: `ToolResultPart(${name})` + }) as any + +const makeToolResultPart = (name: Schema.Top, success: Schema.Constraint, failure: Schema.Constraint) => { const makeSchema = (result: Schema.Constraint, isFailure: boolean) => { const encodedResult = Schema.toEncoded(result) const Common = { id: Schema.String, type: Schema.Literal("tool-result"), isFailure: Schema.Literal(isFailure), - name: Schema.Literal(name) + name } const Decoded = Schema.Struct({ ...Common, @@ -1779,9 +1793,7 @@ export const ToolResultPart = < }) )) } - return Schema.Union([makeSchema(success, false), makeSchema(failure, true)]).annotate({ - identifier: `ToolResultPart(${name})` - }) + return Schema.Union([makeSchema(success, false), makeSchema(failure, true)]) } /** diff --git a/repos/effect/packages/effect/src/ai/Tool.ts b/repos/effect/packages/effect/src/ai/Tool.ts index b194655355..b1abfea674 100644 --- a/repos/effect/packages/effect/src/ai/Tool.ts +++ b/repos/effect/packages/effect/src/ai/Tool.ts @@ -115,7 +115,9 @@ export type DynamicTypeId = "~effect/ai/Tool/Dynamic" * execution will be returned in the error channel of the calling effect. * * If set to `"return"`, errors that occur during tool call handler execution - * will be captured and returned as part of the tool call result. + * will be captured and returned as part of the tool call result. Like + * `Effect.catch`, only the first typed failure is returned, and a cause + * containing a defect is raised as a defect. * * @stability unstable * @category models @@ -246,7 +248,9 @@ export interface Tool< * effect. * * If set to `"return"`, errors that occur during tool call handler execution - * will be captured and returned as part of the tool call result. + * will be captured and returned as part of the tool call result. Like + * `Effect.catch`, only the first typed failure is returned, and a cause + * containing a defect is raised as a defect. */ readonly failureMode: FailureMode @@ -1295,7 +1299,9 @@ export const make = < * execution will be returned in the error channel of the calling effect. * * If set to `"return"`, errors that occur during tool call handler execution - * will be captured and returned as part of the tool call result. + * will be captured and returned as part of the tool call result. Like + * `Effect.catch`, only the first typed failure is returned, and a cause + * containing a defect is raised as a defect. */ readonly failureMode?: Mode /** @@ -1534,7 +1540,9 @@ export const providerDefined = < * execution will be returned in the error channel of the calling effect. * * If set to `"return"`, errors that occur during tool call handler execution - * will be captured and returned as part of the tool call result. + * will be captured and returned as part of the tool call result. Like + * `Effect.catch`, only the first typed failure is returned, and a cause + * containing a defect is raised as a defect. */ readonly failureMode?: Mode | undefined } diff --git a/repos/effect/packages/effect/src/ai/Toolkit.ts b/repos/effect/packages/effect/src/ai/Toolkit.ts index c017c35add..5872416af9 100644 --- a/repos/effect/packages/effect/src/ai/Toolkit.ts +++ b/repos/effect/packages/effect/src/ai/Toolkit.ts @@ -125,7 +125,8 @@ export interface HandlerContext { * **Details** * * Preliminary results are streamed to the caller before the handler completes, - * enabling real-time progress updates for lengthy operations. + * enabling real-time progress updates for lengthy operations. Results are + * buffered without limit unless the caller provides `PreliminaryBuffer`. */ readonly preliminary: (result: Tool.Success) => Effect.Effect } @@ -296,9 +297,60 @@ export const FailureOrigin = Context.Reference("effect/ai/To defaultValue: () => "result" }) +/** + * Limits how many preliminary results `Toolkit.handle` buffers per tool call. + * + * **Details** + * + * Defaults to `"unbounded"`. Provide it as a positive integer around `handle`, + * or around a program that calls it, and `HandlerContext.preliminary` suspends + * once that many results are unconsumed. Any other value is a defect. + * + * The bound only applies to the stream returned by `handle`. Consume that + * stream while the handler runs; waiting for the handler before reading it + * deadlocks once the buffer fills. `LanguageModel` moves results into its own + * buffers, so the bound does not limit memory end to end there. + * + * **Example** (Bounding preliminary results) + * + * ```ts import.meta.vitest + * import { Effect, Schema, Stream } from "effect" + * import { Tool, Toolkit } from "effect/ai" + * + * const toolkit = Toolkit.make(Tool.make("Progress", { success: Schema.Number })) + * + * const program = Effect.gen(function*() { + * const handlers = yield* toolkit + * const stream = yield* handlers.handle("Progress", {}).pipe( + * Effect.provideService(Toolkit.PreliminaryBuffer, 16) + * ) + * const results = yield* Stream.runCollect(stream) + * return results.length + * }).pipe( + * Effect.provide(toolkit.toLayer({ + * Progress: Effect.fnUntraced(function*(_, ctx) { + * for (let i = 0; i < 64; i++) yield* ctx.preliminary(i) + * return 64 + * }) + * })) + * ) + * + * await Effect.runPromise(program) // => 65 + * ``` + * + * @stability unstable + * @category services + * @since 4.1.0 + */ +export const PreliminaryBuffer = Context.Reference("effect/ai/Toolkit/PreliminaryBuffer", { + defaultValue: () => "unbounded" +}) + const failureCause = (error: E, origin: Tool.FailureOrigin): Cause.Cause => Cause.annotate(Cause.fail(error), Context.make(FailureOrigin, origin)) +const failureOrigin = (error: unknown): Tool.FailureOrigin => Schema.isSchemaError(error) ? "result" : "handler" + const Proto = { ...Effectable.Prototype({ label: "Toolkit", @@ -402,11 +454,15 @@ const Proto = { } const decodedParams = decodedParamsResult.success + const capacity = yield* PreliminaryBuffer + if (capacity !== "unbounded" && !(Number.isInteger(capacity) && capacity > 0)) { + return yield* Effect.die(`Tool "${name}": PreliminaryBuffer must be a positive integer, got ${capacity}`) + } const queue = yield* Queue.make<{ readonly result: any readonly isFailure: boolean readonly preliminary: boolean - }, Cause.Done>() + }, Cause.Done>(capacity === "unbounded" ? undefined : { capacity }) const context: HandlerContext = { toolCallId, preliminary: (result) => @@ -448,18 +504,30 @@ const Proto = { } return Stream.fromQueue(queue).pipe( - Stream.catch((error) => { - const normalizedError = normalizeError(error) - const failureOrigin: Tool.FailureOrigin = Schema.isSchemaError(error) ? "result" : "handler" - return tool.failureMode === "error" - ? Stream.failCause(failureCause(normalizedError, failureOrigin)) - : Stream.succeed({ result: normalizedError, isFailure: true, failureOrigin, preliminary: false }) - }), + tool.failureMode === "error" + ? Stream.catchCause((cause) => + Stream.failCause(Cause.map( + Cause.fromReasons(cause.reasons.map((reason) => + Cause.isFailReason(reason) + ? reason.annotate(Context.make(FailureOrigin, failureOrigin(reason.error)), { overwrite: true }) + : reason + )), + normalizeError + )) + ) + : Stream.catch((error) => + Stream.succeed({ + result: normalizeError(error), + isFailure: true, + failureOrigin: failureOrigin(error), + preliminary: false + }) + ), Stream.mapEffect(Effect.fnUntraced(function*(output) { const encodedResult = yield* encodeResult(output.result, output.isFailure) return { ...output, encodedResult } })), - Stream.onEnd(Fiber.interrupt(fiber)) + Stream.ensuring(Fiber.interrupt(fiber)) ) satisfies Stream.Stream, any> }) diff --git a/repos/effect/packages/effect/src/ai/index.ts b/repos/effect/packages/effect/src/ai/index.ts index 30524cc1d4..a855121335 100644 --- a/repos/effect/packages/effect/src/ai/index.ts +++ b/repos/effect/packages/effect/src/ai/index.ts @@ -53,6 +53,18 @@ export * as IdGenerator from "./IdGenerator.ts" */ export * as LanguageModel from "./LanguageModel.ts" +/** + * @stability experimental + * @since 4.0.6 + */ +export * as McpClient from "./McpClient.ts" + +/** + * @stability experimental + * @since 4.0.6 + */ +export * as McpClientError from "./McpClientError.ts" + /** * @stability unstable * @since 4.0.0 diff --git a/repos/effect/packages/effect/src/ai/internal/mcpCore.ts b/repos/effect/packages/effect/src/ai/internal/mcpCore.ts index a7d200d33b..b384c43c02 100644 --- a/repos/effect/packages/effect/src/ai/internal/mcpCore.ts +++ b/repos/effect/packages/effect/src/ai/internal/mcpCore.ts @@ -7,7 +7,6 @@ import * as Arr from "../../Array.ts" import * as Data from "../../Data.ts" import * as Effect from "../../Effect.ts" -import type * as Schema from "../../Schema.ts" import type * as McpProtocol from "../McpProtocol.ts" import * as McpSchema from "../McpSchema.ts" @@ -56,7 +55,7 @@ export interface NegotiatedProtocolProfile>) | undefined } @@ -66,7 +65,7 @@ export interface NegotiatedProtocolProfile> | undefined + readonly experimental?: Readonly>>> | undefined readonly logging?: boolean | undefined readonly completions?: boolean | undefined readonly prompts?: Readonly<{ readonly listChanged?: boolean | undefined }> | undefined @@ -77,7 +76,7 @@ export interface CanonicalServerCapabilities { }> | undefined readonly tools?: Readonly<{ readonly listChanged?: boolean | undefined }> | undefined - readonly extensions?: Schema.JsonObject | undefined + readonly extensions?: Readonly> | undefined } /** @@ -303,7 +302,7 @@ export interface CompletionResult { readonly values: ReadonlyArray readonly total?: number | undefined readonly hasMore?: boolean | undefined - readonly metadata?: Schema.JsonObject | undefined + readonly metadata?: Readonly> | undefined } /** @@ -333,7 +332,7 @@ export type ClientNotification = Data.TaggedEnum<{ readonly progress: number readonly total?: number | undefined readonly message?: string | undefined - readonly metadata?: Schema.JsonObject | undefined + readonly metadata?: Readonly> | undefined } RootsChanged: {} }> @@ -349,7 +348,7 @@ export const ClientNotification = Data.taggedEnum() export interface Cancellation { readonly requestId: string | number readonly reason?: string | undefined - readonly metadata?: Schema.JsonObject | undefined + readonly metadata?: Readonly> | undefined } /** @@ -359,25 +358,25 @@ export type ServerNotification = Data.TaggedEnum<{ Cancelled: { readonly requestId: string | number readonly reason?: string | undefined - readonly metadata?: Schema.JsonObject | undefined + readonly metadata?: Readonly> | undefined } Progress: { readonly progressToken: string | number readonly progress: number readonly total?: number | undefined readonly message?: string | undefined - readonly metadata?: Schema.JsonObject | undefined + readonly metadata?: Readonly> | undefined } LoggingMessage: { readonly level: McpSchema.LoggingLevel readonly logger?: string | undefined readonly data: unknown - readonly metadata?: Schema.JsonObject | undefined + readonly metadata?: Readonly> | undefined } - ResourceUpdated: { readonly uri: string; readonly metadata?: Schema.JsonObject | undefined } - ResourcesChanged: { readonly metadata?: Schema.JsonObject | undefined } - ToolsChanged: { readonly metadata?: Schema.JsonObject | undefined } - PromptsChanged: { readonly metadata?: Schema.JsonObject | undefined } + ResourceUpdated: { readonly uri: string; readonly metadata?: Readonly> | undefined } + ResourcesChanged: { readonly metadata?: Readonly> | undefined } + ToolsChanged: { readonly metadata?: Readonly> | undefined } + PromptsChanged: { readonly metadata?: Readonly> | undefined } ElicitationComplete: { readonly elicitationId: string } }> diff --git a/repos/effect/packages/effect/src/ai/internal/mcpProtocol.ts b/repos/effect/packages/effect/src/ai/internal/mcpProtocol.ts index 13072ba624..dccd8b047b 100644 --- a/repos/effect/packages/effect/src/ai/internal/mcpProtocol.ts +++ b/repos/effect/packages/effect/src/ai/internal/mcpProtocol.ts @@ -19,8 +19,10 @@ import * as PublicMcpSchema from "../McpSchema.ts" import * as McpCore from "./mcpCore.ts" const LEGACY_RESOURCE_NOT_FOUND_ERROR_CODE = -32002 -const BASE64_SENTINEL_PREFIX = "=?base64?" -const BASE64_SENTINEL_SUFFIX = "?=" +/** @internal */ +export const BASE64_SENTINEL_PREFIX = "=?base64?" +/** @internal */ +export const BASE64_SENTINEL_SUFFIX = "?=" // A leading U+FEFF belongs to the header value and must survive comparison with the JSON body. const routingHeaderDecoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }) @@ -120,6 +122,24 @@ export const requireCompleteOperation = ( ? Effect.succeed(outcome.value) : Effect.fail(new McpCore.UnsupportedByProtocol({ protocolVersion, feature: "Client input" })) +/** + * Projects boolean property schemas to equivalent objects for pre-June adapters. + * Their dated tool descriptors require object-valued direct property schemas. + * + * @internal + */ +export const projectLegacyToolInputSchema = (inputSchema: PublicMcpSchema.ToolJson) => ({ + ...inputSchema, + properties: inputSchema.properties === undefined + ? undefined + : Object.fromEntries>>( + Object.entries(inputSchema.properties).map(([name, schema]) => [ + name, + typeof schema === "boolean" ? schema ? {} : { not: {} } : schema + ]) + ) +}) + /** * Unquotes an exact text mirror when the protocol drops string `structuredContent`. * diff --git a/repos/effect/packages/effect/src/ai/internal/mcpProtocol/v2024_11_05.ts b/repos/effect/packages/effect/src/ai/internal/mcpProtocol/v2024_11_05.ts index 6204a33961..123486db34 100644 --- a/repos/effect/packages/effect/src/ai/internal/mcpProtocol/v2024_11_05.ts +++ b/repos/effect/packages/effect/src/ai/internal/mcpProtocol/v2024_11_05.ts @@ -257,7 +257,7 @@ export const protocol = McpProtocol.make({ McpSchema.Tool.make({ name: tool.name, description: tool.description, - inputSchema: tool.inputSchema + inputSchema: McpProtocol.projectLegacyToolInputSchema(tool.inputSchema) }) ) }) diff --git a/repos/effect/packages/effect/src/ai/internal/mcpProtocol/v2025_03_26.ts b/repos/effect/packages/effect/src/ai/internal/mcpProtocol/v2025_03_26.ts index 156c817a43..66558306f8 100644 --- a/repos/effect/packages/effect/src/ai/internal/mcpProtocol/v2025_03_26.ts +++ b/repos/effect/packages/effect/src/ai/internal/mcpProtocol/v2025_03_26.ts @@ -279,7 +279,7 @@ export const protocol = McpProtocol.make({ McpSchema.Tool.make({ name: tool.name, description: tool.description, - inputSchema: tool.inputSchema, + inputSchema: McpProtocol.projectLegacyToolInputSchema(tool.inputSchema), annotations: tool.title === undefined && tool.annotations === undefined ? undefined : McpSchema.ToolAnnotations.make({ diff --git a/repos/effect/packages/effect/src/ai/internal/mcpProtocol/v2025_06_18.ts b/repos/effect/packages/effect/src/ai/internal/mcpProtocol/v2025_06_18.ts index 2318e22dea..1ab071906e 100644 --- a/repos/effect/packages/effect/src/ai/internal/mcpProtocol/v2025_06_18.ts +++ b/repos/effect/packages/effect/src/ai/internal/mcpProtocol/v2025_06_18.ts @@ -122,10 +122,10 @@ const projectResourceContents = ( } const projectStructuredContent = ( - content: Schema.Json | undefined -): Schema.JsonObject | undefined => content === undefined || isJsonObject(content) ? content : undefined + content: unknown +): Readonly> | undefined => content === undefined || isObject(content) ? content : undefined -const isJsonObject = (value: Schema.Json): value is Schema.JsonObject => Predicate.isReadonlyObject(value) +const isObject = (value: unknown): value is Readonly> => Predicate.isReadonlyObject(value) /** * @internal diff --git a/repos/effect/packages/effect/src/ai/internal/mcpProtocol/v2025_11_25.ts b/repos/effect/packages/effect/src/ai/internal/mcpProtocol/v2025_11_25.ts index e6e9846758..487acb57ed 100644 --- a/repos/effect/packages/effect/src/ai/internal/mcpProtocol/v2025_11_25.ts +++ b/repos/effect/packages/effect/src/ai/internal/mcpProtocol/v2025_11_25.ts @@ -112,10 +112,10 @@ const projectContent = Effect.fnUntraced(function*(content: PublicMcpSchema.Cont }) const projectStructuredContent = ( - content: Schema.Json | undefined -): Schema.JsonObject | undefined => content === undefined || isJsonObject(content) ? content : undefined + content: unknown +): Readonly> | undefined => content === undefined || isObject(content) ? content : undefined -const isJsonObject = (value: Schema.Json): value is Schema.JsonObject => Predicate.isReadonlyObject(value) +const isObject = (value: unknown): value is Readonly> => Predicate.isReadonlyObject(value) /** * @internal diff --git a/repos/effect/packages/effect/src/ai/internal/mcpProtocol/v2026_07_28.ts b/repos/effect/packages/effect/src/ai/internal/mcpProtocol/v2026_07_28.ts index 604c356886..11fe397ca4 100644 --- a/repos/effect/packages/effect/src/ai/internal/mcpProtocol/v2026_07_28.ts +++ b/repos/effect/packages/effect/src/ai/internal/mcpProtocol/v2026_07_28.ts @@ -26,7 +26,7 @@ import * as McpCore from "../mcpCore.ts" import * as McpProtocol from "../mcpProtocol.ts" import * as McpSchema from "../mcpSchema/v2026_07_28.ts" -const InputResponses = Schema.Record(Schema.String, Schema.JsonObject) +const InputResponses = Schema.Record(Schema.String, Schema.Record(Schema.String, Schema.Unknown)) const decodeRequestMetadata = Schema.decodeUnknownEffect(McpSchema.RequestMetaObject) const decodeCancellation = Schema.decodeUnknownEffect(McpSchema.CancelledNotification.payloadSchema) const isJson = Schema.is(Schema.Json) @@ -92,8 +92,8 @@ export const profileFromRequestMetadata = Effect.fnUntraced(function*(metadata: const resultMetadata = ( value: ObjectWithUndefined, - serverInfo: Schema.JsonObject -): Schema.JsonObject => { + serverInfo: Readonly> +): Readonly> => { const metadata: unknown = value._meta return { ...(Predicate.isReadonlyObject(metadata) ? metadata : {}), @@ -103,7 +103,7 @@ const resultMetadata = ( const projectInputRequired = ( outcome: McpCore.InputRequiredFields, - serverInfo: Schema.JsonObject + serverInfo: Readonly> ) => omitUndefined({ _meta: { "io.modelcontextprotocol/serverInfo": serverInfo }, @@ -138,8 +138,8 @@ export const projectCallToolOutcome = Effect.fnUntraced(function*( const requiredCapabilitiesForInputRequests = ( inputRequests: McpCore.InputRequiredFields["inputRequests"], capabilities: McpCore.NegotiatedProtocolProfile["clientCapabilities"] -): Record => { - const required: Record> = {} +): Record>> => { + const required: Record> = {} for (const request of Object.values(inputRequests ?? {})) { Match.value(request).pipe( Match.when({ method: "roots/list" }, () => { @@ -264,16 +264,14 @@ const projectError = (error: ProtocolError): McpSchema.McpError => { }) } -const failWithHttpStatus = Effect.fnUntraced(function*(status: number, error: McpProtocol.ProtocolError) { - const request = yield* Effect.serviceOption(HttpServerRequest.HttpServerRequest) - if (Option.isSome(request)) { +const failWithHttpStatus = (status: number, error: McpProtocol.ProtocolError) => + Effect.withFiber((fiber) => { appendPreResponseHandlerUnsafe( - request.value, + fiber.context, (_request, response) => Effect.succeed(HttpServerResponse.setStatus(response, status)) ) - } - return yield* error -}) + return Effect.fail(error) + }) /** * @internal @@ -327,7 +325,7 @@ export const makeHandlers = ( // Older adapters accept arbitrary JSON; this binding requires extension settings objects. const extensions = context.serverInfo.extensions === undefined ? undefined : Record.filter( context.serverInfo.extensions, - (value): value is Schema.JsonObject => Predicate.isReadonlyObject(value) + (value): value is Readonly> => Predicate.isReadonlyObject(value) ) const getDiscovery = Effect.map(context.registrationPresence, (presence) => { const capabilities: Types.Mutable = { completions: {}, logging: {} } diff --git a/repos/effect/packages/effect/src/ai/internal/mcpRuntime.ts b/repos/effect/packages/effect/src/ai/internal/mcpRuntime.ts index 095b741536..de80666556 100644 --- a/repos/effect/packages/effect/src/ai/internal/mcpRuntime.ts +++ b/repos/effect/packages/effect/src/ai/internal/mcpRuntime.ts @@ -31,7 +31,6 @@ const MCP_METHOD_HEADER = "mcp-method" const MCP_NAME_HEADER = "mcp-name" const PROTOCOL_VERSION_METADATA_KEY = "io.modelcontextprotocol/protocolVersion" const CLIENT_CAPABILITIES_METADATA_KEY = "io.modelcontextprotocol/clientCapabilities" -const UNSUPPORTED_PROTOCOL_VERSION_ERROR_CODE = -32022 const asRecord = (input: unknown): Record | undefined => typeof input === "object" && input !== null ? input as Record : undefined @@ -236,7 +235,7 @@ export const make = Effect.fnUntraced(function*( const registry = yield* McpProtocolRegistry.make(protocols) const sessionTerminationListeners = new Set<(binding: RequestBinding) => Effect.Effect>() const unsupportedProtocolVersion = (requested: string) => ({ - code: UNSUPPORTED_PROTOCOL_VERSION_ERROR_CODE, + code: PublicMcpSchema.UNSUPPORTED_PROTOCOL_VERSION_ERROR_CODE, message: `Unsupported protocol version '${requested}'`, data: { supported: protocolVersions, requested } }) @@ -345,7 +344,7 @@ export const make = Effect.fnUntraced(function*( const selected = selectStatefulProtocol(registry.protocols, offeredVersion) if (selected === undefined) { return yield* new McpProtocol.ProtocolError({ - code: UNSUPPORTED_PROTOCOL_VERSION_ERROR_CODE, + code: PublicMcpSchema.UNSUPPORTED_PROTOCOL_VERSION_ERROR_CODE, message: `initialize is not supported by the configured MCP protocols (requested '${offeredVersion}')` }) } @@ -568,7 +567,7 @@ export const make = Effect.fnUntraced(function*( const sessionId = crypto.randomUUID() stateful.registerHttp(sessionId, registration) appendPreResponseHandlerUnsafe( - httpRequest, + fiber.context, (_request, response) => Effect.succeed(HttpServerResponse.setHeaders(response, { [MCP_SESSION_ID_HEADER]: sessionId, diff --git a/repos/effect/packages/effect/src/ai/internal/mcpSchema/v2024_11_05.ts b/repos/effect/packages/effect/src/ai/internal/mcpSchema/v2024_11_05.ts index ed117834fb..a1b5645d24 100644 --- a/repos/effect/packages/effect/src/ai/internal/mcpSchema/v2024_11_05.ts +++ b/repos/effect/packages/effect/src/ai/internal/mcpSchema/v2024_11_05.ts @@ -29,7 +29,14 @@ export const optional = ( encode: SchemaGetter.transformOptional(Option.flatMap(Option.fromUndefinedOr)) }) ) -const JsonObject = Schema.JsonObject + +/** + * Object checked one level deep. Payloads come from `JSON.parse` or are serialized + * through `Schema.toCodecJson`, so deep JSON validation would be redundant. + * + * @internal + */ +export const UnknownObject = Schema.Record(Schema.String, Schema.Unknown) /** * @internal @@ -65,7 +72,7 @@ export const RequestMeta = Schema.Struct({ Schema.Struct({ progressToken: optional(ProgressToken) }), - [Schema.JsonObject] + [UnknownObject] )) }) @@ -73,14 +80,14 @@ export const RequestMeta = Schema.Struct({ * @internal */ export const NotificationMeta = Schema.Struct({ - _meta: optional(JsonObject) + _meta: optional(UnknownObject) }) /** * @internal */ export const ResultMeta = Schema.Struct({ - _meta: optional(JsonObject) + _meta: optional(UnknownObject) }) /** @@ -111,21 +118,21 @@ export const Implementation = Schema.Struct({ * @internal */ export const ClientCapabilities = Schema.Struct({ - extensions: optional(Schema.Record(Schema.TemplateLiteral([Schema.String, "/", Schema.String]), Schema.Json)), - experimental: optional(Schema.Record(Schema.String, JsonObject)), + extensions: optional(Schema.Record(Schema.TemplateLiteral([Schema.String, "/", Schema.String]), Schema.Unknown)), + experimental: optional(Schema.Record(Schema.String, UnknownObject)), roots: optional(Schema.Struct({ listChanged: optional(Schema.Boolean) })), - sampling: optional(JsonObject) + sampling: optional(UnknownObject) }) /** * @internal */ export const ServerCapabilities = Schema.Struct({ - extensions: optional(Schema.Record(Schema.TemplateLiteral([Schema.String, "/", Schema.String]), Schema.Json)), - experimental: optional(Schema.Record(Schema.String, JsonObject)), - logging: optional(JsonObject), + extensions: optional(Schema.Record(Schema.TemplateLiteral([Schema.String, "/", Schema.String]), Schema.Unknown)), + experimental: optional(Schema.Record(Schema.String, UnknownObject)), + logging: optional(UnknownObject), prompts: optional(Schema.Struct({ listChanged: optional(Schema.Boolean) })), @@ -284,7 +291,7 @@ export const PromptMessage = Schema.Struct({ */ export const ToolInputSchema = Schema.Struct({ type: Schema.Literal("object"), - properties: optional(Schema.Record(Schema.String, JsonObject)), + properties: optional(Schema.Record(Schema.String, UnknownObject)), required: optional(Schema.Array(Schema.String)) }) @@ -582,7 +589,7 @@ export class CallTool extends Rpc.make("tools/call", { payload: { ...RequestMeta.fields, name: Schema.String, - arguments: optional(JsonObject) + arguments: optional(UnknownObject) } }) {} @@ -610,7 +617,7 @@ export class CreateMessage extends Rpc.make("sampling/createMessage", { temperature: optional(Schema.Finite), maxTokens: Schema.Finite, stopSequences: optional(Schema.Array(Schema.String)), - metadata: optional(JsonObject) + metadata: optional(UnknownObject) } }) {} diff --git a/repos/effect/packages/effect/src/ai/internal/mcpSchema/v2025_03_26.ts b/repos/effect/packages/effect/src/ai/internal/mcpSchema/v2025_03_26.ts index a8b1587489..285b547571 100644 --- a/repos/effect/packages/effect/src/ai/internal/mcpSchema/v2025_03_26.ts +++ b/repos/effect/packages/effect/src/ai/internal/mcpSchema/v2025_03_26.ts @@ -21,6 +21,8 @@ export const protocolVersion = "2025-03-26" const optional = Previous.optional +const UnknownObject = Previous.UnknownObject + /** * @internal */ @@ -186,7 +188,7 @@ export class CallTool extends Rpc.make("tools/call", { payload: { ...Previous.RequestMeta.fields, name: Schema.String, - arguments: optional(Schema.JsonObject) + arguments: optional(UnknownObject) } }) {} @@ -205,7 +207,7 @@ export class CreateMessage extends Rpc.make("sampling/createMessage", { temperature: optional(Schema.Finite), maxTokens: Schema.Finite, stopSequences: optional(Schema.Array(Schema.String)), - metadata: optional(Schema.JsonObject) + metadata: optional(UnknownObject) } }) {} diff --git a/repos/effect/packages/effect/src/ai/internal/mcpSchema/v2025_06_18.ts b/repos/effect/packages/effect/src/ai/internal/mcpSchema/v2025_06_18.ts index 6fa204820e..a7b37eaf67 100644 --- a/repos/effect/packages/effect/src/ai/internal/mcpSchema/v2025_06_18.ts +++ b/repos/effect/packages/effect/src/ai/internal/mcpSchema/v2025_06_18.ts @@ -19,8 +19,8 @@ export * from "./v2025_03_26.ts" export const protocolVersion = "2025-06-18" const optional = Previous.optional -const JsonObject = Schema.JsonObject -const Meta = optional(JsonObject) +const UnknownObject = Previous.UnknownObject +const Meta = optional(UnknownObject) /** * @internal @@ -218,10 +218,10 @@ export class Initialize extends Rpc.make("initialize", { const ToolJsonSchema = Schema.StructWithRest( Schema.Struct({ type: Schema.Literal("object"), - properties: optional(Schema.Record(Schema.String, JsonObject)), + properties: optional(Schema.Record(Schema.String, Schema.Union([UnknownObject, Schema.Boolean]))), required: optional(Schema.Array(Schema.String)) }), - [Schema.JsonObject] + [UnknownObject] ) /** @@ -243,7 +243,7 @@ export const Tool = Schema.Struct({ export const CallToolResult = Schema.Struct({ ...Previous.ResultMeta.fields, content: Schema.Array(ContentBlock), - structuredContent: optional(JsonObject), + structuredContent: optional(UnknownObject), isError: optional(Schema.Boolean) }) @@ -273,7 +273,7 @@ export class CreateMessage extends Rpc.make("sampling/createMessage", { temperature: optional(Schema.Finite), maxTokens: Schema.Finite, stopSequences: optional(Schema.Array(Schema.String)), - metadata: optional(JsonObject) + metadata: optional(UnknownObject) } }) {} @@ -433,7 +433,7 @@ export class CallTool extends Rpc.make("tools/call", { payload: { ...Previous.RequestMeta.fields, name: Schema.String, - arguments: optional(JsonObject) + arguments: optional(UnknownObject) } }) {} diff --git a/repos/effect/packages/effect/src/ai/internal/mcpSchema/v2025_11_25.ts b/repos/effect/packages/effect/src/ai/internal/mcpSchema/v2025_11_25.ts index dcaf301167..3c2247fc9e 100644 --- a/repos/effect/packages/effect/src/ai/internal/mcpSchema/v2025_11_25.ts +++ b/repos/effect/packages/effect/src/ai/internal/mcpSchema/v2025_11_25.ts @@ -19,8 +19,8 @@ export * from "./v2025_06_18.ts" export const protocolVersion = "2025-11-25" const optional = Previous.optional -const JsonObject = Schema.JsonObject -const Meta = optional(JsonObject) +const UnknownObject = Previous.UnknownObject +const Meta = optional(UnknownObject) /** * @internal @@ -48,12 +48,12 @@ export const Implementation = Schema.Struct({ export const ClientCapabilities = Schema.Struct({ ...Previous.ClientCapabilities.fields, sampling: optional(Schema.Struct({ - context: optional(JsonObject), - tools: optional(JsonObject) + context: optional(UnknownObject), + tools: optional(UnknownObject) })), elicitation: optional(Schema.Struct({ - form: optional(JsonObject), - url: optional(JsonObject) + form: optional(UnknownObject), + url: optional(UnknownObject) })) }) @@ -182,7 +182,7 @@ export const ToolUseContent = Schema.Struct({ type: Schema.Literal("tool_use"), id: Schema.String, name: Schema.String, - input: JsonObject, + input: UnknownObject, _meta: Meta }) @@ -193,7 +193,7 @@ export const ToolResultContent = Schema.Struct({ type: Schema.Literal("tool_result"), toolUseId: Schema.String, content: Schema.Array(ContentBlock), - structuredContent: optional(JsonObject), + structuredContent: optional(UnknownObject), isError: optional(Schema.Boolean), _meta: Meta }) diff --git a/repos/effect/packages/effect/src/ai/internal/mcpSchema/v2026_07_28.ts b/repos/effect/packages/effect/src/ai/internal/mcpSchema/v2026_07_28.ts index b38c929e3c..da64197299 100644 --- a/repos/effect/packages/effect/src/ai/internal/mcpSchema/v2026_07_28.ts +++ b/repos/effect/packages/effect/src/ai/internal/mcpSchema/v2026_07_28.ts @@ -16,7 +16,9 @@ export * from "./v2025_11_25.ts" export const protocolVersion = "2026-07-28" const optional = Previous.optional -const Meta = optional(Schema.JsonObject) + +const UnknownObject = Previous.UnknownObject +const Meta = optional(UnknownObject) /** * @internal @@ -44,19 +46,19 @@ export const Implementation = Previous.Implementation */ export const ClientCapabilities = Schema.StructWithRest( Schema.Struct({ - experimental: optional(Schema.Record(Schema.String, Schema.JsonObject)), + experimental: optional(Schema.Record(Schema.String, UnknownObject)), roots: optional(Schema.Struct({})), sampling: optional(Schema.Struct({ - context: optional(Schema.JsonObject), - tools: optional(Schema.JsonObject) + context: optional(UnknownObject), + tools: optional(UnknownObject) })), elicitation: optional(Schema.Struct({ - form: optional(Schema.JsonObject), - url: optional(Schema.JsonObject) + form: optional(UnknownObject), + url: optional(UnknownObject) })), - extensions: optional(Schema.Record(Schema.String, Schema.JsonObject)) + extensions: optional(Schema.Record(Schema.String, UnknownObject)) }), - [Schema.JsonObject] + [UnknownObject] ) /** @@ -64,18 +66,18 @@ export const ClientCapabilities = Schema.StructWithRest( */ export const ServerCapabilities = Schema.StructWithRest( Schema.Struct({ - experimental: optional(Schema.Record(Schema.String, Schema.JsonObject)), - logging: optional(Schema.JsonObject), - completions: optional(Schema.JsonObject), + experimental: optional(Schema.Record(Schema.String, UnknownObject)), + logging: optional(UnknownObject), + completions: optional(UnknownObject), prompts: optional(Schema.Struct({ listChanged: optional(Schema.Boolean) })), resources: optional(Schema.Struct({ subscribe: optional(Schema.Boolean), listChanged: optional(Schema.Boolean) })), tools: optional(Schema.Struct({ listChanged: optional(Schema.Boolean) })), - extensions: optional(Schema.Record(Schema.String, Schema.JsonObject)) + extensions: optional(Schema.Record(Schema.String, UnknownObject)) }), - [Schema.JsonObject] + [UnknownObject] ) /** @@ -89,7 +91,7 @@ export const RequestMetaObject = Schema.StructWithRest( "io.modelcontextprotocol/clientCapabilities": ClientCapabilities, "io.modelcontextprotocol/logLevel": optional(LoggingLevel) }), - [Schema.JsonObject] + [UnknownObject] ) /** @@ -104,7 +106,7 @@ export const NotificationMetaObject = Schema.StructWithRest( Schema.Struct({ "io.modelcontextprotocol/subscriptionId": optional(RequestId) }), - [Schema.JsonObject] + [UnknownObject] ) /** * @internal @@ -116,15 +118,15 @@ export const NotificationParams = Schema.Struct({ _meta: optional(NotificationMe */ export const ResultMetaObject = Schema.StructWithRest( Schema.Struct({ - "io.modelcontextprotocol/serverInfo": Implementation + "io.modelcontextprotocol/serverInfo": optional(Implementation) }), - [Schema.JsonObject] + [UnknownObject] ) /** * @internal */ export const ResultMeta = { - _meta: ResultMetaObject, + _meta: optional(ResultMetaObject), resultType: Schema.Literal("complete") } /** @@ -133,7 +135,7 @@ export const ResultMeta = { export const McpError = Schema.Struct({ code: Schema.Int, message: Schema.String, - data: optional(Schema.Json) + data: optional(Schema.Unknown) }) /** * @internal @@ -220,11 +222,11 @@ const ToolInputSchema = Schema.StructWithRest( $schema: optional(Schema.String), type: Schema.Literal("object") }), - [Schema.JsonObject] + [UnknownObject] ) const ToolOutputSchema = Schema.StructWithRest( Schema.Struct({ $schema: optional(Schema.String) }), - [Schema.JsonObject] + [UnknownObject] ) /** * @internal @@ -251,7 +253,7 @@ export const ToolResultContent = Schema.Struct({ type: Schema.Literal("tool_result"), toolUseId: Schema.String, content: Schema.Array(ContentBlock), - structuredContent: optional(Schema.Json), + structuredContent: optional(Schema.Unknown), isError: optional(Schema.Boolean), _meta: Meta }) @@ -278,7 +280,7 @@ export const SamplingMessage = Schema.Struct({ */ export const ModelHint = Schema.StructWithRest( Schema.Struct({ name: optional(Schema.String) }), - [Schema.JsonObject] + [UnknownObject] ) const ModelPriority = Schema.Finite.check(Schema.isBetween({ minimum: 0, maximum: 1 })) /** @@ -309,7 +311,7 @@ export const CreateMessageRequest = Schema.Struct({ temperature: optional(Schema.Finite), maxTokens: Schema.Int, stopSequences: optional(Schema.Array(Schema.String)), - metadata: optional(Schema.JsonObject), + metadata: optional(UnknownObject), tools: optional(Schema.Array(Tool)), toolChoice: optional(ToolChoice) }) @@ -444,7 +446,7 @@ export const InputRequests = Schema.Record(Schema.String, InputRequest) */ export const InputResponses = Schema.Record(Schema.String, InputResponse) const InputRequiredResultMeta = { - _meta: ResultMetaObject, + _meta: optional(ResultMetaObject), resultType: Schema.Literal("input_required") } /** @@ -457,7 +459,7 @@ export const InputRequiredResult = Schema.Union([ inputRequests: InputRequests, requestState: optional(Schema.String) }), - [Schema.JsonObject] + [UnknownObject] ), Schema.StructWithRest( Schema.Struct({ @@ -465,7 +467,7 @@ export const InputRequiredResult = Schema.Union([ inputRequests: optional(InputRequests), requestState: Schema.String }), - [Schema.JsonObject] + [UnknownObject] ) ]) /** @@ -510,7 +512,7 @@ export const DiscoverResult = Schema.StructWithRest( capabilities: ServerCapabilities, instructions: optional(Schema.String) }), - [Schema.JsonObject] + [UnknownObject] ) /** * @internal @@ -530,7 +532,7 @@ export const ListResourcesResult = Schema.StructWithRest( ...CacheableResult, resources: Schema.Array(Resource) }), - [Schema.JsonObject] + [UnknownObject] ) /** * @internal @@ -550,7 +552,7 @@ export const ListResourceTemplatesResult = Schema.StructWithRest( ...CacheableResult, resourceTemplates: Schema.Array(ResourceTemplate) }), - [Schema.JsonObject] + [UnknownObject] ) /** * @internal @@ -569,7 +571,7 @@ export const ReadResourceResult = Schema.StructWithRest( ...CacheableResult, contents: Schema.Array(ResourceContents) }), - [Schema.JsonObject] + [UnknownObject] ) /** * @internal @@ -589,7 +591,7 @@ export const ListPromptsResult = Schema.StructWithRest( ...CacheableResult, prompts: Schema.Array(Prompt) }), - [Schema.JsonObject] + [UnknownObject] ) /** * @internal @@ -609,7 +611,7 @@ export const GetPromptResult = Schema.StructWithRest( description: optional(Schema.String), messages: Schema.Array(PromptMessage) }), - [Schema.JsonObject] + [UnknownObject] ) /** * @internal @@ -633,7 +635,7 @@ export const ListToolsResult = Schema.StructWithRest( ...CacheableResult, tools: Schema.Array(Tool) }), - [Schema.JsonObject] + [UnknownObject] ) /** * @internal @@ -651,10 +653,10 @@ export const CallToolResult = Schema.StructWithRest( Schema.Struct({ ...ResultMeta, content: Schema.Array(ContentBlock), - structuredContent: optional(Schema.Json), + structuredContent: optional(Schema.Unknown), isError: optional(Schema.Boolean) }), - [Schema.JsonObject] + [UnknownObject] ) /** * @internal @@ -665,7 +667,7 @@ export class CallTool extends Rpc.make("tools/call", { payload: { ...InputResponseRequestParams, name: Schema.String, - arguments: optional(Schema.JsonObject) + arguments: optional(UnknownObject) } }) {} @@ -689,7 +691,7 @@ export const CompleteResult = Schema.StructWithRest( hasMore: optional(Schema.Boolean) }) }), - [Schema.JsonObject] + [UnknownObject] ) /** * @internal @@ -721,10 +723,10 @@ export const SubscriptionFilter = Schema.Struct({ */ export const SubscriptionsListenResultMetaObject = Schema.StructWithRest( Schema.Struct({ - "io.modelcontextprotocol/serverInfo": Implementation, + "io.modelcontextprotocol/serverInfo": optional(Implementation), "io.modelcontextprotocol/subscriptionId": RequestId }), - [Schema.JsonObject] + [UnknownObject] ) /** * @internal @@ -734,7 +736,7 @@ export const SubscriptionsListenResult = Schema.StructWithRest( _meta: SubscriptionsListenResultMetaObject, resultType: Schema.Literal("complete") }), - [Schema.JsonObject] + [UnknownObject] ) /** * @internal @@ -775,7 +777,7 @@ export class LoggingMessageNotification extends Rpc.make("notifications/message" ...NotificationParams.fields, level: LoggingLevel, logger: optional(Schema.String), - data: Schema.Json + data: Schema.Unknown } }) {} /** diff --git a/repos/effect/packages/effect/src/cli/CliError.ts b/repos/effect/packages/effect/src/cli/CliError.ts index 99afc264f5..eb6051dde5 100644 --- a/repos/effect/packages/effect/src/cli/CliError.ts +++ b/repos/effect/packages/effect/src/cli/CliError.ts @@ -3,10 +3,11 @@ * * CLI errors describe problems such as unknown or duplicate flags, missing * flags or arguments, unexpected positional arguments, invalid values, unknown - * subcommands, user handler failures, and requests to show command help. This - * module includes the `CliError` union, the `isCliError` guard, schema-backed - * error classes with display messages, and the `NonShowHelpErrors` union used - * when parse or validation errors should be shown with help output. + * subcommands, rejected input combinations, user handler failures, and + * requests to show command help. This module includes the `CliError` union, + * the `isCliError` guard, schema-backed error classes with display messages, + * and the `NonShowHelpErrors` union used when parse or validation errors + * should be shown with help output. * * @stability unstable * @since 4.0.0 @@ -83,6 +84,7 @@ export type CliError = | UnexpectedArgument | InvalidValue | UnknownSubcommand + | ValidationError | ShowHelp | UserError @@ -500,6 +502,47 @@ export class UnknownSubcommand extends Schema.TaggedError( } } +/** + * Error for parsed input that fails a rule of your own, such as two flags + * that cannot be used together. + * + * **Details** + * + * Fail with it from `Command.validate`, `Flag.mapEffect` or + * `Argument.mapEffect`. The runner shows it with the command help, like any + * other parse error. Handlers report their own failures with `UserError`. + * + * **Example** (Rejecting a flag combination) + * + * ```ts import.meta.vitest + * import { CliError } from "effect/cli" + * + * const error = new CliError.ValidationError({ + * message: "--use-api and --use-docker are mutually exclusive" + * }) + * + * error._tag // => "ValidationError" + * error.message // => "--use-api and --use-docker are mutually exclusive" + * ``` + * + * @stability unstable + * @category errors + * @since 4.1.0 + */ +export class ValidationError extends Schema.TaggedError( + `${TypeId}/ValidationError` +)("ValidationError", { + message: Schema.String +}) { + /** + * Marks this value as a CLI validation error for runtime guards. + * + * @stability unstable + * @since 4.1.0 + */ + readonly [TypeId] = TypeId +} + /** * Error wrapper for user handler failures in the CLI error channel. * @@ -604,6 +647,7 @@ export const NonShowHelpErrors: Schema.Union< typeof UnexpectedArgument, typeof InvalidValue, typeof UnknownSubcommand, + typeof ValidationError, typeof UserError ] > = Schema.Union([ @@ -614,6 +658,7 @@ export const NonShowHelpErrors: Schema.Union< UnexpectedArgument, InvalidValue, UnknownSubcommand, + ValidationError, UserError ]) diff --git a/repos/effect/packages/effect/src/cli/Command.ts b/repos/effect/packages/effect/src/cli/Command.ts index 2a5c9c01b1..63ad39cae4 100644 --- a/repos/effect/packages/effect/src/cli/Command.ts +++ b/repos/effect/packages/effect/src/cli/Command.ts @@ -30,10 +30,10 @@ import * as Terminal from "../Terminal.ts" import type { Contravariant, Covariant, NoInfer, Simplify } from "../Types.ts" import * as CliConfig from "./CliConfig.ts" import * as CliError from "./CliError.ts" -import * as CliOutput from "./CliOutput.ts" import * as GlobalFlag from "./GlobalFlag.ts" import { checkForDuplicateFlags, makeCommand, makeParser, toImpl, TypeId } from "./internal/command.ts" import { mergeConfig, parseConfig } from "./internal/config.ts" +import { resolveFormatter } from "./internal/formatter.ts" import { getGlobalFlagsForCommandPath, getGlobalFlagsForCommandTree, getHelpForCommandPath } from "./internal/help.ts" import * as Lexer from "./internal/lexer.ts" import * as Parser from "./internal/parser.ts" @@ -735,6 +735,101 @@ export const withHandler: { ): Command> => makeCommand({ ...toImpl(self), handle: handler } as any)) +/** + * Validates the parsed input of a command before the handler runs. + * + * **Details** + * + * Use this for rules that involve several flags or arguments, such as + * mutually exclusive flags. The validation receives the same input as the + * handler, with defaults, prompts and the wizard already applied. A failure + * is shown with the command help, like a parse error, and the handler does + * not run. Fail with `CliError.ValidationError` to supply the message, or + * with any other `CliError`. Validations run in the order they were added and + * stop at the first failure. Adding a handler later with `withHandler` keeps + * them. + * + * A validation sees only the input that exists where it is applied. Add it + * after `withSharedFlags` to see the shared flags. Added after + * `withSubcommands`, it also runs when a subcommand is selected, with the + * parent's shared flags as input. + * + * **Example** (Rejecting mutually exclusive flags) + * + * ```ts import.meta.vitest + * import { Effect, FileSystem, Layer, Path, Stdio, Terminal } from "effect" + * import { CliError, Command, Flag } from "effect/cli" + * import { ChildProcessSpawner } from "effect/process" + * import { TestConsole } from "effect/testing" + * + * const CliTestLayer = Layer.mergeAll( + * FileSystem.layerNoop({}), + * Path.layer, + * Stdio.layerTest({}), + * TestConsole.layer, + * Layer.succeed(Terminal.Terminal, Terminal.make({ + * columns: Effect.succeed(80), + * rows: Effect.succeed(24), + * readInput: Effect.die("unused"), + * readLine: Effect.die("unused"), + * display: () => Effect.void + * })), + * Layer.succeed( + * ChildProcessSpawner.ChildProcessSpawner, + * ChildProcessSpawner.make(() => Effect.die("unused")) + * ) + * ) + * + * const download = Command.make("download", { + * useApi: Flag.Boolean("use-api").pipe(Flag.withDefault(false)), + * useDocker: Flag.Boolean("use-docker").pipe(Flag.withDefault(false)) + * }, () => Effect.void).pipe( + * Command.validate(({ useApi, useDocker }) => + * useApi && useDocker + * ? Effect.fail( + * new CliError.ValidationError({ + * message: "--use-api and --use-docker are mutually exclusive" + * }) + * ) + * : Effect.void + * ) + * ) + * + * const error = await Effect.runPromise( + * Command.runWith(download, { version: "1.0.0" })(["--use-api", "--use-docker"]).pipe( + * Effect.flip, + * Effect.provide(CliTestLayer) + * ) + * ) + * error._tag // => "ShowHelp" + * error.errors[0].message // => "--use-api and --use-docker are mutually exclusive" + * ``` + * + * @stability unstable + * @category combinators + * @since 4.1.0 + */ +export const validate: { + ( + f: (input: Input) => Effect.Effect + ): ( + self: Command + ) => Command + ( + self: Command, + f: (input: Input) => Effect.Effect + ): Command +} = dual(2, ( + self: Command, + f: (input: Input) => Effect.Effect +): Command => { + const impl = toImpl(self) + return makeCommand({ + ...impl, + parse: (raw) => Effect.tap(impl.parse(raw), f) + }) as any +}) + interface SubcommandGroupInternal { readonly group: string | undefined readonly commands: NonEmptyReadonlyArray @@ -954,7 +1049,7 @@ export const withSubcommands: { unlisted: impl.unlisted, annotations: impl.annotations, globalFlags: impl.globalFlags, - examples: impl.examples, + resolveExamples: impl.resolveExamples, service: impl.service, subcommands: normalized.groups, parse, @@ -1055,7 +1150,7 @@ export const withSharedFlags: { unlisted: impl.unlisted, annotations: impl.annotations, globalFlags: impl.globalFlags, - examples: impl.examples, + resolveExamples: impl.resolveExamples, service: impl.service as Context.Key, NextContextInput>, subcommands: impl.subcommands, parse, @@ -1370,6 +1465,10 @@ export const annotateMerge: { * Examples are exposed in structured `HelpDoc` data and rendered by the * default formatter in an `EXAMPLES` section. * + * Pass a function to build examples from the `commandPath` shown in help, so + * they stay correct wherever the command is mounted. The `examples` field + * resolves the function with the command's own name as the path. + * * **Example** (Adding usage examples) * * ```ts import.meta.vitest @@ -1385,22 +1484,48 @@ export const annotateMerge: { * login.examples.map((example) => example.command) // => ["myapp login", "myapp login --token sbp_abc123"] * ``` * + * **Example** (Deriving examples from the command path) + * + * ```ts import.meta.vitest + * import { Command } from "effect/cli" + * + * const deploy = Command.make("deploy").pipe( + * Command.withExamples(({ commandPath }) => [ + * { command: `${commandPath.join(" ")} --yes`, description: "Take every default" } + * ]) + * ) + * + * deploy.examples.map((example) => example.command) // => ["deploy --yes"] + * ``` + * * @stability unstable * @category combinators * @since 4.0.0 */ export const withExamples: { - (examples: ReadonlyArray): ( + ( + examples: + | ReadonlyArray + | ((context: { readonly commandPath: ReadonlyArray }) => ReadonlyArray) + ): ( self: Command ) => Command ( self: Command, - examples: ReadonlyArray + examples: + | ReadonlyArray + | ((context: { readonly commandPath: ReadonlyArray }) => ReadonlyArray) ): Command } = dual(2, ( self: Command, - examples: ReadonlyArray -) => makeCommand({ ...toImpl(self), examples })) + examples: + | ReadonlyArray + | ((context: { readonly commandPath: ReadonlyArray }) => ReadonlyArray) +) => + makeCommand({ + ...toImpl(self), + resolveExamples: Predicate.isFunction(examples) ? (commandPath) => examples({ commandPath }) : () => examples + })) /* ========================================================================== */ /* Providing Services */ @@ -1711,7 +1836,7 @@ const showHelp = ( ): Effect.Effect => Effect.gen(function*() { const { builtIns } = yield* CliConfig.CliConfig - const formatter = yield* CliOutput.Formatter + const formatter = yield* resolveFormatter const helpDoc = yield* getHelpForCommandPath(command, error.commandPath, builtIns) yield* Console.log(formatter.formatHelpDoc(helpDoc)) if (renderErrors && error.errors.length > 0) { @@ -1721,7 +1846,7 @@ const showHelp = ( const showUserError = (error: CliError.UserError): Effect.Effect => Effect.gen(function*() { - const formatter = yield* CliOutput.Formatter + const formatter = yield* resolveFormatter yield* Console.error(formatter.formatError(error)) error[Runtime.errorReported] = false }) diff --git a/repos/effect/packages/effect/src/cli/Completions.ts b/repos/effect/packages/effect/src/cli/Completions.ts index 2b0996af3d..c637764ff1 100644 --- a/repos/effect/packages/effect/src/cli/Completions.ts +++ b/repos/effect/packages/effect/src/cli/Completions.ts @@ -53,6 +53,11 @@ export interface FlagDescriptor { * `=` (`--name=value`), never given as the next word. */ readonly optionalValue?: boolean | undefined + /** + * Long name that sets a boolean flag to `false`. Defaults to `no-`; + * `false` means the flag has no negated form. + */ + readonly negation?: string | false | undefined } /** diff --git a/repos/effect/packages/effect/src/cli/Flag.ts b/repos/effect/packages/effect/src/cli/Flag.ts index 1d7f619f5b..11625df85b 100644 --- a/repos/effect/packages/effect/src/cli/Flag.ts +++ b/repos/effect/packages/effect/src/cli/Flag.ts @@ -546,6 +546,67 @@ export const withOptionalValue: { (self: Flag, value: string): Flag } = dual(2, (self: Flag, value: string) => Param.withOptionalValue(self, value)) +/** + * Renames the negated form of a boolean flag, which defaults to `--no-`. + * + * **Details** + * + * The new name replaces the default, so `--no-` is no longer accepted + * and shell completions offer the new name instead. Has no effect on + * non-boolean flags. + * + * **Example** (Renaming a boolean flag's negation) + * + * ```ts import.meta.vitest + * import { Flag } from "effect/cli" + * + * const validate = Flag.Boolean("validate").pipe( + * Flag.withDefault(true), + * Flag.withNegation("skip-validate") + * ) + * // --validate => true + * // --skip-validate => false + * // --no-validate => unrecognized flag + * validate.kind // => "flag" + * ``` + * + * @stability unstable + * @category combinators + * @since 4.1.0 + */ +export const withNegation: { + (name: string): (self: Flag) => Flag + (self: Flag, name: string): Flag +} = dual(2, (self: Flag, name: string) => Param.withNegation(self, name)) + +/** + * Removes the negated `--no-` form of a boolean flag. + * + * **Details** + * + * The negated form is then rejected as an unrecognized flag and left out of + * shell completions. Has no effect on non-boolean flags. + * + * **Example** (Disabling a boolean flag's negation) + * + * ```ts import.meta.vitest + * import { Flag } from "effect/cli" + * + * const unsafe = Flag.Boolean("unsafe").pipe( + * Flag.withDefault(false), + * Flag.withoutNegation + * ) + * // --unsafe => true + * // --no-unsafe => unrecognized flag + * unsafe.kind // => "flag" + * ``` + * + * @stability unstable + * @category combinators + * @since 4.1.0 + */ +export const withoutNegation = (self: Flag): Flag => Param.withoutNegation(self) + // ------------------------------------------------------------------------------------- // metadata // ------------------------------------------------------------------------------------- diff --git a/repos/effect/packages/effect/src/cli/GlobalFlag.ts b/repos/effect/packages/effect/src/cli/GlobalFlag.ts index f1d8f44801..166f54f9bb 100644 --- a/repos/effect/packages/effect/src/cli/GlobalFlag.ts +++ b/repos/effect/packages/effect/src/cli/GlobalFlag.ts @@ -17,11 +17,11 @@ import * as Context from "../Context.ts" import * as Effect from "../Effect.ts" import type { LogLevel as LogLevelType } from "../LogLevel.ts" import * as Option from "../Option.ts" -import * as CliOutput from "./CliOutput.ts" import type * as Command from "./Command.ts" import * as Completions_ from "./Completions.ts" import * as Flag from "./Flag.ts" import * as CommandDescriptor from "./internal/completions/descriptor.ts" +import { resolveFormatter } from "./internal/formatter.ts" import * as HelpInternal from "./internal/help.ts" /* ========================================================================== */ @@ -169,7 +169,7 @@ export const Help: Action = Action({ Flag.withDefault(false) ), run: Effect.fnUntraced(function*(_, { builtIns, command, commandPath }) { - const formatter = yield* CliOutput.Formatter + const formatter = yield* resolveFormatter const helpDoc = yield* HelpInternal.getHelpForCommandPath(command, commandPath, builtIns) yield* Console.log(formatter.formatHelpDoc(helpDoc)) }) @@ -193,7 +193,7 @@ export const Version: Action = Action({ Flag.withDefault(false) ), run: Effect.fnUntraced(function*(_, { command, version }) { - const formatter = yield* CliOutput.Formatter + const formatter = yield* resolveFormatter yield* Console.log(formatter.formatVersion(command.name, version)) }) }) diff --git a/repos/effect/packages/effect/src/cli/Param.ts b/repos/effect/packages/effect/src/cli/Param.ts index 5fcc31e2f4..793f8e8f71 100644 --- a/repos/effect/packages/effect/src/cli/Param.ts +++ b/repos/effect/packages/effect/src/cli/Param.ts @@ -217,6 +217,11 @@ export interface Single extends Param { * Raw value used for a bare flag. See `withOptionalValue`. */ readonly optionalValue?: string | undefined + /** + * Long name that sets a boolean flag to `false`. `undefined` means the + * default `no-`, `false` means no negated form. See `withNegation`. + */ + readonly negation?: string | false | undefined } /** @@ -367,6 +372,7 @@ export const makeSingle = (params: { readonly aliases?: ReadonlyArray | undefined readonly hidden?: boolean | undefined readonly optionalValue?: string | undefined + readonly negation?: string | false | undefined }): Single => { const parse: Parse = (args) => params.kind === argumentKind @@ -1867,6 +1873,74 @@ export const withOptionalValue: { optionalValue: value }))) +/** + * Renames the negated form of a boolean flag, which defaults to `--no-`. + * + * **Details** + * + * The new name replaces the default, so `--no-` is no longer accepted. + * Has no effect on non-boolean flags. + * + * **Example** (Renaming a boolean flag's negation) + * + * ```ts import.meta.vitest + * import { Param } from "effect/cli" + * + * // --validate sets true, --skip-validate sets false + * const validate = Param.Boolean(Param.flagKind, "validate").pipe( + * Param.withNegation("skip-validate") + * ) + * validate.kind // => "flag" + * ``` + * + * @stability unstable + * @category combinators + * @since 4.1.0 + */ +export const withNegation: { + (name: string): (self: Param) => Param + (self: Param, name: string): Param +} = dual(2, ( + self: Param, + name: string +) => + transformSingle(self, (single) => + makeSingle({ + ...single, + negation: name + }))) + +/** + * Removes the negated `--no-` form of a boolean flag. + * + * **Details** + * + * The negated form is then rejected as an unrecognized flag and left out of + * shell completions. Has no effect on non-boolean flags. + * + * **Example** (Disabling a boolean flag's negation) + * + * ```ts import.meta.vitest + * import { Param } from "effect/cli" + * + * // --unsafe sets true, --no-unsafe is rejected + * const unsafe = Param.Boolean(Param.flagKind, "unsafe").pipe( + * Param.withoutNegation + * ) + * unsafe.kind // => "flag" + * ``` + * + * @stability unstable + * @category combinators + * @since 4.1.0 + */ +export const withoutNegation = (self: Param): Param => + transformSingle(self, (single) => + makeSingle({ + ...single, + negation: false + })) + /** * Validates parsed values against a Schema, providing detailed error messages. * diff --git a/repos/effect/packages/effect/src/cli/TerminalCapabilities.ts b/repos/effect/packages/effect/src/cli/TerminalCapabilities.ts new file mode 100644 index 0000000000..c8a3a0fe72 --- /dev/null +++ b/repos/effect/packages/effect/src/cli/TerminalCapabilities.ts @@ -0,0 +1,156 @@ +/** + * Detection of whether a CLI can color, animate, or prompt. + * + * The default reads `NO_COLOR`, `FORCE_COLOR`, `TERM`, and the TTY state of + * the current process. The layer reads the same variables through + * `ConfigProvider`, plus the TTY state of `Stdio`. Unless a + * `CliOutput.Formatter` is provided, the built-in help, version, and error + * output of `Command` follows `canColor`. + * + * @stability unstable + * @since 4.0.0 + */ +import * as Config from "../Config.ts" +import * as Context from "../Context.ts" +import * as Effect from "../Effect.ts" +import * as Layer from "../Layer.ts" +import * as Option from "../Option.ts" +import * as Stdio from "../Stdio.ts" + +/** + * Terminal rendering flags for the current process. + * + * **Details** + * + * `canColor` says whether styling will be rendered rather than stored as + * bytes. `canAnimate` says whether output is being watched, so pacing means + * something. `canPrompt` says whether a question can be asked and answered. + * Only `canColor` is read by the built-in `Command` output; the other two are + * for application code that renders its own output. + * + * @stability unstable + * @category models + * @since 4.0.0 + */ +export interface TerminalCapabilities { + readonly canColor: boolean + readonly canAnimate: boolean + readonly canPrompt: boolean +} + +/** + * Reference to the terminal capabilities of the current process. + * + * **When to use** + * + * Use when application output should share the built-in help, version, and + * error output's answer. Provide `layer` to detect through + * `ConfigProvider` and `Stdio`, or provide a value to override detection. + * + * **Details** + * + * The default is detected from the current process the first time it is + * read, so later changes to the environment do not affect it. Without a + * `process` global, every flag is `false`. + * + * @stability unstable + * @category services + * @since 4.0.0 + */ +export const TerminalCapabilities: Context.Reference = Context.Reference( + "effect/cli/TerminalCapabilities", + { defaultValue: () => fromProcess() } +) + +const isPresent = (value: string | undefined): value is string => value !== undefined && value.length > 0 + +const isForcingColor = (value: string | undefined): boolean => + isPresent(value) && value !== "0" && value.toLowerCase() !== "false" + +const detect = (input: { + readonly noColor: string | undefined + readonly forceColor: string | undefined + readonly term: string | undefined + readonly stdinIsTerminal: boolean + readonly stdoutIsTerminal: boolean +}): TerminalCapabilities => { + if (input.term === "dumb") { + return { canColor: false, canAnimate: false, canPrompt: false } + } + + return { + canColor: !isPresent(input.noColor) && (isForcingColor(input.forceColor) || input.stdoutIsTerminal), + canAnimate: input.stdoutIsTerminal, + canPrompt: input.stdinIsTerminal && input.stdoutIsTerminal + } +} + +const fromProcess = (): TerminalCapabilities => { + const process = (globalThis as any).process + return detect({ + noColor: process?.env?.NO_COLOR, + forceColor: process?.env?.FORCE_COLOR, + term: process?.env?.TERM, + stdinIsTerminal: process?.stdin?.isTTY === true, + stdoutIsTerminal: process?.stdout?.isTTY === true + }) +} + +const environment = Config.all({ + noColor: Config.option(Config.String("NO_COLOR")), + forceColor: Config.option(Config.String("FORCE_COLOR")), + term: Config.option(Config.String("TERM")) +}) + +/** + * Detects terminal capabilities from `Stdio` and the current `ConfigProvider`. + * + * **Details** + * + * `TERM=dumb` clears all three flags, because a terminal that cannot move the + * cursor can neither animate nor redraw a prompt. Otherwise `NO_COLOR`, when + * set to any non-empty value, disables color and wins over `FORCE_COLOR`. + * `FORCE_COLOR` forces color even off a TTY, except for `0` and `false`, which + * mean "not forcing". Empty variables count as unset, matching + * `ConfigProvider.fromEnv`, so `FORCE_COLOR=` does not force color on. + * `canPrompt` requires both standard input and standard output to be + * terminals. A `ConfigProvider` fault is a defect, so the layer never fails. + * + * **Example** (NO_COLOR wins over FORCE_COLOR) + * + * ```ts import.meta.vitest + * import { ConfigProvider, Effect, Stdio } from "effect" + * import { TerminalCapabilities } from "effect/cli" + * + * const program = Effect.gen(function*() { + * const caps = yield* TerminalCapabilities.TerminalCapabilities + * return caps.canColor + * }).pipe( + * Effect.provide(TerminalCapabilities.layer), + * Effect.provide(Stdio.layerTest({ stdoutIsTerminal: Effect.succeed(true) })), + * Effect.provideService( + * ConfigProvider.ConfigProvider, + * ConfigProvider.fromEnv({ env: { NO_COLOR: "1", FORCE_COLOR: "1" } }) + * ) + * ) + * + * Effect.runSync(program) // => false + * ``` + * + * @stability unstable + * @category layers + * @since 4.0.0 + */ +export const layer: Layer.Layer = Layer.effect( + TerminalCapabilities +)(Effect.gen(function*() { + const stdio = yield* Stdio.Stdio + const env = yield* Effect.orDie(environment) + return detect({ + noColor: Option.getOrUndefined(env.noColor), + forceColor: Option.getOrUndefined(env.forceColor), + term: Option.getOrUndefined(env.term), + stdinIsTerminal: yield* stdio.stdinIsTerminal, + stdoutIsTerminal: yield* stdio.stdoutIsTerminal + }) +})) diff --git a/repos/effect/packages/effect/src/cli/index.ts b/repos/effect/packages/effect/src/cli/index.ts index 4ed3433cab..de545b8c03 100644 --- a/repos/effect/packages/effect/src/cli/index.ts +++ b/repos/effect/packages/effect/src/cli/index.ts @@ -76,3 +76,9 @@ export * as Primitive from "./Primitive.ts" * @since 4.0.0 */ export * as Prompt from "./Prompt.ts" + +/** + * @stability unstable + * @since 4.0.0 + */ +export * as TerminalCapabilities from "./TerminalCapabilities.ts" diff --git a/repos/effect/packages/effect/src/cli/internal/command.ts b/repos/effect/packages/effect/src/cli/internal/command.ts index 7b52da2484..0111fe06dd 100644 --- a/repos/effect/packages/effect/src/cli/internal/command.ts +++ b/repos/effect/packages/effect/src/cli/internal/command.ts @@ -48,6 +48,7 @@ export interface CommandInternal commandPath: ReadonlyArray ) => Effect.Effect readonly buildHelpDoc: (commandPath: ReadonlyArray) => HelpDoc + readonly resolveExamples: (commandPath: ReadonlyArray) => ReadonlyArray } /* ========================================================================== */ @@ -99,7 +100,7 @@ export const makeCommand = | undefined + readonly resolveExamples?: ((commandPath: ReadonlyArray) => ReadonlyArray) | undefined readonly subcommands?: ReadonlyArray | undefined readonly parse?: ((input: ParsedTokens) => Effect.Effect) | undefined readonly parseContext?: @@ -115,6 +116,7 @@ export const makeCommand = = options.examples ?? [] + const examples: ReadonlyArray = resolveExamples(commandPath) return { description: options.description ?? "", @@ -204,7 +206,8 @@ export const makeCommand = => [] + /** * Converts a single flag param into a FlagDoc for help display. * diff --git a/repos/effect/packages/effect/src/cli/internal/completions/bash.ts b/repos/effect/packages/effect/src/cli/internal/completions/bash.ts index 2ccead6ce0..205edd771a 100644 --- a/repos/effect/packages/effect/src/cli/internal/completions/bash.ts +++ b/repos/effect/packages/effect/src/cli/internal/completions/bash.ts @@ -7,7 +7,7 @@ * @internal */ import type * as Completions from "../../Completions.ts" -import { takesSeparateValue } from "./descriptor.ts" +import { negatedName, takesSeparateValue } from "./descriptor.ts" // --------------------------------------------------------------------------- // Helpers @@ -22,8 +22,9 @@ const flagNamesForWordlist = (flag: Completions.FlagDescriptor): Array = for (const alias of flag.aliases) { names.push(alias.length === 1 ? `-${alias}` : `--${alias}`) } - if (flag.type._tag === "Boolean") { - names.push(`--no-${flag.name}`) + const negation = negatedName(flag) + if (negation !== undefined) { + names.push(`--${negation}`) } return names } diff --git a/repos/effect/packages/effect/src/cli/internal/completions/descriptor.ts b/repos/effect/packages/effect/src/cli/internal/completions/descriptor.ts index 378d418b13..b2e87e9ff6 100644 --- a/repos/effect/packages/effect/src/cli/internal/completions/descriptor.ts +++ b/repos/effect/packages/effect/src/cli/internal/completions/descriptor.ts @@ -96,7 +96,8 @@ export const fromCommand = ( aliases: single.aliases, description: Option.getOrUndefined(single.description), type: toFlagType(single as Param.Single<"flag", unknown>), - optionalValue: single.optionalValue !== undefined + optionalValue: single.optionalValue !== undefined, + negation: single.negation }) } } @@ -148,3 +149,12 @@ export const fromCommand = ( */ export const takesSeparateValue = (flag: Completions.FlagDescriptor): boolean => flag.type._tag !== "Boolean" && flag.optionalValue !== true + +/** + * Long name that sets a boolean flag to `false`, or `undefined` when the flag + * has no negated form. + * + * @internal + */ +export const negatedName = (flag: Completions.FlagDescriptor): string | undefined => + flag.type._tag !== "Boolean" || flag.negation === false ? undefined : flag.negation ?? `no-${flag.name}` diff --git a/repos/effect/packages/effect/src/cli/internal/completions/fish.ts b/repos/effect/packages/effect/src/cli/internal/completions/fish.ts index 0aefa710b4..df4fce1b79 100644 --- a/repos/effect/packages/effect/src/cli/internal/completions/fish.ts +++ b/repos/effect/packages/effect/src/cli/internal/completions/fish.ts @@ -7,7 +7,7 @@ * @internal */ import type * as Completions from "../../Completions.ts" -import { takesSeparateValue } from "./descriptor.ts" +import { negatedName, takesSeparateValue } from "./descriptor.ts" // --------------------------------------------------------------------------- // Helpers @@ -62,8 +62,9 @@ const flagContainsOptCondition = (flag: Completions.FlagDescriptor): string => { optArgs.push(alias) } } - if (flag.type._tag === "Boolean") { - optArgs.push(`no-${flag.name}`) + const negation = negatedName(flag) + if (negation !== undefined) { + optArgs.push(negation) } return `not __fish_contains_opt ${optArgs.join(" ")}` } @@ -170,10 +171,11 @@ const generateCompletions = ( lines.push(parts.join(" ")) // Boolean negation - if (flag.type._tag === "Boolean") { + const negation = negatedName(flag) + if (negation !== undefined) { const negParts = [`complete -c ${executableName}`] if (flagCondArg) negParts.push(flagCondArg) - negParts.push(`-l no-${flag.name}`) + negParts.push(`-l ${negation}`) if (flag.description) { negParts.push(`-d '${escapeFishString(`Disable ${flag.name}`)}'`) } @@ -192,7 +194,7 @@ const generateCompletions = ( for (const flag of descriptor.flags) { const bareCondition = `${bareBase}; and ${flagContainsOptCondition(flag)}` - const isBoolean = flag.type._tag === "Boolean" + const negation = negatedName(flag) const parts = [`complete -c ${executableName}`] parts.push(`-n '${bareCondition}'`) @@ -202,10 +204,10 @@ const generateCompletions = ( } lines.push(parts.join(" ")) - if (isBoolean) { + if (negation !== undefined) { const negParts = [`complete -c ${executableName}`] negParts.push(`-n '${bareCondition}'`) - negParts.push(`-f -a '--no-${flag.name}'`) + negParts.push(`-f -a '--${negation}'`) if (flag.description) { negParts.push(`-d '${escapeFishString(`Disable ${flag.name}`)}'`) } diff --git a/repos/effect/packages/effect/src/cli/internal/completions/powershell.ts b/repos/effect/packages/effect/src/cli/internal/completions/powershell.ts index 868d5a26ab..9b1e0a9ac2 100644 --- a/repos/effect/packages/effect/src/cli/internal/completions/powershell.ts +++ b/repos/effect/packages/effect/src/cli/internal/completions/powershell.ts @@ -8,6 +8,7 @@ * @internal */ import type * as Completions from "../../Completions.ts" +import { negatedName } from "./descriptor.ts" /** * Encode non-ASCII and control characters as `[char]` code units. Windows @@ -46,12 +47,15 @@ const valueFields = (type: Completions.FlagType | Completions.ArgumentType): str `values = ${psArray(type._tag === "Choice" ? type.values : [])}; ` + `pathType = ${type._tag === "Path" ? quotePs(type.pathType) : "$null"}` -const flagEntry = (flag: Completions.FlagDescriptor): string => - `@{ name = ${quotePs(flag.name)}; forms = ${psArray(flagForms(flag))}; ` + - `takesValue = $${flag.type._tag !== "Boolean"}; ` + - (flag.optionalValue === true ? `optionalValue = $true; ` : "") + - `${valueFields(flag.type)}; ` + - `description = ${psOptional(flag.description)} }` +const flagEntry = (flag: Completions.FlagDescriptor): string => { + const negation = negatedName(flag) + return `@{ name = ${quotePs(flag.name)}; forms = ${psArray(flagForms(flag))}; ` + + `takesValue = $${flag.type._tag !== "Boolean"}; ` + + (negation === undefined ? "" : `negation = ${quotePs(`--${negation}`)}; `) + + (flag.optionalValue === true ? `optionalValue = $true; ` : "") + + `${valueFields(flag.type)}; ` + + `description = ${psOptional(flag.description)} }` +} const argumentEntry = (argument: Completions.ArgumentDescriptor): string => `@{ name = ${quotePs(argument.name)}; variadic = $${argument.variadic}; ${valueFields(argument.type)}; ` + @@ -154,10 +158,10 @@ const completer = (dataName: string): string => foreach ($form in $flag.forms) { & $add $form 'ParameterName' $flag.description $typed '' } - if (-not $flag.takesValue) { + if ($flag.negation) { $tooltip = $null if ($flag.description) { $tooltip = 'Disable ' + $flag.name } - & $add ('--no-' + $flag.name) 'ParameterName' $tooltip $typed '' + & $add $flag.negation 'ParameterName' $tooltip $typed '' } } } @@ -228,14 +232,19 @@ const completer = (dataName: string): string => foreach ($form in $forms) { $pending = $null $flag = $lookup[$form] - if ($null -eq $flag) { - if ($form.StartsWith('--no-')) { - $name = $form.Substring(5) - $flag = $lookup['--' + $name] - if ($null -ne $flag -and -not $flag.takesValue -and $flag.name -ceq $name) { $used[$name] = $true } + # As in the CLI parser, a negation wins over an alias but not over a + # canonical name. + if ($null -eq $flag -or $form -cne ('--' + $flag.name)) { + $negated = $null + foreach ($candidate in $context.flags) { + if ($candidate.negation -ceq $form) { $negated = $candidate } + } + if ($null -ne $negated) { + $used[$negated.name] = $true + continue } - continue } + if ($null -eq $flag) { continue } $used[$flag.name] = $true # A flag with an optional value only takes a value attached with '='. if (-not $hasValue -and -not $flag.optionalValue) { $pending = $flag } diff --git a/repos/effect/packages/effect/src/cli/internal/completions/zsh.ts b/repos/effect/packages/effect/src/cli/internal/completions/zsh.ts index 334bf42189..5bc91d73ae 100644 --- a/repos/effect/packages/effect/src/cli/internal/completions/zsh.ts +++ b/repos/effect/packages/effect/src/cli/internal/completions/zsh.ts @@ -7,6 +7,7 @@ * @internal */ import type * as Completions from "../../Completions.ts" +import { negatedName } from "./descriptor.ts" // --------------------------------------------------------------------------- // Helpers @@ -22,15 +23,16 @@ const escapeZshChoice = (s: string): string => s.replace(/[^A-Za-z0-9_.,/@%+-]/g const sanitize = (s: string): string => s.replace(/[^a-zA-Z0-9_]/g, "_") /** - * All forms of a flag (--name, -alias, --no-name) for the exclusion group. + * All forms of a flag (--name, -alias, negation) for the exclusion group. */ const allForms = (flag: Completions.FlagDescriptor): Array => { const forms: Array = [`--${flag.name}`] for (const alias of flag.aliases) { forms.push(alias.length === 1 ? `-${alias}` : `--${alias}`) } - if (flag.type._tag === "Boolean") { - forms.push(`--no-${flag.name}`) + const negation = negatedName(flag) + if (negation !== undefined) { + forms.push(`--${negation}`) } return forms } @@ -89,9 +91,10 @@ const flagSpecs = (flag: Completions.FlagDescriptor): Array => { } // Boolean negation - if (flag.type._tag === "Boolean") { + const negation = negatedName(flag) + if (negation !== undefined) { const negDesc = flag.description ? `[${escapeZshExplanation(`Disable ${flag.name}`)}]` : "" - specs.push(`'${excl}--no-${flag.name}${negDesc}'`) + specs.push(`'${excl}--${negation}${negDesc}'`) } return specs diff --git a/repos/effect/packages/effect/src/cli/internal/formatter.ts b/repos/effect/packages/effect/src/cli/internal/formatter.ts new file mode 100644 index 0000000000..60fcf16ecf --- /dev/null +++ b/repos/effect/packages/effect/src/cli/internal/formatter.ts @@ -0,0 +1,18 @@ +import * as Context from "../../Context.ts" +import * as Effect from "../../Effect.ts" +import * as CliOutput from "../CliOutput.ts" +import * as TerminalCapabilities from "../TerminalCapabilities.ts" + +/** + * The formatter for built-in command output. An installed `CliOutput.Formatter` + * wins, otherwise the default formatter colors when `TerminalCapabilities` + * says it can. + * + * @internal + */ +export const resolveFormatter: Effect.Effect = Effect.contextWith((context) => + Effect.succeed( + Context.getOrUndefined(context, CliOutput.Formatter) ?? + CliOutput.defaultFormatter({ colors: Context.get(context, TerminalCapabilities.TerminalCapabilities).canColor }) + ) +) diff --git a/repos/effect/packages/effect/src/cli/internal/lexer.ts b/repos/effect/packages/effect/src/cli/internal/lexer.ts index 71fc7d343b..3f534aa036 100644 --- a/repos/effect/packages/effect/src/cli/internal/lexer.ts +++ b/repos/effect/packages/effect/src/cli/internal/lexer.ts @@ -58,8 +58,12 @@ const lexTokens = (args: ReadonlyArray): ReadonlyArray => { const equalIndex = arg.indexOf("=") if (equalIndex !== -1) { const name = arg.slice(2, equalIndex) - const value = arg.slice(equalIndex + 1) - tokens.push({ _tag: "LongOption", name, raw: arg, value }) + if (name === "") { + tokens.push({ _tag: "Value", value: arg }) + } else { + const value = arg.slice(equalIndex + 1) + tokens.push({ _tag: "LongOption", name, raw: arg, value }) + } } else { tokens.push({ _tag: "LongOption", name: arg.slice(2), raw: arg }) } @@ -67,9 +71,12 @@ const lexTokens = (args: ReadonlyArray): ReadonlyArray => { const flags = arg.slice(1) const equalIndex = flags.indexOf("=") if (equalIndex !== -1) { - const flag = flags.slice(0, equalIndex) - const value = flags.slice(equalIndex + 1) - tokens.push({ _tag: "ShortOption", flag, raw: `-${flag}`, value }) + const chars = Array.from(flags.slice(0, equalIndex)) + const flag = chars.pop() ?? "" + for (const ch of chars) { + tokens.push({ _tag: "ShortOption", flag: ch, raw: `-${ch}` }) + } + tokens.push({ _tag: "ShortOption", flag, raw: `-${flag}`, value: flags.slice(equalIndex + 1) }) } else { for (const ch of flags) { tokens.push({ _tag: "ShortOption", flag: ch, raw: `-${ch}` }) diff --git a/repos/effect/packages/effect/src/cli/internal/parser.ts b/repos/effect/packages/effect/src/cli/internal/parser.ts index 2e77922d4c..4c917fa438 100644 --- a/repos/effect/packages/effect/src/cli/internal/parser.ts +++ b/repos/effect/packages/effect/src/cli/internal/parser.ts @@ -125,6 +125,8 @@ type FlagRegistry = { readonly params: ReadonlyArray /** Maps canonical names and aliases → flag param */ readonly index: Map + /** Maps negated names of boolean flags → flag param */ + readonly negations: Map } /** @@ -229,12 +231,22 @@ const makeCursor = (tokens: ReadonlyArray): TokenCursor => { /* Flag Registry */ /* ========================================================================== */ +/** + * Long name that sets a boolean flag to `false`, or `undefined` when the flag + * has no negated form. + */ +const negatedName = (param: FlagParam): string | undefined => + !Primitive.isBoolean(param.primitiveType) || param.negation === false + ? undefined + : param.negation ?? `no-${param.name}` + /** * Creates a registry for O(1) flag lookup by name or alias. * @internal */ export const createFlagRegistry = (params: ReadonlyArray): FlagRegistry => { const index = new Map() + const negations = new Map() for (const param of params) { if (index.has(param.name)) { @@ -250,9 +262,19 @@ export const createFlagRegistry = (params: ReadonlyArray): FlagRegist } index.set(alias, param) } + + const negation = negatedName(param) + if (negation !== undefined) { + if (negations.has(negation)) { + throw new Error( + `Duplicate negation "${negation}" in command definition (conflicts with "${negations.get(negation)!.name}")` + ) + } + negations.set(negation, param) + } } - return { params, index } + return { params, index, negations } } const buildSubcommandIndex = ( @@ -362,14 +384,11 @@ const resolveFlag = ( } } - if (token._tag === "LongOption" && token.name.startsWith("no-")) { - const canonicalName = token.name.slice(3) - const param = registry.index.get(canonicalName) - if (param && param.name === canonicalName && Primitive.isBoolean(param.primitiveType)) { - return { - param, - negated: true - } + const negated = token._tag === "LongOption" ? registry.negations.get(token.name) : undefined + if (negated) { + return { + param: negated, + negated: true } } @@ -740,8 +759,9 @@ const createUnrecognizedFlagError = ( // that was intentionally kept out of --help. if (p.hidden) continue validNames.push(p.name) - if (Primitive.isBoolean(p.primitiveType)) { - validNames.push(`no-${p.name}`) + const negation = negatedName(p) + if (negation !== undefined) { + validNames.push(negation) } for (const alias of p.aliases) { validNames.push(alias) diff --git a/repos/effect/packages/effect/src/cluster/ClusterCron.ts b/repos/effect/packages/effect/src/cluster/ClusterCron.ts index a71539e554..1c0ea9cbca 100644 --- a/repos/effect/packages/effect/src/cluster/ClusterCron.ts +++ b/repos/effect/packages/effect/src/cluster/ClusterCron.ts @@ -7,9 +7,15 @@ * for distributed maintenance work where the job should be owned by the cluster * rather than by every runner independently. * + * With `calculateNextRunFromPrevious`, every run is processed by a single + * entity per cron job, so runs never overlap and are caught up in order. The + * scheduled time of the current run is available through + * `CurrentScheduledTime`. + * * @stability unstable * @since 4.0.0 */ +import * as Context from "../Context.ts" import * as Cron from "../Cron.ts" import * as DateTime from "../DateTime.ts" import * as Duration from "../Duration.ts" @@ -29,6 +35,23 @@ import * as Entity from "./Entity.ts" import type { Sharding } from "./Sharding.ts" import * as Singleton from "./Singleton.ts" +/** + * Service tag for the scheduled time of the cron run currently executing. + * + * **When to use** + * + * Use to read the time the current run was scheduled for, inside the + * `execute` effect of `make`. + * + * @stability unstable + * @category services + * @since 4.0.0 + */ +export class CurrentScheduledTime extends Context.Service< + CurrentScheduledTime, + DateTime.Utc +>()("effect/cluster/ClusterCron/CurrentScheduledTime") {} + /** * Creates a layer that runs a cron job through the cluster sharding system. * @@ -60,6 +83,11 @@ export const make = (options: { * * Defaults to `false`, meaning the next run will be calculated from the * current time. + * + * When enabled, all runs are processed one at a time by a single entity, so + * missed runs are caught up in order without overlapping. A run that fails + * is retried in place and blocks later runs until it succeeds or is older + * than `skipIfOlderThan`. */ readonly calculateNextRunFromPrevious?: boolean | undefined @@ -76,26 +104,42 @@ export const make = (options: { * Defaults to "1 day". */ readonly skipIfOlderThan?: Duration.Input | undefined -}): Layer.Layer> => { +}): Layer.Layer, Scope>> => { + const fromPrevious = options.calculateNextRunFromPrevious === true + const nextRun = (after: DateTime.Utc) => DateTime.fromDateUnsafe(Cron.next(options.cron, after)) + const CronEntity = Entity.make(`ClusterCron/${options.name}`, [ Rpc.make("run", { payload: CronPayload }) .annotate(Persisted, true) - .annotate(Uninterruptible, true) + .annotate(Uninterruptible, true), + Rpc.make("start").annotate(Persisted, true) ]) .annotate(ClusterSchema.ShardGroup, () => options.shardGroup ?? "default") .annotate(ClusterSchema.ClientTracingEnabled, false) + // with `calculateNextRunFromPrevious`, every run shares the executor address + const send = (dateTime: DateTime.Utc) => + Effect.flatMap(CronEntity.client, (makeClient) => + fromPrevious + ? makeClient(executorId).run(new CronPayload({ dateTime, executor: true }), { discard: true }) + : makeClient(DateTime.formatIso(dateTime)).run(new CronPayload({ dateTime }), { discard: true })) + + const schedule = (dateTime: DateTime.Utc) => send(dateTime).pipe(withRetry("Failed to schedule next run, retrying")) + const InitialRun = Singleton.make( `ClusterCron/${options.name}`, - Effect.gen(function*() { - const now = yield* DateTime.now - const next = DateTime.fromDateUnsafe(Cron.next(options.cron, now)) - const entityId = options.calculateNextRunFromPrevious ? "initial" : DateTime.formatIso(next) - const client = (yield* CronEntity.client)(entityId) - yield* client.run({ dateTime: next }, { discard: true }) - }), + fromPrevious + // the executor schedules the next run if its chain has been lost. + // `start` has no primary key, so each retry is a new message + ? Effect.flatMap(CronEntity.client, (makeClient) => makeClient(executorId).start()).pipe( + withRetry("Failed to start cron job, retrying") + ) + : Effect.flatMap( + DateTime.now, + (now) => send(nextRun(now)).pipe(withRetry("Failed to schedule initial run, retrying")) + ), { shardGroup: options.shardGroup } ) @@ -104,61 +148,76 @@ export const make = (options: { Option.getOrElse(() => Duration.days(1)) ) - const effect = Effect.fnUntraced(function*(dateTime: DateTime.Utc) { + const execute = Effect.fnUntraced(function*(dateTime: DateTime.Utc) { const now = yield* DateTime.now if (DateTime.isLessThan(dateTime, DateTime.subtractDuration(now, skipIfOlderThan))) { return } - return yield* options.execute + return yield* Effect.provideService(options.execute, CurrentScheduledTime, dateTime) }, Effect.orDie) const EntityLayer = CronEntity.toLayer(Effect.gen(function*() { - const makeClient = yield* CronEntity.client + const address = yield* Entity.CurrentAddress + // runs are processed one at a time, so a run processed before `start` + // means the chain is alive + let processedRun = false return { - run: (request) => - Effect.onExitPrimitive( - effect(request.payload.dateTime), - Effect.fnUntraced(function*(exit) { - if (Exit.isFailure(exit)) { - yield* Effect.logWarning(exit.cause) - } - const now = yield* DateTime.now - const next = DateTime.fromDateUnsafe(Cron.next( - options.cron, - options.calculateNextRunFromPrevious ? request.payload.dateTime : now - )) - const client = makeClient(DateTime.formatIso(next)) - return yield* client.run({ dateTime: next }, { discard: true }).pipe( - Effect.tapCause((cause) => Effect.logWarning("Failed to schedule next run, retrying", cause)), - Effect.sandbox, - Effect.retry(retryPolicy), - Effect.orDie - ) - }), + run: (request) => { + const dateTime = request.payload.dateTime + if (fromPrevious && address.entityId !== executorId) { + // a run scheduled by an earlier release joins the executor's chain + return schedule(dateTime) + } + processedRun = true + return Effect.onExitPrimitive( + execute(dateTime), + (exit) => + Effect.andThen( + Exit.isFailure(exit) ? Effect.logWarning(exit.cause) : Effect.void, + fromPrevious ? schedule(nextRun(dateTime)) : Effect.flatMap(DateTime.now, (now) => schedule(nextRun(now))) + ), true ).pipe( Effect.annotateLogs({ module: "effect/cluster/ClusterCron", name: options.name, - dateTime: request.payload.dateTime + dateTime }) ) + }, + start: () => + processedRun + ? Effect.void + // deduplicated against the pending run if the chain is alive + : Effect.flatMap(DateTime.now, (now) => schedule(nextRun(now))) } })) return Layer.merge(InitialRun, EntityLayer) } +const executorId = "executor" + const retryPolicy = Schedule.min([ Schedule.exponential(200, 1.5), Schedule.spaced("1 minute") ]) +const withRetry = (message: string) => (self: Effect.Effect): Effect.Effect => + self.pipe( + Effect.tapCause((cause) => Effect.logWarning(message, cause)), + Effect.sandbox, + Effect.retry(retryPolicy), + Effect.orDie + ) + class CronPayload extends Schema.Class("effect/cluster/ClusterCron/CronPayload")({ - dateTime: Schema.DateTimeUtc + dateTime: Schema.DateTimeUtc, + // set for runs sent to the executor, which share one entity address + executor: Schema.optional(Schema.Boolean) }) { [PrimaryKey.symbol]() { - return "" + return this.executor ? DateTime.formatIso(this.dateTime) : "" } [DeliverAt.symbol]() { return this.dateTime diff --git a/repos/effect/packages/effect/src/cluster/ClusterMetrics.ts b/repos/effect/packages/effect/src/cluster/ClusterMetrics.ts index f9feb76950..4c27779827 100644 --- a/repos/effect/packages/effect/src/cluster/ClusterMetrics.ts +++ b/repos/effect/packages/effect/src/cluster/ClusterMetrics.ts @@ -1,10 +1,11 @@ /** * Standard metric definitions used by the unstable cluster runtime. The gauges - * track runner-local entities, singleton processes, registered runners, - * healthy runners, and acquired shards. + * track runner-local entities, entity mailbox sizes, singleton processes, + * registered runners, healthy runners, acquired shards, and shards being + * released. The timer tracks shard release duration. * * This module only defines the metric handles. The runtime components that - * manage entities, runners, singletons, and sharding update these gauges while + * manage entities, runners, singletons, and sharding update these metrics while * the cluster is running. * * @stability unstable @@ -23,13 +24,13 @@ import * as Metric from "../Metric.ts" * * **Details** * - * Bigint gauge named `effect_cluster_entities`, updated with the entity type as - * a metric tag. + * Bigint gauge named `effect_cluster_entities`, tagged with the entity type. + * Each entity manager registers a series that reads its active entity count + * when metrics are snapshotted, and removes it when the manager shuts down. * * **Gotchas** * - * This gauge is runner-local and sampled by the entity manager loop. Aggregate - * across runners and expect up to roughly one polling interval of lag. + * This gauge is runner-local. Aggregate across runners. * * @see {@link singletons} for singleton process counts on the current runner * @see {@link shards} for shard ownership on the current runner @@ -40,6 +41,86 @@ import * as Metric from "../Metric.ts" */ export const entities = Metric.gauge("effect_cluster_entities", { bigint: true }) +/** + * Represents a gauge tracking the mailbox size of each entity resident on the + * current runner. + * + * **When to use** + * + * Use to find entities whose mailboxes back up, before sends start failing + * with `MailboxFull`. + * + * **Details** + * + * Bigint gauge named `effect_cluster_entity_mailbox_size`, with one series per + * resident entity tagged with `type`, `id` and `shard`. The value is the + * number the mailbox capacity check compares against: requests admitted to the + * entity, whether queued or in flight, excluding internal keep-alive requests. + * + * The value is read when metrics are snapshotted, so it adds no work per + * message. Each series is removed when its entity is released. + * + * **Gotchas** + * + * Off by default. Enable it with `ShardingConfig.entityMailboxMetrics` or the + * `mailboxMetrics` option when registering an entity. One series per resident + * entity can mean many series on busy runners. + * + * An entity keeps reporting while it drains in-flight requests during + * shutdown. `Metric.update` and `Metric.modify` on this gauge do nothing for + * series the runtime registered. + * + * Like the other runner gauges it has no runner attribute: runners sharing a + * registry without distinct metric attributes overwrite each other's series. + * + * @see {@link entities} for the number of resident entities per type + * @see {@link entityMailboxSizeMax} for the always-on per-type maximum + * + * @stability unstable + * @category metrics + * @since 4.0.6 + */ +export const entityMailboxSize = Metric.gauge("effect_cluster_entity_mailbox_size", { bigint: true }) + +/** + * Represents a gauge tracking the largest mailbox size among the entities of + * each type resident on the current runner. + * + * **When to use** + * + * Use to alert on entity types whose mailboxes back up, before sends start + * failing with `MailboxFull`, without paying for a series per entity. + * + * **Details** + * + * Bigint gauge named `effect_cluster_entity_mailbox_size_max`, with one series + * per registered entity type tagged with `type`. The value is the largest + * mailbox size that {@link entityMailboxSize} would report for the type, + * including entities draining in-flight requests during shutdown, or `0` when + * the type has no resident entities. + * + * The value is computed when metrics are snapshotted, by visiting each + * resident entity of the type, so it adds no work per message. Entities + * draining during shutdown keep their slot until they finish, so the visit is + * bounded by `ShardingConfig.maxResidentEntities` across all types, unless it + * is `"unbounded"`. The series is removed when the entity type shuts down. + * + * **Gotchas** + * + * Always on. `Metric.update` and `Metric.modify` on this gauge do nothing for + * series the runtime registered. + * + * Like the other runner gauges it has no runner attribute: runners sharing a + * registry without distinct metric attributes overwrite each other's series. + * + * @see {@link entityMailboxSize} for the opt-in per-entity mailbox sizes + * + * @stability unstable + * @category metrics + * @since 4.0.6 + */ +export const entityMailboxSizeMax = Metric.gauge("effect_cluster_entity_mailbox_size_max", { bigint: true }) + /** * Creates a gauge tracking the number of singleton processes currently running on the * current runner. @@ -116,3 +197,64 @@ export const runnersHealthy = Metric.gauge("effect_cluster_runners_healthy", { b * @since 4.0.0 */ export const shards = Metric.gauge("effect_cluster_shards", { bigint: true }) + +/** + * Represents a gauge tracking the number of shards the current runner is still + * releasing. + * + * **When to use** + * + * Use to detect shard releases that stall during rebalancing, shutdown handoff + * or lock storage failures. + * + * **Details** + * + * Bigint gauge named `effect_cluster_shards_releasing`, updated with the number + * of distinct shards being released. A shard counts from the moment it is no + * longer acquired until its entities and singletons have stopped and its lock + * is released, including waits for lock storage to recover and release + * retries. + * + * **Gotchas** + * + * Positive values are normal while shards move; alert only when the value + * stays positive. It does not reflect lock ownership, and reacquisition can + * stay blocked briefly after it returns to zero, while a forced release + * finishes its bookkeeping. + * + * Like the other runner gauges it assumes one runner per process: runners + * sharing a registry without distinct metric attributes overwrite each other's + * value. + * + * @stability unstable + * @category metrics + * @since 4.1.0 + */ +export const shardsReleasing = Metric.gauge("effect_cluster_shards_releasing", { bigint: true }) + +/** + * Represents a timer tracking how long the current runner takes to release a + * shard after reassignment, lock loss or shutdown handoff. + * + * **When to use** + * + * Use to observe shard handoff latency during rebalancing, shutdown, or lock + * storage failures. + * + * **Details** + * + * Millisecond timer named `effect_cluster_shard_release`. Each sample covers + * entity teardown, retries and any wait for shard lock storage to recover. + * + * **Gotchas** + * + * Only completed releases are recorded. Interrupted releases and forced + * releases after lock storage becomes unhealthy are not. + * + * @stability unstable + * @category metrics + * @since 4.0.0 + */ +export const shardReleaseDuration = Metric.timer("effect_cluster_shard_release", { + description: "Time taken to release a shard, including entity teardown, retries and lock storage recovery" +}) diff --git a/repos/effect/packages/effect/src/cluster/ClusterWorkflowEngine.ts b/repos/effect/packages/effect/src/cluster/ClusterWorkflowEngine.ts index 8e0c3be7bf..aa32bac393 100644 --- a/repos/effect/packages/effect/src/cluster/ClusterWorkflowEngine.ts +++ b/repos/effect/packages/effect/src/cluster/ClusterWorkflowEngine.ts @@ -24,7 +24,7 @@ import * as Rpc from "../rpc/Rpc.ts" import { ClientAbort } from "../rpc/RpcSchema.ts" import * as Schedule from "../Schedule.ts" import * as Schema from "../Schema.ts" -import type * as Scope from "../Scope.ts" +import * as Scope from "../Scope.ts" import * as Semaphore from "../Semaphore.ts" import * as Activity from "../workflow/Activity.ts" import * as DurableClock from "../workflow/DurableClock.ts" @@ -135,11 +135,21 @@ export const make = Effect.gen(function*() { // Rebuilds share an activation; overlapping activations have separate results. // Weak keys do not retain scopes, but contexts captured by clients, activities, // or finalizers can keep a closed scope (and its results) reachable until later. - const pendingByActivation = new WeakMap> - }>() + const pendingByActivation = new WeakMap() + // Closes the scopes retained by suspended runs. The latest run already closed + // its own scope with its real exit, so earlier finalizers observe success and + // skip superseded compensations. Abandonment releases owner-local resources + // without running durable finalizers, so the run can replay elsewhere. + const closeRuns = (state: ActivationState, abandoned: boolean) => + Effect.suspend(() => { + const scope = state.scope + if (abandoned) { + for (const instance of state.instances) instance.abandoned = true + } + state.instances.clear() + state.scope = Scope.makeUnsafe() + return Scope.close(scope, Exit.void) + }) const clients = yield* RcMap.make({ lookup: Effect.fnUntraced(function*(workflowName: string) { const entity = entities.get(workflowName) @@ -374,26 +384,25 @@ export const make = Effect.gen(function*() { // `Suspended`. Replies are only observed on the runner that saves them, so // callers on other runners, and any failure here, leave delivery to the // caller's retry schedule. - const awaitComplete = (workflow: Workflow.Any, executionId: string) => - Effect.gen(function*() { - const address = entityAddressFor({ workflow, entityType: `Workflow/${workflow._tag}`, executionId }) - if (!sharding.hasShardId(address.shardId)) return yield* Effect.never - const requestId = yield* storage.requestIdForPrimaryKey({ address, tag: "run", id: "" }) - if (Option.isNone(requestId)) return yield* Effect.never - const request = Envelope.makeRequest({ - requestId: requestId.value, - address, - tag: "run", - payload: {}, - headers: Headers.empty - }) - // A replay that suspends again re-arms the wait, so it does not spend a - // step of the caller's retry schedule. - let settled = false - while (!settled) { - settled = yield* awaitRunReply(workflow, request, (reply) => !isSuspendedReply(reply)) - } - }).pipe(Effect.catchCause(() => Effect.never)) + const awaitComplete = Effect.fnUntraced(function*(workflow: Workflow.Any, executionId: string) { + const address = entityAddressFor({ workflow, entityType: `Workflow/${workflow._tag}`, executionId }) + if (!sharding.hasShardId(address.shardId)) return yield* Effect.never + const requestId = yield* storage.requestIdForPrimaryKey({ address, tag: "run", id: "" }) + if (Option.isNone(requestId)) return yield* Effect.never + const request = Envelope.makeRequest({ + requestId: requestId.value, + address, + tag: "run", + payload: {}, + headers: Headers.empty + }) + // A replay that suspends again re-arms the wait, so it does not spend a + // step of the caller's retry schedule. + let settled = false + while (!settled) { + settled = yield* awaitRunReply(workflow, request, (reply) => !isSuspendedReply(reply)) + } + }, Effect.catchCause(() => Effect.never)) const interrupt = Effect.fnUntraced( function*(workflow: Workflow.Any, executionId: string) { @@ -465,7 +474,14 @@ export const make = Effect.gen(function*() { (decoded.exit._tag === "Success" ? decoded.exit.value : decoded.exit) as Exit.Exit ) } - pending = { workflowName: workflow._tag, executionId, results } + pending = { + workflowName: workflow._tag, + executionId, + results, + instances: new Set(), + scope: Scope.makeUnsafe() + } + yield* Scope.addFinalizer(activation, closeRuns(pending, true)) pendingByActivation.set(activation, pending) } // Retain awaited names through reply persistence, including suspension. @@ -476,24 +492,34 @@ export const make = Effect.gen(function*() { // Activity awaits must wake the run after suspension without preempting it. readonly activityAwaits: Map } | undefined - // Concurrent wakes share one wait for the current run to publish its reply. - // Explicit wakes cannot be dropped: operator resume bypasses the gate; - // InterruptSignal enters it only on a fresh entity, with no run to wait for. + // Concurrent automatic wakes share one wait for the current run to + // publish its reply. Explicit wakes cannot be dropped, because an + // automatic wake skips the reset of a failure suspension, so they + // wait outside the gate. Operator resume does not use the gate. const resumeGate = Semaphore.makeUnsafe(1) + const waitForCurrentRunReply = Effect.suspend(() => + currentRun ? waitForRunReply(workflow, currentRun.request) : Effect.void + ) const resumeCurrentRun = (mode: WakeMode) => - Effect.suspend(() => - resumeGate.withPermitsIfAvailable(1)( - currentRun ? waitForRunReply(workflow, currentRun.request) : Effect.void - ) - ).pipe( - // Release the gate before reset can start another run, so later - // completions can wake that replay. - Effect.flatMap((waited) => Option.isSome(waited) ? resume(workflow, executionId, mode) : Effect.void), - ensureSuccess + ensureSuccess( + mode === "explicit" + ? Effect.andThen(waitForCurrentRunReply, resume(workflow, executionId, mode)) + : resumeGate.withPermitsIfAvailable(1)(waitForCurrentRunReply).pipe( + // Release the gate before reset can start another run, so later + // completions can wake that replay. + Effect.flatMap((waited) => + Option.isSome(waited) ? resume(workflow, executionId, mode) : Effect.void + ) + ) ) return { run: (request: Entity.Request) => { - const instance = WorkflowEngine.WorkflowInstance.initial(workflow, executionId) + // Each run gets a fresh scope under the activation's parent, so + // replays rebuild compensations while suspended runs keep theirs. + const instance = WorkflowEngine.WorkflowInstance.initial(workflow, executionId, { + scope: Scope.forkUnsafe(pending.scope) + }) + pending.instances.add(instance) currentRun = { request, instance, activityAwaits: new Map() } const parent = (request.payload as any)[payloadParentKey] as | { workflowName: string; executionId: string; detached?: boolean } @@ -533,6 +559,11 @@ export const make = Effect.gen(function*() { ) }), Workflow.intoResult, + Effect.onExit((exit) => + Exit.isSuccess(exit) && exit.value._tag === "Suspended" + ? Effect.void + : closeRuns(pending, instance.abandoned) + ), (effect) => deferredState.trackRun(instance, effect), // Request middleware and local callers may carry another entity's context. Effect.provideService(CurrentActivationScope, activation) @@ -591,8 +622,10 @@ export const make = Effect.gen(function*() { // a later await would require waiting for its reply before reset. // A run completing its own deferred also reads it from the cache // when it awaits later, unless an activity is already parked on it. + // A run only reads InterruptSignal while suspending, so an interrupt + // always waits for the current run's reply before it resets the run. const wake = Effect.suspend(() => - currentRun && !currentRun.instance.awaitedDeferreds.has(payload.name) + currentRun && mode === "automatic" && !currentRun.instance.awaitedDeferreds.has(payload.name) ? currentRun.activityAwaits.has(payload.name) // Hold an activity wake until the run has published its reply. ? resumeCurrentRun(mode) @@ -909,6 +942,16 @@ const DeferredRpc = Rpc.make("deferred", { type WakeMode = "explicit" | "automatic" +interface ActivationState { + readonly workflowName: string + readonly executionId: string + readonly results: Map> + // Every run forks its scope from here, so runs that suspended keep their + // resources and durable finalizers until the execution finishes. + scope: Scope.Closeable + readonly instances: Set +} + const isSuspendedReply = (reply: WithExitEncoded>) => reply.exit._tag === "Success" && reply.exit.value._tag === "Suspended" diff --git a/repos/effect/packages/effect/src/cluster/Entity.ts b/repos/effect/packages/effect/src/cluster/Entity.ts index b9da75bee7..23704305d7 100644 --- a/repos/effect/packages/effect/src/cluster/Entity.ts +++ b/repos/effect/packages/effect/src/cluster/Entity.ts @@ -147,6 +147,7 @@ export interface Entity< readonly maxIdleTime?: Duration.Input | undefined readonly concurrency?: number | "unbounded" | undefined readonly mailboxCapacity?: number | "unbounded" | undefined + readonly mailboxMetrics?: boolean | undefined readonly disableFatalDefects?: boolean | undefined readonly defectRetryPolicy?: Schedule.Schedule | undefined readonly spanAttributes?: Record | undefined @@ -191,6 +192,7 @@ export interface Entity< options?: { readonly maxIdleTime?: Duration.Input | undefined readonly mailboxCapacity?: number | "unbounded" | undefined + readonly mailboxMetrics?: boolean | undefined readonly disableFatalDefects?: boolean | undefined readonly defectRetryPolicy?: Schedule.Schedule | undefined readonly spanAttributes?: Record | undefined @@ -287,6 +289,7 @@ const Proto = { readonly maxIdleTime?: Duration.Input | undefined readonly concurrency?: number | "unbounded" | undefined readonly mailboxCapacity?: number | "unbounded" | undefined + readonly mailboxMetrics?: boolean | undefined readonly disableFatalDefects?: boolean | undefined readonly defectRetryPolicy?: Schedule.Schedule | undefined readonly spanAttributes?: Record | undefined @@ -335,6 +338,7 @@ const Proto = { options?: { readonly maxIdleTime?: Duration.Input | undefined readonly mailboxCapacity?: number | "unbounded" | undefined + readonly mailboxMetrics?: boolean | undefined readonly disableFatalDefects?: boolean | undefined readonly defectRetryPolicy?: Schedule.Schedule | undefined readonly spanAttributes?: Record | undefined diff --git a/repos/effect/packages/effect/src/cluster/MessageStorage.ts b/repos/effect/packages/effect/src/cluster/MessageStorage.ts index b1876c5c0f..f14101ea63 100644 --- a/repos/effect/packages/effect/src/cluster/MessageStorage.ts +++ b/repos/effect/packages/effect/src/cluster/MessageStorage.ts @@ -119,6 +119,13 @@ export class MessageStorage extends Context.Service( message: Message.OutgoingRequest | Message.IncomingRequest diff --git a/repos/effect/packages/effect/src/cluster/Runners.ts b/repos/effect/packages/effect/src/cluster/Runners.ts index 7872bcaee4..002382062f 100644 --- a/repos/effect/packages/effect/src/cluster/Runners.ts +++ b/repos/effect/packages/effect/src/cluster/Runners.ts @@ -265,17 +265,15 @@ export const make: ( // The pump is owned by the shared entry rather than the first caller, so any // waiter can be interrupted without stopping delivery to the others. + // + // `replyFromStorage` registers the request with the storage loop before the + // pump starts, so the pump only re-registers after a stream chunk is acked. const pumpReplies = Effect.fnUntraced(function*( requestId: Snowflake.Snowflake, entry: Omit ) { while (true) { // wait for the storage loop to notify us - entry.latch.closeUnsafe() - const message = entry.messages.values().next().value - if (!message) return - waitingStorageRequests.set(requestId, message) - storageLatch.openUnsafe() yield* entry.latch.await // send the replies back @@ -300,6 +298,13 @@ export const make: ( yield* entry.latch.await } entry.replies = [] + + // ask the storage loop for the next chunk + entry.latch.closeUnsafe() + const message = entry.messages.values().next().value + if (!message) return + waitingStorageRequests.set(requestId, message) + storageLatch.openUnsafe() } }) @@ -316,9 +321,14 @@ export const make: ( } entry = Object.assign(state, { pump: yield* Effect.forkIn(pumpReplies(requestId, state), runnersScope) }) storageRequests.set(requestId, entry) + waitingStorageRequests.set(requestId, message) } const current = entry current.messages.add(message) + // A waiter that joins an existing entry, such as a duplicate, reads + // now. A reply committed without a notification would otherwise wait + // for the next poll. + storageLatch.openUnsafe() return yield* restore(Fiber.join(current.pump)).pipe( Effect.ensuring(Effect.suspend(() => { current.messages.delete(message) @@ -336,6 +346,52 @@ export const make: ( }) ) + // A request that may already be processing has to read its replies back + // from storage. A local reply handler wakes the storage loop, so streams do + // not have to wait for the next poll. + const replyFromStorageWithWake = (message: Message.OutgoingRequest) => + Effect.raceFirst( + replyFromStorage(message), + storage.registerReplyHandler( + new Message.OutgoingRequest({ ...message, respond: () => storageLatch.open }) + ).pipe(Effect.ignore, Effect.andThen(Effect.never)) + ) + + // Stream replies stay on the storage loop, which delivers chunks in order + // and waits for their acks. + const canReplyLocally = (message: Message.OutgoingRequest) => { + const rpc = message.rpc as any as Rpc.AnyWithProps + return storage !== MessageStorage.noop && Context.get(rpc.annotations, Persisted) && + !RpcSchema.isStreamSchema(rpc.successSchema) + } + + // A request can be processed as soon as it is saved, so its reply handler is + // registered first. `registerReplyHandler` registers as soon as it starts, + // so a fresh request gets its reply from the handler without reading it + // back from storage. + const notifyLocalRequest = Effect.fnUntraced(function*( + message: Message.OutgoingRequest, + notifyEntity: (message: Message.Outgoing) => Effect.Effect + ) { + const handler = yield* Effect.forkScoped(storage.registerReplyHandler(message), { startImmediately: true }) + return yield* notifyWith(message, (message, duplicate) => { + const outgoing = message as Message.OutgoingRequest + // A duplicate is answered under its original request id. + if (duplicate) { + return Fiber.interrupt(handler).pipe( + Effect.andThen(notifyEntity(message)), + Effect.andThen(replyFromStorageWithWake(outgoing)) + ) + } + // The handler can be dropped, for example when its shard moves or a + // commit error hides the notification. The reply is then in storage. + return notifyEntity(message).pipe( + Effect.andThen(Fiber.join(handler)), + Effect.catchTag("EntityNotAssignedToRunner", () => replyFromStorage(outgoing)) + ) + }) + }, Effect.scoped) + const storageLatch = Latch.makeUnsafe(false) if (storage !== MessageStorage.noop) { yield* Effect.gen(function*() { @@ -431,28 +487,23 @@ export const make: ( }) }, notifyLocal(options) { - return notifyWith(options.message, (message) => { + const message = options.message + const notifyEntity = (message: Message.Outgoing) => + Effect.catchTag( + options.notify(Message.incomingLocalFromOutgoing(message)), + "EntityNotAssignedToRunner", + () => Effect.void + ) + if (message._tag === "OutgoingRequest" && !options.discard && !options.storageOnly && canReplyLocally(message)) { + return notifyLocalRequest(message, notifyEntity) + } + return notifyWith(message, (message) => { if (options.discard || message._tag === "OutgoingEnvelope") { - return Effect.catchTag( - options.notify(Message.incomingLocalFromOutgoing(message)), - "EntityNotAssignedToRunner", - () => Effect.void - ) + return notifyEntity(message) } - // The request can be processed as soon as it is saved, so replies are - // always read back from storage. The local reply handler only wakes the - // storage loop, so streams do not have to wait for the next poll. - const readReplies = options.storageOnly - ? replyFromStorage(message) - : Effect.raceFirst( - replyFromStorage(message), - storage.registerReplyHandler( - new Message.OutgoingRequest({ ...message, respond: () => storageLatch.open }) - ).pipe(Effect.ignore, Effect.andThen(Effect.never)) - ) - return options.notify(Message.incomingLocalFromOutgoing(message)).pipe( - Effect.catchTag("EntityNotAssignedToRunner", () => Effect.void), - Effect.andThen(readReplies) + return Effect.andThen( + notifyEntity(message), + options.storageOnly ? replyFromStorage(message) : replyFromStorageWithWake(message) ) }) } diff --git a/repos/effect/packages/effect/src/cluster/Sharding.ts b/repos/effect/packages/effect/src/cluster/Sharding.ts index 2e088dfd7b..4142f42bde 100644 --- a/repos/effect/packages/effect/src/cluster/Sharding.ts +++ b/repos/effect/packages/effect/src/cluster/Sharding.ts @@ -27,6 +27,7 @@ import * as HashRing from "../HashRing.ts" import * as InternalRecord from "../internal/record.ts" import * as Latch from "../Latch.ts" import * as Layer from "../Layer.ts" +import * as Metric from "../Metric.ts" import * as MutableHashMap from "../MutableHashMap.ts" import * as MutableHashSet from "../MutableHashSet.ts" import * as MutableRef from "../MutableRef.ts" @@ -141,6 +142,7 @@ export class Sharding extends Context.Service | undefined readonly spanAttributes?: Record | undefined @@ -276,6 +278,17 @@ const make = Effect.gen(function*() { // the active shards are the ones that we have acquired the lock for const acquiredShards = MutableHashSet.empty() + // shards still held by this runner while their entities stop and their locks + // are released + const releasingShards = MutableHashSet.empty() + // The releasing gauge reports to the metric registry and attributes the + // layer was built with. + const updateReleasingShardsGauge = () => + ClusterMetrics.shardsReleasing.updateUnsafe(BigInt(MutableHashSet.size(releasingShards)), services) + const updateShardGauges = () => { + ClusterMetrics.shards.updateUnsafe(BigInt(MutableHashSet.size(acquiredShards)), Context.empty()) + updateReleasingShardsGauge() + } const activeShardsLatch = yield* Latch.make(false) // open once shutdown starts handing every shard off const handoffLatch = Latch.makeUnsafe(false) @@ -343,7 +356,9 @@ const make = Effect.gen(function*() { // This should be shutdown last, when all entities have been shutdown, to // allow them to move to another runner. - const releasingShards = MutableHashSet.empty() + // Keep startup fibers behind singleton initialization without moving resource + // acquisition (and therefore changing the shutdown finalizer order). + const singletonsReady = Latch.makeUnsafe(false) // Hands every held shard off on shutdown, set when this is a runner let handoff: Effect.Effect = Effect.void // Shards whose entities must be force interrupted and locks fully released @@ -352,6 +367,7 @@ const make = Effect.gen(function*() { const initialRunnerAddress = getRunnerAddress() if (initialRunnerAddress) { const selfAddress = initialRunnerAddress + updateReleasingShardsGauge() yield* Scope.addFinalizerExit(shardingScope, () => { // the locks expire over time, so if this fails we ignore it return Effect.ignore(runnerStorage.releaseAll(selfAddress)) @@ -405,6 +421,7 @@ const make = Effect.gen(function*() { for (const shardId of shardIds) { MutableHashSet.remove(releasingShards, shardId) MutableHashSet.remove(forceReleasingShards, shardId) + updateReleasingShardsGauge() // During shutdown, parked waiters are interrupted: this runner will // never observe the reply. During a rebalance the waiters are resumed // with `EntityNotAssignedToRunner`, and the client falls back to @@ -419,22 +436,30 @@ const make = Effect.gen(function*() { const retryShardRelease = (annotations: { readonly fiber: string; readonly shardId?: ShardId }) => (effect: Effect.Effect) => - effect.pipe( - Effect.sandbox, - Effect.tapError((cause) => - Effect.logDebug(`Could not release shards, retrying`, cause).pipe( - Effect.annotateLogs({ - module: "effect/cluster/Sharding", - runner: selfAddress, - ...annotations - }), - // Effect.eventually retries immediately, so space failures to - // avoid hot-looping while storage is unavailable. - Effect.andThen(Effect.sleep(50)) - ) - ), - Effect.eventually - ) + Effect.suspend(() => { + let lastWarning = -Infinity + return effect.pipe( + Effect.sandbox, + Effect.tapError((cause) => + Effect.suspend(() => { + const now = clock.currentTimeMillisUnsafe() + if (now - lastWarning < 30_000) return Effect.void + lastWarning = now + return Effect.logWarning("Could not release shards, retrying", cause) + }).pipe( + Effect.annotateLogs({ + module: "effect/cluster/Sharding", + runner: selfAddress, + ...annotations + }), + // Effect.eventually retries immediately, so space failures to + // avoid hot-looping while storage is unavailable. + Effect.andThen(Effect.sleep(50)) + ) + ), + Effect.eventually + ) + }) const releaseShard = Effect.fnUntraced( function*(shardId: ShardId) { yield* runShardRelease( @@ -446,6 +471,8 @@ const make = Effect.gen(function*() { (effect, shardId) => effect.pipe( retryShardRelease({ fiber: "releaseShard", shardId }), + Effect.timed, + Effect.flatMap(([duration]) => Metric.update(ClusterMetrics.shardReleaseDuration, duration)), FiberMap.run(releaseShardsMap, shardId, { onlyIfMissing: true }) ) ) @@ -454,14 +481,62 @@ const make = Effect.gen(function*() { // normal path while it is pending, otherwise the bulk release wipes a lock // the runner already considers acquired. const forcedShardReleasePending = () => forcedShardReleaseRunning || MutableHashSet.size(forceReleasingShards) > 0 + // Repeat while pending, but no more often than every 30 seconds + const forcedReleaseWarningInterval = Duration.max( + Duration.fromInputUnsafe(config.shardLockExpiration), + Duration.seconds(30) + ) + // Names the shards still blocking reacquisition and the entity types and + // singletons still stopping on them + const warnForcedReleasePending = Effect.suspend(() => { + const entityTypes = Arr.empty() + for (const [entityType, state] of entityManagers) { + for (const shardId of forceReleasingShards) { + if (state.manager.hasServersUnsafe(shardId)) { + entityTypes.push(entityType) + break + } + } + } + const singletonNames = new Set() + for (const shardId of forceReleasingShards) { + for (const [address] of singletons.get(shardId) ?? []) { + if (FiberMap.hasUnsafe(singletonFibers, address)) singletonNames.add(address.name) + } + } + MutableHashMap.forEach(stoppingSingletons, (_, address) => { + if (MutableHashSet.has(forceReleasingShards, address.shardId)) singletonNames.add(address.name) + }) + return Effect.logWarning( + "Forced shard release is still pending; reacquisition is blocked until release completes" + ).pipe(Effect.annotateLogs({ + module: "effect/cluster/Sharding", + runner: selfAddress, + shardIds: Array.from(forceReleasingShards, String), + entityTypes, + singletonNames: [...singletonNames] + })) + }) const releaseForcedShards = Effect.suspend(() => { if (forcedShardReleaseRunning || MutableHashSet.size(forceReleasingShards) === 0) { return Effect.void } forcedShardReleaseRunning = true const shardIds = [...forceReleasingShards] - return runShardRelease(shardIds, true, runnerStorage.releaseAll(selfAddress)).pipe( - retryShardRelease({ fiber: "releaseForcedShards" }), + return Effect.gen(function*() { + // Observe independently: a timeout around release cannot interrupt + // uninterruptible work, and must not allow locks to be reacquired. + const monitor = yield* warnForcedReleasePending.pipe( + Effect.repeat(Schedule.spaced(forcedReleaseWarningInterval)), + Effect.delay(config.shardLockExpiration), + Effect.interruptible, + Effect.forkIn(releaseScope) + ) + yield* runShardRelease(shardIds, true, runnerStorage.releaseAll(selfAddress)).pipe( + retryShardRelease({ fiber: "releaseForcedShards" }), + Effect.ensuring(Fiber.interrupt(monitor)) + ) + }).pipe( Effect.ensuring(Effect.sync(() => { forcedShardReleaseRunning = false activeShardsLatch.openUnsafe() @@ -480,6 +555,7 @@ const make = Effect.gen(function*() { }) yield* Effect.gen(function*() { + yield* singletonsReady.await activeShardsLatch.openUnsafe() while (true) { @@ -494,6 +570,7 @@ const make = Effect.gen(function*() { MutableHashSet.remove(acquiredShards, shardId) MutableHashSet.add(releasingShards, shardId) } + updateShardGauges() if (MutableHashSet.size(releasingShards) > 0 || MutableHashSet.size(forceReleasingShards) > 0) { yield* Effect.forkIn(syncSingletons, shardingScope) @@ -555,13 +632,11 @@ const make = Effect.gen(function*() { } MutableHashSet.add(acquiredShards, shardId) } + updateShardGauges() if (acquired.length > 0) { yield* notifyShardLockWaiters yield* storageReadLatch.open yield* Effect.forkIn(syncSingletons, shardingScope) - - // update metrics - ClusterMetrics.shards.updateUnsafe(BigInt(MutableHashSet.size(acquiredShards)), Context.empty()) } // the shutdown handoff releases the shards acquired here without waiting yield* Effect.raceFirst(Effect.sleep(1000), handoffLatch.await) @@ -589,7 +664,7 @@ const make = Effect.gen(function*() { MutableHashSet.add(releasingShards, shardId) MutableHashSet.add(forceReleasingShards, shardId) } - ClusterMetrics.shards.updateUnsafe(BigInt(0), Context.empty()) + updateShardGauges() activeShardsLatch.openUnsafe() return Effect.gen(function*() { @@ -632,6 +707,7 @@ const make = Effect.gen(function*() { MutableHashSet.add(releasingShards, shardId) } } + updateShardGauges() if (MutableHashSet.size(releasingShards) > 0) { yield* activeShardsLatch.open } @@ -655,6 +731,7 @@ const make = Effect.gen(function*() { yield* Effect.suspend(() => shardLocksHealthyLatch.isOpen() ? refreshShardLocks : probeShardLocks).pipe( Effect.repeat(Schedule.fixed(shardLockInterval)), Effect.forever, + (effect) => Effect.andThen(singletonsReady.await, effect), Effect.forkIn(shardingScope) ) @@ -941,20 +1018,18 @@ const make = Effect.gen(function*() { } removableNotifications.clear() } - if (MutableHashSet.size(resetAddresses) > 0) { - for (const address of resetAddresses) { - yield* Effect.logWarning("Could not find entity manager for address, retrying").pipe( - Effect.annotateLogs({ address }) - ) - yield* Effect.forkIn(storage.resetAddress(address), shardingScope) - } - MutableHashSet.clear(resetAddresses) + for (const address of resetAddresses) { + yield* Effect.logWarning("Could not find entity manager for address, retrying").pipe( + Effect.annotateLogs({ address }) + ) } - // Capture claims skipped because of the entity cap. They are reset in - // one storage operation after releasing the read lock. - let cappedAddressesToReset: Array | undefined - if (MutableHashSet.size(cappedAddresses) > 0) { - cappedAddressesToReset = Arr.fromIterable(cappedAddresses) + // Capture claims for unregistered entity types, and claims skipped + // because of the entity cap. They are reset in one storage operation + // after releasing the read lock, and before the next read starts. + let addressesToReset: Array | undefined + if (MutableHashSet.size(resetAddresses) > 0 || MutableHashSet.size(cappedAddresses) > 0) { + addressesToReset = [...resetAddresses, ...cappedAddresses] + MutableHashSet.clear(resetAddresses) MutableHashSet.clear(cappedAddresses) } @@ -966,8 +1041,8 @@ const make = Effect.gen(function*() { resetRequestIds.length = 0 } - if (cappedAddressesToReset !== undefined) { - yield* Effect.ignore(storage.resetAddresses(cappedAddressesToReset)) + if (addressesToReset !== undefined) { + yield* Effect.ignore(storage.resetAddresses(addressesToReset)) } // A full batch means more messages could be waiting; start the next @@ -1796,6 +1871,8 @@ const make = Effect.gen(function*() { (stops) => Effect.all(stops, { discard: true }) ) + singletonsReady.openUnsafe() + // --- Entities --- const reaper = yield* EntityReaper @@ -1851,6 +1928,9 @@ const make = Effect.gen(function*() { yield* withStorageReadLock(Effect.sync(() => { entityRegistrationStartMillis ??= clock.currentTimeMillisUnsafe() entityManagers.set(entity.type, state) + // persisted messages read before this registration were reset, so + // read storage again instead of waiting for the next poll interval + storageReadLatch.openUnsafe() if (entityManagerLatches.has(entity.type)) { entityManagerLatches.get(entity.type)!.openUnsafe() entityManagerLatches.delete(entity.type) diff --git a/repos/effect/packages/effect/src/cluster/ShardingConfig.ts b/repos/effect/packages/effect/src/cluster/ShardingConfig.ts index 4b8b8474c9..8120e7df58 100644 --- a/repos/effect/packages/effect/src/cluster/ShardingConfig.ts +++ b/repos/effect/packages/effect/src/cluster/ShardingConfig.ts @@ -85,6 +85,8 @@ export class ShardingConfig extends Context.Service = Config.all({ Config.withDefault(defaults.entityMailboxCapacity) // Config.withDescription("The default capacity of the mailbox for entities.") ), + entityMailboxMetrics: Config.Boolean("entityMailboxMetrics").pipe( + Config.withDefault(defaults.entityMailboxMetrics) + // Config.withDescription("Report the mailbox size of each resident entity.") + ), maxResidentEntities: Config.schema( Schema.Int.check(Schema.isGreaterThan(0)), "maxResidentEntities" diff --git a/repos/effect/packages/effect/src/cluster/SqlMessageStorage.ts b/repos/effect/packages/effect/src/cluster/SqlMessageStorage.ts index 74fc016a37..98dee0f73e 100644 --- a/repos/effect/packages/effect/src/cluster/SqlMessageStorage.ts +++ b/repos/effect/packages/effect/src/cluster/SqlMessageStorage.ts @@ -277,13 +277,42 @@ const makeEncodedStorage = Effect.fnUntraced(function*( const sqlFalse = sql.literal(supportsBooleans ? "FALSE" : "0") const sqlTrue = sql.literal(supportsBooleans ? "TRUE" : "1") + // Read SQLite IDs as text to avoid rounding on number-returning drivers + // such as Durable Objects. + const idsAsText = sql.onDialectOrElse({ + sqlite: () => true, + orElse: () => false + }) + const idAs = (column: string, alias: string) => + idsAsText ? `CAST(${column} AS TEXT) AS ${alias}` : `${column} AS ${alias}` + const replyJoinColumns = "r.kind as reply_kind, r.payload as reply_payload, r.sequence as reply_sequence" + const messageColumns = idsAsText + ? `${idAs("m.id", "id")}, m.message_id, m.shard_id, m.entity_type, m.entity_id, m.kind, m.tag, m.payload, ` + + `m.headers, m.trace_id, m.span_id, m.sampled, m.processed, ${idAs("m.request_id", "request_id")}, ` + + `${idAs("m.reply_id", "reply_id")}, ${idAs("m.last_reply_id", "last_reply_id")}, m.last_read, m.deliver_at` + : "m.*" + + const idColumn = sql.literal(idAs("id", "id")) + const replyColumns = sql.literal( + `${idAs("id", "id")}, kind, ${idAs("request_id", "request_id")}, payload, sequence` + ) + const duplicateColumns = sql.literal( + `${idAs("m.id", "id")}, ${idAs("r.id", "reply_id")}, ${replyJoinColumns}` + ) + const messageJoinColumns = sql.literal( + `${messageColumns}, ${idAs("r.id", "reply_reply_id")}, ${replyJoinColumns}` + ) + const selectByMessageId = (message_id: string): Effect.Effect, SqlError> => sql` - SELECT m.id, r.id as reply_id, r.kind as reply_kind, r.payload as reply_payload, r.sequence as reply_sequence + SELECT ${duplicateColumns} FROM ${messagesTableSql} m LEFT JOIN ${repliesTableSql} r ON r.id = m.last_reply_id WHERE m.message_id = ${message_id} ` + // an insert that returned no rows hit an existing message_id + const selectUnlessInserted = (message_id: string) => (inserted: ReadonlyArray) => + inserted.length > 0 ? Effect.succeed([]) : selectByMessageId(message_id) const insertEnvelope: ( row: MessageRow, @@ -294,11 +323,7 @@ const makeEncodedStorage = Effect.fnUntraced(function*( INSERT INTO ${messagesTableSql} ${sql.insert(row)} ON CONFLICT (message_id) DO NOTHING RETURNING id - `.pipe(Effect.flatMap((rows) => { - // inserted a new row - if (rows.length > 0) return Effect.succeed([]) - return selectByMessageId(message_id) - })), + `.pipe(Effect.flatMap(selectUnlessInserted(message_id))), mysql: () => (row, message_id) => Effect.flatMap( sql`INSERT IGNORE INTO ${messagesTableSql} ${sql.insert(row)}`.raw, @@ -316,37 +341,8 @@ const makeEncodedStorage = Effect.fnUntraced(function*( ON target.message_id = source.message_id WHEN NOT MATCHED THEN INSERT ${sql.insert(row)} - OUTPUT - inserted.id, - CASE - WHEN inserted.id IS NULL THEN ( - SELECT r.id, r.kind, r.payload - FROM ${repliesTableSql} r - WHERE r.id = target.last_reply_id - ) - END as reply_id, - CASE - WHEN inserted.id IS NULL THEN ( - SELECT r.kind - FROM ${repliesTableSql} r - WHERE r.id = target.last_reply_id - ) - END as reply_kind, - CASE - WHEN inserted.id IS NULL THEN ( - SELECT r.payload - FROM ${repliesTableSql} r - WHERE r.id = target.last_reply_id - ) - END as reply_payload, - CASE - WHEN inserted.id IS NULL THEN ( - SELECT r.sequence - FROM ${repliesTableSql} r - WHERE r.id = target.last_reply_id - ) - END as reply_sequence; - `, + OUTPUT inserted.id; + `.pipe(Effect.flatMap(selectUnlessInserted(message_id))), orElse: () => (row, message_id) => selectByMessageId(message_id).pipe( Effect.tap(sql`INSERT OR IGNORE INTO ${messagesTableSql} ${sql.insert(row)}`), @@ -374,9 +370,15 @@ const makeEncodedStorage = Effect.fnUntraced(function*( orElse: () => (s: string) => `'${s}'` }) const forUpdate = sql.onDialectOrElse({ + mssql: () => sql.literal(""), sqlite: () => sql.literal(""), orElse: () => sql.literal("FOR UPDATE") }) + // mssql locks claimed rows with a table hint instead of FOR UPDATE + const updateLock = sql.onDialectOrElse({ + mssql: () => sql.literal("WITH (UPDLOCK)"), + orElse: () => sql.literal("") + }) const emptyFragment = sql.literal("") // mssql only limits with OFFSET/FETCH, which requires the ORDER BY that @@ -459,8 +461,8 @@ const makeEncodedStorage = Effect.fnUntraced(function*( `, orElse: () => (shardIds: ReadonlyArray, now: number, filters: UnprocessedFilters) => sql` - SELECT m.*, r.id as reply_reply_id, r.kind as reply_kind, r.payload as reply_payload, r.sequence as reply_sequence - FROM ${messagesTableSql} m + SELECT ${messageJoinColumns} + FROM ${messagesTableSql} m ${updateLock} LEFT JOIN ${repliesTableSql} r ON r.id = m.last_reply_id WHERE m.shard_id IN (${sql.literal(shardIds.map(wrapString).join(","))}) ${filters.addressFilter} @@ -630,11 +632,11 @@ const makeEncodedStorage = Effect.fnUntraced(function*( requestIdForPrimaryKey: (primaryKey) => messageIdForPrimaryKey(primaryKey).pipe( Effect.flatMap((messageId) => - sql<{ id: string | bigint }>`SELECT id FROM ${messagesTableSql} WHERE message_id = ${messageId}` + sql<{ id: string | bigint }>`SELECT ${idColumn} FROM ${messagesTableSql} WHERE message_id = ${messageId}` ), Effect.flatMap((rows) => rows.length === 0 && mayHaveLegacyRow(primaryKey) - ? sql<{ id: string | bigint }>`SELECT id FROM ${messagesTableSql} WHERE message_id = ${primaryKey}` + ? sql<{ id: string | bigint }>`SELECT ${idColumn} FROM ${messagesTableSql} WHERE message_id = ${primaryKey}` : Effect.succeed(rows) ), Effect.map((rows) => Option.map(Option.fromNullishOr(rows[0]?.id), Snowflake.Snowflake)), @@ -649,7 +651,7 @@ const makeEncodedStorage = Effect.fnUntraced(function*( // - the kind is WithExit // - or the kind is Chunk and has not been acked yet sql` - SELECT id, kind, request_id, payload, sequence + SELECT ${replyColumns} FROM ${repliesTableSql} WHERE request_id IN (${sql.literal(requestIds.join(","))}) AND ( @@ -669,7 +671,7 @@ const makeEncodedStorage = Effect.fnUntraced(function*( repliesForUnfiltered: (requestIds) => sql` - SELECT id, kind, request_id, payload, sequence + SELECT ${replyColumns} FROM ${repliesTableSql} WHERE request_id IN (${sql.literal(requestIds.join(","))}) ORDER BY rowid ASC @@ -705,7 +707,7 @@ const makeEncodedStorage = Effect.fnUntraced(function*( unprocessedMessagesById(ids, now) { const idArr = Array.from(ids, (id) => String(id)) return sql` - SELECT m.*, r.id as reply_reply_id, r.kind as reply_kind, r.payload as reply_payload, r.sequence as reply_sequence + SELECT ${messageJoinColumns} FROM ${messagesTableSql} m LEFT JOIN ${repliesTableSql} r ON r.id = m.last_reply_id WHERE m.id IN (${sql.literal(idArr.join(","))}) @@ -840,6 +842,8 @@ export const migrations = (options?: { const messagesTableSql = sql(messagesTable) const repliesTableSql = sql(repliesTable) + // mssql UNIQUE constraints treat NULLs as equal, so nullable keys use + // filtered unique indexes instead yield* sql.onDialectOrElse({ mssql: () => sql` @@ -853,8 +857,8 @@ export const migrations = (options?: { entity_id VARCHAR(255) NOT NULL, kind INT NOT NULL, tag VARCHAR(50), - payload TEXT, - headers TEXT, + payload NVARCHAR(MAX), + headers NVARCHAR(MAX), trace_id VARCHAR(32), span_id VARCHAR(16), sampled BIT, @@ -864,7 +868,7 @@ export const migrations = (options?: { last_reply_id BIGINT, last_read DATETIME, deliver_at BIGINT, - UNIQUE (message_id) + INDEX ${sql(messagesTable + "_message_id_idx")} UNIQUE (message_id) WHERE message_id IS NOT NULL ) `, mysql: () => @@ -967,17 +971,16 @@ export const migrations = (options?: { CREATE INDEX ${sql(requestIdLookupIndex)} ON ${messagesTableSql} (request_id); `.unprepared.pipe(Effect.ignore), - pg: () => - Effect.gen(function*() { - yield* sql` + pg: Effect.fnUntraced(function*() { + yield* sql` CREATE INDEX IF NOT EXISTS ${sql(shardLookupIndex)} ON ${messagesTableSql} (shard_id, processed, last_read, deliver_at) ` - yield* sql` + yield* sql` CREATE INDEX IF NOT EXISTS ${sql(requestIdLookupIndex)} ON ${messagesTableSql} (request_id) ` - }), + }), orElse: () => // sqlite Effect.all([ @@ -1001,11 +1004,11 @@ export const migrations = (options?: { rowid BIGINT IDENTITY(1,1), kind INT, request_id BIGINT NOT NULL, - payload TEXT NOT NULL, + payload NVARCHAR(MAX) NOT NULL, sequence INT, acked BIT NOT NULL DEFAULT 0, - CONSTRAINT ${sql(repliesTable + "_one_exit")} UNIQUE (request_id, kind), - CONSTRAINT ${sql(repliesTable + "_sequence")} UNIQUE (request_id, sequence) + INDEX ${sql(repliesTable + "_one_exit")} UNIQUE (request_id, kind) WHERE kind IS NOT NULL, + INDEX ${sql(repliesTable + "_sequence")} UNIQUE (request_id, sequence) WHERE sequence IS NOT NULL ) `, mysql: () => diff --git a/repos/effect/packages/effect/src/cluster/SqlRunnerStorage.ts b/repos/effect/packages/effect/src/cluster/SqlRunnerStorage.ts index f5a807823d..95fafaec81 100644 --- a/repos/effect/packages/effect/src/cluster/SqlRunnerStorage.ts +++ b/repos/effect/packages/effect/src/cluster/SqlRunnerStorage.ts @@ -6,6 +6,8 @@ * shard. This module migrates the required runner and lock tables, supports an * optional table prefix, uses advisory locks for PostgreSQL and MySQL when * enabled, and provides constructors and layers for the storage service. + * PGlite clients always use lock table rows on the shared client, so the + * storage never holds their only connection. * * `layer`, `layerWith`, and `make` run the migrations before building the * storage. To run migrations with a different connection, such as a schema @@ -53,9 +55,13 @@ const makeStorage = Effect.fnUntraced(function*(options: { const config = yield* ShardingConfig.ShardingConfig const shardGroups = ShardingConfig.shardGroupConfig(config) const availableShardGroups = Array.from(shardGroups.available) - const disableAdvisoryLocks = config.shardLockDisableAdvisory const lockOperationInterval = effectiveInterval(config) const sql = (yield* SqlClient.SqlClient).withoutTransforms() + // PGlite runs every statement on one session, so advisory locks are shared by + // every user of the client and a reserved connection would block all other + // statements. Use table leases on the shared client instead. + const singleSession = sql.driver === "pglite" + const disableAdvisoryLocks = config.shardLockDisableAdvisory || singleSession const layerScope = yield* Effect.scope const prefix = options?.prefix ?? "cluster" const table = (name: string) => `${prefix}_${name}` @@ -65,7 +71,7 @@ const makeStorage = Effect.fnUntraced(function*(options: { // Keep all PostgreSQL and MySQL shard-lock operations on a rebuildable // reserved connection, including when advisory locks are disabled. - const acquireLockConn = sql.onDialectOrElse({ + const acquireLockConn = singleSession ? undefined : sql.onDialectOrElse({ pg: () => Effect.fnUntraced(function*(scope: Scope.Scope) { const conn = yield* Effect.orDie(sql.reserve).pipe( @@ -667,8 +673,8 @@ const makeStorage = Effect.fnUntraced(function*(options: { * permission to create tables. When `prefix` is omitted, `make` uses the * `cluster` prefix, creating `cluster_runners`, `cluster_locks`, and the * `cluster_runner_migrations` history table. PostgreSQL and MySQL use advisory - * locks unless `ShardingConfig.shardLockDisableAdvisory` is enabled; other - * dialects use rows in the locks table. + * locks unless `ShardingConfig.shardLockDisableAdvisory` is enabled or the + * client is PGlite; other dialects and PGlite use rows in the locks table. * * **Gotchas** * diff --git a/repos/effect/packages/effect/src/cluster/internal/entityManager.ts b/repos/effect/packages/effect/src/cluster/internal/entityManager.ts index f36da9ae53..4b1df918d3 100644 --- a/repos/effect/packages/effect/src/cluster/internal/entityManager.ts +++ b/repos/effect/packages/effect/src/cluster/internal/entityManager.ts @@ -8,7 +8,7 @@ import * as Effect from "../../Effect.ts" import * as Equal from "../../Equal.ts" import * as Exit from "../../Exit.ts" import * as Fiber from "../../Fiber.ts" -import { identity } from "../../Function.ts" +import { constVoid, identity } from "../../Function.ts" import { scopeAddFinalizerUnsafe, scopeRemoveFinalizerUnsafe } from "../../internal/effect.ts" import * as Latch from "../../Latch.ts" import * as Metric from "../../Metric.ts" @@ -67,6 +67,9 @@ export interface EntityManager { readonly interruptShard: (shardId: ShardId, options?: { readonly force?: boolean }) => Effect.Effect + // Whether any of the shard's entities are still open, including those that + // `interruptShard` is waiting on + readonly hasServersUnsafe: (shardId: ShardId) => boolean readonly activeEntityCount: Effect.Effect } @@ -137,6 +140,7 @@ export const make = Effect.fnUntraced(function*< readonly maxIdleTime?: Input | undefined readonly concurrency?: number | "unbounded" | undefined readonly mailboxCapacity?: number | "unbounded" | undefined + readonly mailboxMetrics?: boolean | undefined readonly disableFatalDefects?: boolean | undefined readonly defectRetryPolicy?: Schedule.Schedule | undefined readonly spanAttributes?: Record | undefined @@ -147,6 +151,7 @@ export const make = Effect.fnUntraced(function*< const managerScope = yield* Effect.scope const storageEnabled = options.storage !== MessageStorage.noop const mailboxCapacity = options.mailboxCapacity ?? config.entityMailboxCapacity + const mailboxMetrics = options.mailboxMetrics ?? config.entityMailboxMetrics const clock = yield* Clock const context = yield* Effect.context | Rpc.Middleware | RX>() const defectRetryPolicy = options.defectRetryPolicy @@ -155,6 +160,22 @@ export const make = Effect.fnUntraced(function*< const retryDriver = yield* Schedule.toStepWithSleep(defectRetryPolicy) const entityRpcs = new Map(entity.protocol.requests) + // The series reads the mailbox size on snapshot, so delivery does no extra work. + const registerMailboxSize = (address: EntityAddress, activeRequests: EntityState["activeRequests"]) => + Metric.registerUnsafe( + Metric.withAttributes(ClusterMetrics.entityMailboxSize, { + type: entity.type, + id: address.entityId, + shard: address.shardId.toString() + }), + context as Context.Context, + { + get: () => ({ value: BigInt(activeRequests.size) }), + update: constVoid, + modify: constVoid + } + ) + // add internal rpcs entityRpcs.set(KeepAliveRpc._tag, KeepAliveRpc as any) @@ -162,6 +183,34 @@ export const make = Effect.fnUntraced(function*< // Entities finishing their in-flight requests after removal, which still // accept interrupts and acks const drainingServers = new Map() + // Mailboxes of every resident entity, including draining ones. Unlike the + // server maps it is not keyed by entity id, which can be resident on several + // shards at once. + const residentMailboxes = new Set() + + // Registered before the entities resource map, so the series is removed + // only after every entity has drained on shutdown. + yield* Effect.acquireRelease( + Effect.sync(() => + Metric.registerUnsafe( + Metric.withAttributes(ClusterMetrics.entityMailboxSizeMax, { type: entity.type }), + context as Context.Context, + { + get: () => { + let max = 0 + for (const mailbox of residentMailboxes) { + if (mailbox.size > max) max = mailbox.size + } + return { value: BigInt(max) } + }, + update: constVoid, + modify: constVoid + } + ) + ), + (registration) => Effect.sync(registration.remove) + ) + const serverCloseLatches = new Map { releaseEntity(address) + mailboxSeries?.remove() + residentMailboxes.delete(activeRequests) }) ) @@ -213,7 +268,6 @@ export const make = Effect.fnUntraced(function*< } ) - const activeRequests: EntityState["activeRequests"] = new Map() const retired = Latch.makeUnsafe() const isActive = () => !retired.isOpen() @@ -488,6 +542,9 @@ export const make = Effect.fnUntraced(function*< yield* state.write(0, { _tag: "Eof" }) }).pipe( Effect.andThen(endLatch.await), + // A defect restart that finds the entity retired opens `endLatch` + // without building handlers, so the writes above may never land. + Effect.raceFirst(endLatch.await), Effect.timeoutOption(config.entityTerminationTimeout), Effect.interruptible ), @@ -505,6 +562,10 @@ export const make = Effect.fnUntraced(function*< // Do not make shard interruption wait for an entity that is still building. serverCloseLatches.set(address, closeLatches) activeServers.set(address.entityId, state) + residentMailboxes.add(activeRequests) + if (mailboxMetrics) { + mailboxSeries = registerMailboxSize(address, activeRequests) + } return state }, Effect.provideService(CurrentLogAnnotations, {}))) @@ -521,15 +582,18 @@ export const make = Effect.fnUntraced(function*< }) } - // update metrics for active servers - const typeAttributes = Metric.CurrentMetricAttributes.context({ type: entity.type }) - yield* Effect.sync(() => { - ClusterMetrics.entities.updateUnsafe(BigInt(activeServers.size), typeAttributes) - }).pipe( - Effect.andThen(Effect.sleep(1000)), - Effect.forever, - Effect.forkIn(managerScope) + // The series reads the active entity count on snapshot, so an idle runner + // does no work. + const entitiesSeries = Metric.registerUnsafe( + Metric.withAttributes(ClusterMetrics.entities, { type: entity.type }), + context as Context.Context, + { + get: () => ({ value: BigInt(activeServers.size) }), + update: constVoid, + modify: constVoid + } ) + yield* Scope.addFinalizer(managerScope, Effect.sync(entitiesSeries.remove)) function sendEnvelope(server: EntityState, message: Message.IncomingEnvelope): Effect.Effect { const entry = server.activeRequests.get(message.envelope.requestId) @@ -752,6 +816,12 @@ export const make = Effect.fnUntraced(function*< clearProcessed() { processedRequestIds.clear() }, + hasServersUnsafe(shardId) { + for (const address of serverCloseLatches.keys()) { + if (shardId[Equal.symbol](address.shardId)) return true + } + return false + }, isResidentUnsafe: (address) => entities.hasUnsafe(address), residentAddressesUnsafe: () => entities.keysUnsafe(), sendLocal, diff --git a/repos/effect/packages/effect/src/devtools/DevTools.ts b/repos/effect/packages/effect/src/devtools/DevTools.ts index 48434199ed..8c6eaec00c 100644 --- a/repos/effect/packages/effect/src/devtools/DevTools.ts +++ b/repos/effect/packages/effect/src/devtools/DevTools.ts @@ -56,8 +56,9 @@ export const layerWebSocket = ( * **Gotchas** * * This layer only installs the client-side tracer; it does not start a devtools - * server, so the configured WebSocket endpoint must already be reachable. It - * relies on `globalThis.WebSocket` being available in the runtime. + * server. The client retries failed connections every second, so the configured + * endpoint can become reachable after startup. It relies on + * `globalThis.WebSocket` being available in the runtime. * * @see {@link layerWebSocket} for installing the devtools tracer with an explicit `WebSocketConstructor` requirement * @see {@link layerSocket} for installing the devtools tracer over an existing `Socket` transport diff --git a/repos/effect/packages/effect/src/devtools/DevToolsClient.ts b/repos/effect/packages/effect/src/devtools/DevToolsClient.ts index 4fd8879ee4..eb42bde650 100644 --- a/repos/effect/packages/effect/src/devtools/DevToolsClient.ts +++ b/repos/effect/packages/effect/src/devtools/DevToolsClient.ts @@ -9,7 +9,6 @@ * @stability unstable * @since 4.0.0 */ -import * as Cause from "../Cause.ts" import * as Context from "../Context.ts" import * as Deferred from "../Deferred.ts" import * as Effect from "../Effect.ts" @@ -18,6 +17,7 @@ import * as Fiber from "../Fiber.ts" import * as Layer from "../Layer.ts" import * as Metric from "../Metric.ts" import * as Queue from "../Queue.ts" +import * as Schedule from "../Schedule.ts" import * as Schema from "../Schema.ts" import type * as Scope from "../Scope.ts" import * as Socket from "../socket/Socket.ts" @@ -49,7 +49,20 @@ const makeEffect = Effect.gen(function*() { const socket = yield* Socket.Socket const services = yield* Effect.context() const requests = yield* Queue.unbounded() + yield* Queue.offer(requests, { _tag: "Ping" }) const connected = yield* Deferred.make() + let closing = false + let writable = false + const setWritable = (value: boolean) => + Effect.sync(() => { + writable = value + }) + // Tracks whether the current connection has a writer that can drain the + // queue on shutdown. + const transport = Socket.make({ + reader: socket.reader, + writer: Effect.tap(socket.writer, Effect.acquireRelease(setWritable(true), () => setWritable(false))) + }) const offerMetricsSnapshot = Effect.sync(() => { Queue.offerUnsafe(requests, toMetricsSnapshot(services)) @@ -70,24 +83,28 @@ const makeEffect = Effect.gen(function*() { const fiber = yield* Stream.fromQueue(requests).pipe( Stream.pipeThroughChannel( - Ndjson.duplexSchemaString(Socket.toChannelString(socket), { + Ndjson.duplexSchemaString(Socket.toChannelString(transport), { inputSchema: RequestSchema, outputSchema: ResponseSchema }) ), Stream.onFirst(() => Deferred.completeWith(connected, Effect.void)), + // Each run closes its stream scope, including the socket reader and writer, + // before retrying with a fresh connection and the same request queue. Stream.runForEach(handleResponse), + Effect.tapError((error) => Effect.logDebug("DevTools connection failed", error)), + Effect.retry({ schedule: Schedule.spaced("1 second"), while: () => !closing }), Effect.forkDetach ) - yield* Effect.addFinalizer(() => - offerMetricsSnapshot.pipe( - Effect.andThen( - Effect.flatMap(Effect.fiberId, (id) => Queue.failCause(requests, Cause.interrupt(id))) - ), - Effect.andThen(Fiber.await(fiber)) - ) - ) + yield* Effect.addFinalizer(Effect.fnUntraced(function*() { + closing = true + yield* offerMetricsSnapshot + yield* Queue.interrupt(requests) + // A ready writer drains the queue before stopping. Failed or pending + // acquisition cannot observe the queue interruption, so stop it directly. + yield* writable ? Fiber.await(fiber) : Fiber.interrupt(fiber) + })) yield* Effect.suspend(() => Queue.offer(requests, { _tag: "Ping" })).pipe( Effect.delay("3 seconds"), diff --git a/repos/effect/packages/effect/src/encoding/Base64Url.ts b/repos/effect/packages/effect/src/encoding/Base64Url.ts index 92486cb5fe..31d87c1767 100644 --- a/repos/effect/packages/effect/src/encoding/Base64Url.ts +++ b/repos/effect/packages/effect/src/encoding/Base64Url.ts @@ -32,6 +32,7 @@ import { EncodingError } from "./EncodingError.ts" * Base64Url.encode(bytes) // => "SGVsbG8_" * ``` * + * @see {@link encodePadded} for output that keeps `=` padding * @see {@link decode} for decoding URL-safe Base64 to bytes * @see {@link decodeString} for decoding URL-safe Base64 to UTF-8 text * @@ -42,6 +43,36 @@ import { EncodingError } from "./EncodingError.ts" export const encode: (input: Uint8Array | string) => string = (input) => Base64.encode(input).replace(/=/g, "").replace(/\+/g, "-").replace(/\//g, "_") +/** + * Encodes the given value into a padded URL-safe Base64 string. + * + * **When to use** + * + * Use when a consumer requires `=` padding on URL-safe Base64, such as Go's + * `base64.URLEncoding`. + * + * **Details** + * + * Same as {@link encode}, but keeps the `=` padding. {@link decode} accepts + * both forms. + * + * **Example** (Encoding padded URL-safe Base64) + * + * ```ts import.meta.vitest + * import { Base64Url } from "effect/encoding" + * + * Base64Url.encodePadded("fo") // => "Zm8=" + * ``` + * + * @see {@link encode} for unpadded output + * + * @stability stable + * @category encoding + * @since 4.0.6 + */ +export const encodePadded: (input: Uint8Array | string) => string = (input) => + Base64.encode(input).replace(/\+/g, "-").replace(/\//g, "_") + /** * Decodes a padded or unpadded URL-safe Base64 string into bytes safely. * diff --git a/repos/effect/packages/effect/src/encoding/Ini.ts b/repos/effect/packages/effect/src/encoding/Ini.ts index 235d35fc16..937e503a22 100644 --- a/repos/effect/packages/effect/src/encoding/Ini.ts +++ b/repos/effect/packages/effect/src/encoding/Ini.ts @@ -77,6 +77,9 @@ const unquote = (input: string | undefined): string => { output += character } } + if (escaped) { + output += "\\" + } return output.trim() } diff --git a/repos/effect/packages/effect/src/encoding/JsonArray.ts b/repos/effect/packages/effect/src/encoding/JsonArray.ts new file mode 100644 index 0000000000..fc5e2dc3f4 --- /dev/null +++ b/repos/effect/packages/effect/src/encoding/JsonArray.ts @@ -0,0 +1,849 @@ +/** + * Encodes and decodes a single JSON array as a stream of its elements. + * + * **Details** + * + * Use `JsonArray` when a large response or file is one JSON document, such as + * `[...]` or `{"items":[...],"total":n}`, and its elements should be processed + * without buffering the whole document. Use `Ndjson` when the producer writes + * one JSON value per line. This module is unrelated to `Schema.JsonArray`, the + * type of a JSON array value. + * + * Decoders stream the elements of one array, the root array by default or the + * array selected by an RFC 6901 JSON Pointer. Each element is buffered only + * until it is complete and then parsed with `JSON.parse`. The rest of the + * document is validated and discarded. Encoders write a stream of values as + * one JSON array. + * + * **Example** (Streaming the items of a large HTTP response) + * + * ```ts + * import { Effect, Schema, Stream } from "effect" + * import { JsonArray } from "effect/encoding" + * import { HttpClient, HttpClientResponse } from "effect/http" + * + * const User = Schema.Struct({ id: Schema.Number, name: Schema.String }) + * + * // The response body is {"data":[{"id":1,"name":"Alice"},...],"total":...} + * const program = HttpClientResponse.stream(HttpClient.get("https://example.com/users")).pipe( + * Stream.pipeThroughChannel(JsonArray.decodeSchema(User)({ pointer: "/data" })), + * Stream.runForEach((user) => Effect.log(user.name)) + * ) + * ``` + * + * @stability unstable + * @since 4.0.6 + */ +import * as Arr from "../Array.ts" +import * as Cause from "../Cause.ts" +import * as Channel from "../Channel.ts" +import * as ChannelSchema from "../ChannelSchema.ts" +import * as Data from "../Data.ts" +import * as Effect from "../Effect.ts" +import * as JsonPointer from "../JsonPointer.ts" +import * as Predicate from "../Predicate.ts" +import * as Pull from "../Pull.ts" +import type * as Schema from "../Schema.ts" + +const JsonArrayErrorTypeId = "~effect/encoding/JsonArray/JsonArrayError" + +const encoder = new TextEncoder() + +/** + * Error raised when JSON array encoding or decoding fails. + * + * **Details** + * + * - `Syntax`: the input is not valid JSON, including an empty input or + * content after the document. + * - `Limit`: a value exceeds `maxDepth` or `maxElementSize`, whether it is an + * element, a key, or a value outside the selected array. + * - `Pointer`: the pointer target is missing or not an array, a value along the + * pointer is not an object or array, or an object along the pointer repeats + * the key that the pointer follows. + * - `Encode`: a value has no JSON representation. + * + * Decoding errors set `offset`, a zero-based offset in UTF-16 code units into + * the decoded text after any leading BOM. When a value is invalid or exceeds a + * limit, the offset is the start of that value. `Encode` errors have no + * offset. When `JSON.parse` or `JSON.stringify` throws, `Syntax` and `Encode` + * errors retain the original thrown value as `cause`. + * + * @stability unstable + * @category errors + * @since 4.0.6 + */ +export class JsonArrayError extends Data.TaggedError("JsonArrayError")<{ + readonly kind: "Syntax" | "Limit" | "Pointer" | "Encode" + readonly message: string + readonly offset?: number | undefined + readonly cause?: unknown +}> { + /** + * Marks this value as a JSON array encoding or decoding error for runtime + * guards. + * + * @stability unstable + * @since 4.0.6 + */ + readonly [JsonArrayErrorTypeId] = JsonArrayErrorTypeId +} + +/** + * Checks whether a value is a `JsonArrayError`. + * + * @stability unstable + * @category guards + * @since 4.0.6 + */ +export const isJsonArrayError = (u: unknown): u is JsonArrayError => Predicate.hasProperty(u, JsonArrayErrorTypeId) + +/** + * JSON array decoding options. + * + * **Details** + * + * - `pointer`: an RFC 6901 JSON Pointer to the array to stream, such as + * `"/data"`. Defaults to `""`, the root. Tokens select object members by + * key and array elements by index. + * - `maxDepth`: the maximum nesting of arrays and objects in one value, where + * an array or object element has depth 1. Defaults to 1000. + * - `maxElementSize`: the maximum length of one value in UTF-16 code units. + * Defaults to 16 MiB; `Infinity` disables the limit. + * + * Both limits also apply to values outside the selected array, which are + * buffered and validated before being discarded. Invalid options throw when + * the channel is created. + * + * @stability unstable + * @category configuration + * @since 4.0.6 + */ +export interface DecodeOptions { + readonly pointer?: string | undefined + readonly maxDepth?: number | undefined + readonly maxElementSize?: number | undefined +} + +interface Config { + readonly tokens: ReadonlyArray + // The array index selected by each token, or -1 when it is not an index. + readonly indices: ReadonlyArray + readonly maxDepth: number + readonly maxElementSize: number +} + +const makeConfig = (options: DecodeOptions | undefined): Config => { + const pointer = options?.pointer ?? "" + if (pointer !== "" && (!pointer.startsWith("/") || /~(?:[^01]|$)/.test(pointer))) { + throw new Error(`JsonArray: pointer must be an RFC 6901 JSON Pointer, got ${JSON.stringify(pointer)}`) + } + const maxDepth = options?.maxDepth ?? 1000 + if (!(Number.isInteger(maxDepth) && maxDepth >= 0)) { + throw new Error(`JsonArray: maxDepth must be a non-negative integer, got ${maxDepth}`) + } + const maxElementSize = options?.maxElementSize ?? 16 * 1024 * 1024 + if (!(maxElementSize >= 1)) { + throw new Error(`JsonArray: maxElementSize must be at least 1, got ${maxElementSize}`) + } + const tokens = pointer === "" ? [] : pointer.slice(1).split("/").map(JsonPointer.unescapeToken) + return { + tokens, + indices: tokens.map((token) => /^(?:0|[1-9]\d*)$/.test(token) ? Number(token) : -1), + maxDepth, + maxElementSize + } +} + +const TAB = 9 +const LF = 10 +const CR = 13 +const SPACE = 32 +const QUOTE = 34 +const COMMA = 44 +const COLON = 58 +const LBRACKET = 91 +const BACKSLASH = 92 +const RBRACKET = 93 +const LBRACE = 123 +const RBRACE = 125 + +const isWhitespace = (c: number): boolean => c === SPACE || c === LF || c === CR || c === TAB + +const indexOf = (text: string, search: string, i: number): number => { + const index = text.indexOf(search, i) + return index === -1 ? text.length : index +} + +const isValueStart = (c: number): boolean => + c === QUOTE || c === LBRACKET || c === LBRACE || c === 45 || (c >= 48 && c <= 57) || c === 102 || c === 110 || + c === 116 + +// The scanner walks the containers along the pointer and the selected array. +// Every other value is captured whole and parsed with `JSON.parse`: elements +// are emitted, keys are matched against the pointer, and the rest is +// discarded. + +// Structural states. +const VALUE = 0 // the value at the current pointer prefix +const OBJECT_START = 1 // after `{`: a key or `}` +const OBJECT_KEY = 2 // after `,`: a key +const OBJECT_COLON = 3 // after a key: `:` +const OBJECT_VALUE = 4 // after `:` for a key the pointer does not follow +const OBJECT_NEXT = 5 // after a member: `,` or `}` +const ARRAY_START = 6 // after `[`: a value or `]` +const ARRAY_VALUE = 7 // after `,`: a value +const ARRAY_NEXT = 8 // after an element: `,` or `]` +const END = 9 // after the document + +// Capture modes. +const NONE = 0 +const SKIP = 1 +const ELEMENT = 2 +const KEY = 3 + +interface Frame { + readonly object: boolean + // Whether this is the selected array. + readonly target: boolean + // Whether the pointer token for this container has been found. + found: boolean + index: number +} + +class Scanner { + readonly config: Config + state = VALUE + readonly frames: Array = [] + key = "" + started = false + // The offset of the start of the current chunk. + base = 0 + + // The value being captured, which may span chunks. + capture = NONE + start = 0 + buffer = "" + scalar = false + depth = 0 + inString = false + escape = false + + // Cached quote and backslash positions; the chunk length means not found. + quote = -1 + backslash = -1 + // Absolute end of a batch that failed to parse. The precise scan handles + // text before it, so the batch is not rescanned for every element. + slow = 0 + + constructor(config: Config) { + this.config = config + } + + fail(kind: "Syntax" | "Limit" | "Pointer", message: string, offset: number, cause?: unknown): JsonArrayError { + return new JsonArrayError({ kind, message, offset, cause }) + } + + pointerPrefix(length: number, ...rest: Array): string { + return JSON.stringify( + this.config.tokens.slice(0, length).concat(rest).map((token) => "/" + JsonPointer.escapeToken(token)).join("") + ) + } + + // Names the value being captured, for limit errors. + subject(): string { + if (this.capture === ELEMENT) return "Element" + const depth = this.frames.length - 1 + if (this.capture === KEY) return `Key in ${this.pointerPrefix(depth)}` + const frame = this.frames[depth] + const token = frame.object ? this.key : String(frame.index - 1) + return `Value outside the selected array at ${this.pointerPrefix(depth, token)}` + } + + feed(text: string, out: Array): void { + if (!this.started && text.length > 0) { + this.started = true + if (text.charCodeAt(0) === 0xFEFF) text = text.slice(1) + } + this.quote = -1 + this.backslash = -1 + let i = this.capture === NONE ? 0 : this.scan(text, 0, out) + while (i < text.length) { + const c = text.charCodeAt(i) + i = isWhitespace(c) ? i + 1 : this.step(text, i, c, out) + } + if (this.capture !== NONE) { + this.buffer += text.slice(Math.max(this.start - this.base, 0)) + this.checkSize(this.buffer.length) + } + this.base += text.length + } + + end(): void { + if (this.state !== END || this.capture !== NONE) { + throw this.fail("Syntax", "Unexpected end of input", this.base) + } + } + + // Handles one structural character and returns the next index. + step(text: string, i: number, c: number, out: Array): number { + const frame = this.frames[this.frames.length - 1] + switch (this.state) { + case VALUE: { + const target = this.frames.length === this.config.tokens.length + if (c === LBRACKET || (c === LBRACE && !target)) { + this.frames.push({ object: c === LBRACE, target, found: false, index: 0 }) + this.state = c === LBRACE ? OBJECT_START : ARRAY_START + return i + 1 + } + if (isValueStart(c)) { + const expected = target ? "an array" : "an object or array" + throw this.fail( + "Pointer", + `Expected ${expected} at ${this.pointerPrefix(this.frames.length)}`, + this.base + i + ) + } + break + } + case OBJECT_START: + case OBJECT_KEY: { + if (c === QUOTE) { + this.state = OBJECT_COLON + return this.begin(KEY, text, i, out) + } + if (c === RBRACE && this.state === OBJECT_START) return this.close(i) + break + } + case OBJECT_COLON: { + if (c !== COLON) break + if (this.key === this.config.tokens[this.frames.length - 1]) { + if (frame.found) { + throw this.fail("Pointer", `Duplicate key ${JSON.stringify(this.key)} along the pointer`, this.base + i) + } + frame.found = true + this.state = VALUE + } else { + this.state = OBJECT_VALUE + } + return i + 1 + } + case OBJECT_VALUE: { + if (!isValueStart(c)) break + this.state = OBJECT_NEXT + return this.begin(SKIP, text, i, out) + } + case OBJECT_NEXT: + case ARRAY_NEXT: { + if (c === COMMA) { + this.state = this.state === OBJECT_NEXT ? OBJECT_KEY : ARRAY_VALUE + return i + 1 + } + if (c === (this.state === OBJECT_NEXT ? RBRACE : RBRACKET)) return this.close(i) + break + } + case ARRAY_START: + case ARRAY_VALUE: { + if (c === RBRACKET && this.state === ARRAY_START) return this.close(i) + if (!isValueStart(c)) break + if (frame.target) { + const next = this.scanElements(text, i, out) + if (next > i) return next + this.state = ARRAY_NEXT + return this.begin(ELEMENT, text, i, out) + } + this.state = ARRAY_NEXT + if (frame.index++ === this.config.indices[this.frames.length - 1]) { + frame.found = true + this.state = VALUE + return i + } + return this.begin(SKIP, text, i, out) + } + } + throw this.fail( + "Syntax", + this.state === END + ? "Unexpected content after the document" + : `Unexpected character ${JSON.stringify(String.fromCharCode(c))}`, + this.base + i + ) + } + + close(i: number): number { + const frame = this.frames.pop()! + if (!frame.target && !frame.found) { + throw this.fail("Pointer", `No value at ${this.pointerPrefix(this.frames.length + 1)}`, this.base + i) + } + const parent = this.frames[this.frames.length - 1] + this.state = parent === undefined ? END : parent.object ? OBJECT_NEXT : ARRAY_NEXT + return i + 1 + } + + // Starts capturing the value that begins at `i`. + begin(mode: number, text: string, i: number, out: Array): number { + const c = text.charCodeAt(i) + this.capture = mode + this.start = this.base + i + this.inString = c === QUOTE + this.escape = false + this.depth = c === LBRACKET || c === LBRACE ? 1 : 0 + this.scalar = !this.inString && this.depth === 0 + if (this.depth > this.config.maxDepth) this.failDepth() + return this.scan(text, i + 1, out) + } + + // Continues the captured value from `i`, completing it if it ends in this + // chunk. Returns the index after the value, or the chunk length. + scan(text: string, i: number, out: Array): number { + const end = this.scalar ? this.scanScalar(text, i) : this.scanNested(text, i) + if (end === -1) return text.length + const source = this.buffer + text.slice(Math.max(this.start - this.base, 0), end) + this.checkSize(source.length) + const mode = this.capture + this.capture = NONE + this.buffer = "" + let value: unknown + try { + value = JSON.parse(source) + } catch (cause) { + throw this.fail("Syntax", "Invalid JSON value", this.start, cause) + } + if (mode === ELEMENT) out.push(value) + else if (mode === KEY) this.key = value as string + return end + } + + // A scalar ends at the first delimiter, which `JSON.parse` then validates. + scanScalar(text: string, i: number): number { + for (; i < text.length; i++) { + const c = text.charCodeAt(i) + if (isWhitespace(c) || c === COMMA || c === RBRACKET || c === RBRACE) return i + } + return -1 + } + + // Tracks only string, escape and nesting state; `JSON.parse` validates the + // rest. Returns the index after the value, or -1. + scanNested(text: string, i: number): number { + if (this.inString) { + i = this.scanString(text, i, this.escape) + if (i === -1) return -1 + this.inString = false + if (this.depth === 0) return i + } + let depth = this.depth + while (i < text.length) { + const c = text.charCodeAt(i++) + if (c === QUOTE) { + i = this.scanString(text, i, false) + if (i === -1) { + this.inString = true + break + } + } else if (c === LBRACKET || c === LBRACE) { + if (++depth > this.config.maxDepth) this.failDepth() + } else if ((c === RBRACKET || c === RBRACE) && --depth === 0) { + this.depth = 0 + return i + } + } + this.depth = depth + return -1 + } + + // Scans string contents from `i`; returns the index after the closing quote, + // or -1 if the string continues into the next chunk, recording `escape` for + // `scanNested` to resume with. + scanString(text: string, i: number, escape: boolean): number { + let plain = 0 + while (i < text.length) { + const c = text.charCodeAt(i++) + if (escape) { + escape = false + } else if (c === BACKSLASH) { + escape = true + } else if (c === QUOTE) { + return i + } else if (++plain === 16) { + // A long string: jump to the next quote or backslash. + if (this.quote < i) this.quote = indexOf(text, "\"", i) + if (this.backslash < i) this.backslash = indexOf(text, "\\", i) + i = Math.min(this.quote, this.backslash) + plain = 0 + } + } + this.escape = escape + return -1 + } + + // Parses the elements from `i` up to the last comma or closing bracket at + // the array's level with one `JSON.parse`. Wrapped in brackets, that text + // parses only when every element and separator in it is valid. Returns `i` + // to leave anything else, including errors, to the precise scan. + scanElements(text: string, i: number, out: Array): number { + if (this.base + i < this.slow) return i + const { maxDepth, maxElementSize } = this.config + let cut = -1 + let start = i + let close = false + let depth = 0 + let j = i + while (j < text.length) { + const c = text.charCodeAt(j++) + if (c === QUOTE) { + j = this.scanString(text, j, false) + if (j === -1) break + } else if (c === LBRACKET || c === LBRACE) { + if (++depth > maxDepth) break + } else if (c === RBRACKET || c === RBRACE) { + if (depth-- > 0) continue + // Including surrounding whitespace makes this a conservative size check. + if (c === RBRACKET && j - 1 - start <= maxElementSize) { + cut = j - 1 + close = true + } + break + } else if (c === COMMA && depth === 0) { + if (j - 1 - start > maxElementSize) break + cut = j - 1 + start = j + } + } + let values: Array | undefined + if (cut !== -1) { + try { + values = JSON.parse("[" + text.slice(i, cut) + "]") + } catch { + this.slow = this.base + cut + } + } + // Rescanning may start before the cached matches, so discard them. + this.quote = -1 + this.backslash = -1 + if (values === undefined) return i + for (let k = 0; k < values.length; k++) out.push(values[k]) + if (close) return this.close(cut) + this.state = ARRAY_VALUE + return cut + 1 + } + + checkSize(size: number): void { + if (size > this.config.maxElementSize) { + throw this.fail("Limit", `${this.subject()} exceeds maxElementSize of ${this.config.maxElementSize}`, this.start) + } + } + + failDepth(): never { + throw this.fail("Limit", `${this.subject()} exceeds maxDepth of ${this.config.maxDepth}`, this.start) + } +} + +/** + * Creates a channel that parses the text of one JSON document and emits the + * elements of one of its arrays. + * + * **When to use** + * + * Use when a large JSON array, or an object that contains one, arrives as + * string chunks and its elements should be processed as they arrive. + * + * **Details** + * + * Each output chunk holds the elements completed by one input chunk. Elements + * may span input chunks and are buffered only until they are complete. A + * leading BOM is stripped. + * + * **Gotchas** + * + * - The document is validated as it streams, so elements from earlier + * successfully processed input chunks may already have been emitted when a + * later error fails the channel. Elements accumulated in the failing chunk + * are not emitted. A failure in content after the selected array signals + * that the document was invalid, but cannot guard against using earlier + * output. + * - The pointer follows the first occurrence of a key, where `JSON.parse` + * keeps the last. A repeated followed key fails with a `Pointer` error, as + * does a first occurrence that is not an object or array where the pointer + * needs one. When the repeat comes after the selected array, elements from + * earlier successfully processed chunks may already have been emitted. + * - A root that is not an array fails with a `Pointer` error unless the + * pointer selects an array inside it. + * + * **Example** (Streaming an array inside an envelope) + * + * ```ts import.meta.vitest + * import { Effect, Stream } from "effect" + * import { JsonArray } from "effect/encoding" + * + * const items = Stream.make("{\"total\":2,\"items\":[{\"id\":1},{\"i", "d\":2}]}").pipe( + * Stream.pipeThroughChannel(JsonArray.decodeString({ pointer: "/items" })), + * Stream.runCollect + * ) + * await Effect.runPromise(items) // => [{ id: 1 }, { id: 2 }] + * ``` + * + * @see decode + * @see decodeSchemaString + * @stability unstable + * @category decoding + * @since 4.0.6 + */ +export const decodeString = (options?: DecodeOptions): Channel.Channel< + Arr.NonEmptyReadonlyArray, + IE | JsonArrayError, + Done, + Arr.NonEmptyReadonlyArray, + IE, + Done +> => { + const config = makeConfig(options) + return Channel.fromTransform((upstream, _scope) => + Effect.sync(() => { + const scanner = new Scanner(config) + let done: Pull.Pull | undefined + type Out = Pull.Pull, IE | JsonArrayError, Done> + + const emit = (run: (out: Array) => void): Out => { + const out: Array = [] + try { + run(out) + } catch (error) { + if (isJsonArrayError(error)) return Effect.fail(error) + throw error + } + return Arr.isArrayNonEmpty(out) ? Effect.succeed(out) : pull + } + + const pull: Out = Effect.suspend(() => { + if (done !== undefined) return done + return Pull.matchEffect(upstream, { + onSuccess: (texts) => + emit((out) => { + for (const text of texts) scanner.feed(text, out) + }), + onFailure: Effect.failCause, + onDone: (value) => { + done = Cause.done(value) + return emit(() => scanner.end()) + } + }) + }) + return pull + }) + ) +} + +/** + * Creates a channel that parses UTF-8 bytes of one JSON document and emits the + * elements of one of its arrays. + * + * **Details** + * + * Multi-byte characters may span input chunks, and invalid UTF-8 is replaced + * with U+FFFD. + * + * @see decodeString + * @stability unstable + * @category decoding + * @since 4.0.6 + */ +export const decode = (options?: DecodeOptions): Channel.Channel< + Arr.NonEmptyReadonlyArray, + IE | JsonArrayError, + Done, + Arr.NonEmptyReadonlyArray, + IE, + Done +> => Channel.pipeTo(Channel.decodeText("utf-8", { ignoreBOM: true }), decodeString(options)) + +/** + * Creates a channel that parses the text of one JSON document and decodes the + * elements of one of its arrays with a schema. + * + * @see decodeString + * @stability unstable + * @category decoding + * @since 4.0.6 + */ +export const decodeSchemaString = ( + schema: S +) => +(options?: DecodeOptions): Channel.Channel< + Arr.NonEmptyReadonlyArray, + Schema.SchemaError | JsonArrayError | IE, + Done, + Arr.NonEmptyReadonlyArray, + IE, + Done, + S["DecodingServices"] +> => Channel.pipeTo(decodeString(options), ChannelSchema.decodeUnknown(schema)()) + +/** + * Creates a channel that parses UTF-8 bytes of one JSON document and decodes + * the elements of one of its arrays with a schema. + * + * @see decodeSchemaString + * @stability unstable + * @category decoding + * @since 4.0.6 + */ +export const decodeSchema = ( + schema: S +) => +(options?: DecodeOptions): Channel.Channel< + Arr.NonEmptyReadonlyArray, + Schema.SchemaError | JsonArrayError | IE, + Done, + Arr.NonEmptyReadonlyArray, + IE, + Done, + S["DecodingServices"] +> => Channel.pipeTo(decode(options), ChannelSchema.decodeUnknown(schema)()) + +const stringify = (value: unknown): string => { + let output: string | undefined + try { + output = JSON.stringify(value) + } catch (cause) { + throw new JsonArrayError({ kind: "Encode", message: cause instanceof Error ? cause.message : String(cause), cause }) + } + if (output === undefined) { + throw new JsonArrayError({ kind: "Encode", message: `Value of type ${typeof value} cannot be represented as JSON` }) + } + return output +} + +/** + * Creates a channel that writes a stream of values as the text of one JSON + * array. + * + * **Details** + * + * Each input chunk produces one string. An empty stream produces `"[]"`. + * Values are written with `JSON.stringify`. + * + * **Gotchas** + * + * A value that `JSON.stringify` cannot write, such as `undefined`, a function + * or a `bigint`, fails with an `Encode` error after the preceding text has + * been emitted. + * + * **Example** (Writing values as a JSON array) + * + * ```ts import.meta.vitest + * import { Effect, Stream } from "effect" + * import { JsonArray } from "effect/encoding" + * + * const json = Stream.make({ id: 1 }, { id: 2 }).pipe( + * Stream.pipeThroughChannel(JsonArray.encodeString()), + * Stream.mkString + * ) + * await Effect.runPromise(json) // => '[{"id":1},{"id":2}]' + * ``` + * + * @see encode + * @see encodeSchemaString + * @stability unstable + * @category encoding + * @since 4.0.6 + */ +export const encodeString = (): Channel.Channel< + Arr.NonEmptyReadonlyArray, + IE | JsonArrayError, + Done, + Arr.NonEmptyReadonlyArray, + IE, + Done +> => + Channel.fromTransform((upstream, _scope) => + Effect.sync(() => { + let started = false + let done: Pull.Pull | undefined + type Out = Pull.Pull, IE | JsonArrayError, Done> + const pull: Out = Effect.suspend((): Out => { + if (done !== undefined) return done + return Pull.matchEffect(upstream, { + onSuccess: (values): Out => { + let text: string + try { + text = (started ? "," : "[") + values.map(stringify).join(",") + } catch (error) { + if (isJsonArrayError(error)) return Effect.fail(error) + throw error + } + started = true + return Effect.succeed(Arr.of(text)) + }, + onFailure: Effect.failCause, + onDone: (value) => { + done = Cause.done(value) + return Effect.succeed(Arr.of(started ? "]" : "[]")) + } + }) + }) + return pull + }) + ) + +/** + * Creates a channel that writes a stream of values as UTF-8 bytes of one JSON + * array. + * + * @see encodeString + * @stability unstable + * @category encoding + * @since 4.0.6 + */ +export const encode = (): Channel.Channel< + Arr.NonEmptyReadonlyArray, + IE | JsonArrayError, + Done, + Arr.NonEmptyReadonlyArray, + IE, + Done +> => Channel.map(encodeString(), Arr.map((text) => encoder.encode(text))) + +/** + * Creates a channel that encodes values with a schema and writes them as the + * text of one JSON array. + * + * @see encodeString + * @stability unstable + * @category encoding + * @since 4.0.6 + */ +export const encodeSchemaString = ( + schema: S +) => +(): Channel.Channel< + Arr.NonEmptyReadonlyArray, + JsonArrayError | Schema.SchemaError | IE, + Done, + Arr.NonEmptyReadonlyArray, + IE, + Done, + S["EncodingServices"] +> => Channel.pipeTo(ChannelSchema.encode(schema)(), encodeString()) + +/** + * Creates a channel that encodes values with a schema and writes them as UTF-8 + * bytes of one JSON array. + * + * @see encodeSchemaString + * @stability unstable + * @category encoding + * @since 4.0.6 + */ +export const encodeSchema = ( + schema: S +) => +(): Channel.Channel< + Arr.NonEmptyReadonlyArray, + JsonArrayError | Schema.SchemaError | IE, + Done, + Arr.NonEmptyReadonlyArray, + IE, + Done, + S["EncodingServices"] +> => Channel.pipeTo(ChannelSchema.encode(schema)(), encode()) diff --git a/repos/effect/packages/effect/src/encoding/Ndjson.ts b/repos/effect/packages/effect/src/encoding/Ndjson.ts index d692a5864e..9e1c2ab8b5 100644 --- a/repos/effect/packages/effect/src/encoding/Ndjson.ts +++ b/repos/effect/packages/effect/src/encoding/Ndjson.ts @@ -9,17 +9,20 @@ * @since 4.0.0 */ import * as Arr from "../Array.ts" +import * as Cause from "../Cause.ts" import * as Channel from "../Channel.ts" import * as ChannelSchema from "../ChannelSchema.ts" import * as Data from "../Data.ts" import * as Effect from "../Effect.ts" -import { dual, identity } from "../Function.ts" +import { dual } from "../Function.ts" import * as Predicate from "../Predicate.ts" +import * as Pull from "../Pull.ts" import type * as Schema from "../Schema.ts" const NdjsonErrorTypeId = "~effect/encoding/Ndjson/NdjsonError" const encoder = new TextEncoder() +const utf8Decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }) /** * Error raised when NDJSON encoding or decoding fails. @@ -29,12 +32,18 @@ const encoder = new TextEncoder() * The `kind` field identifies whether the failure happened while packing or * unpacking, and `cause` preserves the original error. * + * Decoders also set `reason` to say why a line was rejected, and + * `unterminated` when that line is the final line of the input and has no + * line terminator. + * * @stability unstable * @category errors * @since 4.0.0 */ export class NdjsonError extends Data.TaggedError("NdjsonError")<{ readonly kind: "Pack" | "Unpack" + readonly reason?: NdjsonErrorReason | undefined + readonly unterminated?: boolean | undefined readonly cause: unknown }> { /** @@ -56,6 +65,21 @@ export class NdjsonError extends Data.TaggedError("NdjsonError")<{ } } +/** + * Why a decoder rejected a line. + * + * **Details** + * + * - `InvalidJson`: the line is not valid JSON. + * - `InvalidUtf8`: the line is not valid UTF-8 (byte decoders only). + * - `LineTooLarge`: the line is longer than `maxLineBytes` (byte decoders only). + * + * @stability unstable + * @category errors + * @since 4.0.6 + */ +export type NdjsonErrorReason = "InvalidJson" | "InvalidUtf8" | "LineTooLarge" + /** * Checks whether a value is an `NdjsonError`. * @@ -169,6 +193,59 @@ export const encodeSchemaString = ( S["EncodingServices"] > => Channel.pipeTo(ChannelSchema.encode(schema)(), encodeString()) +// Collects the values of one input chunk into `out`. A failure stops the +// chunk, and the values collected before it are still emitted. +type DecodeStep = (input: In, out: Array) => NdjsonError | undefined + +const makeDecodePull = ( + upstream: Pull.Pull, + step: DecodeStep, + flush: (out: Array) => NdjsonError | undefined +): Pull.Pull, IE | NdjsonError, Done> => { + let failure: NdjsonError | undefined + let done: { readonly leftover: Done } | undefined + + const emit = (out: Array, error: NdjsonError | undefined) => { + failure = error + if (Arr.isArrayNonEmpty(out)) return Effect.succeed(out) + return error !== undefined ? Effect.fail(error) : undefined + } + + const pull: Pull.Pull, IE | NdjsonError, Done> = Effect.suspend(() => { + if (failure !== undefined) return Effect.fail(failure) + if (done !== undefined) return Cause.done(done.leftover) + return Pull.matchEffect(upstream, { + onSuccess: (input) => { + const out: Array = [] + return emit(out, step(input, out)) ?? pull + }, + onFailure: Effect.failCause, + onDone: (leftover): Pull.Pull, NdjsonError, Done> => { + done = { leftover } + const out: Array = [] + return emit(out, flush(out)) ?? Cause.done(leftover) + } + }) + }) + return pull +} + +const parseLine = ( + line: string, + out: Array, + ignoreEmptyLines: boolean, + unterminated: boolean +): NdjsonError | undefined => { + if (ignoreEmptyLines && line.length === 0) return + try { + out.push(JSON.parse(line)) + } catch (cause) { + return new NdjsonError({ kind: "Unpack", reason: "InvalidJson", unterminated, cause }) + } +} + +const flushNothing = () => undefined + /** * Creates a channel that parses NDJSON string chunks into values. * @@ -179,7 +256,9 @@ export const encodeSchemaString = ( * * **Details** * - * Lines may span input chunks. + * Lines may span input chunks. `\n`, `\r\n` and a bare `\r` end a line. The + * first line that fails to parse fails the channel with + * `reason: "InvalidJson"`, after the lines before it have been emitted. * * **Gotchas** * @@ -199,28 +278,53 @@ export const decodeString = (options?: { Arr.NonEmptyReadonlyArray, IE, Done -> => { - const lines = Channel.splitLines().pipe( - options?.ignoreEmptyLines === true ? - Channel.filterArray((line) => line.length > 0) : - identity - ) - return Channel.mapEffect(lines, (chunk) => { - try { - return Effect.succeed(Arr.map(chunk, (line) => JSON.parse(line))) - } catch (cause) { - return Effect.fail(new NdjsonError({ kind: "Unpack", cause })) - } +> => + Channel.fromTransform((upstream, scope) => { + const ignoreEmptyLines = options?.ignoreEmptyLines === true + // splitLines only emits a line after its input has ended when that line + // has no terminator + let ended = false + const input = Pull.matchEffect(upstream, { + onSuccess: Effect.succeed, + onFailure: Effect.failCause, + onDone: (leftover) => { + ended = true + return Cause.done(leftover) + } + }) + return Effect.map( + Channel.toTransform(Channel.splitLines())(input, scope), + (lines) => + makeDecodePull(lines, (chunk, out) => { + for (const line of chunk) { + const error = parseLine(line, out, ignoreEmptyLines, ended) + if (error !== undefined) return error + } + }, flushNothing) + ) }) -} + +const LF = 0x0a +const CR = 0x0d +const emptyBytes = new Uint8Array(0) /** * Creates a channel that decodes UTF-8 byte chunks and parses them as NDJSON. * * **Details** * - * Lines may span input chunks, and `ignoreEmptyLines` controls whether blank - * lines are skipped before JSON parsing. + * Lines may span input chunks. `\n`, `\r\n` and a bare `\r` end a line, and a + * leading UTF-8 BOM is skipped. `ignoreEmptyLines` controls whether blank lines + * are skipped before JSON parsing. + * + * The first bad line fails the channel after the lines before it have been + * emitted. A line that is not valid UTF-8 fails with `reason: "InvalidUtf8"` + * and a line that is not valid JSON fails with `reason: "InvalidJson"`. + * + * `maxLineBytes` limits the UTF-8 bytes of a line, not counting its + * terminator. A longer line fails with `reason: "LineTooLarge"` as soon as the + * limit is passed, without reading the rest of the line. Lines are unbounded + * by default. * * @stability unstable * @category constructors @@ -228,6 +332,7 @@ export const decodeString = (options?: { */ export const decode = (options?: { readonly ignoreEmptyLines?: boolean | undefined + readonly maxLineBytes?: number | undefined }): Channel.Channel< Arr.NonEmptyReadonlyArray, IE | NdjsonError, @@ -235,9 +340,88 @@ export const decode = (options?: { Arr.NonEmptyReadonlyArray, IE, Done -> => { - return Channel.pipeTo(Channel.decodeText(), decodeString(options)) -} +> => + Channel.fromTransform((upstream, _scope) => + Effect.sync(() => { + const ignoreEmptyLines = options?.ignoreEmptyLines === true + const maxLineBytes = options?.maxLineBytes ?? Number.POSITIVE_INFINITY + // The first decoder strips a leading BOM. `\n` and `\r` never occur + // inside a multi-byte character, so each line is decoded on its own. + let decoder = new TextDecoder("utf-8", { fatal: true }) + // Bytes of the current line from earlier chunks + let pending: Array = [] + let pendingBytes = 0 + // The previous byte was `\r`, so a `\n` right after it ends nothing + let afterCR = false + + const takeLine = (chunk: Uint8Array, start: number, end: number): Uint8Array => { + if (pendingBytes === 0) return chunk.subarray(start, end) + const line = new Uint8Array(pendingBytes + end - start) + let offset = 0 + for (const bytes of pending) { + line.set(bytes, offset) + offset += bytes.length + } + line.set(chunk.subarray(start, end), offset) + pending = [] + pendingBytes = 0 + return line + } + + const decodeLine = (bytes: Uint8Array, out: Array, unterminated: boolean) => { + let line: string + try { + line = decoder.decode(bytes) + } catch (cause) { + return new NdjsonError({ kind: "Unpack", reason: "InvalidUtf8", unterminated, cause }) + } + const firstLine = decoder !== utf8Decoder + decoder = utf8Decoder + // A stripped BOM alone is not an unterminated record. + if (firstLine && unterminated && line.length === 0) return + return parseLine(line, out, ignoreEmptyLines, unterminated) + } + + const step: DecodeStep> = (chunks, out) => { + for (const chunk of chunks) { + let start = 0 + for (let i = 0; i < chunk.length; i++) { + const byte = chunk[i] + if (byte === LF || byte === CR) { + if (byte === LF && afterCR) { + start = i + 1 + } else { + const error = decodeLine(takeLine(chunk, start, i), out, false) + if (error !== undefined) return error + start = i + 1 + } + afterCR = byte === CR + } else { + afterCR = false + if (pendingBytes + i + 1 - start > maxLineBytes) { + return new NdjsonError({ + kind: "Unpack", + reason: "LineTooLarge", + unterminated: false, + cause: new RangeError(`NDJSON line exceeds maxLineBytes of ${maxLineBytes}`) + }) + } + } + } + if (start < chunk.length) { + // Copy, so a reused upstream buffer cannot change the line + pending.push(chunk.slice(start)) + pendingBytes += chunk.length - start + } + } + } + + const flush = (out: Array) => + pendingBytes === 0 ? undefined : decodeLine(takeLine(emptyBytes, 0, 0), out, true) + + return makeDecodePull(upstream, step, flush) + }) + ) /** * Creates an NDJSON byte decoder channel for values of a schema. @@ -256,6 +440,7 @@ export const decodeSchema = ( ) => (options?: { readonly ignoreEmptyLines?: boolean | undefined + readonly maxLineBytes?: number | undefined }): Channel.Channel< Arr.NonEmptyReadonlyArray, Schema.SchemaError | NdjsonError | IE, @@ -308,6 +493,7 @@ export const decodeSchemaString = ( export const duplex: { (options?: { readonly ignoreEmptyLines?: boolean | undefined + readonly maxLineBytes?: number | undefined }): ( self: Channel.Channel< Arr.NonEmptyReadonlyArray, @@ -339,6 +525,7 @@ export const duplex: { >, options?: { readonly ignoreEmptyLines?: boolean | undefined + readonly maxLineBytes?: number | undefined } ): Channel.Channel< Arr.NonEmptyReadonlyArray, @@ -361,6 +548,7 @@ export const duplex: { >, options?: { readonly ignoreEmptyLines?: boolean | undefined + readonly maxLineBytes?: number | undefined } ): Channel.Channel< Arr.NonEmptyReadonlyArray, @@ -478,6 +666,7 @@ export const duplexSchema: { readonly inputSchema: In readonly outputSchema: Out readonly ignoreEmptyLines?: boolean | undefined + readonly maxLineBytes?: number | undefined } ): ( self: Channel.Channel< @@ -512,6 +701,7 @@ export const duplexSchema: { readonly inputSchema: In readonly outputSchema: Out readonly ignoreEmptyLines?: boolean | undefined + readonly maxLineBytes?: number | undefined } ): Channel.Channel< Arr.NonEmptyReadonlyArray, @@ -536,6 +726,7 @@ export const duplexSchema: { readonly inputSchema: In readonly outputSchema: Out readonly ignoreEmptyLines?: boolean | undefined + readonly maxLineBytes?: number | undefined } ): Channel.Channel< Arr.NonEmptyReadonlyArray, diff --git a/repos/effect/packages/effect/src/encoding/SchemaBinary.ts b/repos/effect/packages/effect/src/encoding/SchemaBinary.ts index a8cf71fb8e..4e0eae5bed 100644 --- a/repos/effect/packages/effect/src/encoding/SchemaBinary.ts +++ b/repos/effect/packages/effect/src/encoding/SchemaBinary.ts @@ -13,6 +13,7 @@ */ import * as Arr from "../Array.ts" import * as BigDecimal from "../BigDecimal.ts" +import * as Bytes from "../Bytes.ts" import * as Cause from "../Cause.ts" import * as Channel from "../Channel.ts" import * as Chunk from "../Chunk.ts" @@ -43,7 +44,7 @@ import * as SchemaTransformation from "../SchemaTransformation.ts" * * The default mode supports compatible schema evolution. `fingerprint: true` * uses positional layouts and an 8-byte layout hash for smaller frames, but - * requires peers to use the same schema definition. + * requires peers to use an identical encoded layout. * * @stability unstable * @category models @@ -92,6 +93,14 @@ const directFingerprintCodecCache = new WeakMap> * Encoded results are arena-backed views and may share a larger buffer. Use * `bytes.slice()` when independent ownership is required. * + * **Gotchas** + * + * Bytes carry the layout of the schema that wrote them. In the default mode, + * decode them with that schema or a compatible evolution of it; fingerprint + * mode requires an identical encoded layout. Most union codecs prefix values + * with a member selector, so they cannot read bytes written by a member's own + * codec. + * * **Example** (Deriving a binary codec) * * ```ts @@ -248,16 +257,7 @@ export function encodeManyUnknownSync( } function concatFrames(frames: ReadonlyArray>): Uint8Array { - if (frames.length === 1) return frames[0] - let length = 0 - for (const frame of frames) length += frame.length - const out = new Uint8Array(length) - let offset = 0 - for (const frame of frames) { - out.set(frame, offset) - offset += frame.length - } - return out + return frames.length === 1 ? frames[0] : Bytes.concat(frames) } /** @@ -1074,14 +1074,6 @@ function pushU64(out: Array, n: bigint) { } } -function compareBytes(a: Uint8Array, b: Uint8Array): number { - const len = Math.min(a.length, b.length) - for (let i = 0; i < len; i++) { - if (a[i] !== b[i]) return a[i] - b[i] - } - return a.length - b.length -} - class IssueError extends Error { readonly issue: SchemaIssue.Issue constructor(issue: SchemaIssue.Issue) { @@ -2050,7 +2042,7 @@ function sentinelSetHash(sentinels: ReadonlyArray): number { const bn = typeof b.key === "number" if (an !== bn) return an ? -1 : 1 if (an) return (a.key as number) - (b.key as number) - return compareBytes(utf8Encode.encode(a.key as string), utf8Encode.encode(b.key as string)) + return Bytes.Order(utf8Encode.encode(a.key as string), utf8Encode.encode(b.key as string)) }) const out: Array = [] for (const sentinel of sorted) { @@ -3171,7 +3163,7 @@ function extraPairs(ctx: EncodeContext, layout: StructLayout, obj: Record a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0) } else { for (const pair of pairs) pair[2] = utf8Encode.encode(pair[0]) - pairs.sort((a, b) => compareBytes(a[2]!, b[2]!)) + pairs.sort((a, b) => Bytes.Order(a[2]!, b[2]!)) } if (persistent) { for (const pair of pairs) pair[2] ??= utf8Encode.encode(pair[0]) diff --git a/repos/effect/packages/effect/src/encoding/Tar.ts b/repos/effect/packages/effect/src/encoding/Tar.ts new file mode 100644 index 0000000000..6ee58b732d --- /dev/null +++ b/repos/effect/packages/effect/src/encoding/Tar.ts @@ -0,0 +1,1417 @@ +/** + * Encodes and decodes tar archives as streams in Effect channels. + * + * `decode` streams entry bodies from the archive input; `encode` streams them + * back into archive bytes. Neither buffers an entire entry body. + * + * The decoder reads the formats produced by common tools: v7, USTAR, PAX + * extended headers, and GNU long names and base-256 numbers. The encoder writes + * USTAR, adding PAX extended headers for fields that do not fit or for custom + * metadata. It does not access the file system or compress archives. + * + * @stability unstable + * @since 4.0.5 + */ +import * as Arr from "../Array.ts" +import * as Cause from "../Cause.ts" +import * as Channel from "../Channel.ts" +import * as Data from "../Data.ts" +import * as Effect from "../Effect.ts" +import * as Exit from "../Exit.ts" +import * as Predicate from "../Predicate.ts" +import * as Pull from "../Pull.ts" +import * as Scope from "../Scope.ts" +import * as Semaphore from "../Semaphore.ts" +import * as Stream from "../Stream.ts" +import type * as Types from "../Types.ts" + +const TarErrorTypeId = "~effect/encoding/Tar/TarError" + +/** + * Error raised when a tar archive cannot be encoded or decoded. + * + * **Details** + * + * The `kind` field identifies the failure: + * + * - `InvalidHeader`: a header has a bad checksum, a malformed field, or a value + * that cannot be written. + * - `Truncated`: the input ended inside a header or an entry body. + * - `SizeMismatch`: an entry body produced a byte count other than its declared + * `size`. + * - `Unsupported`: the archive uses a feature this module does not read, such + * as GNU sparse files. + * - `BodyConsumed`: an entry body was read after the decoder had moved past it. + * + * @stability unstable + * @category errors + * @since 4.0.5 + */ +export class TarError extends Data.TaggedError("TarError")<{ + readonly kind: "InvalidHeader" | "Truncated" | "SizeMismatch" | "Unsupported" | "BodyConsumed" + readonly message: string +}> { + /** + * Marks this value as a tar error for runtime guards. + * + * @stability unstable + * @since 4.0.5 + */ + readonly [TarErrorTypeId] = TarErrorTypeId +} + +/** + * Checks whether a value is a `TarError`. + * + * @stability unstable + * @category guards + * @since 4.0.5 + */ +export const isTarError = (u: unknown): u is TarError => Predicate.hasProperty(u, TarErrorTypeId) + +/** + * Kind of filesystem object described by a tar entry. + * + * **Details** + * + * `Other` covers every typeflag without a dedicated kind, such as GNU volume + * labels; the raw flag is kept in `Header.typeflag`. + * + * @stability unstable + * @category models + * @since 4.0.5 + */ +export type EntryType = + | "File" + | "Directory" + | "Symlink" + | "Hardlink" + | "CharDevice" + | "BlockDevice" + | "Fifo" + | "Other" + +/** + * Metadata of a tar entry. + * + * **Details** + * + * `path` and `linkTarget` are archive paths; hard link targets name another + * entry of the same archive. `mode` holds the permission bits, including + * setuid, setgid and sticky; file type bits, as in a `stat` mode, are dropped + * when decoding and encoding. `pax` holds the PAX extended header records, + * including those of global headers, that are not reflected in the other + * fields, such as `SCHILY.xattr.*`, `atime` or the `comment` of `git archive`. + * `devMajor` and `devMinor` are set for device entries only. + * + * @stability unstable + * @category models + * @since 4.0.5 + */ +export interface Header { + readonly path: string + readonly type: EntryType + readonly typeflag: string + readonly size: number + readonly mode: number + readonly uid: number + readonly gid: number + readonly uname: string + readonly gname: string + readonly mtime: Date + readonly linkTarget?: string | undefined + readonly devMajor?: number | undefined + readonly devMinor?: number | undefined + readonly pax: Readonly> +} + +/** + * Tar entry: a header and a stream of exactly `header.size` body bytes. + * + * @stability unstable + * @category models + * @since 4.0.5 + */ +export interface Entry { + readonly header: Header + readonly body: Stream.Stream +} + +/** + * Options for the entry constructors. + * + * **Details** + * + * Unset fields default to mode `0o644` (`0o755` for directories, `0o777` for + * symlinks), uid and gid `0`, empty user and group names, an mtime at the Unix + * epoch, and no extra PAX records. The defaults make archives reproducible. + * + * @stability unstable + * @category models + * @since 4.0.5 + */ +export interface EntryOptions { + readonly mode?: number | undefined + readonly uid?: number | undefined + readonly gid?: number | undefined + readonly uname?: string | undefined + readonly gname?: string | undefined + readonly mtime?: Date | undefined + readonly pax?: Readonly> | undefined +} + +const makeHeader = ( + path: string, + type: "File" | "Directory" | "Symlink" | "Hardlink", + size: number, + options: EntryOptions | undefined, + linkTarget?: string +): Header => ({ + path, + type, + typeflag: typeflags[type], + size, + mode: options?.mode ?? (type === "Directory" ? 0o755 : type === "Symlink" ? 0o777 : 0o644), + uid: options?.uid ?? 0, + gid: options?.gid ?? 0, + uname: options?.uname ?? "", + gname: options?.gname ?? "", + mtime: options?.mtime ?? new Date(0), + ...(linkTarget === undefined || linkTarget === "" ? undefined : { linkTarget }), + pax: options?.pax ?? {} +}) + +/** + * Creates a regular file entry from in-memory content. + * + * **Details** + * + * String content is encoded as UTF-8, and `size` is the byte length of the + * content. + * + * @see {@link fileStream} for content that is streamed instead of held in memory + * @see {@link encode} for writing entries to an archive + * + * @stability unstable + * @category constructors + * @since 4.0.5 + */ +export const file = (path: string, content: string | Uint8Array, options?: EntryOptions): Entry => { + const bytes = typeof content === "string" ? textEncoder.encode(content) : content + const header = makeHeader(path, "File", bytes.length, options) + if (bytes.length === 0) return { header, body: Stream.empty } + const body = Stream.succeed(bytes) + inMemoryBodies.set(body, bytes) + return { header, body } +} + +// Key by body identity so replacing an entry body bypasses the encode fast path. +const inMemoryBodies = new WeakMap, Uint8Array>() + +/** + * Creates a regular file entry whose content is streamed from `body`. + * + * **When to use** + * + * Use when the content is too large to hold in memory, or is read from a file + * or another stream, and its byte length is known up front. + * + * **Details** + * + * The defaults match `file`: mode `0o644`, uid and gid `0`, and an mtime at the + * Unix epoch. `body` is not run until the entry is encoded. + * + * **Gotchas** + * + * `size` must be the exact byte length of `body`: `encode` fails with a + * `SizeMismatch` `TarError` when the body produces fewer or more bytes. + * + * @see {@link file} for in-memory content + * @see {@link encode} for writing entries to an archive + * + * @stability unstable + * @category constructors + * @since 4.0.5 + */ +export const fileStream = ( + path: string, + size: number, + body: Stream.Stream, + options?: EntryOptions +): Entry => ({ + header: makeHeader(path, "File", size, options), + body +}) + +/** + * Creates a directory entry. + * + * **Details** + * + * A trailing `/` is added to a non-empty `path` when missing, as tar tools + * expect for directories. + * + * @stability unstable + * @category constructors + * @since 4.0.5 + */ +export const directory = (path: string, options?: EntryOptions): Entry => ({ + // An empty path stays empty, so that `encode` rejects it + header: makeHeader(path === "" || path.endsWith("/") ? path : `${path}/`, "Directory", 0, options), + body: Stream.empty +}) + +/** + * Creates a symbolic link entry pointing at `target`. + * + * @stability unstable + * @category constructors + * @since 4.0.5 + */ +export const symlink = (path: string, target: string, options?: EntryOptions): Entry => ({ + header: makeHeader(path, "Symlink", 0, options, target), + body: Stream.empty +}) + +/** + * Creates a hard link entry pointing at another entry of the archive. + * + * **Details** + * + * `target` is the archive path of an entry written earlier in the same + * archive. + * + * @stability unstable + * @category constructors + * @since 4.0.5 + */ +export const hardlink = (path: string, target: string, options?: EntryOptions): Entry => ({ + header: makeHeader(path, "Hardlink", 0, options, target), + body: Stream.empty +}) + +/** + * Creates a channel that decodes tar archive bytes into entries. + * + * **Details** + * + * Each output chunk holds one entry, and each entry body streams exactly + * `header.size` bytes straight from the input. Pulling the next entry skips + * whatever is left of the current body, like Go's `tar.Reader`. The decoder + * reads v7, USTAR, PAX extended headers (applying `path`, `linkpath`, `size`, + * `uid`, `gid`, `uname`, `gname` and `mtime`), GNU long names and links, and + * base-256 numbers. The records of a PAX global header apply to every entry + * that follows, except `path`, `linkpath` and `size`, which describe a single + * entry. Anything after the end-of-archive marker is ignored. + * + * **Gotchas** + * + * - Read each body before pulling the next entry: a body read afterwards fails + * with a `BodyConsumed` `TarError`. Operators that pull ahead, such as + * `Stream.buffer`, therefore can't sit between `decode` and the body reads. + * - Read bodies while the entry stream runs, for example in `Stream.mapEffect`: + * once it has ended, as after `Stream.runCollect`, `Stream.runHead` or + * `Stream.take`, every body fails with a `BodyConsumed` `TarError`. + * - A body reads the entry's bytes once: running it again continues where the + * previous run stopped. + * - Directories, links, devices and FIFOs have no body, and their `size` is + * reported as `0`. + * - Paths and link targets are returned as written in the archive, so they can + * be absolute or contain `..`, and a symlink entry can redirect a later entry + * outside the target directory. Validate both before writing entries to disk. + * - Names that are not valid UTF-8 are decoded with replacement characters. + * - PAX extended headers and GNU long names of more than 1 MiB fail with an + * `InvalidHeader` `TarError`. + * - The channel completes only when its input ends: input after the + * end-of-archive marker is read and discarded, so that the upstream done + * value can be passed on. + * - GNU sparse entries fail with an `Unsupported` `TarError`. + * + * @see {@link encode} for the inverse operation + * + * @stability unstable + * @category decoding + * @since 4.0.5 + */ +export const decode = (): Channel.Channel< + Arr.NonEmptyReadonlyArray>, + TarError | IE, + Done, + Arr.NonEmptyReadonlyArray, + IE, + Done +> => + Channel.fromTransform((upstream, scope) => + Effect.suspend(() => { + const decoder = new Decoder() + let done: { readonly value: Done } | undefined + let pulls = 0 + + const pullUpstream = upstream.pipe( + Effect.map((chunks) => { + pulls++ + for (const chunk of chunks) decoder.push(chunk) + }), + Pull.catchDone((value) => { + pulls++ + done = { value: value as Done } + decoder.end() + return Effect.void + }) + ) + + // Serialize entry and body pulls; reuse input pulled while waiting for the lock. + const lock = Semaphore.makeUnsafe(1) + const pullInput = (seen: number) => + lock.withPermit(Effect.suspend(() => pulls === seen ? pullUpstream : Effect.void)) + + const run = (f: () => A | undefined | typeof NeedInput, end: () => D): Pull.Pull => + Effect.suspend(function loop(): Pull.Pull { + const seen = pulls + let result: A | undefined | typeof NeedInput + try { + result = f() + } catch (error) { + return isTarError(error) ? Effect.fail(error) : Effect.die(error) + } + if (result === NeedInput) return Effect.flatMap(pullInput(seen), loop) + return result === undefined ? Cause.done(end()) : Effect.succeed(result) + }) + + const body = (id: number): Stream.Stream => + Stream.fromPull(Effect.succeed(run(() => decoder.read(id), () => undefined))) + + const next = run(() => { + const header = decoder.next() + return header === NeedInput || header === undefined + ? header + : Arr.of({ header, body: body(decoder.generation) }) + }, () => done!.value) + + return Effect.as(Scope.addFinalizer(scope, Effect.sync(() => decoder.close())), next) + }) + ) + +/** + * Creates a channel that encodes entries into tar archive bytes. + * + * **Details** + * + * Each entry is written as a USTAR header, with the path split into the USTAR + * prefix and name fields when it is longer than 100 bytes. A PAX extended + * header is added only for what USTAR cannot hold: a long or non-ASCII path, + * link target, user or group name, a size of 8 GiB or more, a uid or gid above + * `2097151`, an mtime outside the USTAR range, or custom `pax` records. The + * archive ends with two zero blocks. + * + * Headers, padding and body chunks below 16 KiB are gathered into output chunks + * of up to 64 KiB, and larger body chunks pass through without copying, so + * bodies stream without being held in memory. Output is handed out before + * waiting for more entries, but up to 64 KiB of it can be held while a body is + * pulled. + * + * **Gotchas** + * + * - A body that produces any byte count other than `header.size` fails with a + * `SizeMismatch` `TarError`, so the size must be known before the body is + * written. + * - `header.pax` may contain custom metadata such as `SCHILY.xattr.*`. Keys + * `path`, `linkpath`, `size`, `uid`, `gid`, `uname`, `gname` and `mtime` must + * be set through the corresponding header fields. These keys and + * `GNU.sparse.*` records fail with an `InvalidHeader` `TarError`. + * - A `File` entry whose path ends with `/` fails with an `InvalidHeader` + * `TarError`, because other tools read it as a directory. + * - Entries other than `File` and `Other` must have `size` 0, and `devMajor` + * and `devMinor` must not exceed `2097151`; otherwise encoding fails with an + * `InvalidHeader` `TarError`. + * - mtime is rounded down to whole seconds. + * - `typeflag` is only written for `Other` entries; every other type writes its + * standard flag. + * + * @see {@link decode} for the inverse operation + * @see {@link file} for building entries from in-memory content + * @see {@link fileStream} for building entries from streamed content + * + * @stability unstable + * @category encoding + * @since 4.0.5 + */ +export const encode = = Entry>(): Channel.Channel< + Arr.NonEmptyReadonlyArray, + TarError | IE | (In extends Entry ? E : never), + Done, + Arr.NonEmptyReadonlyArray, + IE, + Done, + In extends Entry ? R : never +> => { + // Distribute over entry unions to infer body errors and services. + type E = In extends Entry ? E : never + type R = In extends Entry ? R : never + return Channel.fromTransform((upstream: Pull.Pull, IE, Done>, scope) => + Effect.sync(() => { + const encoder = new Encoder() + let entries: ReadonlyArray> = [] + let index = 0 + let done: { readonly value: Done } | undefined + let body: Pull.Pull, E, void, R> | undefined + let bodyScope: Scope.Closeable | undefined + + const write = (f: () => void): Effect.Effect => { + try { + f() + } catch (error) { + return isTarError(error) ? Effect.fail(error) : Effect.die(error) + } + return Effect.void + } + + return Effect.gen(function*() { + while (true) { + if (encoder.buffered >= SLAB_SIZE) return encoder.flush() + if (body !== undefined) { + const chunks = yield* Pull.catchDone(body, () => Effect.undefined) + if (chunks !== undefined) { + yield* write(() => { + for (const chunk of chunks) encoder.body(chunk) + }) + continue + } + body = undefined + yield* Scope.close(bodyScope!, Exit.void) + yield* write(() => encoder.end()) + continue + } + if (index < entries.length) { + const entry = entries[index++] + // Directories, links and in-memory files have nothing to stream + const bytes = entry.body === Stream.empty ? emptyBody : inMemoryBodies.get(entry.body) + yield* write(() => { + encoder.begin(entry.header) + if (bytes !== undefined) { + encoder.body(bytes) + encoder.end() + } + }) + if (bytes !== undefined) continue + bodyScope = Scope.forkUnsafe(scope) + body = yield* Channel.toPullScoped(entry.body.channel, bodyScope) + continue + } + if (encoder.buffered > 0) return encoder.flush() + if (done !== undefined) return yield* Cause.done(done.value) + const next = yield* Pull.catchDone(upstream, (value) => { + done = { value: value as Done } + encoder.finish() + return Effect.undefined + }) + if (next !== undefined) { + entries = next + index = 0 + } + } + }) + }) + ) +} + +const invalidHeader = (message: string) => new TarError({ kind: "InvalidHeader", message }) + +const truncated = (at: number) => + new TarError({ kind: "Truncated", message: `unexpected end of tar archive at byte ${at}` }) + +const unsupported = (message: string) => new TarError({ kind: "Unsupported", message }) + +const sizeMismatch = (header: Header, written: number) => + new TarError({ + kind: "SizeMismatch", + message: `body of ${JSON.stringify(header.path)} has ${ + written > header.size ? "at least" : "only" + } ${written} bytes, but its size is ${header.size}` + }) + +const BLOCK = 512 +const PERMISSION_BITS = 0o7777 + +const padding = (size: number): number => (BLOCK - (size % BLOCK)) % BLOCK + +const isZeroBlock = (block: Uint8Array): boolean => { + for (let i = 0; i < BLOCK; i++) { + if (block[i] !== 0) return false + } + return true +} + +const sumBlock = (block: Uint8Array): number => { + // Four sums let the loop run faster than one + let a = 0 + let b = 0 + let c = 0 + let d = 0 + for (let i = 0; i < BLOCK; i += 4) { + a += block[i] + b += block[i + 1] + c += block[i + 2] + d += block[i + 3] + } + return a + b + c + d +} + +const USTAR_MAGIC = [0x75, 0x73, 0x74, 0x61, 0x72, 0x00] // "ustar\0" +const GNU_MAGIC = [0x75, 0x73, 0x74, 0x61, 0x72, 0x20, 0x20, 0x00] // "ustar \0" +const STAR_TRAILER = [0x74, 0x61, 0x72, 0x00] // "tar\0" +const USTAR_MAGIC_VERSION = new Uint8Array([0x75, 0x73, 0x74, 0x61, 0x72, 0x00, 0x30, 0x30]) // "ustar\0" "00" + +const hasBytes = (block: Uint8Array, offset: number, bytes: ReadonlyArray): boolean => { + for (let i = 0; i < bytes.length; i++) { + if (block[offset + i] !== bytes[i]) return false + } + return true +} + +const typeflags: Record, string> = { + File: "0", + Hardlink: "1", + Symlink: "2", + CharDevice: "3", + BlockDevice: "4", + Directory: "5", + Fifo: "6" +} + +const types: Record = { + ...Object.fromEntries(Object.entries(typeflags).map(([type, typeflag]) => [typeflag, type])), + // Contiguous files + "7": "File" +} + +const typeOf = (typeflag: string, path: string): EntryType => { + // Legacy archives mark directories with a trailing slash only + if (typeflag === "\0") return path.endsWith("/") ? "Directory" : "File" + return types[typeflag] ?? "Other" +} + +const typeflagOf = (header: Header): string => header.type === "Other" ? header.typeflag : typeflags[header.type] + +// Entries of these types never carry body bytes, whatever their size field says +const isHeaderOnly = (type: EntryType): boolean => type !== "File" && type !== "Other" + +const textEncoder = new TextEncoder() +// Keeps a leading byte order mark as part of the decoded field +const textDecoder = new TextDecoder("utf-8", { ignoreBOM: true }) + +const isAscii = (s: string): boolean => { + for (let i = 0; i < s.length; i++) { + const c = s.charCodeAt(i) + if (c === 0 || c >= 0x80) return false + } + return true +} + +// Keeps the ASCII characters of a value that is also stored in a PAX record +const toAscii = (s: string): string => { + let out = "" + for (let i = 0; i < s.length; i++) { + const c = s.charCodeAt(i) + if (c > 0 && c < 0x80) out += s[i] + } + return out +} + +const fits = (value: string, length: number): boolean => value.length <= length && isAscii(value) + +const paxStringKeys = new Set(["path", "linkpath", "uname", "gname"]) +const paxHeaderKeys = new Set(["path", "linkpath", "size", "uid", "gid", "uname", "gname", "mtime"]) + +const noPax: Readonly> = Object.freeze({}) + +// Parses records of the form " =\n", where the length counts +// the bytes of the whole record +const parsePax = (data: Uint8Array): Map => { + const records = new Map() + let offset = 0 + while (offset < data.length) { + let space = offset + while (space < data.length && space - offset < 16 && data[space] !== 0x20) space++ + const lengthField = String.fromCharCode.apply(null, data.subarray(offset, space) as unknown as Array) + const length = /^\d+$/.test(lengthField) ? Number(lengthField) : Number.NaN + const end = offset + length + if (!(length >= 5) || end > data.length || data[space] !== 0x20 || space + 1 >= end || data[end - 1] !== 0x0a) { + throw invalidHeader("malformed PAX record") + } + const record = data.subarray(space + 1, end - 1) + const equals = record.indexOf(0x3d) + if (equals <= 0) { + throw invalidHeader("malformed PAX record") + } + const key = textDecoder.decode(record.subarray(0, equals)) + const value = textDecoder.decode(record.subarray(equals + 1)) + if (key.includes("\0") || (paxStringKeys.has(key) && value.includes("\0"))) { + throw invalidHeader(`invalid PAX record ${key}`) + } + records.set(key, value) + offset = end + } + return records +} + +const paxInteger = (key: string, value: string): number => { + const n = /^\d+$/.test(value) ? Number(value) : Number.NaN + if (!Number.isSafeInteger(n)) { + throw invalidHeader(`invalid PAX ${key} ${JSON.stringify(value)}`) + } + return n +} + +// Parses a PAX time of decimal seconds into milliseconds, rounded down. The +// digits are added up separately, because `Number(value) * 1000` is inexact. +const paxMillis = (value: string): number => { + const match = /^(-?)(\d+)(?:\.(\d+))?$/.exec(value) + if (match === null) { + throw invalidHeader(`invalid PAX mtime ${JSON.stringify(value)}`) + } + const [, sign, seconds, fraction = ""] = match + const millis = Number(seconds) * 1000 + Number(fraction.slice(0, 3).padEnd(3, "0")) + if (sign === "") return millis + // A negative time with digits below a millisecond rounds down past them + return -millis - (/[1-9]/.test(fraction.slice(3)) ? 1 : 0) +} + +// Applies the records that override header fields and returns the rest. An +// empty value keeps the field from the header block. +const applyPax = (header: RawHeader, records: Map): Record => { + const rest: Record = {} + for (const [key, value] of records) { + if (key.startsWith("GNU.sparse.")) { + throw unsupported("GNU sparse files are not supported") + } + if (!paxHeaderKeys.has(key)) { + // Defined rather than assigned, so that a `__proto__` key stays a record + Object.defineProperty(rest, key, { value, enumerable: true, writable: true, configurable: true }) + continue + } + if (value === "") continue + switch (key) { + case "path": + header.path = value + break + case "linkpath": + header.linkTarget = value + break + case "uname": + case "gname": + header[key] = value + break + case "uid": + case "gid": + case "size": + header[key] = paxInteger(key, value) + break + case "mtime": + header.mtimeMillis = paxMillis(value) + } + } + return rest +} + +const paxRecord = (key: string, value: string): Uint8Array => { + const rest = textEncoder.encode(` ${key}=${value}\n`) + // The length counts its own digits + const length = rest.length + String(rest.length + String(rest.length).length).length + const out = new Uint8Array(length) + writeString(out, 0, length, String(length)) + out.set(rest, length - rest.length) + return out +} + +const paxHeaderName = (path: string): string => { + const trimmed = path.replace(/\/+$/, "") + const i = trimmed.lastIndexOf("/") + const name = i === -1 ? `PaxHeaders.0/${trimmed}` : `${trimmed.slice(0, i)}/PaxHeaders.0/${trimmed.slice(i + 1)}` + return toAscii(name).slice(0, 100) +} + +const readString = (block: Uint8Array, offset: number, length: number): string => { + let end = offset + let ascii = true + const limit = offset + length + while (end < limit && block[end] !== 0) { + if (block[end] >= 0x80) ascii = false + end++ + } + if (end === offset) return "" + if (!ascii || end - offset > 256) return textDecoder.decode(block.subarray(offset, end)) + // Most header strings are short ASCII, which is cheaper to build directly + return String.fromCharCode.apply(null, block.subarray(offset, end) as unknown as Array) +} + +const readOctal = (block: Uint8Array, offset: number, length: number): number => { + let start = offset + let end = offset + length + while (start < end && (block[start] === 0 || block[start] === 0x20)) start++ + while (end > start && (block[end - 1] === 0 || block[end - 1] === 0x20)) end-- + let value = 0 + for (let i = start; i < end; i++) { + const c = block[i] + if (c < 0x30 || c > 0x37) { + throw invalidHeader(`invalid octal number in the header field at offset ${offset}`) + } + value = value * 8 + (c - 0x30) + } + return value +} + +// GNU base-256: the high bit of the first byte is set, and the next bit is the +// sign of a two's complement big-endian number +const readNumber = (block: Uint8Array, offset: number, length: number): number => { + if ((block[offset] & 0x80) === 0) return readOctal(block, offset, length) + const negative = (block[offset] & 0x40) !== 0 + // Negative numbers are read as the bitwise inverse of their magnitude minus one + let value = 0 + for (let i = 0; i < length; i++) { + let c = block[offset + i] + if (negative) c ^= 0xff + if (i === 0) c &= 0x7f + value = value * 256 + c + if (value > Number.MAX_SAFE_INTEGER) { + throw invalidHeader(`base-256 number in the header field at offset ${offset} is out of range`) + } + } + return negative ? -value - 1 : value +} + +const readUnsigned = (block: Uint8Array, offset: number, length: number, field: string): number => { + const value = readNumber(block, offset, length) + if (value < 0) { + throw invalidHeader(`negative ${field}`) + } + return value +} + +// The checksum is the sum of the block bytes with the checksum field counted as +// spaces. Some old tar writers summed signed bytes, which is accepted too. +const verifyChecksum = (block: Uint8Array): boolean => { + const stored = readOctal(block, 148, 8) + let unsigned = sumBlock(block) + 8 * 0x20 + for (let i = 148; i < 156; i++) unsigned -= block[i] + if (stored === unsigned) return true + let high = 0 + for (let i = 0; i < BLOCK; i++) { + if (block[i] > 0x7f && (i < 148 || i >= 156)) high++ + } + return stored === unsigned - 0x100 * high +} + +interface RawHeader { + readonly typeflag: string + path: string + linkTarget: string + size: number + readonly mode: number + uid: number + gid: number + uname: string + gname: string + mtimeMillis: number + devMajor: number + devMinor: number +} + +const parseHeader = (block: Uint8Array): RawHeader => { + if (!verifyChecksum(block)) { + throw invalidHeader("header checksum mismatch") + } + const isUstar = hasBytes(block, 257, USTAR_MAGIC) + const isGnu = !isUstar && hasBytes(block, 257, GNU_MAGIC) + const header: RawHeader = { + typeflag: String.fromCharCode(block[156]), + path: readString(block, 0, 100), + linkTarget: readString(block, 157, 100), + size: readUnsigned(block, 124, 12, "size"), + mode: readUnsigned(block, 100, 8, "mode"), + uid: readUnsigned(block, 108, 8, "uid"), + gid: readUnsigned(block, 116, 8, "gid"), + mtimeMillis: readNumber(block, 136, 12) * 1000, + uname: "", + gname: "", + devMajor: 0, + devMinor: 0 + } + if (!isUstar && !isGnu) return header + header.uname = readString(block, 265, 32) + header.gname = readString(block, 297, 32) + header.devMajor = readUnsigned(block, 329, 8, "devMajor") + header.devMinor = readUnsigned(block, 337, 8, "devMinor") + if (isUstar) { + // star archives end the block with "tar\0" and keep times after a shorter prefix + const prefix = hasBytes(block, 508, STAR_TRAILER) ? readString(block, 345, 131) : readString(block, 345, 155) + if (prefix.length > 0) header.path = `${prefix}/${header.path}` + } + return header +} + +const MAX_OCTAL_7 = 0o7777777 // 8-byte numeric fields +const MAX_OCTAL_11 = 0o77777777777 // 12-byte numeric fields + +const writeString = (block: Uint8Array, offset: number, length: number, value: string): void => { + const n = Math.min(value.length, length) + for (let i = 0; i < n; i++) block[offset + i] = value.charCodeAt(i) +} + +// Writes `length - 1` zero-padded octal digits; the value must fit +const writeOctal = (block: Uint8Array, offset: number, length: number, value: number): void => { + let i = offset + length - 2 + // Bit operations are faster, but only cover 32 bits + for (; value > 0x3fffffff && i >= offset; i--) { + block[i] = 0x30 + (value % 8) + value = Math.floor(value / 8) + } + for (; i >= offset; i--) { + block[i] = 0x30 + (value & 7) + value >>>= 3 + } +} + +// Split at the last slash that fits the 155-byte USTAR prefix. +const splitPath = (path: string): readonly [prefix: string, name: string] | undefined => { + if (path.length <= 100 || !isAscii(path)) return undefined + let length = path.length + if (length > 156) length = 156 + else if (path[length - 1] === "/") length-- + const i = path.lastIndexOf("/", length - 1) + if (i <= 0 || path.length - i - 1 > 100) return undefined + return [path.slice(0, i), path.slice(i + 1)] +} + +interface UstarFields { + readonly name: string + readonly prefix: string + readonly typeflag: string + readonly size: number + readonly mode: number + readonly uid: number + readonly gid: number + readonly mtime: number + readonly linkname: string + readonly uname: string + readonly gname: string + readonly devMajor: number + readonly devMinor: number +} + +const paxHeaderFields: UstarFields = { + name: "", + prefix: "", + typeflag: "x", + size: 0, + mode: 0o644, + uid: 0, + gid: 0, + mtime: 0, + linkname: "", + uname: "", + gname: "", + devMajor: 0, + devMinor: 0 +} + +const writeUstarBlock = (block: Uint8Array, fields: UstarFields): void => { + writeString(block, 0, 100, fields.name) + writeOctal(block, 100, 8, fields.mode) + writeOctal(block, 108, 8, fields.uid) + writeOctal(block, 116, 8, fields.gid) + writeOctal(block, 124, 12, fields.size) + writeOctal(block, 136, 12, fields.mtime) + block[156] = fields.typeflag.charCodeAt(0) + writeString(block, 157, 100, fields.linkname) + block.set(USTAR_MAGIC_VERSION, 257) + writeString(block, 265, 32, fields.uname) + writeString(block, 297, 32, fields.gname) + writeOctal(block, 329, 8, fields.devMajor) + writeOctal(block, 337, 8, fields.devMinor) + writeString(block, 345, 155, fields.prefix) + block.fill(0x20, 148, 156) + // Six digits, a NUL and the space left from the fill + writeOctal(block, 148, 7, sumBlock(block)) + block[154] = 0 +} + +const checkNumber = (field: string, value: number | undefined, max: number): number => { + const n = value ?? 0 + if (!Number.isSafeInteger(n) || n < 0 || n > max) { + throw invalidHeader(`invalid ${field} ${n}`) + } + return n +} + +const checkNoNul = (field: string, value: string): string => { + if (value.includes("\0")) { + throw invalidHeader(`${field} must not contain NUL characters`) + } + return value +} + +// Writes the header blocks of an entry, with a PAX header first when needed, into +// zeroed memory from `alloc`. Throws before allocating when the header is invalid. +const encodeHeader = (header: Header, alloc: (size: number) => Uint8Array): void => { + const path = checkNoNul("path", header.path) + if (path === "") { + throw invalidHeader("path must not be empty") + } + // GNU tar and node-tar read such an entry as a directory + if (header.type === "File" && path.endsWith("/")) { + throw invalidHeader(`File entry ${JSON.stringify(path)} must not end with "/"`) + } + const typeflag = typeflagOf(header) + if (typeflag.length !== 1 || typeflag.charCodeAt(0) >= 0x80 || "xgLKS".includes(typeflag)) { + throw invalidHeader(`invalid typeflag ${JSON.stringify(typeflag)}`) + } + const size = checkNumber("size", header.size, Number.MAX_SAFE_INTEGER) + if (size > 0 && isHeaderOnly(header.type)) { + throw invalidHeader(`${header.type} entry ${JSON.stringify(path)} must have size 0`) + } + const mode = checkNumber("mode", header.mode, Number.MAX_SAFE_INTEGER) & PERMISSION_BITS + const uid = checkNumber("uid", header.uid, Number.MAX_SAFE_INTEGER) + const gid = checkNumber("gid", header.gid, Number.MAX_SAFE_INTEGER) + const devMajor = checkNumber("devMajor", header.devMajor, MAX_OCTAL_7) + const devMinor = checkNumber("devMinor", header.devMinor, MAX_OCTAL_7) + const millis = header.mtime.getTime() + if (Number.isNaN(millis)) { + throw invalidHeader("mtime must be a valid date") + } + const mtime = Math.floor(millis / 1000) + + // Records of the PAX header, created when one is needed. Their keys are + // unique, as custom records cannot use the keys of header fields. + let pax: Array | undefined + for (const key of Object.keys(header.pax)) { + if (key.startsWith("GNU.sparse.")) { + throw invalidHeader(`PAX record ${JSON.stringify(key)} is not supported`) + } + if (paxHeaderKeys.has(key)) { + throw invalidHeader(`PAX record ${JSON.stringify(key)} must be set through its header field`) + } + if (key === "" || key.includes("=") || key.includes("\0")) { + throw invalidHeader(`invalid PAX record ${JSON.stringify(key)}`) + } + pax ??= [] + pax.push([key, header.pax[key]]) + } + + // PAX holds overflow values; USTAR keeps truncated ASCII strings or zero. + const split = splitPath(path) + const linkTarget = checkNoNul("linkTarget", header.linkTarget ?? "") + const uname = checkNoNul("uname", header.uname) + const gname = checkNoNul("gname", header.gname) + const pathFits = split !== undefined || fits(path, 100) + const linkTargetFits = fits(linkTarget, 100) + const unameFits = fits(uname, 32) + const gnameFits = fits(gname, 32) + if (!pathFits) (pax ??= []).push(["path", path]) + if (!linkTargetFits) (pax ??= []).push(["linkpath", linkTarget]) + if (!unameFits) (pax ??= []).push(["uname", uname]) + if (!gnameFits) (pax ??= []).push(["gname", gname]) + if (size > MAX_OCTAL_11) (pax ??= []).push(["size", String(size)]) + if (uid > MAX_OCTAL_7) (pax ??= []).push(["uid", String(uid)]) + if (gid > MAX_OCTAL_7) (pax ??= []).push(["gid", String(gid)]) + if (mtime < 0 || mtime > MAX_OCTAL_11) (pax ??= []).push(["mtime", String(mtime)]) + + const ustar: UstarFields = { + name: split !== undefined ? split[1] : pathFits ? path : toAscii(path).slice(0, 100), + prefix: split !== undefined ? split[0] : "", + typeflag, + size: size > MAX_OCTAL_11 ? 0 : size, + mode, + uid: uid > MAX_OCTAL_7 ? 0 : uid, + gid: gid > MAX_OCTAL_7 ? 0 : gid, + mtime: mtime < 0 || mtime > MAX_OCTAL_11 ? 0 : mtime, + linkname: linkTargetFits ? linkTarget : toAscii(linkTarget).slice(0, 100), + uname: unameFits ? uname : toAscii(uname).slice(0, 32), + gname: gnameFits ? gname : toAscii(gname).slice(0, 32), + devMajor, + devMinor + } + + if (pax === undefined) { + writeUstarBlock(alloc(BLOCK), ustar) + return + } + const records = pax.sort(([a], [b]) => a < b ? -1 : 1).map(([key, value]) => paxRecord(key, value)) + let dataLength = 0 + for (const record of records) dataLength += record.length + const paxBlocks = 1 + Math.ceil(dataLength / BLOCK) + const out = alloc(BLOCK * (paxBlocks + 1)) + writeUstarBlock(out, { ...paxHeaderFields, name: paxHeaderName(path), size: dataLength }) + let offset = BLOCK + for (const record of records) { + out.set(record, offset) + offset += record.length + } + writeUstarBlock(out.subarray(BLOCK * paxBlocks), ustar) +} + +const MAX_META_SIZE = 1024 * 1024 // PAX and GNU long name data +const MAX_DATE_MILLIS = 8.64e15 // the range of `Date` + +// A queue of byte chunks that hands out views without copying where possible. +// Consumed chunks are released right away. +class ByteQueue { + private chunks: Array = [] + private head = 0 + private offset = 0 + length = 0 + consumed = 0 + + push(chunk: Uint8Array): void { + if (chunk.length === 0) return + this.chunks.push(chunk) + this.length += chunk.length + } + + takeSome(max: number): Uint8Array { + const chunk = this.chunks[this.head]! + const available = chunk.length - this.offset + if (max < available) { + const out = chunk.subarray(this.offset, this.offset + max) + this.offset += max + this.length -= max + this.consumed += max + return out + } + const out = this.offset === 0 ? chunk : chunk.subarray(this.offset) + this.chunks[this.head++] = undefined + this.length -= available + this.consumed += available + this.offset = 0 + if (this.head === this.chunks.length) { + this.chunks = [] + this.head = 0 + } else if (this.head >= 1024) { + this.chunks = this.chunks.slice(this.head) + this.head = 0 + } + return out + } + + // Takes exactly `n` bytes; the queue must hold at least `n` + take(n: number): Uint8Array { + if (n === 0) return new Uint8Array(0) + const first = this.takeSome(n) + if (first.length === n) return first + const out = new Uint8Array(n) + out.set(first) + let filled = first.length + while (filled < n) { + const part = this.takeSome(n - filled) + out.set(part, filled) + filled += part.length + } + return out + } + + clear(): void { + this.chunks = [] + this.head = 0 + this.offset = 0 + this.length = 0 + } +} + +// Returned by the decoder when the buffered input is not enough to continue +const NeedInput: unique symbol = Symbol("effect/encoding/Tar/NeedInput") + +// `next` skips unread body bytes; `read` consumes the current body. +// Both preserve progress across `NeedInput` retries. +class Decoder { + private readonly queue = new ByteQueue() + private ended = false + private finished = false + private closed = false + // Incremented whenever the current body is invalidated + generation = 0 + private remaining = 0 + private pad = 0 + // Bytes to discard before the next header + private skip = 0 + private zeroBlock = false + // Archive offset of the last header block read, for error messages + private at = 0 + // An extended header whose data is still to be read + private meta: RawHeader | undefined + private pax: Map | undefined + // Records of the PAX global headers read so far + private global: Map | undefined + private longName = "" + private longLink = "" + + push(chunk: Uint8Array): void { + if (!this.finished) this.queue.push(chunk) + } + + end(): void { + this.ended = true + } + + close(): void { + this.closed = true + this.generation++ + this.queue.clear() + } + + // Returns the next entry header, or `undefined` once the archive and the + // input have ended. Input after the end of the archive is discarded. + next(): Header | undefined | typeof NeedInput { + try { + return this.advance() + } catch (error) { + // Header errors are thrown where the archive offset is not known + if (isTarError(error) && (error.kind === "InvalidHeader" || error.kind === "Unsupported")) { + throw new TarError({ kind: error.kind, message: `${error.message} (header at byte ${this.at})` }) + } + throw error + } + } + + private advance(): Header | undefined | typeof NeedInput { + this.generation++ + this.skip += this.remaining + this.pad + this.remaining = 0 + this.pad = 0 + const queue = this.queue + while (true) { + if (this.finished) return this.finish() + while (this.skip > 0) { + if (queue.length === 0) return this.needInput() + this.skip -= queue.takeSome(this.skip).length + } + if (this.meta !== undefined) { + const { size, typeflag } = this.meta + const total = size + padding(size) + if (queue.length < total) return this.needInput() + const data = queue.take(total).subarray(0, size) + this.meta = undefined + if (typeflag === "x") this.pax = parsePax(data) + else if (typeflag === "g") this.addGlobal(parsePax(data)) + else if (typeflag === "L") this.longName = readString(data, 0, size) + else this.longLink = readString(data, 0, size) + continue + } + if (queue.length < BLOCK) { + if (queue.length === 0 && this.ended) return this.finish() + return this.needInput() + } + this.at = queue.consumed + const block = queue.take(BLOCK) + if (isZeroBlock(block)) { + if (this.zeroBlock) return this.finish() + this.zeroBlock = true + continue + } + if (this.zeroBlock) { + throw invalidHeader("non-zero block after an end-of-archive block") + } + const raw = parseHeader(block) + switch (raw.typeflag) { + case "x": + case "g": + case "L": + case "K": { + if (raw.size > MAX_META_SIZE) { + throw invalidHeader(`extended header of ${raw.size} bytes exceeds the limit`) + } + this.meta = raw + continue + } + case "S": { + throw unsupported("GNU sparse files are not supported") + } + } + const header = this.toHeader(raw) + this.remaining = header.size + this.pad = padding(header.size) + return header + } + } + + // Returns the buffered body bytes of the entry from generation `id`, or + // `undefined` once the whole body has been read + read(id: number): Arr.NonEmptyArray | undefined | typeof NeedInput { + if (id !== this.generation) { + throw new TarError({ + kind: "BodyConsumed", + message: this.closed ? + "tar entry body read after the archive stream ended" : + "tar entry body read after moving to the next entry" + }) + } + if (this.remaining === 0) return undefined + if (this.queue.length === 0) return this.needInput() + const chunks = Arr.of(this.queue.takeSome(this.remaining)) + this.remaining -= chunks[0].length + while (this.remaining > 0 && this.queue.length > 0) { + const chunk = this.queue.takeSome(this.remaining) + this.remaining -= chunk.length + chunks.push(chunk) + } + return chunks + } + + private needInput(): typeof NeedInput { + if (this.ended) throw truncated(this.queue.consumed + this.queue.length) + return NeedInput + } + + private finish(): undefined | typeof NeedInput { + // An extended header must be followed by the entry it describes + if (!this.finished && (this.pax !== undefined || this.longName !== "" || this.longLink !== "")) { + throw truncated(this.queue.consumed) + } + this.finished = true + this.queue.clear() + return this.ended ? undefined : NeedInput + } + + // Global records apply to every entry that follows, except the ones that only + // describe a single entry, and an empty value removes a record + private addGlobal(records: Map): void { + for (const [key, value] of records) { + if (key === "path" || key === "linkpath" || key === "size") continue + if (value !== "") (this.global ??= new Map()).set(key, value) + else this.global?.delete(key) + } + if (this.global?.size === 0) this.global = undefined + } + + private toHeader(raw: RawHeader): Header { + // The records of the entry's own extended header override global ones + const records = this.global === undefined + ? this.pax + : this.pax === undefined + ? this.global + : new Map([...this.global, ...this.pax]) + const rest = records === undefined ? noPax : applyPax(raw, records) + // Empty GNU long names are ignored, as in Go + if (this.longName !== "") raw.path = this.longName + if (this.longLink !== "") raw.linkTarget = this.longLink + this.pax = undefined + this.longName = "" + this.longLink = "" + if (!(Math.abs(raw.mtimeMillis) <= MAX_DATE_MILLIS)) { + throw invalidHeader("mtime is out of range") + } + const type = typeOf(raw.typeflag, raw.path) + const header: Types.Mutable
= { + path: raw.path, + type, + typeflag: raw.typeflag, + size: isHeaderOnly(type) ? 0 : raw.size, + mode: raw.mode & PERMISSION_BITS, + uid: raw.uid, + gid: raw.gid, + uname: raw.uname, + gname: raw.gname, + mtime: new Date(raw.mtimeMillis), + pax: rest + } + if (raw.linkTarget !== "") header.linkTarget = raw.linkTarget + if (type === "CharDevice" || type === "BlockDevice") { + header.devMajor = raw.devMajor + header.devMinor = raw.devMinor + } + return header + } +} + +const SLAB_SIZE = 64 * 1024 + +const emptyBody = new Uint8Array(0) + +// Bodies below this size are copied next to their header instead of being +// handed out as chunks of their own +const COPY_LIMIT = 16 * 1024 + +// Pack headers, padding and small body chunks into slabs; pass large chunks +// through without copying. Flushed slab regions are never overwritten. +class Encoder { + private slab = new Uint8Array(0) + // Start of the bytes in `slab` that have not been flushed, and their end + private start = 0 + private offset = 0 + private chunks: Array = [] + buffered = 0 + // The entry whose body is being written, and the body bytes written so far + private header: Header | undefined + private written = 0 + + begin(header: Header): void { + encodeHeader(header, this.alloc) + this.header = header + this.written = 0 + } + + body(chunk: Uint8Array): void { + const header = this.header! + this.written += chunk.length + if (this.written > header.size) { + throw sizeMismatch(header, this.written) + } + if (chunk.length < COPY_LIMIT) { + // `reserve` can start a new slab, so it must run before `slab` is read + const offset = this.reserve(chunk.length) + this.slab.set(chunk, offset) + return + } + this.cut() + this.chunks.push(chunk) + this.buffered += chunk.length + } + + end(): void { + const header = this.header! + this.header = undefined + if (this.written !== header.size) { + throw sizeMismatch(header, this.written) + } + this.reserve(padding(header.size)) + } + + finish(): void { + this.reserve(BLOCK * 2) + } + + flush(): Arr.NonEmptyArray { + this.cut() + const chunks = this.chunks as Arr.NonEmptyArray + this.chunks = [] + this.buffered = 0 + return chunks + } + + // Ends the current run of slab bytes as a chunk + private cut(): void { + if (this.offset > this.start) { + this.chunks.push(this.slab.subarray(this.start, this.offset)) + this.start = this.offset + } + } + + // Returns `size` zeroed bytes of the output + private alloc = (size: number): Uint8Array => { + const offset = this.reserve(size) + return this.slab.subarray(offset, offset + size) + } + + // Reserves `size` zeroed bytes of the output in `slab`, and returns their offset + private reserve(size: number): number { + if (this.offset + size > this.slab.length) { + this.cut() + this.slab = new Uint8Array(Math.max(SLAB_SIZE, size)) + this.start = 0 + this.offset = 0 + } + const offset = this.offset + this.offset += size + this.buffered += size + return offset + } +} diff --git a/repos/effect/packages/effect/src/encoding/Toml.ts b/repos/effect/packages/effect/src/encoding/Toml.ts index b95d711e10..f9ff653b15 100644 --- a/repos/effect/packages/effect/src/encoding/Toml.ts +++ b/repos/effect/packages/effect/src/encoding/Toml.ts @@ -169,9 +169,9 @@ class TomlParser { this.advance() } key = this.input.slice(start, this.index) - } - if (key.length === 0) { - this.fail("Expected a key") + if (key.length === 0) { + this.fail("Expected a key") + } } keys.push(key) this.skipInlineWhitespace() diff --git a/repos/effect/packages/effect/src/encoding/Xml.ts b/repos/effect/packages/effect/src/encoding/Xml.ts new file mode 100644 index 0000000000..3c55b738d6 --- /dev/null +++ b/repos/effect/packages/effect/src/encoding/Xml.ts @@ -0,0 +1,2037 @@ +/** + * Decodes and encodes XML 1.0 documents as streams of events in Effect + * channels. + * + * `decode` and `decodeString` turn UTF-8 bytes or text into events, and + * `encode` and `encodeString` write events back as XML. `elements` collects + * repeated elements, such as the items of a feed, into small trees, so that a + * large document can be processed one record at a time. + * + * The decoder checks that documents are well-formed. It never reads external + * resources and never expands entities declared in a DOCTYPE, so external + * entity (XXE) and entity expansion attacks do not apply. It does not validate + * documents against a DTD or schema, and reports names as written, without + * resolving namespaces. + * + * @stability experimental + * @since 4.0.6 + */ +import * as Arr from "../Array.ts" +import * as Cause from "../Cause.ts" +import * as Channel from "../Channel.ts" +import * as Data from "../Data.ts" +import * as Effect from "../Effect.ts" +import * as Predicate from "../Predicate.ts" +import * as Pull from "../Pull.ts" + +const XmlErrorTypeId = "~effect/encoding/Xml/XmlError" + +/** + * Error raised when XML cannot be decoded or encoded. + * + * **Details** + * + * The `kind` field identifies the failure: + * + * - `Syntax`: the input is not well-formed XML or not valid UTF-8. + * - `Unsupported`: the input uses a feature this module does not process, such + * as an entity declared in a DOCTYPE or a byte encoding other than UTF-8. + * - `Limit`: the input exceeds `maxTokenSize`, `maxDepth` or `maxElementSize`. + * - `InvalidEvent`: the events do not form a well-formed document, such as an + * end tag that does not match its start tag. + * + * Errors from `decode` and `decodeString` carry the one-based `line` and + * `column` where the problem was found, also appended to `message`. Columns + * count UTF-16 code units, and lines end at LF, CR or CRLF. + * + * @stability experimental + * @category errors + * @since 4.0.6 + */ +export class XmlError extends Data.TaggedError("XmlError")<{ + readonly kind: "Syntax" | "Unsupported" | "Limit" | "InvalidEvent" + readonly message: string + readonly line?: number | undefined + readonly column?: number | undefined +}> { + /** + * Marks this value as an XML error for runtime guards. + * + * @stability experimental + * @since 4.0.6 + */ + readonly [XmlErrorTypeId] = XmlErrorTypeId +} + +/** + * Checks whether a value is an `XmlError`. + * + * @stability experimental + * @category guards + * @since 4.0.6 + */ +export const isXmlError = (u: unknown): u is XmlError => Predicate.hasProperty(u, XmlErrorTypeId) + +/** + * Event for the XML declaration at the start of a document, such as + * ``. + * + * **Details** + * + * `encoding` and `standalone` are present only when the declaration sets them. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export interface Declaration { + readonly _tag: "Declaration" + readonly version: string + readonly encoding?: string | undefined + readonly standalone?: boolean | undefined +} + +/** + * Event for a document type declaration, such as ``. + * + * **Details** + * + * `publicId` and `systemId` are present only when the declaration names an + * external DTD; the DTD is never read. Whitespace in `publicId` is collapsed + * to single spaces and trimmed. The internal subset is not reported. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export interface Doctype { + readonly _tag: "Doctype" + readonly name: string + readonly publicId?: string | undefined + readonly systemId?: string | undefined +} + +/** + * Event for a start tag, with its attributes in document order. + * + * **Details** + * + * `name` is the qualified name as written and is the only element-name field + * the encoder reads. + * + * Attribute values have their references replaced and whitespace normalized as + * XML requires. Decoded attributes inherit nothing, so only the attributes of + * the tag are present and `__proto__` is an ordinary key. Their key order is + * document order, as XML names are never integer-like. A `structuredClone` of + * a decoded map gets `Object.prototype` back. A self-closing tag such as + * `
` is decoded as a `StartElement` followed by an `EndElement`. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export interface StartElement { + readonly _tag: "StartElement" + readonly name: string + readonly attributes: Readonly> +} + +/** + * Event for an end tag. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export interface EndElement { + readonly _tag: "EndElement" + readonly name: string +} + +/** + * Event for character data inside the root element. + * + * **Details** + * + * Adjacent text, references and CDATA sections are decoded into one `Text` + * event that ends at the next tag, comment or processing instruction, however + * the input is chunked. Line endings are normalized to `\n`. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export interface Text { + readonly _tag: "Text" + readonly text: string +} + +/** + * Event for a comment, holding the text between ``. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export interface Comment { + readonly _tag: "Comment" + readonly text: string +} + +/** + * Event for a processing instruction, such as + * ``. + * + * **Details** + * + * `data` is the text after the target, without leading whitespace. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export interface ProcessingInstruction { + readonly _tag: "ProcessingInstruction" + readonly target: string + readonly data: string +} + +/** + * Event produced by the decoders and consumed by the encoders. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export type Event = + | Declaration + | Doctype + | StartElement + | EndElement + | Text + | Comment + | ProcessingInstruction + +/** + * Element collected by `elements`, with its attributes and children. + * + * **Details** + * + * Children are elements and strings of character data, in document order. + * Adjacent text is merged into one string; comments and processing + * instructions are dropped. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export interface Element { + readonly name: string + readonly attributes: Readonly> + readonly children: ReadonlyArray +} + +/** + * Options for the XML decoders. + * + * **Details** + * + * - `maxTokenSize`: the maximum length of a single token in UTF-16 code units, + * as written in the input: a tag with its attributes, a comment, a processing + * instruction, a DOCTYPE, or a text node including its references and CDATA + * sections. Defaults to 10 MiB; `Infinity` disables the limit. + * - `maxDepth`: the maximum number of nested elements. Defaults to 1024; + * `Infinity` disables the limit. + * + * Memory per token and for open element names is bounded by `maxTokenSize`. + * Invalid options throw when the channel is created. + * + * @stability experimental + * @category configuration + * @since 4.0.6 + */ +export interface DecodeOptions { + readonly maxTokenSize?: number | undefined + readonly maxDepth?: number | undefined +} + +/** + * Options for `elements`. + * + * **Details** + * + * `maxElementSize` is the maximum size of one collected element, counted as + * the UTF-16 code units of its names, attribute names and values, and text. + * Defaults to 16 MiB; `Infinity` disables the limit. Invalid options throw when + * the channel is created. + * + * @stability experimental + * @category configuration + * @since 4.0.6 + */ +export interface ElementsOptions { + readonly maxElementSize?: number | undefined +} + +/** + * Creates a channel that decodes XML text chunks into events. + * + * **Details** + * + * - Tokens can be split anywhere between chunks. Each output chunk holds the + * events completed by one input chunk. + * - The document must have one root element. Whitespace outside it is not + * reported. + * - A leading byte order mark is skipped, and the encoding named by the XML + * declaration is ignored. + * - Events completed before an error are emitted before the channel fails. + * + * **Gotchas** + * + * - End of input inside the root fails with a `Syntax` error ("unclosed + * element"); interruption ends the stream without this check. + * - A DOCTYPE is reported as a `Doctype` event, and its internal subset is + * skipped without checking its declarations. They are not applied, so + * default attribute values are not added, and parameter entities are not + * expanded. + * - References to entities other than `<`, `>`, `&`, `'` and + * `"` fail with an `Unsupported` `XmlError` when the document has a + * DOCTYPE, and with a `Syntax` `XmlError` otherwise. Character references + * such as `é` are always decoded. + * - Names are reported as written, including namespace prefixes, and `xmlns` + * declarations are ordinary attributes. + * - Tokens or aggregate open element names longer than `maxTokenSize`, and + * elements nested deeper than `maxDepth`, fail with a `Limit` `XmlError`. A + * long token can instead fail with an error found in its first + * `maxTokenSize + 1` characters. + * - Strings in events can share memory with their input chunk, so keeping a + * short string, such as an id from each record, can keep the whole chunk in + * memory. Copy strings that are kept long, for example with + * `structuredClone`. + * + * **Example** (Decoding events) + * + * ```ts import.meta.vitest + * import { Effect, Stream } from "effect" + * import { Xml } from "effect/encoding" + * + * const events = Stream.make("Hi & bye
").pipe( + * Stream.pipeThroughChannel(Xml.decodeString()), + * Stream.map((event) => event._tag), + * Stream.runCollect + * ) + * await Effect.runPromise(events) // => ["StartElement", "Text", "StartElement", "EndElement", "EndElement"] + * ``` + * + * @see {@link decode} for UTF-8 byte input + * @see {@link elements} for collecting repeated elements into trees + * @see {@link encodeString} for the inverse operation + * + * @stability experimental + * @category decoding + * @since 4.0.6 + */ +export const decodeString = (options?: DecodeOptions): Channel.Channel< + Arr.NonEmptyReadonlyArray, + XmlError | IE, + Done, + Arr.NonEmptyReadonlyArray, + IE, + Done +> => { + const parser = makeParser(options, false) + return makeChannel(() => new TextInput(parser())) +} + +/** + * Creates a channel that decodes UTF-8 XML bytes into events. + * + * **Details** + * + * Works like `decodeString` after decoding the bytes as UTF-8. Multi-byte + * characters can be split between chunks, and a leading byte order mark is + * skipped. + * + * **Gotchas** + * + * - Invalid or truncated UTF-8 fails with a `Syntax` `XmlError`. + * - An initial UTF-16 byte order mark or an XML declaration naming an encoding + * other than UTF-8 or US-ASCII fails with an `Unsupported` `XmlError`. + * Decode such bytes with a `TextDecoder` for that encoding and use `decodeString`. + * + * @see {@link decodeString} for text input and the shared gotchas + * + * @stability experimental + * @category decoding + * @since 4.0.6 + */ +export const decode = (options?: DecodeOptions): Channel.Channel< + Arr.NonEmptyReadonlyArray, + XmlError | IE, + Done, + Arr.NonEmptyReadonlyArray, + IE, + Done +> => { + const parser = makeParser(options, true) + return makeChannel(() => new Utf8Input(parser())) +} + +/** + * Creates a channel that collects the elements at a path into trees. + * + * **When to use** + * + * Use to process large documents such as feeds, sitemaps or exports one record + * at a time, without holding the whole document in memory. + * + * **Details** + * + * `path` lists element names from the root, and `"*"` matches any name. Each + * element whose ancestors and own name match the path is emitted once its end + * tag arrives, with all of its descendants; everything outside the matched + * elements is skipped. Because paths start at the root, an element nested in a + * matched element is part of its tree and is not emitted separately. + * + * **Gotchas** + * + * - A path that does not start at the root element matches nothing. + * - Whitespace between child elements, such as indentation, is kept as string + * children. + * - A collected element larger than `maxElementSize` fails with a `Limit` + * `XmlError`. The size counts characters, not memory: an element with many + * small children can take many times its size in memory, so use a lower + * limit for untrusted input. + * - Events that end inside an element, or end an element that was never + * started, fail with an `InvalidEvent` `XmlError`. + * - A path that is empty or contains an invalid name throws when the channel is + * created. + * + * **Example** (Reading feed items) + * + * ```ts import.meta.vitest + * import { Effect, Stream } from "effect" + * import { Xml } from "effect/encoding" + * + * const feed = `News + * First + * Second + * ` + * + * const items = await Effect.runPromise( + * Stream.make(feed).pipe( + * Stream.pipeThroughChannel(Xml.decodeString()), + * Stream.pipeThroughChannel(Xml.elements(["rss", "channel", "item"])), + * Stream.runCollect + * ) + * ) + * items.length // => 2 + * items[1] // => { name: "item", attributes: {}, children: [{ name: "title", attributes: {}, children: ["Second"] }] } + * ``` + * + * @see {@link decode} for producing events from bytes + * + * @stability experimental + * @category decoding + * @since 4.0.6 + */ +export const elements = ( + path: ReadonlyArray, + options?: ElementsOptions +): Channel.Channel< + Arr.NonEmptyReadonlyArray, + XmlError | IE, + Done, + Arr.NonEmptyReadonlyArray, + IE, + Done +> => { + if (path.length === 0) { + throw new Error("Xml: elements path must not be empty") + } + for (const name of path) { + if (name !== "*" && !isName(name)) { + throw new Error(`Xml: invalid element name in path: ${JSON.stringify(name)}`) + } + } + const maxElementSize = checkLimit("maxElementSize", options?.maxElementSize, defaultMaxElementSize) + const segments = Array.from(path) + return makeChannel(() => new Collector(segments, maxElementSize)) +} + +/** + * Creates a channel that writes events as XML text. + * + * **Details** + * + * - Each input chunk is written as one string. + * - Text escapes `&`, `<` and `>`. Attribute values are double-quoted and + * escape `&`, `<`, `"`, tabs and line breaks. Carriage returns are written + * as ` ` so that they survive decoding. + * - Start tags are written completely on their event. Empty elements use + * explicit closing tags, such as `

`, never self-closing tags. + * - Nothing is added between events, so whitespace comes only from `Text` + * events. + * + * **Gotchas** + * + * Events that do not form one well-formed document fail with an + * `InvalidEvent` `XmlError`, such as an end tag that does not match the open + * element, a second root element, non-whitespace text outside the root + * element, invalid names or characters, `--` in a comment, `?>` in processing + * instruction data, or a declaration that is not the first event. The stream + * must end with the root element closed. + * + * **Example** (Writing a document) + * + * ```ts import.meta.vitest + * import { Effect, Stream } from "effect" + * import { Xml } from "effect/encoding" + * + * const events: ReadonlyArray = [ + * { _tag: "StartElement", name: "a", attributes: { href: "x?a=1&b=2" } }, + * { _tag: "Text", text: "1 < 2" }, + * { _tag: "EndElement", name: "a" } + * ] + * + * const xml = Stream.fromIterable(events).pipe( + * Stream.pipeThroughChannel(Xml.encodeString()), + * Stream.mkString + * ) + * await Effect.runPromise(xml) // => '1 < 2' + * ``` + * + * @see {@link encode} for UTF-8 byte output + * @see {@link decodeString} for the inverse operation + * + * @stability experimental + * @category encoding + * @since 4.0.6 + */ +export const encodeString = (): Channel.Channel< + Arr.NonEmptyReadonlyArray, + XmlError | IE, + Done, + Arr.NonEmptyReadonlyArray, + IE, + Done +> => makeChannel(() => new Writer(false)) + +/** + * Creates a channel that writes events as UTF-8 XML bytes. + * + * **Details** + * + * Works like `encodeString`, with one byte array per input chunk. + * + * **Gotchas** + * + * A `Declaration` event naming an encoding other than UTF-8 fails + * with an `InvalidEvent` `XmlError`. + * + * @see {@link encodeString} for text output and the shared gotchas + * + * @stability experimental + * @category encoding + * @since 4.0.6 + */ +export const encode = (): Channel.Channel< + Arr.NonEmptyReadonlyArray, + XmlError | IE, + Done, + Arr.NonEmptyReadonlyArray, + IE, + Done +> => + Channel.map(makeChannel(() => new Writer(true)), Arr.map((text) => textEncoder.encode(text))) + +const defaultMaxTokenSize = 10 * 1024 * 1024 +const defaultMaxDepth = 1024 +const defaultMaxElementSize = 16 * 1024 * 1024 + +const checkLimit = (name: string, value: number | undefined, defaultValue: number): number => { + const limit = value ?? defaultValue + if (!(limit > 0)) { + throw new Error(`Xml: ${name} must be positive, got ${limit}`) + } + return limit +} + +// The per-run state of a channel. `push` and `end` add output to `out`, and +// throw `XmlError`s. +interface Machine { + out: Array + push(input: In): void + end(): void +} + +const makeChannel = ( + make: () => Machine +): Channel.Channel< + Arr.NonEmptyReadonlyArray, + XmlError | IE, + Done, + Arr.NonEmptyReadonlyArray, + IE, + Done +> => + Channel.fromTransform((upstream: Pull.Pull, IE, Done>, _scope) => + Effect.sync(() => { + const machine = make() + let done: Pull.Pull | undefined + let failure: XmlError | undefined + + // Output completed before an error is emitted before the failure + const emit = ( + inputs: ReadonlyArray | undefined + ): Pull.Pull, XmlError | IE, Done> => { + try { + if (inputs === undefined) machine.end() + else for (let i = 0; i < inputs.length; i++) machine.push(inputs[i]) + } catch (error) { + if (!isXmlError(error)) throw error + failure = error + } + const out = machine.out + if (Arr.isArrayNonEmpty(out)) { + machine.out = [] + return Effect.succeed(out) + } + return pull + } + + const step = Pull.matchEffect(upstream, { + onSuccess: emit, + onFailure: Effect.failCause, + onDone: (value) => { + done = Cause.done(value) + return emit(undefined) + } + }) + const pull: Pull.Pull, XmlError | IE, Done> = Effect.suspend(() => + failure !== undefined ? Effect.fail(failure) : done ?? step + ) + return pull + }) + ) + +const makeParser = (options: DecodeOptions | undefined, bytes: boolean): () => Parser => { + const maxTokenSize = checkLimit("maxTokenSize", options?.maxTokenSize, defaultMaxTokenSize) + const maxDepth = checkLimit("maxDepth", options?.maxDepth, defaultMaxDepth) + return () => new Parser(maxTokenSize, maxDepth, bytes) +} + +const textEncoder = new TextEncoder() + +const TAB = 9 +const LF = 10 +const CR = 13 +const SPACE = 32 +const BANG = 33 +const DQUOTE = 34 +const HASH = 35 +const AMPERSAND = 38 +const SQUOTE = 39 +const DASH = 45 +const SLASH = 47 +const LT = 60 +const EQUALS = 61 +const GT = 62 +const QUESTION = 63 +const LBRACKET = 91 +const RBRACKET = 93 +const LOWER_X = 120 + +const isSpace = (c: number): boolean => c === SPACE || c === LF || c === TAB || c === CR + +// Characters outside the XML `Char` production, including unpaired surrogates +// oxlint-disable-next-line no-control-regex +const invalidChar = /[\0-\x08\x0B\x0C\x0E-\x1F\uFFFE\uFFFF\uD800-\uDFFF]/u + +// Decoded UTF-8 has no unpaired surrogates, and without the `u` flag the +// search runs several times faster on non-ASCII text +// oxlint-disable-next-line no-control-regex +const invalidDecodedChar = /[\0-\x08\x0B\x0C\x0E-\x1F\uFFFE\uFFFF]/ + +// `isWellFormed` (ES2024) finds unpaired surrogates much faster than a `u` +// regex, which is slow on non-ASCII text +const isWellFormed: ((s: string) => boolean) | undefined = + typeof (String.prototype as { isWellFormed?: unknown }).isWellFormed === "function" ? + (s) => (s as unknown as { isWellFormed(): boolean }).isWellFormed() : + undefined + +// Returns the index of the first character outside the XML `Char` production, +// or -1 +const searchInvalid = (s: string): number => + isWellFormed !== undefined && s.search(invalidDecodedChar) === -1 && isWellFormed(s) ? -1 : s.search(invalidChar) + +const isChar = (c: number): boolean => + c === TAB || c === LF || c === CR || (c >= 0x20 && c <= 0xD7FF) || (c >= 0xE000 && c <= 0xFFFD) || + (c >= 0x10000 && c <= 0x10FFFF) + +const NAME_START = 1 +const NAME_CHAR = 2 +const asciiName = new Uint8Array(128) +for (let c = 0; c < 128; c++) { + if ((c >= 65 && c <= 90) || (c >= 97 && c <= 122) || c === 95 || c === 58) asciiName[c] = NAME_START | NAME_CHAR + else if ((c >= 48 && c <= 57) || c === DASH || c === 46) asciiName[c] = NAME_CHAR +} + +// Name characters of XML 1.0 fifth edition. Supplementary characters are +// accepted by their surrogates, which are paired once input is validated. +const isNameStartCode = (c: number): boolean => + c < 128 ? + (asciiName[c] & NAME_START) !== 0 : + (c >= 0xC0 && c <= 0xD6) || (c >= 0xD8 && c <= 0xF6) || (c >= 0xF8 && c <= 0x2FF) || + (c >= 0x370 && c <= 0x37D) || (c >= 0x37F && c <= 0x1FFF) || (c >= 0x200C && c <= 0x200D) || + (c >= 0x2070 && c <= 0x218F) || (c >= 0x2C00 && c <= 0x2FEF) || (c >= 0x3001 && c <= 0xD7FF) || + (c >= 0xF900 && c <= 0xFDCF) || (c >= 0xFDF0 && c <= 0xFFFD) || (c >= 0xD800 && c <= 0xDB7F) + +const isNameCode = (c: number): boolean => + c < 128 ? + (asciiName[c] & NAME_CHAR) !== 0 : + isNameStartCode(c) || c === 0xB7 || (c >= 0x300 && c <= 0x36F) || (c >= 0x203F && c <= 0x2040) || + (c >= 0xDC00 && c <= 0xDFFF) + +// Returns the end of the name starting at `i`, or `i` when there is none +const scanName = (s: string, i: number, end: number): number => { + if (i >= end || !isNameStartCode(s.charCodeAt(i))) return i + i++ + while (i < end && isNameCode(s.charCodeAt(i))) i++ + return i +} + +// Name characters exclude every invalid character but unpaired surrogates +const isName = (s: string): boolean => + s.length > 0 && scanName(s, 0, s.length) === s.length && + (isWellFormed !== undefined ? isWellFormed(s) : !invalidChar.test(s)) + +const skipSpace = (s: string, i: number, end: number): number => { + while (i < end && isSpace(s.charCodeAt(i))) i++ + return i +} + +const lineEndings = /\r\n?/g +const attributeWhitespace = /\r\n|[\t\n\r]/g +// Characters of an attribute value that need decoding, or are not allowed +const attributeSpecial = /[&<\t\n\r]/ +const attributeWhitespaceChar = /[\t\n\r]/ +// Attribute values shorter than this are scanned with a loop +const shortValue = 24 +const publicIdWhitespace = /[ \r\n]+/g +const decimalDigits = /^[0-9]+$/ +const hexDigits = /^[0-9a-fA-F]+$/ +const versionPattern = /^1\.[0-9]+$/ +const encodingPattern = /^[A-Za-z][A-Za-z0-9._-]*$/ + +const normalizeLines = (s: string): string => s.indexOf("\r") === -1 ? s : s.replace(lineEndings, "\n") + +// Normalizes the line endings and whitespace of a literal attribute value part +const normalizeAttribute = (s: string): string => s.replace(attributeWhitespace, " ") + +const isReservedTarget = (target: string): boolean => target.length === 3 && target.toLowerCase() === "xml" + +const isUtf8 = (encoding: string): boolean => { + const lower = encoding.toLowerCase() + return lower === "utf-8" || lower === "us-ascii" +} + +const declarationPattern = + /^[ \t\r\n]+version[ \t\r\n]*=[ \t\r\n]*(?:"(1\.[0-9]+)"|'(1\.[0-9]+)')(?:[ \t\r\n]+encoding[ \t\r\n]*=[ \t\r\n]*(?:"([A-Za-z][A-Za-z0-9._-]*)"|'([A-Za-z][A-Za-z0-9._-]*)'))?(?:[ \t\r\n]+standalone[ \t\r\n]*=[ \t\r\n]*(?:"(yes|no)"|'(yes|no)'))?[ \t\r\n]*$/ + +const pubidPattern = /^[ \r\na-zA-Z0-9\-'()+,./:=?;!*#@$_%]*$/ + +// States of `Scanner`, which finds where a token split between chunks ends +const S_START = 0 +const S_TEXT = 1 +const S_LT = 2 // after "<" +const S_BANG = 3 // after "= 2) { + if (this.state === S_COMMENT) return i + 1 + this.reset(S_SUBSET) + } else { + this.aux = 0 + } + break + } + case S_CDATA: { + if (c === RBRACKET) this.aux++ + else if (c === GT && this.aux >= 2) return i + 1 + else this.aux = 0 + break + } + case S_PI: + case S_SUBSET_PI: { + if (c === GT && this.aux === 1) { + if (this.state === S_PI) return i + 1 + this.reset(S_SUBSET) + } else { + this.aux = c === QUESTION ? 1 : 0 + } + break + } + case S_TAG: { + // Attribute values cannot contain "<" + if (c === LT) return i + if (this.aux === 0) { + if (c === GT) return i + 1 + if (c === DQUOTE || c === SQUOTE) this.aux = c + } else if (c === this.aux) { + this.aux = 0 + } + break + } + case S_DOCTYPE: + case S_DECLARATION: { + if (this.aux === 0) { + if (c === LT) return i + if (c === GT) { + if (this.state === S_DOCTYPE) return i + 1 + this.reset(S_SUBSET) + } else if (c === LBRACKET && this.state === S_DOCTYPE) { + this.reset(S_SUBSET) + } else if (c === DQUOTE || c === SQUOTE) { + this.aux = c + } + } else if (c === this.aux) { + this.aux = 0 + } + break + } + case S_SUBSET: { + if (c === RBRACKET) this.state = S_AFTER_SUBSET + else if (c === LT) this.state = S_SUBSET_LT + else if (c === DQUOTE || c === SQUOTE || c === GT) return i + break + } + case S_SUBSET_LT: { + if (c === BANG) this.state = S_SUBSET_BANG + else if (c === QUESTION) this.reset(S_SUBSET_PI) + else return i + break + } + case S_SUBSET_BANG: { + if (c === DASH) { + this.state = S_SUBSET_BANG_DASH + } else { + this.reset(S_DECLARATION) + continue + } + break + } + case S_SUBSET_BANG_DASH: { + if (c !== DASH) return i + this.reset(S_SUBSET_COMMENT) + break + } + case S_AFTER_SUBSET: { + if (c === GT) return i + 1 + if (!isSpace(c)) return i + break + } + } + i++ + } + return -1 + } +} + +const PROLOG = 0 +const CONTENT = 1 +const EPILOG = 2 + +const describeToken = (s: string, i: number): string => + s.startsWith("", i + 4) + if (end === -1) return -1 + const dashes = s.indexOf("--", i + 4) + if (dashes < end) throw this.syntax("\"--\" is not allowed in comments", dashes) + this.checkSize(i, end + 3) + this.flushText() + this.out.push({ _tag: "Comment", text: normalizeLines(s.slice(i + 4, end)) }) + return end + 3 + } + + private processingInstruction(s: string, i: number): number { + const end = s.indexOf("?>", i + 2) + if (end === -1) return -1 + const t = scanName(s, i + 2, end) + if (t === i + 2) throw this.syntax("invalid processing instruction target", i + 2) + const target = s.slice(i + 2, t) + if (isReservedTarget(target)) { + if (target === "xml" && this.base + i === 0) return this.declaration(s, i, t, end) + throw this.syntax( + target === "xml" ? + "the XML declaration is only allowed at the start of the document" : + `processing instruction target ${target} is reserved`, + i + ) + } + let data = "" + if (t < end) { + if (!isSpace(s.charCodeAt(t))) throw this.syntax("expected whitespace after the target", t) + data = normalizeLines(s.slice(skipSpace(s, t, end), end)) + } + this.checkSize(i, end + 2) + this.flushText() + this.out.push({ _tag: "ProcessingInstruction", target, data }) + return end + 2 + } + + private declaration(s: string, i: number, t: number, end: number): number { + this.checkSize(i, end + 2) + const match = declarationPattern.exec(s.slice(t, end)) + if (match === null) throw this.syntax("invalid XML declaration", i) + const encoding = match[3] ?? match[4] + const standalone = match[5] ?? match[6] + if (this.bytes && encoding !== undefined && !isUtf8(encoding)) { + throw this.error( + "Unsupported", + `encoding ${encoding} is not supported for byte input, decode the bytes to text first`, + this.base + i + ) + } + this.ascii = this.bytes && encoding?.toLowerCase() === "us-ascii" + const event: { -readonly [K in keyof Declaration]: Declaration[K] } = { + _tag: "Declaration", + version: match[1] ?? match[2] + } + if (encoding !== undefined) event.encoding = encoding + if (standalone !== undefined) event.standalone = standalone === "yes" + this.out.push(event) + return end + 2 + } + + private cdata(s: string, i: number): number { + const end = s.indexOf("]]>", i + 9) + if (end === -1) return -1 + if (this.state !== CONTENT) throw this.syntax("CDATA sections are only allowed inside the root element", i) + this.addText(i, end + 3 - i) + this.text += normalizeLines(s.slice(i + 9, end)) + return end + 3 + } + + private doctypeDeclaration(s: string, i: number): number { + if (this.state !== PROLOG || this.doctype) { + throw this.syntax("a DOCTYPE is only allowed once, before the root element", i) + } + const scanner = new Scanner() + scanner.reset(S_DOCTYPE) + const end = scanner.scan(s, i + 9) + if (end === -1) return -1 + this.checkSize(i, end) + this.doctype = true + let j = this.space(s, i + 9, end) + const nameEnd = this.name(s, j, end) + const event: { -readonly [K in keyof Doctype]: Doctype[K] } = { _tag: "Doctype", name: s.slice(j, nameEnd) } + j = skipSpace(s, nameEnd, end) + if (j > nameEnd && (s.startsWith("PUBLIC", j) || s.startsWith("SYSTEM", j))) { + if (s.charCodeAt(j) === 80) { + // Whitespace in public identifiers is normalized for matching + event.publicId = this.publicLiteral(s, this.space(s, j + 6, end), end).replace(publicIdWhitespace, " ").trim() + j = this.literalEnd + } + event.systemId = normalizeLines( + this.literal(s, this.space(s, j + (event.publicId === undefined ? 6 : 0), end), end) + ) + j = skipSpace(s, this.literalEnd, end) + } + if (s.charCodeAt(j) === LBRACKET) { + // The scanner skips the internal subset, whose declarations are not + // checked, and ends the token early only at malformed markup + if (scanner.state !== S_AFTER_SUBSET) throw this.syntax("invalid markup in the internal subset", end) + j = skipSpace(s, s.lastIndexOf("]", end - 1) + 1, end) + } + if (j !== end - 1 || s.charCodeAt(j) !== GT) throw this.syntax("invalid DOCTYPE", j) + this.out.push(event) + return end + } + + // Skips the whitespace that is required at `i` + private space(s: string, i: number, end: number): number { + if (i >= end || !isSpace(s.charCodeAt(i))) throw this.syntax("expected whitespace", i) + return skipSpace(s, i, end) + } + + // Returns the end of the name that is required at `i` + private name(s: string, i: number, end: number): number { + const nameEnd = scanName(s, i, end) + if (nameEnd === i) throw this.syntax("expected a name", i) + return nameEnd + } + + // Reads the quoted literal at `i`, and sets `literalEnd` to the index after it + private literal(s: string, i: number, end: number): string { + const quote = s.charCodeAt(i) + if (quote !== DQUOTE && quote !== SQUOTE) throw this.syntax("expected a quoted literal", i) + const close = s.indexOf(quote === DQUOTE ? "\"" : "'", i + 1) + // Unreachable while the scanner ends a DOCTYPE only after its literals close + if (close === -1 || close >= end) throw this.syntax("unterminated literal", i) + this.literalEnd = close + 1 + return s.slice(i + 1, close) + } + + private publicLiteral(s: string, i: number, end: number): string { + const literal = this.literal(s, i, end) + if (!pubidPattern.test(literal)) throw this.syntax("invalid public identifier", i) + return literal + } +} + +const indexOrEnd = (s: string, search: string, from: number): number => { + const i = s.indexOf(search, from) + return i === -1 ? s.length : i +} + +class TextInput implements Machine { + private readonly parser: Parser + // A high surrogate at the end of a chunk, which pairs with the next chunk + private surrogate = "" + + constructor(parser: Parser) { + this.parser = parser + } + + get out(): Array { + return this.parser.out + } + set out(out: Array) { + this.parser.out = out + } + + push(chunk: string): void { + if (this.surrogate !== "") { + chunk = this.surrogate + chunk + this.surrogate = "" + } + const last = chunk.charCodeAt(chunk.length - 1) + if (last >= 0xD800 && last <= 0xDBFF) { + this.surrogate = chunk.slice(-1) + chunk = chunk.slice(0, -1) + } + this.parser.feed(chunk) + } + + end(): void { + // An unpaired high surrogate fails as an invalid character + this.parser.feed(this.surrogate) + this.parser.end() + } +} + +const utf8Length = (lead: number): number => lead >= 0xF0 ? 4 : lead >= 0xE0 ? 3 : lead >= 0xC0 ? 2 : 1 + +// Returns the length of the valid UTF-8 prefix of `bytes` +const validUtf8 = (bytes: Uint8Array): number => { + let i = 0 + while (i < bytes.length) { + const lead = bytes[i] + if (lead < 0x80) { + i++ + continue + } + const length = lead >= 0xC2 && lead <= 0xF4 ? utf8Length(lead) : 0 + if (length === 0 || i + length > bytes.length) return i + let code = lead & (0xFF >> (length + 1)) + for (let k = 1; k < length; k++) { + const c = bytes[i + k] + if ((c & 0xC0) !== 0x80) return i + code = (code << 6) | (c & 0x3F) + } + const min = length === 2 ? 0x80 : length === 3 ? 0x800 : 0x10000 + if (code < min || code > 0x10FFFF || (code >= 0xD800 && code <= 0xDFFF)) return i + i += length + } + return i +} + +const utf8Decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }) + +// Decodes UTF-8 without streaming state, so that the position of invalid bytes +// is known: an incomplete sequence at the end of a chunk is carried over. +class Utf8Input implements Machine { + private readonly parser: Parser + private carry: Uint8Array | undefined + private started = false + + constructor(parser: Parser) { + this.parser = parser + } + + get out(): Array { + return this.parser.out + } + set out(out: Array) { + this.parser.out = out + } + + push(chunk: Uint8Array): void { + if (!this.started && chunk.length > 0) { + const first = this.carry === undefined ? chunk[0] : this.carry[0] + const second = this.carry === undefined ? chunk[1] : chunk[0] + if ((first === 0xFE && second === 0xFF) || (first === 0xFF && second === 0xFE)) { + throw this.parser.error( + "Unsupported", + "UTF-16 is not supported for byte input, decode the bytes to text first", + 0 + ) + } + // Keep a lone BOM prefix pending in the UTF-8 carry below + this.started = second !== undefined || (first !== 0xFE && first !== 0xFF) + } + let start = 0 + if (this.carry !== undefined) { + const need = utf8Length(this.carry[0]) - this.carry.length + const head = new Uint8Array(this.carry.length + Math.min(need, chunk.length)) + head.set(this.carry) + head.set(chunk.subarray(0, need), this.carry.length) + if (chunk.length < need) { + this.carry = head + return + } + this.carry = undefined + this.decode(head) + start = need + } + const n = chunk.length + let end = n + for (let k = 1; k <= 3 && n - k >= start; k++) { + const b = chunk[n - k] + if ((b & 0xC0) === 0x80) continue + if (b >= 0xC0 && k < utf8Length(b)) end = n - k + break + } + if (end < n) this.carry = chunk.slice(end) + if (end > start) this.decode(start === 0 && end === n ? chunk : chunk.subarray(start, end)) + } + + end(): void { + // The carried bytes may also be an invalid sequence, such as a single 0xFF + if (this.carry !== undefined) { + this.parser.feed("", true) + throw this.parser.error("Syntax", "invalid UTF-8", this.parser.received) + } + this.parser.end() + } + + private decode(bytes: Uint8Array): void { + let text: string + try { + text = utf8Decoder.decode(bytes) + } catch { + this.parser.feed(utf8Decoder.decode(bytes.subarray(0, validUtf8(bytes))), true) + throw this.parser.error("Syntax", "invalid UTF-8", this.parser.received) + } + this.parser.feed(text) + } +} + +interface MutableElement { + readonly name: string + readonly attributes: Readonly> + readonly children: Array +} + +class Collector implements Machine { + out: Array = [] + private depth = 0 + // Number of open elements, from the root, that match the path + private matched = 0 + // The element being collected and its open descendants + private readonly stack: Array = [] + private size = 0 + private readonly path: ReadonlyArray + private readonly maxElementSize: number + + constructor(path: ReadonlyArray, maxElementSize: number) { + this.path = path + this.maxElementSize = maxElementSize + } + + push(event: Event): void { + switch (event._tag) { + case "StartElement": { + const depth = this.depth++ + if (this.stack.length === 0) { + const segment = this.path[depth] + if (this.matched !== depth || (segment !== "*" && segment !== event.name)) return + if (++this.matched < this.path.length) return + this.size = 0 + } + const element: MutableElement = { name: event.name, attributes: event.attributes, children: [] } + if (this.stack.length > 0) this.stack[this.stack.length - 1].children.push(element) + this.stack.push(element) + let size = event.name.length + for (const key in event.attributes) size += key.length + event.attributes[key].length + this.grow(size) + return + } + case "EndElement": { + if (this.depth === 0) throw invalidEvent(`end tag without a start tag`) + if (this.stack.length > 0) { + const element = this.stack.pop()! + if (this.stack.length === 0) this.out.push(element) + } + if (this.matched === this.depth) this.matched-- + this.depth-- + return + } + case "Text": { + if (this.stack.length === 0 || event.text.length === 0) return + this.grow(event.text.length) + const children = this.stack[this.stack.length - 1].children + const last = children.length - 1 + if (last >= 0 && typeof children[last] === "string") children[last] += event.text + else children.push(event.text) + return + } + } + } + + end(): void { + if (this.depth > 0) throw invalidEvent("events ended inside an element") + } + + private grow(size: number): void { + this.size += size + if (this.size > this.maxElementSize) { + throw positionless("Limit", `element <${this.stack[0].name}> exceeds maxElementSize of ${this.maxElementSize}`) + } + } +} + +// JavaScriptCore gives every Error its own source `line` and `column`, which +// would pass for an XML position, so errors without one set both to undefined +const positionless = (kind: XmlError["kind"], message: string) => + new XmlError({ kind, message, line: undefined, column: undefined }) + +const invalidEvent = (message: string) => positionless("InvalidEvent", message) + +const textSpecial = /[&<>\r]/ +const attributeSpecials = /[&<"\t\n\r]/ + +// The regex finds the first special character, which is fast when there is +// none; the rest is copied in runs between escapes. Invalid characters are +// kept, so that the writer can check them in the output. +const escapeText = (s: string): string => { + const first = s.search(textSpecial) + if (first === -1) return s + let out = "" + let last = 0 + for (let i = first; i < s.length; i++) { + const c = s.charCodeAt(i) + const escaped = c === AMPERSAND ? "&" : c === LT ? "<" : c === GT ? ">" : c === CR ? " " : undefined + if (escaped !== undefined) { + out += s.slice(last, i) + escaped + last = i + 1 + } + } + return out + s.slice(last) +} + +const escapeAttribute = (s: string): string => { + const first = s.search(attributeSpecials) + if (first === -1) return s + let out = "" + let last = 0 + for (let i = first; i < s.length; i++) { + const c = s.charCodeAt(i) + const escaped = c === AMPERSAND ? + "&" : + c === LT ? + "<" : + c === DQUOTE ? + """ : + c === TAB ? + " " : + c === LF ? + " " : + c === CR ? + " " : + undefined + if (escaped !== undefined) { + out += s.slice(last, i) + escaped + last = i + 1 + } + } + return out + s.slice(last) +} + +const checkChars = (what: string, s: string): void => { + const invalid = searchInvalid(s) + if (invalid !== -1) { + const code = s.codePointAt(invalid)!.toString(16).toUpperCase().padStart(4, "0") + throw invalidEvent(`${what} contains the invalid character U+${code}`) + } +} + +const checkName = (what: string, name: string): void => { + if (typeof name !== "string" || !isName(name)) throw invalidEvent(`invalid ${what} ${JSON.stringify(name)}`) +} + +const isWhitespace = (s: string): boolean => { + for (let i = 0; i < s.length; i++) { + if (!isSpace(s.charCodeAt(i))) return false + } + return true +} + +const NOTHING_WRITTEN = 3 + +class Writer implements Machine { + out: Array = [] + private readonly stack: Array = [] + private state = NOTHING_WRITTEN + private doctype = false + private readonly bytes: boolean + // Element and attribute names already checked, which repeat in most + // documents. Cleared when full, so that it stays small. + private readonly names = new Set() + + constructor(bytes: boolean) { + this.bytes = bytes + } + + private checkName(what: string, name: string): void { + if (!this.isName(name)) checkName(what, name) + } + + private isName(name: string): boolean { + if (this.names.has(name)) return true + if (typeof name !== "string" || !isName(name)) return false + if (this.names.size >= 1024) this.names.clear() + this.names.add(name) + return true + } + + // Checks attributes one by one, in order, after a batched check has failed + private checkAttributes(attributes: Readonly>): void { + for (const name in attributes) { + if (!Object.hasOwn(attributes, name)) continue + const value = attributes[name] + this.checkName("attribute name", name) + if (typeof value !== "string") throw invalidEvent(`attribute ${name} must be a string`) + checkChars(`attribute ${name}`, value) + } + } + + push(event: Event): void { + const text = this.write(event) + if (text.length > 0) { + const out = this.out + if (out.length > 0) out[0] += text + else out.push(text) + } + } + + end(): void { + if (this.stack.length > 0) throw invalidEvent(`unclosed element <${this.stack[this.stack.length - 1]}>`) + if (this.state !== EPILOG) throw invalidEvent("no root element") + } + + private write(event: Event): string { + if (event._tag === "EndElement") { + const open = this.stack.length > 0 ? this.stack[this.stack.length - 1] : undefined + if (open === undefined || event.name !== open) { + // Only valid names are written into messages + checkName("element name", event.name) + throw invalidEvent( + open === undefined + ? `end tag without a start tag` + : `end tag does not match <${open}>` + ) + } + this.stack.pop() + if (this.stack.length === 0) this.state = EPILOG + return `` + } + switch (event._tag) { + case "StartElement": { + this.checkName("element name", event.name) + if (this.state === EPILOG) throw invalidEvent(`second root element <${event.name}>`) + let tag = "<" + event.name + let valid = true + for (const name in event.attributes) { + if (!Object.hasOwn(event.attributes, name)) continue + const value = event.attributes[name] + if (typeof value !== "string" || !this.isName(name)) { + valid = false + break + } + tag += ` ${name}="${escapeAttribute(value)}"` + } + // Escaping keeps invalid characters, so one search of the tag checks + // every value; the first error is then found in attribute order + if (!valid || (tag.length > event.name.length + 1 && searchInvalid(tag) !== -1)) { + this.checkAttributes(event.attributes) + } + this.stack.push(event.name) + this.state = CONTENT + return tag + ">" + } + case "Text": { + if (this.state !== CONTENT) { + checkChars("text", event.text) + if (!isWhitespace(event.text)) throw invalidEvent("text outside the root element must be whitespace") + if (this.state === NOTHING_WRITTEN && event.text.length > 0) this.state = PROLOG + return event.text + } + checkChars("text", event.text) + return escapeText(event.text) + } + case "Comment": { + checkChars("comment", event.text) + if (event.text.includes("--") || event.text.endsWith("-")) { + throw invalidEvent("comments cannot contain \"--\" or end with \"-\"") + } + if (this.state === NOTHING_WRITTEN) this.state = PROLOG + return `` + } + case "ProcessingInstruction": { + checkName("processing instruction target", event.target) + if (isReservedTarget(event.target)) { + throw invalidEvent(`processing instruction target ${event.target} is reserved`) + } + checkChars("processing instruction", event.data) + if (event.data.includes("?>")) throw invalidEvent("processing instruction data cannot contain \"?>\"") + if (this.state === NOTHING_WRITTEN) this.state = PROLOG + return `` + } + case "Declaration": { + if (this.state !== NOTHING_WRITTEN) throw invalidEvent("the XML declaration must be the first event") + if (!versionPattern.test(event.version)) { + throw invalidEvent(`invalid XML version ${JSON.stringify(event.version)}`) + } + let declaration = `" + } + case "Doctype": { + if (this.state === CONTENT || this.state === EPILOG || this.doctype) { + throw invalidEvent("a DOCTYPE must appear once, before the root element") + } + checkName("DOCTYPE name", event.name) + let doctype = `" + } + } + } +} diff --git a/repos/effect/packages/effect/src/encoding/Yaml.ts b/repos/effect/packages/effect/src/encoding/Yaml.ts index e553d226bc..66de408429 100644 --- a/repos/effect/packages/effect/src/encoding/Yaml.ts +++ b/repos/effect/packages/effect/src/encoding/Yaml.ts @@ -4,9 +4,13 @@ * Supports YAML 1.2 block and flow collections, indentless sequence values, * multiline plain and quoted scalars, block scalars, anchors, and aliases. * + * Unlike YAML 1.2, the closing bracket of a multiline flow collection may sit + * at its parent's indentation. + * * Unsupported: compact nested block sequences (`- - value` on one line), - * multiline flow collections, explicit complex keys, tag/directive processing, - * recursive aliases, and document streams. Mapping keys are JavaScript strings. + * multiline quoted scalars in flow collections, explicit complex keys, + * tag/directive processing, recursive aliases, and document streams. Mapping + * keys are JavaScript strings. * * @stability unstable * @since 4.0.0 @@ -37,6 +41,7 @@ type Line = { readonly text: string readonly indent: number readonly number: number + readonly tabIndent: boolean } const hasOwn = Object.prototype.hasOwnProperty @@ -63,8 +68,8 @@ const closingQuote = (input: string, quote: string, start: number): number => { return -1 } -const findIndicator = (input: string, indicator: ":" | "#"): number => { - let depth = 0 +// `depth` is the number of flow collections already open before `input`. +const findIndicator = (input: string, indicator: ":" | "#", depth = 0): number => { let tokenStart = true for (let index = 0; index < input.length; index++) { const character = input[index] @@ -95,8 +100,8 @@ const findIndicator = (input: string, indicator: ":" | "#"): number => { return -1 } -const stripComment = (input: string): string => { - const index = findIndicator(input, "#") +const stripComment = (input: string, depth = 0): string => { + const index = findIndicator(input, "#", depth) return index === -1 ? input : input.slice(0, index).trimEnd() } @@ -106,6 +111,13 @@ const isSequenceEntry = (text: string): boolean => text === "-" || text.startsWi const foldLineBreak = (blankLines: number): string => blankLines === 0 ? " " : "\n".repeat(blankLines) +// Folds the line breaks of a multiline plain flow scalar. +const foldFlowLines = (input: string): string => + input.replace( + /[ \t]*\n((?:[ \t]*\n)*)[ \t]*/g, + (_, blankLines: string) => foldLineBreak(blankLines.split("\n").length - 1) + ) + const endsWithEscape = (text: string): boolean => /\\*$/.exec(text)![0].length % 2 === 1 // Escaped trailing spaces are content, not folding whitespace. @@ -205,13 +217,16 @@ const parseKey = (input: string): string => { } class FlowParser { - private readonly input: string - private readonly anchors: ReadonlyMap + private input: string + private readonly anchors: Map + private readonly nextLine: (depth: number) => string | undefined private index = 0 + private depth = 0 - constructor(input: string, anchors: ReadonlyMap) { + constructor(input: string, anchors: Map, nextLine: (depth: number) => string | undefined) { this.input = input this.anchors = anchors + this.nextLine = nextLine } parse(): unknown { @@ -226,8 +241,12 @@ class FlowParser { private parseValue(): unknown { this.skipWhitespace() const character = this.peek() - if (character === "[") return this.parseSequence() - if (character === "{") return this.parseMapping() + if (character === "[" || character === "{") { + this.depth++ + const collection = character === "[" ? this.parseSequence() : this.parseMapping() + this.depth-- + return collection + } if (character === "\"" || character === "'") return parseScalar(this.readQuoted(character)) if (character === "*") { this.index++ @@ -235,7 +254,17 @@ class FlowParser { if (!this.anchors.has(name)) this.fail(`Unknown alias '*${name}'`) return this.anchors.get(name) } - return parseScalar(this.readUntil(/[,\]}]/).trim()) + if (character === "&") { + this.index++ + const name = this.readUntil(/[\s,[\]{}]/) + if (name === "" || /[[{]/.test(this.peek())) this.fail("Invalid anchor") + this.skipWhitespace() + if (this.peek() === "&") this.fail("Multiple anchors on one node") + const value = this.parseValue() + this.anchors.set(name, value) + return value + } + return parseScalar(foldFlowLines(this.readUntil(/[,\]}]/)).trim()) } private parseSequence(): Array { @@ -296,7 +325,7 @@ class FlowParser { if (character === "\"" || character === "'") { return parseKey(this.readQuoted(character)) } - return this.readUntil(/:/).trim() + return foldFlowLines(this.readUntil(/:/)).trim() } private readQuoted(quote: string): string { @@ -309,7 +338,7 @@ class FlowParser { private readUntil(stop: RegExp): string { const start = this.index - while (this.index < this.input.length && !stop.test(this.peek())) this.index++ + while (!stop.test(this.peek()) && this.index < this.input.length) this.index++ return this.input.slice(start, this.index) } @@ -323,6 +352,11 @@ class FlowParser { } private peek(): string { + // Inside a collection, the end of the line continues on the next line. + if (this.index === this.input.length && this.depth > 0) { + const line = this.nextLine(this.depth) + if (line !== undefined) this.input += `\n${line}` + } return this.input[this.index] ?? "" } @@ -441,10 +475,24 @@ class YamlParser { if (!this.anchors.has(name)) throw new SyntaxError(`Unknown YAML alias '*${name}'`) return this.anchors.get(name) } - if (text.startsWith("[") || text.startsWith("{")) return new FlowParser(text, this.anchors).parse() + if (text.startsWith("[") || text.startsWith("{")) return this.parseFlowValue(text, parentIndent) return this.parsePlainValue(text, parentIndent) } + private parseFlowValue(text: string, parentIndent: number): unknown { + return new FlowParser(text, this.anchors, (depth) => { + const next = this.lines[this.index] + if (next === undefined || (next.indent === 0 && /^(?:---|\.\.\.)(?:\s|$)/.test(next.text))) return undefined + // Continuation lines are more indented than the parent; a closing bracket may sit at its indentation. + const content = next.text.trim().length > 0 + if (content && next.indent < parentIndent) return undefined + if (content && next.indent === parentIndent && !/^[\]}]/.test(next.text)) return undefined + this.index++ + // Strip comments again knowing the open collections, so quoted `#` after a comma is kept. + return stripComment(next.raw.slice(next.indent), depth) + }).parse() + } + private parsePlainValue(value: string, parentIndent: number): unknown { const first = this.lines[this.index - 1] if (/:(?:\s|$)/.test(value)) this.fail(first, "Invalid colon in plain scalar") @@ -454,12 +502,14 @@ class YamlParser { const blankLines = this.skipBlankLines() const next = this.lines[this.index] if (next === undefined || next.indent <= parentIndent) break + this.rejectTabIndent(next) // Quotes in plain continuations are literal. const content = next.raw.slice(next.indent) const comment = content.search(/(?:^|[ \t])#/) const text = (comment === -1 ? content : content.slice(0, comment)).trimEnd() if (text.length === 0 || (next.indent === 0 && /^(?:---|\.\.\.)(?:\s|$)/.test(text))) break if (/:(?:\s|$)/.test(text)) this.fail(next, "Invalid colon in plain scalar") + this.rejectTabIndentedBlankLines(blankLines, parentIndent) output += foldLineBreak(blankLines) + text this.index++ terminated = comment !== -1 @@ -483,10 +533,13 @@ class YamlParser { if (quote === "\"" && endsWithEscape(line)) { output += line.slice(0, -1) } else { - output += trimFoldedEnd(line, quote) + foldLineBreak(this.skipBlankLines()) + const blankLines = this.skipBlankLines() + this.rejectTabIndentedBlankLines(blankLines, parentIndent) + output += trimFoldedEnd(line, quote) + foldLineBreak(blankLines) } const next = this.lines[this.index++] if (next === undefined) throw new SyntaxError("Unterminated quoted YAML scalar") + this.rejectTabIndent(next) if (next.raw.trim().length > 0 && next.indent <= parentIndent) { this.fail(next, "Invalid quoted scalar indentation") } @@ -500,27 +553,35 @@ class YamlParser { const explicitIndent = Number.parseInt(indicator.replace(/[^1-9]/g, ""), 10) const start = this.index let end = start - let contentIndent = Number.isNaN(explicitIndent) ? Number.POSITIVE_INFINITY : parentIndent + explicitIndent + let contentIndent = Number.isNaN(explicitIndent) ? undefined : parentIndent + explicitIndent + let hasContent = false while (end < this.lines.length) { const line = this.lines[end] - if (line.raw.trim().length !== 0 && line.indent <= parentIndent) break - if (line.raw.trim().length !== 0) contentIndent = Math.min(contentIndent, line.indent) + // Only spaces make a line empty; a tab after the indentation is content. + if (/[^ ]/.test(line.raw)) { + if (contentIndent === undefined && line.indent > parentIndent) { + // The first non-empty line sets the indentation, which no leading empty line may exceed. + const deeper = this.lines.slice(start, end).find((empty) => empty.indent > line.indent) + if (deeper !== undefined) { + this.fail(deeper, "Leading empty lines cannot be more indented than the block scalar content") + } + contentIndent = line.indent + } + if (contentIndent !== undefined && line.indent >= contentIndent) { + hasContent = true + } else { + this.rejectTabIndent(line) + // A less indented line or comment ends the scalar. + if (line.indent <= parentIndent || line.text.length === 0) break + this.fail(line, `Expected block scalar indentation of ${contentIndent} spaces`) + } + } end++ } - if (contentIndent === Number.POSITIVE_INFINITY) contentIndent = parentIndent + 1 - const content: Array = [] - for (let index = start; index < end; index++) { - const line = this.lines[index] - if (line.raw.trim().length === 0) { - content.push("") - } else if (line.indent < contentIndent) { - this.fail(line, `Expected block scalar indentation of ${contentIndent} spaces`) - } else { - content.push(line.raw.slice(contentIndent)) - } - } + // Whitespace past the indentation is content only once the scalar has a content line. + const content = this.lines.slice(start, end).map((line) => hasContent ? line.raw.slice(contentIndent) : "") this.index = end let contentEnd = content.length @@ -539,7 +600,7 @@ class YamlParser { if (line.length === 0) { blankLines++ } else { - const moreIndented = line.startsWith(" ") + const moreIndented = line.startsWith(" ") || line.startsWith("\t") const hardBreak = moreIndented || previousMoreIndented === true if (previousMoreIndented === undefined) output += "\n".repeat(blankLines) else if (blankLines === 0) output += hardBreak ? "\n" : " " @@ -554,6 +615,17 @@ class YamlParser { return chomp === "clip" && contentEnd > 0 && trailingBreaks > 0 ? `${output}\n` : output } + private rejectTabIndent(line: Line): void { + if (line.tabIndent) this.fail(line, "Tabs cannot be used for YAML indentation") + } + + private rejectTabIndentedBlankLines(count: number, parentIndent: number): void { + for (let index = this.index - count; index < this.index; index++) { + const line = this.lines[index] + if (line.indent <= parentIndent) this.rejectTabIndent(line) + } + } + private skipBlankLines(): number { const start = this.index while (this.index < this.lines.length && this.lines[this.index].raw.trim().length === 0) this.index++ @@ -577,6 +649,7 @@ class YamlParser { this.fail(this.lines[this.index], "Multiple YAML documents are not supported") } } else { + this.rejectTabIndent(line) return } } @@ -603,14 +676,13 @@ export const parse = (input: string): unknown => { const source = input.replace(/^\uFEFF/, "").replace(/\r\n?/g, "\n") const lines = source.split("\n").map((raw, index): Line => { const indentation = /^( *)/.exec(raw)![1].length - if (raw.slice(0, indentation + 1).includes("\t")) { - throw new SyntaxError(`Tabs cannot be used for YAML indentation at line ${index + 1}`) - } return { raw, text: stripComment(raw.slice(indentation)), indent: indentation, - number: index + 1 + number: index + 1, + // Rejected where the line is read as structure; block scalar content may hold tabs. + tabIndent: raw[indentation] === "\t" } }) return new YamlParser(lines).parse() diff --git a/repos/effect/packages/effect/src/encoding/Zip.ts b/repos/effect/packages/effect/src/encoding/Zip.ts new file mode 100644 index 0000000000..cb9eadd459 --- /dev/null +++ b/repos/effect/packages/effect/src/encoding/Zip.ts @@ -0,0 +1,1558 @@ +/** + * Encodes and decodes zip archives as streams in Effect channels. + * + * `encode` writes each entry as it arrives and adds the central directory once + * the input ends; `decode` reads entries in archive order from their local + * headers. `decode` never buffers an entire entry body, and `encode` only + * collects deflated bodies with a `size` of at most 64 KiB. + * + * Entries are stored or compressed with deflate, the methods every zip tool + * reads. Compression uses the runtime's `CompressionStream("deflate-raw")`; + * decompression is implemented here, so that `decode` can find the end of + * entries whose sizes follow their data. ZIP64 fields are written when sizes, + * offsets or the entry count need them, and read from local headers and data + * descriptors. + * Encryption, other compression methods and archives split across files are + * not supported. + * + * This module is experimental and may receive breaking changes in patch + * releases. + * + * @stability experimental + * @since 4.0.6 + */ +import * as Arr from "../Array.ts" +import * as Cause from "../Cause.ts" +import * as Channel from "../Channel.ts" +import * as Data from "../Data.ts" +import * as Effect from "../Effect.ts" +import * as Exit from "../Exit.ts" +import { ByteQueue, InflateError, InflateMemory, Inflater, NeedInput } from "../internal/inflate.ts" +import * as Predicate from "../Predicate.ts" +import * as Pull from "../Pull.ts" +import * as Scope from "../Scope.ts" +import * as Semaphore from "../Semaphore.ts" +import * as Stream from "../Stream.ts" + +const ZipErrorTypeId = "~effect/encoding/Zip/ZipError" + +/** + * Error raised when a zip archive cannot be encoded or decoded. + * + * **Details** + * + * The `kind` field identifies the failure: + * + * - `InvalidHeader`: a header has an unknown signature or a malformed field, or + * an entry has a value that cannot be written. + * - `InvalidData`: the compressed data of an entry is corrupt. + * - `Truncated`: the input ended inside an entry or before the central + * directory. + * - `Checksum`: the data of an entry does not match its CRC-32. + * - `SizeMismatch`: an entry body produced a byte count other than its declared + * `size`, or the data of an entry does not match the sizes in the archive. + * - `Unsupported`: the archive uses a feature this module does not read, such + * as encryption or a compression method other than store and deflate. + * - `BodyConsumed`: an entry body was read after the decoder had moved past it. + * + * @stability experimental + * @category errors + * @since 4.0.6 + */ +export class ZipError extends Data.TaggedError("ZipError")<{ + readonly kind: + | "InvalidHeader" + | "InvalidData" + | "Truncated" + | "Checksum" + | "SizeMismatch" + | "Unsupported" + | "BodyConsumed" + readonly message: string +}> { + /** + * Marks this value as a zip error for runtime guards. + * + * @stability experimental + * @since 4.0.6 + */ + readonly [ZipErrorTypeId] = ZipErrorTypeId +} + +/** + * Checks whether a value is a `ZipError`. + * + * @stability experimental + * @category guards + * @since 4.0.6 + */ +export const isZipError = (u: unknown): u is ZipError => Predicate.hasProperty(u, ZipErrorTypeId) + +/** + * Kind of object described by a zip entry. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export type EntryType = "File" | "Directory" + +/** + * Compression method of a zip entry. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export type Method = "Store" | "Deflate" + +/** + * Metadata of a zip entry. + * + * **Details** + * + * `path` is the archive path, with `/` separators; directory paths end with + * `/`. `size` is the uncompressed byte length when it is known before the body: + * decoded entries whose sizes follow their data have no `size`. `mode` holds + * Unix permission bits and is written to the central directory, which `decode` + * does not read, so decoded headers have no `mode`. `mtime` has a resolution of + * one second when the archive has an extended timestamp and two seconds + * otherwise. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export interface Header { + readonly path: string + readonly type: EntryType + readonly method: Method + readonly mtime: Date + readonly size?: number | undefined + readonly mode?: number | undefined +} + +/** + * Zip entry: a header and a stream of the uncompressed body bytes. + * + * **Details** + * + * `decode` returns these, and `fileStream` builds them. `encode` writes the + * sizes and CRC-32 of a streamed body in a data descriptor after the data. + * + * @see {@link ContentEntry} for entries whose content is held in memory + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export interface Entry { + readonly header: Header + readonly body: Stream.Stream + readonly content?: undefined +} + +/** + * Zip entry whose uncompressed content is held in memory. + * + * **Details** + * + * `file` and `directory` build these. Because the content is known before the + * entry is written, `encode` puts its sizes and CRC-32 in the local header and + * writes no data descriptor, which every zip reader accepts. `content` and + * `body` exclude each other, so an entry whose body is replaced must drop + * `content`. + * + * @see {@link Entry} for entries whose content is streamed + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export interface ContentEntry { + readonly header: Header + readonly content: Uint8Array + readonly body?: undefined +} + +/** + * Options for the entry constructors. + * + * **Details** + * + * Unset fields default to an mtime of 1980-01-01T00:00:00Z, the earliest time + * a zip header can hold, and mode `0o644` (`0o755` for directories). The + * defaults make archives reproducible. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export interface EntryOptions { + readonly mtime?: Date | undefined + readonly mode?: number | undefined +} + +/** + * Options for file entries. + * + * **Details** + * + * `method` defaults to `"Deflate"`. Use `"Store"` for content that is already + * compressed, such as images or archives. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export interface FileOptions extends EntryOptions { + readonly method?: Method | undefined +} + +/** + * Options for file entries whose content is streamed. + * + * **Details** + * + * `size` is the exact byte length of the body. It is required for `Store`, + * because streaming readers can only find the end of stored data from its + * size, and optional for `Deflate`, where it is checked against the body. + * + * @stability experimental + * @category models + * @since 4.0.6 + */ +export type FileStreamOptions = + & EntryOptions + & ( + | { readonly method?: "Deflate" | undefined; readonly size?: number | undefined } + | { readonly method: "Store"; readonly size: number } + ) + +const MIN_MTIME = Date.UTC(1980, 0, 1) + +const makeHeader = ( + path: string, + type: EntryType, + method: Method, + options: EntryOptions | undefined, + size: number | undefined +): Header => ({ + path, + type, + method, + mtime: options?.mtime ?? new Date(MIN_MTIME), + mode: options?.mode ?? (type === "Directory" ? 0o755 : 0o644), + ...(size === undefined ? undefined : { size }) +}) + +/** + * Creates a file entry from in-memory content. + * + * **Details** + * + * String content is encoded as UTF-8, and `size` is the byte length of the + * content. Because the content is known up front, its sizes and CRC-32 are + * written in the local header, which every zip reader accepts. + * + * @see {@link fileStream} for content that is streamed instead of held in memory + * @see {@link encode} for writing entries to an archive + * + * @stability experimental + * @category constructors + * @since 4.0.6 + */ +export const file = (path: string, content: string | Uint8Array, options?: FileOptions): ContentEntry => { + const bytes = typeof content === "string" ? textEncoder.encode(content) : content + return { header: makeHeader(path, "File", options?.method ?? "Deflate", options, bytes.length), content: bytes } +} + +/** + * Creates a file entry whose content is streamed from `body`. + * + * **When to use** + * + * Use when the content is too large to hold in memory, or is read from a file + * or another stream. + * + * **Details** + * + * `body` is not run until the entry is encoded. Its sizes and CRC-32 are + * written in a data descriptor after the data. + * + * **Gotchas** + * + * - A stored entry without `size` fails with an `InvalidHeader` `ZipError`. + * - When `size` is set, `encode` fails with a `SizeMismatch` `ZipError` if the + * body produces fewer or more bytes. + * - Non-empty stored entries are followed by a data descriptor, which some + * streaming readers mishandle. Java's `ZipInputStream` rejects them, and + * libarchive reading from a pipe (as `bsdtar` does) ends them at the first + * bytes that look like a descriptor, so content can cut the entry short and + * add entries. Use `Deflate` or an in-memory `file` for untrusted content + * and for those readers. + * + * @see {@link file} for in-memory content + * @see {@link encode} for writing entries to an archive + * + * @stability experimental + * @category constructors + * @since 4.0.6 + */ +export const fileStream = ( + path: string, + body: Stream.Stream, + options?: FileStreamOptions +): Entry => ({ + header: makeHeader(path, "File", options?.method ?? "Deflate", options, options?.size), + body +}) + +/** + * Creates a directory entry. + * + * **Details** + * + * A trailing `/` is added to a non-empty `path` when missing, as zip tools + * expect for directories. + * + * @stability experimental + * @category constructors + * @since 4.0.6 + */ +export const directory = (path: string, options?: EntryOptions): ContentEntry => ({ + // An empty path stays empty, so that `encode` rejects it + header: makeHeader(path === "" || path.endsWith("/") ? path : `${path}/`, "Directory", "Store", options, 0), + content: emptyBody +}) + +/** + * Creates a channel that decodes zip archive bytes into entries. + * + * **Details** + * + * Entries are read in archive order from their local headers, so archives can + * be decoded as they arrive, for example from an HTTP response. Each output + * chunk holds one entry, and each body streams the uncompressed bytes of the + * entry. Pulling the next entry skips whatever is left of the current body. + * Stored and deflated entries are read, including ZIP64 sizes and deflated + * entries whose sizes follow their data in a data descriptor. Every body is + * checked against its CRC-32 and sizes. Reading stops at the central directory. + * + * Paths use the UTF-8 flag when set, and otherwise the first Info-ZIP Unicode + * path field when it matches the name, as Info-ZIP and 7-Zip do; other paths + * are read as UTF-8 when valid and as CP437 otherwise. Times are read from the + * extended timestamp field when present, and otherwise from the DOS time and + * date, which have no time zone and are read as UTC. + * + * **Gotchas** + * + * - Read each body before pulling the next entry: a body read afterwards fails + * with a `BodyConsumed` `ZipError`. Operators that pull ahead, such as + * `Stream.buffer`, therefore can't sit between `decode` and the body reads. + * - Read bodies while the entry stream runs, for example in `Stream.mapEffect`: + * once it has ended, as after `Stream.runCollect`, `Stream.runHead` or + * `Stream.take`, every body fails with a `BodyConsumed` `ZipError`. + * - A body reads the entry's bytes once: running it again continues where the + * previous run stopped. + * - The central directory is not read, so `mode`, comments and entries that + * only the central directory lists are not reported. Tools that read the + * central directory can see other entries and paths, for example in + * archives updated by appending. + * - Archives that do not start with an entry or a central directory, such as + * self-extracting archives, fail with an `InvalidHeader` `ZipError`. + * - Stored entries whose sizes only follow their data fail with an + * `Unsupported` `ZipError`. Python's `zipfile` writes them for stored + * entries on unseekable output, and Info-ZIP's `zip` for empty files when + * writing to a pipe. Searching for their end would let the content of an + * entry forge the entries after it. + * - A body that grows past its `size` fails with a `SizeMismatch` `ZipError` + * before the extra bytes are emitted, so `size` can be checked before reading. + * - Deflated data can expand about a thousand times, and pulling the next + * entry inflates the rest of the current body to find its end. To bound the + * work on untrusted archives, check `size`, count the bytes of bodies + * without one, and end the stream at an entry that is too large rather than + * skip it. + * - Every `ZipError` ends the archive: later entry and body reads fail with + * the same error. + * - The channel completes only when its input ends: input after the central + * directory is read and discarded, so that the upstream done value can be + * passed on. + * - Paths are returned as written in the archive: they can be absolute, + * contain `..`, drive letters, control characters or backslashes, which some + * tools read as separators, and can repeat. Validate them before writing + * entries to disk; `encode` rejects absolute paths, drive letters and + * backslashes. Paths with a NUL character, which other tools take as the end + * of the name, fail with an `InvalidHeader` `ZipError`. + * + * @see {@link encode} for the inverse operation + * + * @stability experimental + * @category decoding + * @since 4.0.6 + */ +export const decode = (): Channel.Channel< + Arr.NonEmptyReadonlyArray>, + ZipError | IE, + Done, + Arr.NonEmptyReadonlyArray, + IE, + Done +> => + Channel.fromTransform((upstream, scope) => + Effect.suspend(() => { + const decoder = new Decoder() + let done: { readonly value: Done } | undefined + let pulls = 0 + + const pullUpstream = upstream.pipe( + Effect.map((chunks) => { + pulls++ + for (const chunk of chunks) decoder.push(chunk) + }), + Pull.catchDone((value) => { + pulls++ + done = { value: value as Done } + decoder.end() + return Effect.void + }) + ) + + // Serialize entry and body pulls; reuse input pulled while waiting for the lock. + const lock = Semaphore.makeUnsafe(1) + const pullInput = (seen: number) => + lock.withPermit(Effect.suspend(() => pulls === seen ? pullUpstream : Effect.void)) + + const run = (f: () => A | undefined | typeof NeedInput, end: () => D): Pull.Pull => + Effect.suspend(function loop(): Pull.Pull { + const seen = pulls + let result: A | undefined | typeof NeedInput + try { + result = f() + } catch (error) { + return isZipError(error) ? Effect.fail(error) : Effect.die(error) + } + if (result === NeedInput) return Effect.flatMap(pullInput(seen), loop) + return result === undefined ? Cause.done(end()) : Effect.succeed(result) + }) + + const body = (id: number): Stream.Stream => + Stream.fromPull(Effect.succeed(run(() => decoder.read(id), () => undefined))) + + const next = run(() => { + const header = decoder.next() + return header === NeedInput || header === undefined + ? header + : Arr.of({ header, body: body(decoder.generation) }) + }, () => done!.value) + + return Effect.as(Scope.addFinalizer(scope, Effect.sync(() => decoder.close())), next) + }) + ) + +/** + * Creates a channel that encodes entries into zip archive bytes. + * + * **Details** + * + * Each entry is written as soon as it arrives: a local header, the stored or + * deflated body and, for entries with a streamed `body`, a data descriptor + * with the CRC-32 and sizes. A `ContentEntry` has them in its local header. + * The central directory is written once the input ends. Paths are written as + * UTF-8, with the UTF-8 flag set when they are not ASCII. `mtime` + * is written both as a DOS time in UTC, which keeps two-second resolution and + * covers 1980 to 2107, and as an extended timestamp in whole seconds when it + * falls between 1970 and 2038. ZIP64 fields are added when a size, an offset + * or the entry count does not fit the classic fields, and to the local header + * of a deflated entry streamed without a `size` or with one near 4 GiB, so + * that readers know the width of the sizes after its data. + * + * Headers and body chunks below 16 KiB are gathered into output chunks of + * about 64 KiB, and larger body chunks pass through without copying. Output is + * handed out before waiting for more entries, but up to 64 KiB of it can be + * held while a body is pulled. Each body chunk of a streamed `Deflate` entry is + * a separate compressor job, so larger chunks compress faster. A `Deflate` + * body with a `size` of at most 64 KiB is instead collected and compressed in + * one job, and still gets a data descriptor; omit `size` to stream it. + * + * **Gotchas** + * + * - Deflated entries need `CompressionStream("deflate-raw")`. On a runtime + * without it, encoding a deflated entry dies with the runtime's error, as + * it does when the compressor fails: check for the API or use `Store`. + * - Deflated bytes come from the runtime's `CompressionStream`, so they can + * differ between runtimes. Archives of stored entries are byte-identical + * everywhere. + * - A body that produces a byte count other than a set `header.size`, or any + * bytes for a directory, fails with a `SizeMismatch` `ZipError`. + * - Directories are always stored, whatever their `method`. + * - Empty paths, paths starting with `/` or a drive letter such as `C:`, paths + * containing `\` or NUL, `File` paths ending with `/`, `Directory` paths not + * ending with `/`, and invalid modes, sizes or times fail with an + * `InvalidHeader` `ZipError`. Other paths are written as given, including + * `..` segments, so validate untrusted paths before encoding them. + * + * **Example** (Writing and reading an archive) + * + * ```ts import.meta.vitest + * import { Effect, Stream } from "effect" + * import { Zip } from "effect/encoding" + * + * const archive = Stream.make( + * Zip.directory("docs"), + * Zip.file("docs/readme.txt", "Hello, zip!") + * ).pipe(Stream.pipeThroughChannel(Zip.encode())) + * + * const listing = archive.pipe( + * Stream.pipeThroughChannel(Zip.decode()), + * Stream.mapEffect((entry) => + * Stream.mkString(Stream.decodeText(entry.body)).pipe( + * Effect.map((text) => `${entry.header.path} ${JSON.stringify(text)}`) + * ) + * ), + * Stream.runCollect + * ) + * await Effect.runPromise(listing) // => ['docs/ ""', 'docs/readme.txt "Hello, zip!"'] + * ``` + * + * @see {@link decode} for the inverse operation + * @see {@link file} for building entries from in-memory content + * @see {@link fileStream} for building entries from streamed content + * + * @stability experimental + * @category encoding + * @since 4.0.6 + */ +export const encode = < + IE = never, + Done = unknown, + In extends Entry | ContentEntry = Entry | ContentEntry +>(): Channel.Channel< + Arr.NonEmptyReadonlyArray, + ZipError | IE | (In extends Entry ? E : never), + Done, + Arr.NonEmptyReadonlyArray, + IE, + Done, + In extends Entry ? R : never +> => { + // Distribute over entry unions to infer body errors and services. + type E = In extends Entry ? E : never + type R = In extends Entry ? R : never + return Channel.fromTransform((upstream: Pull.Pull, IE, Done>, scope) => + Effect.sync(() => { + const writer = new Writer() + let entries: ReadonlyArray | ContentEntry> = [] + let index = 0 + let done: { readonly value: Done } | undefined + let body: Pull.Pull, E | ZipError, void, R> | undefined + let bodyScope: Scope.Closeable | undefined + // In-memory entries are deflated ahead of the one being written, so the + // deflates overlap + let deflates: Array = [] + + const startDeflates = (from: number): void => { + for (let i = from; i < Math.min(entries.length, from + DEFLATE_AHEAD); i++) { + if (deflates[i] !== undefined) continue + const { content, header } = entries[i] + if (header.type !== "File" || header.method !== "Deflate" || content === undefined) continue + if (i > from && content.length > DEFLATE_AHEAD_SIZE) continue + const deflating: Deflating = { promise: deflateBytes(content), data: undefined } + // Failures surface when the entry is written; deflates left behind by + // a failed encode finish on their own + deflating.promise.then((data) => { + deflating.data = data + }, () => {}) + deflates[i] = deflating + } + } + + const write = (f: () => void): Effect.Effect => { + try { + f() + } catch (error) { + return isZipError(error) ? Effect.fail(error) : Effect.die(error) + } + return Effect.void + } + + return Effect.gen(function*() { + while (true) { + if (writer.buffered >= SLAB_SIZE) return writer.flush() + if (body !== undefined) { + const chunks = yield* Pull.catchDone(body, () => Effect.undefined) + if (chunks !== undefined) { + yield* write(() => writer.data(chunks)) + continue + } + body = undefined + yield* Scope.close(bodyScope!, Exit.void) + yield* write(() => writer.end()) + continue + } + if (index < entries.length) { + const entry = entries[index++] + const { header } = entry + yield* write(() => validate(header)) + const compress = header.type === "File" && header.method === "Deflate" + if (entry.content !== undefined) { + const bytes = entry.content + let data = bytes + if (compress) { + startDeflates(index - 1) + const deflating = deflates[index - 1]! + deflates[index - 1] = undefined + data = deflating.data ?? (yield* Effect.promise(() => deflating.promise)) + } + yield* write(() => writer.entry(header, bytes, data)) + continue + } + yield* write(() => writer.begin(header)) + if (compress && header.size !== undefined && header.size <= DEFLATE_AHEAD_SIZE) { + // A small body costs less to deflate in one piece than as a stream + const size = header.size + const chunks: Array = [] + let length = 0 + yield* Stream.runForEachArray(entry.body, (array) => + write(() => { + for (const chunk of array) { + chunks.push(chunk) + length += chunk.length + } + if (length > size) throw sizeMismatch(header.path, length, size) + writer.update(array) + })) + const data = yield* Effect.promise(() => deflateBytes(concat(chunks, length))) + yield* write(() => { + writer.data([data]) + writer.end() + }) + continue + } + const raw = Stream.mapArray(entry.body, (chunks) => { + writer.update(chunks) + return chunks + }) + bodyScope = Scope.forkUnsafe(scope) + body = yield* Channel.toPullScoped((compress ? deflate(raw) : raw).channel, bodyScope) + continue + } + if (writer.buffered > 0) return writer.flush() + if (done !== undefined) return yield* Cause.done(done.value) + const next = yield* Pull.catchDone(upstream, (value) => { + done = { value: value as Done } + return Effect.undefined + }) + if (next === undefined) { + writer.finish() + continue + } + entries = next + index = 0 + deflates = [] + } + }) + }) + ) +} + +// Compressor failures, including a runtime without `deflate-raw`, are defects +const deflate = (input: Stream.Stream): Stream.Stream => + Stream.unwrap(Effect.map( + Effect.sync(() => new CompressionStream("deflate-raw")), + (compressor) => { + const writer = compressor.writable.getWriter() + const feed = Stream.runForEach( + input, + (chunk) => Effect.promise(() => writer.write(compressible(chunk))) + ).pipe( + Effect.andThen(Effect.promise(() => writer.close())), + Effect.onError(() => + Effect.sync(() => { + writer.abort().catch(() => {}) + }) + ) + ) + const output = Stream.orDie(Stream.fromReadableStream({ + evaluate: () => compressor.readable, + onError: (cause) => cause + })) + return Stream.merge(output, Stream.fromEffectDrain(feed)) + } + )) + +// CompressionStream rejects views of a SharedArrayBuffer +const compressible = (bytes: Uint8Array): Uint8Array => + bytes.buffer instanceof ArrayBuffer ? bytes as Uint8Array : bytes.slice() + +// How many in-memory entries are deflated at once, and the largest one that +// is deflated before its turn +const DEFLATE_AHEAD = 16 +const DEFLATE_AHEAD_SIZE = 64 * 1024 + +// A deflate started ahead, with its data once it is done +interface Deflating { + readonly promise: Promise + data: Uint8Array | undefined +} + +// One promise per in-memory entry costs less than a stream +const deflateBytes = async (bytes: Uint8Array): Promise => { + const compressor = new CompressionStream("deflate-raw") + const writer = compressor.writable.getWriter() + // A failed write errors the readable, so the reads report it + if (bytes.length > 0) writer.write(compressible(bytes)).catch(() => {}) + writer.close().catch(() => {}) + const reader = compressor.readable.getReader() + const chunks: Array = [] + let length = 0 + while (true) { + const { done, value } = await reader.read() + if (done) break + chunks.push(value) + length += value.length + } + return concat(chunks, length) +} + +const concat = (chunks: ReadonlyArray, length: number): Uint8Array => { + if (chunks.length === 1) return chunks[0] + const out = new Uint8Array(length) + let at = 0 + for (const chunk of chunks) { + out.set(chunk, at) + at += chunk.length + } + return out +} + +const invalidHeader = (message: string) => new ZipError({ kind: "InvalidHeader", message }) + +const invalidData = (message: string) => new ZipError({ kind: "InvalidData", message }) + +const truncated = (at: number) => + new ZipError({ kind: "Truncated", message: `unexpected end of zip archive at byte ${at}` }) + +const unsupported = (message: string) => new ZipError({ kind: "Unsupported", message }) + +const sizeMismatch = (path: string, actual: number, expected: number) => + new ZipError({ + kind: "SizeMismatch", + message: `body of ${JSON.stringify(path)} has ${ + actual > expected ? "at least" : "only" + } ${actual} bytes, but its size is ${expected}` + }) + +const LOCAL = 0x04034b50 +const CENTRAL = 0x02014b50 +const DESCRIPTOR = 0x08074b50 +const END = 0x06054b50 +const END64 = 0x06064b50 +const LOCATOR64 = 0x07064b50 + +const FLAG_ENCRYPTED = 0x0001 +const FLAG_DESCRIPTOR = 0x0008 +const FLAG_STRONG_ENCRYPTION = 0x0040 +const FLAG_UTF8 = 0x0800 + +const EXTRA_ZIP64 = 0x0001 +const EXTRA_TIMESTAMP = 0x5455 +const EXTRA_UNICODE_PATH = 0x7075 + +const VERSION = 20 +const VERSION_ZIP64 = 45 +const CREATOR_UNIX = 3 + +const MAX16 = 0xffff +const MAX32 = 0xffffffff + +const S_IFDIR = 0o040000 +const S_IFREG = 0o100000 +const DOS_DIRECTORY = 0x10 + +const textEncoder = new TextEncoder() +// Paths keep a leading byte order mark, as other zip tools do +const textDecoder = new TextDecoder("utf-8", { ignoreBOM: true }) +const strictTextDecoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }) + +const emptyBody = new Uint8Array(0) + +// May exceed 2^53 - 1; callers check or compare +const readUint64 = (view: DataView, at: number): number => + view.getUint32(at, true) + view.getUint32(at + 4, true) * 0x100000000 + +const getUint64 = (view: DataView, at: number): number => { + const value = readUint64(view, at) + if (!Number.isSafeInteger(value)) throw invalidHeader("a 64-bit field exceeds 2^53 - 1") + return value +} + +const setUint64 = (view: DataView, at: number, value: number): void => { + view.setUint32(at, value % 0x100000000, true) + view.setUint32(at + 4, Math.floor(value / 0x100000000), true) +} + +// DOS times have two-second resolution and cover 1980 to 2107 +const toDos = (mtime: Date): { readonly time: number; readonly date: number } => { + const millis = mtime.getTime() + if (millis < MIN_MTIME) return { time: 0, date: (1 << 5) | 1 } + const date = new Date(millis) + const year = date.getUTCFullYear() + if (year > 2107) return { time: (23 << 11) | (59 << 5) | 29, date: (127 << 9) | (12 << 5) | 31 } + return { + time: (date.getUTCHours() << 11) | (date.getUTCMinutes() << 5) | (date.getUTCSeconds() >> 1), + date: ((year - 1980) << 9) | ((date.getUTCMonth() + 1) << 5) | date.getUTCDate() + } +} + +const fromDos = (time: number, date: number): number => + Date.UTC( + (date >> 9) + 1980, + ((date >> 5) & 0xf) - 1, + date & 0x1f, + time >> 11, + (time >> 5) & 0x3f, + (time & 0x1f) * 2 + ) + +const CP437 = "ÇüéâäàåçêëèïîìÄÅ" + + "ÉæÆôöòûùÿÖÜ¢£¥₧ƒ" + + "áíóúñѪº¿⌐¬½¼¡«»" + + "░▒▓│┤╡╢╖╕╣║╗╝╜╛┐" + + "└┴┬├─┼╞╟╚╔╩╦╠═╬╧" + + "╨╤╥╙╘╒╓╫╪┘┌█▄▌▐▀" + + "αßΓπΣσµτΦΘΩδ∞φε∩" + + "≡±≥≤⌠⌡÷≈°∙·√ⁿ²■ " + +const decodeName = (bytes: Uint8Array, utf8: boolean): string => { + if (utf8) return textDecoder.decode(bytes) + try { + return strictTextDecoder.decode(bytes) + } catch { + let name = "" + for (let i = 0; i < bytes.length; i++) { + name += bytes[i] < 0x80 ? String.fromCharCode(bytes[i]) : CP437[bytes[i] - 0x80] + } + return name + } +} + +// CRC-32 (IEEE) lookup tables for sixteen bytes at a time +const CRC_TABLE = (() => { + const table = new Int32Array(16 * 256) + for (let n = 0; n < 256; n++) { + let c = n + for (let k = 0; k < 8; k++) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1 + table[n] = c + } + for (let n = 0; n < 256; n++) { + let c = table[n] + for (let k = 1; k < 16; k++) { + c = table[c & 0xff] ^ (c >>> 8) + table[k * 256 + n] = c + } + } + return table +})() + +const littleEndian = new Uint8Array(new Uint32Array([1]).buffer)[0] === 1 + +const crc32 = (crc: number, bytes: Uint8Array): number => { + const t = CRC_TABLE + let c = ~crc + let i = 0 + const length = bytes.length + // Reading whole words is faster than combining bytes, but needs aligned, + // little-endian words + if (littleEndian && length >= 64) { + for (; ((bytes.byteOffset + i) & 3) !== 0; i++) c = t[(c ^ bytes[i]) & 0xff] ^ (c >>> 8) + const count = ((length - i) >>> 4) << 2 + const words = new Int32Array(bytes.buffer, bytes.byteOffset + i, count) + for (let j = 0; j < count; j += 4) { + const a = c ^ words[j] + const b = words[j + 1] + const d = words[j + 2] + const e = words[j + 3] + c = t[3840 + (a & 0xff)] ^ t[3584 + ((a >>> 8) & 0xff)] ^ t[3328 + ((a >>> 16) & 0xff)] ^ + t[3072 + (a >>> 24)] ^ t[2816 + (b & 0xff)] ^ t[2560 + ((b >>> 8) & 0xff)] ^ + t[2304 + ((b >>> 16) & 0xff)] ^ t[2048 + (b >>> 24)] ^ t[1792 + (d & 0xff)] ^ + t[1536 + ((d >>> 8) & 0xff)] ^ t[1280 + ((d >>> 16) & 0xff)] ^ t[1024 + (d >>> 24)] ^ + t[768 + (e & 0xff)] ^ t[512 + ((e >>> 8) & 0xff)] ^ t[256 + ((e >>> 16) & 0xff)] ^ t[e >>> 24] + } + i += count * 4 + } + for (; i + 16 <= length; i += 16) { + const a = c ^ (bytes[i] | (bytes[i + 1] << 8) | (bytes[i + 2] << 16) | (bytes[i + 3] << 24)) + c = t[3840 + (a & 0xff)] ^ t[3584 + ((a >>> 8) & 0xff)] ^ t[3328 + ((a >>> 16) & 0xff)] ^ + t[3072 + (a >>> 24)] ^ t[2816 + bytes[i + 4]] ^ t[2560 + bytes[i + 5]] ^ t[2304 + bytes[i + 6]] ^ + t[2048 + bytes[i + 7]] ^ t[1792 + bytes[i + 8]] ^ t[1536 + bytes[i + 9]] ^ t[1280 + bytes[i + 10]] ^ + t[1024 + bytes[i + 11]] ^ t[768 + bytes[i + 12]] ^ t[512 + bytes[i + 13]] ^ t[256 + bytes[i + 14]] ^ + t[bytes[i + 15]] + } + for (; i < length; i++) c = t[(c ^ bytes[i]) & 0xff] ^ (c >>> 8) + return ~c >>> 0 +} + +const validate = (header: Header): void => { + const { method, mode, mtime, path, size, type } = header + if (path === "") throw invalidHeader("path is empty") + const quoted = JSON.stringify(path) + if (path.startsWith("/")) throw invalidHeader(`path ${quoted} is absolute`) + // The zip format forbids these, and libarchive reads them as separators and roots + if (path.includes("\\")) throw invalidHeader(`path ${quoted} contains a backslash`) + if (/^[A-Za-z]:/.test(path)) throw invalidHeader(`path ${quoted} starts with a drive letter`) + if (path.includes("\0")) throw invalidHeader(`path ${quoted} contains a NUL character`) + if (type === "File") { + if (path.endsWith("/")) throw invalidHeader(`file path ${quoted} ends with "/"`) + if (method !== "Store" && method !== "Deflate") throw invalidHeader(`unknown method for ${quoted}`) + } else if (type === "Directory") { + if (!path.endsWith("/")) throw invalidHeader(`directory path ${quoted} does not end with "/"`) + if (size !== undefined && size !== 0) throw invalidHeader(`directory ${quoted} has a size`) + } else { + throw invalidHeader(`unknown entry type for ${quoted}`) + } + if (size !== undefined && !(Number.isSafeInteger(size) && size >= 0)) { + throw invalidHeader(`size of ${quoted} is not a non-negative safe integer`) + } + if (mode !== undefined && !(Number.isInteger(mode) && mode >= 0 && mode <= 0o7777)) { + throw invalidHeader(`mode of ${quoted} is not a permission mode`) + } + if (Number.isNaN(mtime.getTime())) { + throw invalidHeader(`mtime of ${quoted} is not a valid date`) + } +} + +interface EntryRecord { + readonly name: Uint8Array + readonly flags: number + readonly method: number + readonly time: number + readonly date: number + readonly seconds: number | undefined + readonly external: number + readonly offset: number + // Whether the local header has ZIP64 sizes, which then also follow the data + zip64: boolean + crc: number + compressedSize: number + size: number +} + +// An entry whose data is being written; `record` counts its bytes +interface Streaming { + readonly header: Header + readonly record: EntryRecord + readonly descriptor: boolean +} + +const timestampLength = (record: EntryRecord): number => record.seconds === undefined ? 0 : 9 + +const writeTimestamp = (bytes: Uint8Array, view: DataView, at: number, record: EntryRecord): void => { + if (record.seconds === undefined) return + view.setUint16(at, EXTRA_TIMESTAMP, true) + view.setUint16(at + 2, 5, true) + bytes[at + 4] = 1 + view.setUint32(at + 5, record.seconds, true) +} + +const localHeader = (record: EntryRecord): Uint8Array => { + const { zip64 } = record + const extraLength = (zip64 ? 20 : 0) + timestampLength(record) + const bytes = new Uint8Array(30 + record.name.length + extraLength) + const view = new DataView(bytes.buffer) + view.setUint32(0, LOCAL, true) + view.setUint16(4, zip64 ? VERSION_ZIP64 : VERSION, true) + view.setUint16(6, record.flags, true) + view.setUint16(8, record.method, true) + view.setUint16(10, record.time, true) + view.setUint16(12, record.date, true) + view.setUint32(14, record.crc, true) + view.setUint32(18, zip64 ? MAX32 : record.compressedSize, true) + view.setUint32(22, zip64 ? MAX32 : record.size, true) + view.setUint16(26, record.name.length, true) + view.setUint16(28, extraLength, true) + bytes.set(record.name, 30) + let at = 30 + record.name.length + if (zip64) { + view.setUint16(at, EXTRA_ZIP64, true) + view.setUint16(at + 2, 16, true) + setUint64(view, at + 4, record.size) + setUint64(view, at + 12, record.compressedSize) + at += 20 + } + writeTimestamp(bytes, view, at, record) + return bytes +} + +const dataDescriptor = (record: EntryRecord): Uint8Array => { + const wide = record.zip64 + const bytes = new Uint8Array(wide ? 24 : 16) + const view = new DataView(bytes.buffer) + view.setUint32(0, DESCRIPTOR, true) + view.setUint32(4, record.crc, true) + if (wide) { + setUint64(view, 8, record.compressedSize) + setUint64(view, 16, record.size) + } else { + view.setUint32(8, record.compressedSize, true) + view.setUint32(12, record.size, true) + } + return bytes +} + +const centralHeader = (record: EntryRecord): Uint8Array => { + // ZIP64 holds only the fields that overflow, in this order + const wide: Array = [] + if (record.size >= MAX32) wide.push(record.size) + if (record.compressedSize >= MAX32) wide.push(record.compressedSize) + if (record.offset >= MAX32) wide.push(record.offset) + const zip64Length = wide.length === 0 ? 0 : 4 + 8 * wide.length + const extraLength = zip64Length + timestampLength(record) + const version = wide.length === 0 && !record.zip64 ? VERSION : VERSION_ZIP64 + const bytes = new Uint8Array(46 + record.name.length + extraLength) + const view = new DataView(bytes.buffer) + view.setUint32(0, CENTRAL, true) + view.setUint16(4, (CREATOR_UNIX << 8) | version, true) + view.setUint16(6, version, true) + view.setUint16(8, record.flags, true) + view.setUint16(10, record.method, true) + view.setUint16(12, record.time, true) + view.setUint16(14, record.date, true) + view.setUint32(16, record.crc, true) + view.setUint32(20, Math.min(record.compressedSize, MAX32), true) + view.setUint32(24, Math.min(record.size, MAX32), true) + view.setUint16(28, record.name.length, true) + view.setUint16(30, extraLength, true) + view.setUint32(38, record.external, true) + view.setUint32(42, Math.min(record.offset, MAX32), true) + bytes.set(record.name, 46) + let at = 46 + record.name.length + if (wide.length > 0) { + view.setUint16(at, EXTRA_ZIP64, true) + view.setUint16(at + 2, 8 * wide.length, true) + for (let i = 0; i < wide.length; i++) setUint64(view, at + 4 + 8 * i, wide[i]) + at += zip64Length + } + writeTimestamp(bytes, view, at, record) + return bytes +} + +const SLAB_SIZE = 64 * 1024 + +// Room for deflate to grow data that does not compress, when choosing ZIP64 +const DEFLATE_HEADROOM = 16 * 1024 * 1024 + +// Chunks below this size are copied together instead of being handed out alone +const COPY_LIMIT = 16 * 1024 + +class Writer { + private chunks: Array = [] + buffered = 0 + private offset = 0 + private readonly records: Array = [] + private current: Streaming | undefined + + // Hands out the buffered output, joining runs of small chunks; needs `buffered > 0` + flush(): Arr.NonEmptyArray { + const out: Array = [] + let run: Array = [] + let runLength = 0 + const join = () => { + if (run.length === 1) { + out.push(run[0]) + } else if (run.length > 1) { + const joined = new Uint8Array(runLength) + let at = 0 + for (const chunk of run) { + joined.set(chunk, at) + at += chunk.length + } + out.push(joined) + } + run = [] + runLength = 0 + } + for (const chunk of this.chunks) { + if (chunk.length >= COPY_LIMIT) { + join() + out.push(chunk) + } else { + run.push(chunk) + runLength += chunk.length + } + } + join() + this.chunks = [] + this.buffered = 0 + return out as Arr.NonEmptyArray + } + + private push(bytes: Uint8Array): void { + if (bytes.length === 0) return + this.chunks.push(bytes) + this.offset += bytes.length + this.buffered += bytes.length + } + + private record(header: Header, flags: number): EntryRecord { + const name = textEncoder.encode(header.path) + if (name.length > MAX16) throw invalidHeader(`path ${JSON.stringify(header.path)} is longer than 65535 bytes`) + const { date, time } = toDos(header.mtime) + const seconds = Math.floor(header.mtime.getTime() / 1000) + const directory = header.type === "Directory" + const mode = header.mode ?? (directory ? 0o755 : 0o644) + return { + name, + // The UTF-8 byte length equals the string length only for ASCII + flags: name.length === header.path.length ? flags : flags | FLAG_UTF8, + method: header.type === "File" && header.method === "Deflate" ? 8 : 0, + time, + date, + seconds: seconds >= 0 && seconds <= 0x7fffffff ? seconds : undefined, + external: (((directory ? S_IFDIR : S_IFREG) | mode) * 0x10000 + (directory ? DOS_DIRECTORY : 0)) >>> 0, + offset: this.offset, + zip64: false, + crc: 0, + compressedSize: 0, + size: 0 + } + } + + // Writes an entry whose content is known, with its sizes in the local header + entry(header: Header, content: Uint8Array, data: Uint8Array): void { + const size = header.type === "Directory" ? 0 : header.size + if (size !== undefined && size !== content.length) throw sizeMismatch(header.path, content.length, size) + const record = this.record(header, 0) + record.crc = crc32(0, content) + record.size = content.length + record.compressedSize = data.length + record.zip64 = record.size >= MAX32 || record.compressedSize >= MAX32 + this.push(localHeader(record)) + if (data.length > 0) this.push(data) + this.records.push(record) + } + + // Starts an entry whose sizes and CRC-32 follow its data + begin(header: Header): void { + const directory = header.type === "Directory" + const stored = directory || header.method === "Store" + const size = directory ? 0 : header.size + if (stored && size === undefined) { + throw invalidHeader(`stored file ${JSON.stringify(header.path)} is streamed without a size`) + } + // An empty stored entry needs no descriptor: readers would take its zero + // sizes for missing ones + const descriptor = !(stored && size === 0) + const record = this.record(header, descriptor ? FLAG_DESCRIPTOR : 0) + // Streaming readers find the end of stored data from its size + if (stored) { + record.size = size! + record.compressedSize = size! + } + // The local header fixes the width of the sizes after the data, so declare + // ZIP64 whenever they could reach 4 GiB + record.zip64 = size === undefined || size >= (stored ? MAX32 : MAX32 - DEFLATE_HEADROOM) + this.push(localHeader(record)) + record.size = 0 + record.compressedSize = 0 + this.current = { header, record, descriptor } + } + + // Accounts for uncompressed body bytes + update(chunks: ReadonlyArray): void { + const { record } = this.current! + for (const chunk of chunks) { + record.crc = crc32(record.crc, chunk) + record.size += chunk.length + } + } + + // Writes stored or compressed body bytes + data(chunks: ReadonlyArray): void { + const { header, record } = this.current! + for (const chunk of chunks) { + this.push(chunk) + record.compressedSize += chunk.length + } + const size = header.type === "Directory" ? 0 : header.size + if (size !== undefined && record.size > size) throw sizeMismatch(header.path, record.size, size) + if (!record.zip64 && record.compressedSize >= MAX32) { + throw new ZipError({ + kind: "SizeMismatch", + message: `compressed data of ${JSON.stringify(header.path)} reached 4 GiB, ` + + `but its local header has no ZIP64 sizes` + }) + } + } + + end(): void { + const { descriptor, header, record } = this.current! + this.current = undefined + if (header.size !== undefined && record.size !== header.size) { + throw sizeMismatch(header.path, record.size, header.size) + } + if (descriptor) this.push(dataDescriptor(record)) + this.records.push(record) + } + + finish(): void { + const start = this.offset + for (const record of this.records) this.push(centralHeader(record)) + const size = this.offset - start + const count = this.records.length + if (count >= MAX16 || size >= MAX32 || start >= MAX32) { + const end64 = new Uint8Array(56) + const view = new DataView(end64.buffer) + view.setUint32(0, END64, true) + setUint64(view, 4, 44) + view.setUint16(12, (CREATOR_UNIX << 8) | VERSION_ZIP64, true) + view.setUint16(14, VERSION_ZIP64, true) + setUint64(view, 24, count) + setUint64(view, 32, count) + setUint64(view, 40, size) + setUint64(view, 48, start) + const locator = new Uint8Array(20) + const locatorView = new DataView(locator.buffer) + locatorView.setUint32(0, LOCATOR64, true) + setUint64(locatorView, 8, this.offset) + locatorView.setUint32(16, 1, true) + this.push(end64) + this.push(locator) + } + const end = new Uint8Array(22) + const view = new DataView(end.buffer) + view.setUint32(0, END, true) + view.setUint16(8, Math.min(count, MAX16), true) + view.setUint16(10, Math.min(count, MAX16), true) + view.setUint32(12, Math.min(size, MAX32), true) + view.setUint32(16, Math.min(start, MAX32), true) + this.push(end) + } +} + +interface Current { + readonly header: Header + readonly inflater: Inflater | undefined + readonly descriptor: boolean + readonly zip64: boolean + // The values of the local header; sizes are unknown when they follow the data + readonly crc: number + readonly compressedSize: number | undefined + readonly size: number | undefined + // Queue position where the data starts + readonly start: number + remaining: number + dataDone: boolean + actualCrc: number + actualSize: number + actualCompressedSize: number +} + +// `next` skips the rest of the current body; `read` consumes it. Both preserve +// progress across `NeedInput` retries. +class Decoder { + private readonly queue = new ByteQueue() + private ended = false + private finished = false + private closed = false + // Incremented whenever the current body is invalidated + generation = 0 + private current: Current | undefined + private inflateMemory: InflateMemory | undefined + // The error that ended the archive, repeated by every later call + private failure: ZipError | undefined + + push(chunk: Uint8Array): void { + if (!this.finished) this.queue.push(chunk) + } + + end(): void { + this.ended = true + } + + close(): void { + this.closed = true + this.generation++ + this.queue.clear() + } + + // Returns the next entry header, or `undefined` once the entries and the + // input have ended. Input after the entries is discarded. + next(): Header | undefined | typeof NeedInput { + this.generation++ + return this.guard(() => this.advance()) + } + + private guard(f: () => A): A { + if (this.failure !== undefined) throw this.failure + try { + return f() + } catch (error) { + if (isZipError(error)) this.failure = error + throw error + } + } + + private advance(): Header | undefined | typeof NeedInput { + while (this.current !== undefined) { + if (this.readCurrent() === NeedInput) return NeedInput + } + if (this.finished) return this.finish() + const queue = this.queue + if (queue.length < 4) return this.needInput() + const at = queue.consumed + const head = queue.peek(Math.min(queue.length, 30)) + const view = new DataView(head.buffer) + const signature = view.getUint32(0, true) + if (signature === CENTRAL || signature === END || signature === END64) return this.finish() + if (signature !== LOCAL) { + throw invalidHeader(`unexpected signature 0x${signature.toString(16).padStart(8, "0")} at byte ${at}`) + } + if (head.length < 30) return this.needInput() + const length = 30 + view.getUint16(26, true) + view.getUint16(28, true) + if (queue.length < length) return this.needInput() + try { + this.current = this.parseLocal(queue.take(length)) + } catch (error) { + if (isZipError(error)) { + throw new ZipError({ kind: error.kind, message: `${error.message} (header at byte ${at})` }) + } + throw error + } + return this.current.header + } + + // Returns the buffered body bytes of the entry from generation `id`, or + // `undefined` once the whole body has been read and checked + read(id: number): Arr.NonEmptyArray | undefined | typeof NeedInput { + if (id !== this.generation) { + throw new ZipError({ + kind: "BodyConsumed", + message: this.closed ? + "zip entry body read after the archive stream ended" : + "zip entry body read after moving to the next entry" + }) + } + return this.guard(() => this.readCurrent()) + } + + private readCurrent(): Arr.NonEmptyArray | undefined | typeof NeedInput { + const current = this.current + if (current === undefined) return undefined + const queue = this.queue + if (!current.dataDone) { + if (current.inflater === undefined) { + if (current.remaining > 0) { + if (queue.length === 0) return this.needInput() + const chunks: Array = [] + while (current.remaining > 0 && queue.length > 0) { + const chunk = queue.takeSome(current.remaining) + current.remaining -= chunk.length + current.actualCrc = crc32(current.actualCrc, chunk) + current.actualSize += chunk.length + chunks.push(chunk) + } + current.actualCompressedSize = current.actualSize + return chunks as Arr.NonEmptyArray + } + } else { + let chunk: Uint8Array | undefined | typeof NeedInput + try { + chunk = current.inflater.pull() + } catch (error) { + throw error instanceof InflateError ? invalidData(error.message) : error + } + if (chunk === NeedInput) return this.needInput() + if (chunk !== undefined) { + current.actualSize += chunk.length + // Fail before handing out bytes past the reported size, so that + // consumers can limit what a small compressed entry inflates to + const { path, size } = current.header + if (size !== undefined && current.actualSize > size) throw sizeMismatch(path, current.actualSize, size) + current.actualCrc = crc32(current.actualCrc, chunk) + return [chunk] + } + current.actualCompressedSize = queue.consumed - current.start + } + current.dataDone = true + } + if (current.descriptor) { + const descriptor = this.descriptor(current) + if (descriptor === NeedInput) return NeedInput + this.check(current, descriptor.crc, descriptor.compressedSize, descriptor.size) + } else { + this.check(current, current.crc, current.compressedSize!, current.size!) + } + this.current = undefined + return undefined + } + + private check(current: Current, crc: number, compressedSize: number, size: number): void { + if (current.actualCrc === crc && current.actualCompressedSize === compressedSize && current.actualSize === size) { + return + } + const path = JSON.stringify(current.header.path) + if (current.actualCrc !== crc) { + throw new ZipError({ + kind: "Checksum", + message: `CRC-32 of ${path} is ${hex(current.actualCrc)}, but the archive records ${hex(crc)}` + }) + } + throw new ZipError({ + kind: "SizeMismatch", + message: `${path} has ${current.actualSize} bytes (${current.actualCompressedSize} compressed), ` + + `but the archive records ${size} (${compressedSize} compressed)` + }) + } + + // Reads a data descriptor, whose signature is optional and whose sizes are + // 4 bytes unless the local header has a ZIP64 field. Writers that widen the + // sizes only on overflow are handled by matching each layout against the + // data read; among several matches, one followed by a header signature wins. + private descriptor( + current: Current + ): { readonly crc: number; readonly compressedSize: number; readonly size: number } | typeof NeedInput { + const queue = this.queue + if (queue.length < DESCRIPTOR_PEEK && !this.ended) return NeedInput + const bytes = queue.peek(Math.min(queue.length, DESCRIPTOR_PEEK)) + const view = new DataView(bytes.buffer) + const candidates: Array<{ length: number; crc: number; compressedSize: number; size: number }> = [] + const offsets = bytes.length >= 4 && view.getUint32(0, true) === DESCRIPTOR ? [4, 0] : [0] + for (const offset of offsets) { + for (const wide of current.zip64 ? [true, false] : [false, true]) { + const length = offset + (wide ? 20 : 12) + if (length > bytes.length) continue + candidates.push({ + length, + crc: view.getUint32(offset, true), + compressedSize: wide ? readUint64(view, offset + 4) : view.getUint32(offset + 4, true), + size: wide ? readUint64(view, offset + 12) : view.getUint32(offset + 8, true) + }) + } + } + const matches = candidates.filter((candidate) => + candidate.crc === current.actualCrc && candidate.compressedSize === current.actualCompressedSize && + candidate.size === current.actualSize + ) + const match = + matches.find((candidate) => + candidate.length + 4 <= bytes.length && isHeaderSignature(view.getUint32(candidate.length, true)) + ) ?? matches[0] + if (match !== undefined) { + queue.skip(match.length) + return match + } + // A complete archive has at least an end record after the descriptor + if (candidates.length === 0 || (this.ended && bytes.length < DESCRIPTOR_PEEK)) { + throw truncated(queue.consumed + queue.length) + } + queue.skip(candidates[0].length) + return candidates[0] + } + + private parseLocal(bytes: Uint8Array): Current { + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.length) + const flags = view.getUint16(6, true) + const method = view.getUint16(8, true) + const crc = view.getUint32(14, true) + let compressedSize = view.getUint32(18, true) + let size = view.getUint32(22, true) + const time = view.getUint16(10, true) + const date = view.getUint16(12, true) + const nameLength = view.getUint16(26, true) + const rawName = bytes.subarray(30, 30 + nameLength) + const utf8 = (flags & FLAG_UTF8) !== 0 + let path = decodeName(rawName, utf8) + // As in Info-ZIP and 7-Zip, only the first Unicode path field is read, and + // only without the UTF-8 flag + let readUnicodePath = !utf8 + let seconds: number | undefined + let zip64 = false + let at = 30 + nameLength + while (at + 4 <= bytes.length) { + const id = view.getUint16(at, true) + const start = at + 4 + const end = start + view.getUint16(at + 2, true) + if (end > bytes.length) throw invalidHeader("extra field overflows the header") + if (id === EXTRA_ZIP64) { + zip64 = true + let field = start + if (size === MAX32 && field + 8 <= end) { + size = getUint64(view, field) + field += 8 + } + if (compressedSize === MAX32 && field + 8 <= end) { + compressedSize = getUint64(view, field) + } + } else if (id === EXTRA_TIMESTAMP && end - start >= 5 && (bytes[start] & 1) !== 0) { + // Info-ZIP writes times before 1970 as negative numbers, and others + // write times after 2038 as unsigned ones; the DOS date tells them apart + const value = view.getUint32(start + 1, true) + seconds = value >= 0x80000000 && (date >> 9) + 1980 < 2038 ? value - 0x100000000 : value + } else if (id === EXTRA_UNICODE_PATH && readUnicodePath) { + readUnicodePath = false + if (end - start >= 5 && bytes[start] === 1 && view.getUint32(start + 1, true) === crc32(0, rawName)) { + path = textDecoder.decode(bytes.subarray(start + 5, end)) + } + } + at = end + } + const quoted = JSON.stringify(path) + // Other tools end the name at a NUL character, or reject it + if (path.includes("\0")) throw invalidHeader(`path ${quoted} contains a NUL character`) + if ((flags & (FLAG_ENCRYPTED | FLAG_STRONG_ENCRYPTION)) !== 0) throw unsupported(`entry ${quoted} is encrypted`) + if (method !== 0 && method !== 8) throw unsupported(`entry ${quoted} uses compression method ${method}`) + const descriptor = (flags & FLAG_DESCRIPTOR) !== 0 + // Writers that put sizes after the data usually leave zeros here + const known = !descriptor || compressedSize !== 0 || size !== 0 + // Searching for the descriptor of stored data would let its content forge + // the entries after it + if (method === 0 && !known) throw unsupported(`stored entry ${quoted} has no size before its data`) + if (method === 0 && compressedSize !== size) { + throw invalidHeader(`stored entry ${quoted} has different sizes`) + } + const header: Header = { + path, + type: path.endsWith("/") ? "Directory" : "File", + method: method === 8 ? "Deflate" : "Store", + mtime: new Date(seconds === undefined ? fromDos(time, date) : seconds * 1000), + ...(known && (method === 0 || !descriptor) ? { size } : undefined) + } + return { + header, + inflater: method === 8 + ? new Inflater(this.queue, this.inflateMemory ??= new InflateMemory()) + : undefined, + descriptor, + zip64, + crc, + compressedSize: known ? compressedSize : undefined, + size: known ? size : undefined, + start: this.queue.consumed, + remaining: method === 0 ? compressedSize : 0, + dataDone: false, + actualCrc: 0, + actualSize: 0, + actualCompressedSize: 0 + } + } + + private needInput(): typeof NeedInput { + if (this.ended) throw truncated(this.queue.consumed + this.queue.length) + return NeedInput + } + + private finish(): undefined | typeof NeedInput { + this.finished = true + this.queue.clear() + return this.ended ? undefined : NeedInput + } +} + +const hex = (n: number): string => `0x${n.toString(16).padStart(8, "0")}` + +// A ZIP64 data descriptor with its signature, and the signature after it +const DESCRIPTOR_PEEK = 28 + +const isHeaderSignature = (signature: number): boolean => + signature === LOCAL || signature === CENTRAL || signature === END || signature === END64 diff --git a/repos/effect/packages/effect/src/encoding/index.ts b/repos/effect/packages/effect/src/encoding/index.ts index 7ac4657b0c..079e985c72 100644 --- a/repos/effect/packages/effect/src/encoding/index.ts +++ b/repos/effect/packages/effect/src/encoding/index.ts @@ -35,6 +35,12 @@ export * as Hex from "./Hex.ts" */ export * as Ini from "./Ini.ts" +/** + * @stability unstable + * @since 4.0.6 + */ +export * as JsonArray from "./JsonArray.ts" + /** * @stability unstable * @since 4.0.0 @@ -53,14 +59,32 @@ export * as SchemaBinary from "./SchemaBinary.ts" */ export * as Sse from "./Sse.ts" +/** + * @stability unstable + * @since 4.0.5 + */ +export * as Tar from "./Tar.ts" + /** * @stability unstable * @since 4.0.0 */ export * as Toml from "./Toml.ts" +/** + * @stability experimental + * @since 4.0.6 + */ +export * as Xml from "./Xml.ts" + /** * @stability unstable * @since 4.0.0 */ export * as Yaml from "./Yaml.ts" + +/** + * @stability experimental + * @since 4.0.6 + */ +export * as Zip from "./Zip.ts" diff --git a/repos/effect/packages/effect/src/eventlog/EventJournal.ts b/repos/effect/packages/effect/src/eventlog/EventJournal.ts index caaadd0db3..ca5624664e 100644 --- a/repos/effect/packages/effect/src/eventlog/EventJournal.ts +++ b/repos/effect/packages/effect/src/eventlog/EventJournal.ts @@ -14,20 +14,24 @@ */ import * as Arr from "../Array.ts" import type { Brand } from "../Brand.ts" +import * as Clock from "../Clock.ts" import * as Context from "../Context.ts" import * as Data from "../Data.ts" import * as DateTime from "../DateTime.ts" import * as Effect from "../Effect.ts" import * as SchemaBinary from "../encoding/SchemaBinary.ts" +import { constVoid } from "../Function.ts" import * as Uuid from "../internal/uuid.ts" import * as Layer from "../Layer.ts" import type { Option } from "../Option.ts" import * as Order from "../Order.ts" import * as PubSub from "../PubSub.ts" import * as Schema from "../Schema.ts" +import type * as SchemaAST from "../SchemaAST.ts" import type { Scope } from "../Scope.ts" import * as Semaphore from "../Semaphore.ts" import type { StoreId } from "./EventLogMessage.ts" +import { ConflictIndex, entryIdOrder } from "./internal/conflicts.ts" /** * Context service for storing and replaying event journal entries. @@ -37,6 +41,9 @@ import type { StoreId } from "./EventLogMessage.ts" * The service writes local entries, imports entries from remote journals, exposes * a stream of local changes, and provides per-store locking. * + * Entries written by `EventLog` carry payloads encoded with their event's + * `payloadSchemaBinary`; see {@link Entry} for how to decode them. + * * @stability unstable * @category services * @since 4.0.0 @@ -47,6 +54,14 @@ export class EventJournal extends Context.Service, EventJournalError> + /** + * Read the stored entries with the given ids. The result is aligned with + * `ids`, with `undefined` for ids that are not in the journal. + */ + readonly entriesById: ( + ids: ReadonlyArray + ) => Effect.Effect, EventJournalError> + /** * Write an event to the journal, performing an effect before committing the * event. @@ -58,12 +73,42 @@ export class EventJournal extends Context.Service Effect.Effect }) => Effect.Effect + /** + * Write several events to the journal, performing an effect before + * committing them. + * + * **Details** + * + * `effect` runs once with every entry, in input order and with ascending + * ids. The entries are committed together and then published to `changes` + * only after it succeeds. If it fails, nothing is committed or published. + */ + readonly writeMany: (options: { + readonly entries: Arr.NonEmptyReadonlyArray<{ + readonly event: string + readonly primaryKey: string + readonly payload: Uint8Array + }> + readonly effect: (entries: Arr.NonEmptyReadonlyArray) => Effect.Effect + }) => Effect.Effect + /** * Write events from a remote source to the journal. * * **Details** * - * Effects run sequentially in compaction bracket order. + * `effect` runs once with every new entry after compaction, sorted by + * `Entry.Order`. Each entry is paired with the stored entries for the same + * event and primary key whose id is strictly greater, in id order. Entries + * in the batch never conflict with each other: applying them in id order + * gives the same result as delivering them one at a time. A compaction + * replacement written by `EventLog` takes the place and conflicts of the + * source entry it is anchored to, not of its own id. + * + * All conflicts are read before `effect` runs and the batch is committed + * after it succeeds, with no journal I/O while it runs. It is not called + * when the batch has no new entries. If `effect` fails, no entry from the + * batch is committed. */ readonly writeFromRemote: ( options: { @@ -72,10 +117,12 @@ export class EventJournal extends Context.Service) => Effect.Effect, EventJournalError>) | undefined - readonly effect: (options: { - readonly entry: Entry - readonly conflicts: ReadonlyArray - }) => Effect.Effect + readonly effect: ( + entries: ReadonlyArray<{ + readonly entry: Entry + readonly conflicts: ReadonlyArray + }> + ) => Effect.Effect } ) => Effect.Effect<{ readonly duplicateEntries: ReadonlyArray @@ -93,6 +140,16 @@ export class EventJournal extends Context.Service) => Effect.Effect ) => Effect.Effect, EventJournalError | E, R> + /** + * Records upload acknowledgements, advancing `nextRemoteSequence` only across + * contiguous sequences from the current cursor or `firstSequence`. + */ + readonly writeRemoteAcks: (options: { + readonly remoteId: RemoteId + readonly acks: ReadonlyArray + readonly firstSequence?: number | undefined + }) => Effect.Effect + /** * Retrieve the first unused sequence number for a remote source. */ @@ -240,14 +297,57 @@ export const EntryId = (Schema.Uint8Array as Schema.instanceOf((a, b) => { - for (let i = 0; i < 16; i++) { - if (a[i] !== b[i]) { - return (a[i] - b[i]) < 0 ? -1 : 1 - } +export const EntryIdOrder: Order.Order = entryIdOrder + +const maxEntryIdMillis = 2 ** 48 - 1 + +// The millisecond and 12-bit counter of the last id in a sequence of local ids. +interface EntryIdSequence { + millis: number + counter: number +} + +const processEntryIds: EntryIdSequence = { millis: 0, counter: 0 } + +const nextEntryId = (sequence: EntryIdSequence, now: number): EntryId => { + if (now > sequence.millis) { + sequence.millis = now + sequence.counter = 0 + } else if (++sequence.counter > 0xfff) { + sequence.millis++ + sequence.counter = 0 } - return 0 -}) + // The sequence only moves forward, so once exhausted it stays exhausted. + if (sequence.millis > maxEntryIdMillis) { + throw new RangeError("The entry id sequence has passed the last UUID v7 millisecond") + } + const bytes = Uuid.v7Bytes(sequence.millis) + // The counter replaces the 12 random `rand_a` bits after the version. + bytes[6] = 0x70 | (sequence.counter >> 8) + bytes[7] = sequence.counter & 0xff + return bytes as EntryId +} + +// The process sequence, or a copy advanced to `floor` when that is later. The +// copy is never written back, so a floor read from uncommitted rows leaves the +// process sequence alone. +const sequenceAfter = (floor: EntryId | undefined): EntryIdSequence => { + if (floor === undefined) return processEntryIds + const millis = entryIdMillis(floor) + // Continuing from the floor's `rand_a` sorts the next id after it. + const counter = ((floor[6] & 0x0f) << 8) | floor[7] + return millis > processEntryIds.millis || (millis === processEntryIds.millis && counter > processEntryIds.counter) + ? { millis, counter } + : processEntryIds +} + +const maxEntryId = (ids: Iterable): EntryId | undefined => { + let latest: EntryId | undefined + for (const id of ids) { + if (latest === undefined || EntryIdOrder(id, latest) === 1) latest = id + } + return latest +} /** * Generates a UUID v7 `EntryId`, optionally using the supplied millisecond @@ -258,17 +358,58 @@ export const EntryIdOrder = Order.make((a, b) => { * Use when generating an event-log entry id internally and the UUID v7 bytes * are trusted to satisfy the brand. * + * **Details** + * + * Ids taken from the clock are strictly ascending within the current process. + * Ids created in the same millisecond carry an incrementing 12-bit counter. If + * the clock stalls or moves backwards, ids keep the last millisecond, and + * after 4096 ids in one millisecond they move to the next one. + * + * The journals generate local ids from this sequence, or from a copy of it + * advanced past the latest id in their store, so a local id sorts after every + * entry committed to that store, even when the local clock is behind. + * * **Gotchas** * * This is unsafe because the generated UUID bytes are cast to the brand without * schema validation. * + * Throws a `RangeError` once the sequence passes the last UUID v7 millisecond. + * Every later call throws too, because any id it could return would sort before + * one already generated. + * + * Ids created with an explicit `msecs` are not part of the sequence, and clamp + * to the UUID v7 range. + * * @stability unstable * @category unsafe * @since 4.0.0 */ export const makeEntryIdUnsafe = (options: { msecs?: number } = {}): EntryId => - Uuid.v7Bytes(options.msecs ?? DateTime.nowUnsafe().epochMilliseconds) as EntryId + options.msecs !== undefined + ? Uuid.v7Bytes(options.msecs) as EntryId + : nextEntryId(processEntryIds, Date.now()) + +// Imported ids further ahead of the local clock than this are logged. +const maxClockSkewMillis = 24 * 60 * 60 * 1000 + +/** + * Logs a warning when the latest of the imported `ids` is more than 24 hours + * ahead of the supplied `clock`. Memory and IndexedDB journals call this after + * commit; SQL calls it at the end of its transaction body, so a later rollback + * (including an enclosing transaction rollback) can still leave a warning. + * + * @internal + */ +export const warnIfAhead = (clock: Clock.Clock, ids: Iterable): Effect.Effect => { + const latest = maxEntryId(ids) + if (latest === undefined) return Effect.void + const ahead = entryIdMillis(latest) - clock.currentTimeMillisUnsafe() + if (ahead <= maxClockSkewMillis) return Effect.void + return Effect.logWarning( + `Imported an entry id ${Math.round(ahead / 3_600_000)} hours ahead of the local clock` + ).pipe(Effect.annotateLogs({ service: "EventJournal", entryId: Uuid.stringify(latest) })) +} /** * Extracts the millisecond timestamp encoded in a UUID v7 `EntryId`. @@ -277,11 +418,10 @@ export const makeEntryIdUnsafe = (options: { msecs?: number } = {}): EntryId => * @category getters * @since 4.0.0 */ -export const entryIdMillis = (entryId: EntryId): number => { - const bytes = new Uint8Array(8) - bytes.set(entryId.subarray(0, 6), 2) - return Number(new DataView(bytes.buffer).getBigUint64(0)) -} +export const entryIdMillis = (entryId: EntryId): number => + // Multiply instead of shifting: bitwise operators are signed 32-bit. + ((((entryId[0] * 256 + entryId[1]) * 256 + entryId[2]) * 256 + entryId[3]) * 256 + entryId[4]) * 256 + + entryId[5] /** * Schema for a committed event journal entry. @@ -291,6 +431,14 @@ export const entryIdMillis = (entryId: EntryId): number => { * An entry records its ID, event tag, primary key, and SchemaBinary-encoded * payload, with helpers for array SchemaBinary encoding and creation timestamps. * + * **Gotchas** + * + * `EventLog` encodes `payload` with the `payloadSchemaBinary` of the event whose + * tag is `entry.event`, so decode it with that event's codec. A codec with an + * incompatible wire layout, such as a union of a group's tagged payload schemas, + * cannot read it. Entries from remote journals can carry tags the local schema + * does not define. + * * @stability unstable * @category schemas * @since 4.0.0 @@ -323,7 +471,11 @@ export class Entry extends Schema.Class("effect/eventlog/EventJournal/Ent * @stability unstable * @since 4.0.0 */ - static decodeArray = Schema.decodeUnknownEffect(Entry.arraySchemaBinary) + static decodeArray = ( + input: unknown, + options?: SchemaAST.ParseOptions + ): Effect.Effect, Schema.SchemaError> => + Effect.map(decodeEntryFieldsArray(input, options), Arr.map(entryFromFields)) /** * Ordering for committed entries by their event journal entry id. @@ -333,6 +485,8 @@ export class Entry extends Schema.Class("effect/eventlog/EventJournal/Ent */ static Order = Order.make((a, b) => EntryIdOrder(a.id, b.id)) + #idString: string | undefined + /** * String representation of the entry id. * @@ -340,7 +494,7 @@ export class Entry extends Schema.Class("effect/eventlog/EventJournal/Ent * @since 4.0.0 */ get idString(): string { - return Uuid.stringify(this.id) + return this.#idString ??= Uuid.stringify(this.id) } /** @@ -364,6 +518,18 @@ export class Entry extends Schema.Class("effect/eventlog/EventJournal/Ent } } +/** + * A local entry's remote-assigned sequence. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export interface RemoteAck { + readonly entryId: EntryId + readonly sequence: number +} + /** * Schema for an event journal entry received from a remote source. * @@ -380,6 +546,44 @@ export class RemoteEntry extends Schema.Class("effect/eventlog/Even entry: Entry }) {} +// Decoding frames into the plain field structs and then constructing the +// classes from the already validated fields skips the per-instance class +// decode, which costs more than reading the frame. The wire format is the same. + +/** @internal */ +export const EntryFields = Schema.Struct(Entry.fields) + +/** @internal */ +export const entryFromFields = (fields: typeof EntryFields.Type): Entry => Schema.makeClassTrusted(Entry, fields) + +const decodeEntryFieldsArray = Schema.decodeUnknownEffect(Schema.Array(SchemaBinary.toCodec(EntryFields))) + +/** + * Creates local entries with ids after every id in `after`, normally the + * latest id in the journal's store. + * + * @internal + */ +export const makeEntries = ( + entries: Arr.NonEmptyReadonlyArray< + { readonly event: string; readonly primaryKey: string; readonly payload: Uint8Array } + >, + after: Iterable +): Effect.Effect, EventJournalError> => + Effect.try({ + try: () => { + const sequence = sequenceAfter(maxEntryId(after)) + return Arr.map(entries, ({ event, payload, primaryKey }) => + new Entry({ + id: nextEntryId(sequence, Date.now()), + event, + primaryKey, + payload + }, { disableChecks: true })) + }, + catch: (cause) => new EventJournalError({ method: "writeMany", cause }) + }) + /** * Creates an in-memory `EventJournal` service. * @@ -394,6 +598,17 @@ export class RemoteEntry extends Schema.Class("effect/eventlog/Even */ export const makeMemory: Effect.Effect = Effect.gen(function*() { const journal: Array = [] + // Local writes append in commit order, which unserialized direct callers can + // make differ from id order, leaving the journal unordered until the next + // import re-sorts it. Remote backlogs are subsequences of the journal, so one + // flag covers them too. + let journalSorted = true + // The latest committed id, the floor for local ids. + let latestId: EntryId | undefined + const commitLatest = (ids: Iterable) => { + latestId = maxEntryId(latestId === undefined ? ids : [latestId, ...ids]) + } + const clock = yield* Clock.Clock const byId = new Map() const remotes = new Map }>() const pubsub = yield* PubSub.unbounded() @@ -416,86 +631,82 @@ export const makeMemory: Effect.Effect = Effect.gen(fun return created } - return EventJournal.of({ - entries: Effect.sync(() => journal.slice()), - write({ effect, event, payload, primaryKey }) { - return Effect.acquireUseRelease( - Effect.sync(() => - new Entry({ - id: makeEntryIdUnsafe(), - event, - primaryKey, - payload - }, { disableChecks: true }) - ), - effect, - (entry, exit) => - Effect.suspend(() => { - if (exit._tag === "Failure" || byId.has(entry.idString)) return Effect.void + const writeMany: EventJournal["Service"]["writeMany"] = ({ effect, entries }) => + Effect.acquireUseRelease( + Effect.suspend(() => makeEntries(entries, latestId === undefined ? [] : [latestId])), + effect, + (entries, exit) => + Effect.suspend(() => { + if (exit._tag === "Failure") return Effect.void + for (const entry of entries) { + const last = journal[journal.length - 1] + if (last !== undefined && Entry.Order(last, entry) > 0) { + journalSorted = false + } journal.push(entry) byId.set(entry.idString, entry) remotes.forEach((remote) => { remote.missing.push(entry) }) - return PubSub.publish(pubsub, entry) - }) - ) - }, + } + commitLatest(entries.map((entry) => entry.id)) + return PubSub.publishAll(pubsub, entries) + }) + ) + + return EventJournal.of({ + entries: Effect.sync(() => journal.slice()), + write: ({ effect, ...entry }) => writeMany({ entries: [entry], effect: (entries) => effect(entries[0]) }), + writeMany, writeFromRemote: Effect.fnUntraced(function*(options) { const remote = ensureRemote(options.remoteId) const uncommittedRemotes: Array = [] - const uncommitted: Array = [] const duplicateEntries: Array = [] + // Overlapping batches can repeat an entry; the first copy wins. + const incomingIds = new Set() for (const remoteEntry of options.entries) { - if (byId.has(remoteEntry.entry.idString)) { + const idString = remoteEntry.entry.idString + if (byId.has(idString)) { duplicateEntries.push(remoteEntry.entry) - if (remoteEntry.remoteSequence >= remote.sequence) { - remote.sequence = remoteEntry.remoteSequence + 1 - } - continue + } else if (!incomingIds.has(idString)) { + incomingIds.add(idString) + uncommittedRemotes.push(remoteEntry) } - uncommittedRemotes.push(remoteEntry) - uncommitted.push(remoteEntry.entry) } const compacted = options.compact ? yield* options.compact(uncommittedRemotes) - : uncommitted - - for (const originEntry of compacted) { - const entryMillis = entryIdMillis(originEntry.id) - const conflicts: Array = [] - for (let i = journal.length - 1; i >= -1; i--) { - const entry = journal[i] - if (entry !== undefined && entry.createdAtMillis > entryMillis) { - continue - } - for (let j = i + 1; j < journal.length; j++) { - const scannedEntry = journal[j]! - if (scannedEntry.event === originEntry.event && scannedEntry.primaryKey === originEntry.primaryKey) { - conflicts.push(scannedEntry) - } - } - yield* options.effect({ entry: originEntry, conflicts }) - break + : uncommittedRemotes.map((remoteEntry) => remoteEntry.entry) + + if (Arr.isReadonlyArrayNonEmpty(compacted)) { + const index = new ConflictIndex(compacted) + const oldest = index.oldest + // Only the suffix with greater ids can conflict, unless local writes + // left the journal unordered. + let i = journalSorted ? journal.length : 0 + while (i > 0 && Entry.Order(journal[i - 1], oldest) > 0) i-- + for (; i < journal.length; i++) { + index.add(journal[i]) } + yield* options.effect(index.toBatch()) } for (const remoteEntry of uncommittedRemotes) { - journal.push(remoteEntry.entry) byId.set(remoteEntry.entry.idString, remoteEntry.entry) - remotes.forEach((target) => { - if (target !== remote) { - target.missing.push(remoteEntry.entry) - } - }) - if (remoteEntry.remoteSequence >= remote.sequence) { - remote.sequence = remoteEntry.remoteSequence + 1 - } } - journal.sort((a, b) => a.createdAtMillis - b.createdAtMillis) - remotes.forEach((remote) => { - remote.missing.sort((a, b) => a.createdAtMillis - b.createdAtMillis) + // Sequences advance with the commit, so a failed batch is redelivered. + remote.sequence = nextSequenceAfter(options.entries, remote.sequence) + const imported = uncommittedRemotes.map((remoteEntry) => remoteEntry.entry).sort(Entry.Order) + insertOrdered(journal, journalSorted, imported) + remotes.forEach((target) => { + if (target !== remote) { + insertOrdered(target.missing, journalSorted, imported) + } else if (!journalSorted) { + target.missing.sort(Entry.Order) + } }) + journalSorted = true + commitLatest(imported.map((entry) => entry.id)) + yield* warnIfAhead(clock, imported.map((entry) => entry.id)) return { duplicateEntries } @@ -518,10 +729,18 @@ export const makeMemory: Effect.Effect = Effect.gen(fun } }) ), + writeRemoteAcks: ({ acks, firstSequence, remoteId }) => + Effect.sync(() => { + const remote = ensureRemote(remoteId) + remote.sequence = advanceSequence(remote.sequence, acks, firstSequence) + }), nextRemoteSequence: (remoteId) => Effect.sync(() => ensureRemote(remoteId).sequence), + entriesById: (ids) => Effect.sync(() => ids.map((id) => byId.get(Uuid.stringify(id)))), changes: PubSub.subscribe(pubsub), destroy: Effect.sync(() => { journal.length = 0 + journalSorted = true + latestId = undefined byId.clear() remotes.clear() }), @@ -529,6 +748,28 @@ export const makeMemory: Effect.Effect = Effect.gen(fun }) }) +// Same result as appending `additions` and sorting by id. +const insertOrdered = (entries: Array, sorted: boolean, additions: ReadonlyArray): void => { + let i = entries.length - 1 + for (let n = 0; n < additions.length; n++) { + entries.push(additions[n]) + } + if (!sorted) { + entries.sort(Entry.Order) + return + } + // Merge from the back, so additions sorting after the whole array cost O(additions). + let j = additions.length - 1 + let k = entries.length - 1 + while (j >= 0) { + if (i >= 0 && Entry.Order(entries[i], additions[j]) > 0) { + entries[k--] = entries[i--] + } else { + entries[k--] = additions[j--] + } + } +} + /** * Layer that provides an in-memory `EventJournal`. * @@ -561,20 +802,55 @@ export const makeIndexedDb = (options?: { }): Effect.Effect => Effect.gen(function*() { const database = options?.database ?? "effect_event_journal" - const openRequest = indexedDB.open(database, 1) - openRequest.onupgradeneeded = () => { + const openRequest = indexedDB.open(database, 2) + openRequest.onupgradeneeded = (event) => { const db = openRequest.result + const tx = openRequest.transaction! + + if (event.oldVersion < 1) { + const entries = db.createObjectStore("entries", { keyPath: "id" }) + entries.createIndex("id", "id", { unique: true }) + entries.createIndex("event", "event") + db.createObjectStore("remoteEntryId", { keyPath: ["remoteId"] }).createIndex("id", "remoteId", { unique: true }) + } - const entries = db.createObjectStore("entries", { keyPath: "id" }) - entries.createIndex("id", "id", { unique: true }) - entries.createIndex("event", "event") + const entries = tx.objectStore("entries") + // Bounds conflict reads to one event and primary key. + entries.createIndex("conflict", ["event", "primaryKey", "id"]) + // Entry ids order creation, not insertion, so uploads checkpoint an + // insertion sequence instead. + const sequence = db.createObjectStore("entrySequence", { autoIncrement: true }) + // The remote each imported entry came from, so uploads skip it. + const sources = db.createObjectStore("entrySource") + if (event.oldVersion < 1) return - const remotes = db.createObjectStore("remotes", { keyPath: ["remoteId", "entryId"] }) - remotes.createIndex("id", ["remoteId", "entryId"], { unique: true }) - remotes.createIndex("sequence", ["remoteId", "sequence"], { unique: true }) + // Number the existing entries in creation order. + const entryIds = entries.openKeyCursor() + entryIds.onsuccess = () => { + const cursor = entryIds.result + if (!cursor) return + sequence.add(cursor.primaryKey) + cursor.continue() + } - const remoteEntryId = db.createObjectStore("remoteEntryId", { keyPath: ["remoteId"] }) - remoteEntryId.createIndex("id", "remoteId", { unique: true }) + // Version 1 kept a `remotes` row per entry per remote and recorded upload + // progress as an entry id, which cannot become a checkpoint. Keep one + // source per entry and, walking each remote's rows in sequence order, its + // download cursor; then drop the store. + const progress = tx.objectStore("remoteEntryId") + progress.clear() + const remotes = tx.objectStore("remotes").index("sequence").openCursor() + remotes.onsuccess = () => { + const cursor = remotes.result + if (!cursor) { + db.deleteObjectStore("remotes") + return + } + const row = cursor.value as { remoteId: IDBValidKey; entryId: IDBValidKey; sequence: number } + sources.put(row.remoteId, row.entryId) + progress.put({ remoteId: row.remoteId, nextSequence: row.sequence + 1 }) + cursor.continue() + } } const db = yield* Effect.acquireRelease( @@ -583,6 +859,45 @@ export const makeIndexedDb = (options?: { ) const pubsub = yield* PubSub.unbounded() + const clock = yield* Clock.Clock + + // Other tabs share the database but not the entry id sequence. + const latestEntryId = Effect.callback, EventJournalError>((resume) => { + const tx = db.transaction("entries", "readonly") + const request = tx.objectStore("entries").openKeyCursor(null, "prev") + onTransactionDone(tx, "writeMany", resume, () => { + const key = request.result?.primaryKey as ArrayBuffer | ArrayBufferView | undefined + if (key === undefined) return [] + return [ + (ArrayBuffer.isView(key) + ? new Uint8Array(key.buffer, key.byteOffset, key.byteLength) + : new Uint8Array(key)) as EntryId + ] + }) + return Effect.sync(() => tx.abort()) + }) + + const writeMany: EventJournal["Service"]["writeMany"] = ({ effect, entries }) => + Effect.uninterruptibleMask((restore) => + latestEntryId.pipe( + Effect.flatMap((latest) => makeEntries(entries, latest)), + Effect.flatMap((committed) => + restore(effect(committed)).pipe( + Effect.tap(Effect.callback((resume) => { + const tx = db.transaction(["entries", "entrySequence"], "readwrite") + const store = tx.objectStore("entries") + const sequence = tx.objectStore("entrySequence") + for (const entry of committed) { + store.put(encodeEntryIdb(entry)) + sequence.add(entry.id as IDBValidKey) + } + onTransactionDone(tx, "writeMany", resume, constVoid) + })), + Effect.tap(PubSub.publishAll(pubsub, committed)) + ) + ) + ) + ) return EventJournal.of({ entries: idbReq("entries", () => @@ -590,188 +905,174 @@ export const makeIndexedDb = (options?: { .objectStore("entries") .index("id") .getAll()).pipe( - Effect.flatMap((_) => - decodeEntryIdbArray(_).pipe( - Effect.mapError((cause) => new EventJournalError({ method: "entries", cause })) - ) - ) + Effect.map((rows) => rows.map(entryFromIdb)) ), - write: ({ effect, event, payload, primaryKey }) => - Effect.uninterruptibleMask((restore) => { - const entry = new Entry({ - id: makeEntryIdUnsafe(), - event, - primaryKey, - payload - }, { disableChecks: true }) - return restore(effect(entry)).pipe( - Effect.tap( - idbReq("write", () => - db.transaction("entries", "readwrite") - .objectStore("entries") - .put(encodeEntryIdb(entry))) - ), - Effect.tap(PubSub.publish(pubsub, entry)) - ) + entriesById: (ids) => + Effect.callback, EventJournalError>((resume) => { + const entries: Array = ids.map(() => undefined) + const tx = db.transaction("entries", "readonly") + const store = tx.objectStore("entries") + for (let i = 0; i < ids.length; i++) { + const request = store.get(ids[i] as IDBValidKey) + request.onsuccess = () => { + if (request.result !== undefined) entries[i] = entryFromIdb(request.result) + } + } + onTransactionDone(tx, "entriesById", resume, () => entries) + return Effect.sync(() => tx.abort()) }), + write: ({ effect, ...entry }) => writeMany({ entries: [entry], effect: (entries) => effect(entries[0]) }), + writeMany, writeFromRemote: Effect.fnUntraced(function*(options) { - const uncommitted: Array = [] const uncommittedRemotes: Array = [] const duplicateEntries: Array = [] + // Overlapping batches can repeat an entry; the first copy wins. + const incomingIds = new Set() yield* Effect.callback((resume) => { - const tx = db.transaction(["entries", "remotes"], "readwrite") + const tx = db.transaction("entries", "readonly") const entries = tx.objectStore("entries") - const remotes = tx.objectStore("remotes") - const iterator = options.entries[Symbol.iterator]() - const handleNext = (state: IteratorResult) => { - if (state.done) return - const remoteEntry = state.value - const entry = remoteEntry.entry - const entryIdKey = entry.id as IDBValidKey - entries.get(entryIdKey).onsuccess = (event) => { - if (event.target && "result" in event.target && event.target.result) { - duplicateEntries.push(entry) - remotes.put({ - remoteId: options.remoteId, - entryId: entry.id, - sequence: remoteEntry.remoteSequence - }) - handleNext(iterator.next()) - return + // Requests in a transaction complete in the order they were issued. + for (const remoteEntry of options.entries) { + const request = entries.getKey(remoteEntry.entry.id as IDBValidKey) + request.onsuccess = () => { + const idString = remoteEntry.entry.idString + if (request.result !== undefined) { + duplicateEntries.push(remoteEntry.entry) + } else if (!incomingIds.has(idString)) { + incomingIds.add(idString) + uncommittedRemotes.push(remoteEntry) } - uncommitted.push(entry) - uncommittedRemotes.push(remoteEntry) - handleNext(iterator.next()) } } - handleNext(iterator.next()) - tx.oncomplete = () => resume(Effect.void) - tx.onerror = () => resume(Effect.fail(new EventJournalError({ method: "writeFromRemote", cause: tx.error }))) + onTransactionDone(tx, "writeFromRemote", resume, constVoid) return Effect.sync(() => tx.abort()) }) const compacted = options.compact ? yield* options.compact(uncommittedRemotes) - : uncommitted + : uncommittedRemotes.map((remoteEntry) => remoteEntry.entry) - for (const originEntry of compacted) { - const conflicts: Array = [] + if (Arr.isReadonlyArrayNonEmpty(compacted)) { + const index = new ConflictIndex(compacted) yield* Effect.callback((resume) => { const tx = db.transaction("entries", "readonly") - const entries = tx.objectStore("entries") - const cursorRequest = entries.index("id").openCursor( - IDBKeyRange.lowerBound(originEntry.id as IDBValidKey, true), - "next" - ) - cursorRequest.onsuccess = () => { - const cursor = cursorRequest.result - if (!cursor) return - const decodedEntry = decodeEntryIdb(cursor.value) - if ( - decodedEntry.event === originEntry.event && - decodedEntry.primaryKey === originEntry.primaryKey - ) { - conflicts.push(decodedEntry) + const conflicts = tx.objectStore("entries").index("conflict") + // One read per group of the entries with a greater id than its + // oldest, in id order. + for (const group of index.groups()) { + const request = conflicts.getAll(IDBKeyRange.bound( + [group.event, group.primaryKey, group.oldest.id as IDBValidKey], + [group.event, group.primaryKey, entryIdUpperBound], + true + )) + request.onsuccess = () => { + for (const row of request.result) index.add(entryFromIdb(row)) } - cursor.continue() } - tx.oncomplete = () => resume(Effect.void) - tx.onerror = () => - resume(Effect.fail(new EventJournalError({ method: "writeFromRemote", cause: tx.error }))) + onTransactionDone(tx, "writeFromRemote", resume, constVoid) return Effect.sync(() => tx.abort()) }) - - yield* options.effect({ entry: originEntry, conflicts }) + yield* options.effect(index.toBatch()) } yield* Effect.callback((resume) => { - const tx = db.transaction(["entries", "remotes"], "readwrite") + const tx = db.transaction(["entries", "entrySource", "entrySequence", "remoteEntryId"], "readwrite") const entries = tx.objectStore("entries") - const remotes = tx.objectStore("remotes") + const sources = tx.objectStore("entrySource") + const sequence = tx.objectStore("entrySequence") for (const remoteEntry of uncommittedRemotes) { entries.add(encodeEntryIdb(remoteEntry.entry)) - remotes.put({ - remoteId: options.remoteId, - entryId: remoteEntry.entry.id, - sequence: remoteEntry.remoteSequence - }) + sequence.add(remoteEntry.entry.id as IDBValidKey) + sources.put(options.remoteId, remoteEntry.entry.id as IDBValidKey) } - tx.oncomplete = () => resume(Effect.void) - tx.onerror = () => resume(Effect.fail(new EventJournalError({ method: "writeFromRemote", cause: tx.error }))) + // Sequences advance with the commit, so a failed batch is redelivered. + advanceProgress( + tx.objectStore("remoteEntryId"), + options.remoteId, + "nextSequence", + () => nextSequenceAfter(options.entries) + ) + onTransactionDone(tx, "writeFromRemote", resume, constVoid) return Effect.sync(() => tx.abort()) }) + yield* warnIfAhead(clock, uncommittedRemotes.map((remoteEntry) => remoteEntry.entry.id)) return { duplicateEntries } }), - withRemoteUncommited: (remoteId, f) => - Effect.callback, EventJournalError>((resume) => { - const entries: Array = [] - const tx = db.transaction(["entries", "remotes", "remoteEntryId"], "readwrite") - - const entriesStore = tx.objectStore("entries") - const remotesStore = tx.objectStore("remotes") - const remoteEntryIdStore = tx.objectStore("remoteEntryId") - - const remoteIdKey = remoteId as IDBValidKey - const request = remoteEntryIdStore.get(remoteIdKey) as IDBRequest<{ entryId: IDBValidKey } | undefined> - request.onsuccess = () => { - const startEntryId = request.result?.entryId - const entryCursor = entriesStore.index("id").openCursor( - startEntryId ? IDBKeyRange.lowerBound(startEntryId, true) : null, - "next" - ) - entryCursor.onsuccess = () => { - const cursor = entryCursor.result - if (!cursor) return - const entry = decodeEntryIdb(cursor.value) - remotesStore.get([remoteIdKey, entry.id as IDBValidKey]).onsuccess = (event) => { - if (!(event.target && "result" in event.target && event.target.result)) entries.push(entry) + withRemoteUncommited: (remoteId, f) => { + const remoteIdKey = remoteId as IDBValidKey + return Effect.callback<{ readonly entries: Array; readonly sequence: number }, EventJournalError>( + (resume) => { + const entries: Array = [] + // The last insertion sequence visited, or 0 when nothing follows the checkpoint. + let sequence = 0 + const tx = db.transaction(["entries", "entrySource", "remoteEntryId", "entrySequence"], "readonly") + const entriesStore = tx.objectStore("entries") + const sourcesStore = tx.objectStore("entrySource") + const checkpoint = tx.objectStore("remoteEntryId").get([remoteIdKey]) as IDBRequest< + RemoteProgress | undefined + > + checkpoint.onsuccess = () => { + const cursorRequest = tx.objectStore("entrySequence").openCursor( + IDBKeyRange.lowerBound(checkpoint.result?.sequence ?? 0, true) + ) + cursorRequest.onsuccess = () => { + const cursor = cursorRequest.result + if (!cursor) return + sequence = cursor.key as number + const entryId = cursor.value as IDBValidKey + const source = sourcesStore.get(entryId) + source.onsuccess = () => { + if (source.result !== undefined && indexedDB.cmp(source.result, remoteIdKey) === 0) return + const entry = entriesStore.get(entryId) + entry.onsuccess = () => { + entries.push(entryFromIdb(entry.result)) + } + } cursor.continue() } } + onTransactionDone(tx, "withRemoteUncommited", resume, () => ({ entries, sequence })) + return Effect.sync(() => tx.abort()) } - - tx.oncomplete = () => resume(Effect.succeed(entries)) - tx.onerror = () => - resume(Effect.fail(new EventJournalError({ method: "withRemoteUncommited", cause: tx.error }))) - return Effect.sync(() => tx.abort()) - }).pipe( - Effect.flatMap((entries) => { - if (!Arr.isReadonlyArrayNonEmpty(entries)) return Effect.succeedNone - const entryId = entries[entries.length - 1].id + ).pipe( + Effect.flatMap(({ entries, sequence }) => { + if (sequence === 0) return Effect.succeedNone + entries.sort(byEntryId) return Effect.uninterruptibleMask((restore) => - restore(f(entries)).pipe( - Effect.tap( - idbReq("withRemoteUncommited", () => - db.transaction("remoteEntryId", "readwrite").objectStore("remoteEntryId").put({ - remoteId, - entryId - })) - ), - Effect.asSome + restore(Arr.isReadonlyArrayNonEmpty(entries) ? Effect.asSome(f(entries)) : Effect.succeedNone).pipe( + Effect.tap(Effect.callback((resume) => { + // Uploads can complete out of order; the checkpoint only moves forward. + const tx = db.transaction("remoteEntryId", "readwrite") + advanceProgress(tx.objectStore("remoteEntryId"), remoteId, "sequence", () => sequence) + onTransactionDone(tx, "withRemoteUncommited", resume, constVoid) + })) ) ) }) - ), + ) + }, + writeRemoteAcks: ({ acks, firstSequence, remoteId }) => + Effect.callback((resume) => { + const tx = db.transaction("remoteEntryId", "readwrite") + advanceProgress( + tx.objectStore("remoteEntryId"), + remoteId, + "nextSequence", + (next) => advanceSequence(next, acks, firstSequence) + ) + onTransactionDone(tx, "writeRemoteAcks", resume, constVoid) + return Effect.sync(() => tx.abort()) + }), nextRemoteSequence: (remoteId) => Effect.callback((resume) => { - const tx = db.transaction("remotes", "readonly") - let sequence = 0 - const remoteIdKey = remoteId as IDBValidKey - const cursorRequest = tx.objectStore("remotes").index("sequence").openCursor( - IDBKeyRange.bound([remoteIdKey, 0], [remoteIdKey, Infinity]), - "prev" - ) - cursorRequest.onsuccess = () => { - const cursor = cursorRequest.result - if (!cursor) return - sequence = cursor.value.sequence + 1 - } - tx.oncomplete = () => resume(Effect.succeed(sequence)) - tx.onerror = () => - resume(Effect.fail(new EventJournalError({ method: "nextRemoteSequence", cause: tx.error }))) + const tx = db.transaction("remoteEntryId", "readonly") + const progress = tx.objectStore("remoteEntryId").get([remoteId as IDBValidKey]) as IDBRequest< + RemoteProgress | undefined + > + onTransactionDone(tx, "nextRemoteSequence", resume, () => progress.result?.nextSequence ?? 0) return Effect.sync(() => tx.abort()) }), changes: PubSub.subscribe(pubsub), @@ -782,6 +1083,37 @@ export const makeIndexedDb = (options?: { }) }) +/** + * The sequence following every entry in a delivered batch. + * + * @internal + */ +export const nextSequenceAfter = (entries: ReadonlyArray, next = 0): number => { + for (const remoteEntry of entries) { + if (remoteEntry.remoteSequence >= next) next = remoteEntry.remoteSequence + 1 + } + return next +} + +/** + * Advances across contiguous acknowledgements without skipping unseen entries. + * + * @internal + */ +export const advanceSequence = ( + next: number, + acks: ReadonlyArray, + firstSequence = 0 +): number => { + next = Math.max(next, firstSequence) + const sequences = acks.map((ack) => ack.sequence).sort((a, b) => a - b) + for (const sequence of sequences) { + if (sequence === next) next++ + else if (sequence > next) break + } + return next +} + const makeBrowserWithLock = Effect.fnUntraced(function*(key: string) { if (typeof navigator !== "undefined" && "locks" in navigator) { return (storeId: StoreId) => (self: Effect.Effect) => @@ -806,10 +1138,50 @@ const makeBrowserWithLock = Effect.fnUntraced(function*(key: string) { } }) -const decodeEntryIdb = Schema.decodeSync(Entry) +// Progress for a remote: `sequence` is the upload checkpoint and `nextSequence` +// the download cursor. +interface RemoteProgress { + readonly sequence?: number + readonly nextSequence?: number +} + +// Raises one progress field within the store's transaction; progress only moves forward. +const advanceProgress = ( + store: IDBObjectStore, + remoteId: RemoteId, + field: keyof RemoteProgress, + next: (current: number) => number +): void => { + const request = store.get([remoteId as IDBValidKey]) as IDBRequest + request.onsuccess = () => { + const current = request.result?.[field] ?? 0 + const value = next(current) + if (value > current) store.put({ ...request.result, remoteId, [field]: value }) + } +} + +const byEntryId = (a: Entry, b: Entry) => indexedDB.cmp(a.id as IDBValidKey, b.id as IDBValidKey) + +// Greater than or equal to every entry id. +const entryIdUpperBound = new Uint8Array(16).fill(0xff) + +interface EntryIdb { + readonly id: EntryId + readonly event: string + readonly primaryKey: string + readonly payload: Uint8Array +} + +// Rows are written by `encodeEntryIdb`, so they are not validated again. +const entryFromIdb = (row: EntryIdb): Entry => + new Entry({ + id: row.id, + event: row.event, + primaryKey: row.primaryKey, + payload: row.payload + }, { disableChecks: true }) + const encodeEntryIdb = Schema.encodeSync(Entry) -const EntryIdbArray = Schema.Array(Entry) -const decodeEntryIdbArray = Schema.decodeUnknownEffect(EntryIdbArray) /** * Provides `EventJournal` using the IndexedDB-backed implementation created by @@ -827,6 +1199,18 @@ export const layerIndexedDb = (options?: { makeIndexedDb(options) ) +// Resumes when the transaction commits, and fails if it errors or aborts. An +// abort, such as a failed commit, does not always dispatch an error first. +const onTransactionDone = ( + tx: IDBTransaction, + method: string, + resume: (effect: Effect.Effect) => void, + onComplete: () => A +): void => { + tx.oncomplete = () => resume(Effect.succeed(onComplete())) + tx.onerror = tx.onabort = () => resume(Effect.fail(new EventJournalError({ method, cause: tx.error }))) +} + const idbReq = (method: string, evaluate: () => IDBRequest) => Effect.callback((resume) => { const request = evaluate() diff --git a/repos/effect/packages/effect/src/eventlog/EventLog.ts b/repos/effect/packages/effect/src/eventlog/EventLog.ts index 4b94239c48..a9909f6814 100644 --- a/repos/effect/packages/effect/src/eventlog/EventLog.ts +++ b/repos/effect/packages/effect/src/eventlog/EventLog.ts @@ -10,6 +10,8 @@ * @stability unstable * @since 4.0.0 */ +import * as Arr from "../Array.ts" +import * as Cause from "../Cause.ts" import * as Context from "../Context.ts" import * as Effect from "../Effect.ts" import * as FiberMap from "../FiberMap.ts" @@ -25,6 +27,8 @@ import { Reactivity } from "../reactivity/Reactivity.ts" import * as ReactivityLayer from "../reactivity/Reactivity.ts" import type * as Record from "../Record.ts" import * as Redacted from "../Redacted.ts" +import { CurrentLogAnnotations } from "../References.ts" +import * as Result from "../Result.ts" import * as Schedule from "../Schedule.ts" import * as Schema from "../Schema.ts" import * as SchemaGetter from "../SchemaGetter.ts" @@ -36,6 +40,7 @@ import { Entry, EventJournal, type EventJournalError, makeEntryIdUnsafe, type Re import * as EventLogEncryption from "./EventLogEncryption.ts" import { StoreId } from "./EventLogMessage.ts" import type { EventLogRemote } from "./EventLogRemote.ts" +import { setAnchor } from "./internal/conflicts.ts" /** * Service for writing typed event-log events through registered handlers. @@ -59,6 +64,27 @@ export class EventLog extends Context.Service, Tag>, Event.ErrorWithTag, Tag> | EventJournalError > + /** + * Write several events under one journal lock and one `HandlerTransaction`, + * returning each handler's result in input order. If any handler fails, + * none of the events are committed. + */ + readonly writeMany: < + Groups extends EventGroup.Any, + // Pairs each event tag with its own payload. + Item extends { + readonly [Tag in Event.Tag>]: { + readonly event: Tag + readonly payload: Event.PayloadWithTag, Tag> + } + }[Event.Tag>] + >(options: { + readonly schema: EventLogSchema + readonly events: Arr.NonEmptyReadonlyArray + }) => Effect.Effect< + Arr.NonEmptyArray, Item["event"]>>, + Event.ErrorWithTag, Item["event"]> | EventJournalError + > readonly entries: Effect.Effect, EventJournalError> readonly destroy: Effect.Effect }>()("effect/eventlog/EventLog") {} @@ -83,7 +109,9 @@ export class Registry extends Context.Service readonly effect: (options: { readonly entries: ReadonlyArray - readonly write: (entry: Entry) => Effect.Effect + // Pass an input entry as source to anchor replacements during remote replay. + // Without it, matching IDs retain their position; new IDs use the group start. + readonly write: (entry: Entry, source?: Entry) => Effect.Effect }) => Effect.Effect }) => Effect.Effect @@ -91,7 +119,7 @@ export class Registry extends Context.Service readonly effect: (options: { readonly entries: ReadonlyArray - readonly write: (entry: Entry) => Effect.Effect + readonly write: (entry: Entry, source?: Entry) => Effect.Effect }) => Effect.Effect }> @@ -119,7 +147,7 @@ export const layerRegistry = Layer.effect( readonly events: ReadonlySet readonly effect: (options: { readonly entries: ReadonlyArray - readonly write: (entry: Entry) => Effect.Effect + readonly write: (entry: Entry, source?: Entry) => Effect.Effect }) => Effect.Effect }>() @@ -437,6 +465,68 @@ export class CurrentStoreId extends Context.Reference("effect/eventlog/ defaultValue: constant(StoreId.make("default")) }) {} +/** + * Context reference for a transaction that wraps the event handlers of each + * batch replayed from a remote and of each local `write` or `writeMany`. + * + * **When to use** + * + * Use when handlers project into a store with transactions, such as + * `sql.withTransaction` or an IndexedDB transaction over the projection + * tables, so a batch of N events commits once instead of N times. + * + * **Details** + * + * The journal performs no I/O while the transaction is open: it reads replay + * conflicts before the transaction starts and commits the batch after it + * completes. Defaults to running the handlers without a transaction. + * + * If the transaction fails, the journal commits nothing from the batch. For a + * replayed batch, the remote is subscribed again after a backoff of up to 10 + * seconds and the batch is delivered again, so later batches from that remote + * wait until it succeeds; local writes and other remotes are not blocked. A + * local write fails with the error and is not retried. + * + * **Gotchas** + * + * - A replayed entry that fails to decode, or whose handler fails or dies, + * fails the whole batch, which is retried as above. A failure that repeats + * on every attempt, such as a payload written by a newer schema, stops sync + * from that remote until the app is updated to handle it. + * - In a local write, an error returned by a handler fails the transaction + * and the write, and the journal commits none of its events. + * - Failures that abort the underlying transaction fail the whole batch, even + * when the handler recovers from them. In IndexedDB, any failed request that + * is not prevented aborts the whole transaction, and in PostgreSQL so does a + * failed statement outside a savepoint. + * - Failing the batch does not undo side effects of handlers that already ran. + * Only changes made inside a transaction that rolls back are undone. + * - Replay handlers must tolerate running more than once for the same entry. + * - Unless the transaction and the journal share a database connection, the + * transaction commits before the journal does, and the two commits are not + * atomic. A crash between them replays a remote batch on restart, but a + * local write is lost from the journal while its projection changes remain. + * - An IndexedDB transaction commits once it has no pending requests and + * control returns to the event loop. A custom wrapper must stop the fiber + * from yielding to the scheduler while it is open, by providing + * `References.PreventSchedulerYield` as `withTransaction` from + * `@effect/platform-browser` does. Otherwise it can commit partway through a + * batch, and the remaining handlers fail. + * - The transaction stays open for the whole batch, so other read-write + * transactions on the same tables wait for it. + * - The transaction must not change the error or success of the wrapped + * effect. Convert its own failures into defects. + * + * @stability unstable + * @category services + * @since 4.0.6 + */ +export class HandlerTransaction extends Context.Reference< + (effect: Effect.Effect) => Effect.Effect +>("effect/eventlog/EventLog/HandlerTransaction", { + defaultValue: constant(identity) +}) {} + const RedactedUint8Array = Schema.Uint8ArrayFromBase64.pipe( Schema.decodeTo(Schema.Redacted(Schema.Uint8Array), { decode: SchemaGetter.transform((value) => Redacted.make(value)), @@ -581,28 +671,19 @@ export const group = ( }) ) as any -/** - * Registers a compaction handler for an event group. - * - * **Details** - * - * During remote replay, matching entries are decoded, grouped by primary key, and - * passed to the compaction effect, which may write replacement entries. - * - * @stability unstable - * @category compaction - * @since 4.0.0 - */ -export const groupCompaction = ( +const registerBatchCompaction = ( group: EventGroup.EventGroup, effect: (options: { - readonly primaryKey: string - readonly entries: ReadonlyArray - readonly events: ReadonlyArray> - readonly write: >( + readonly entries: ReadonlyArray<{ + readonly entry: Entry + readonly event: Event.TaggedPayload | undefined + }> + readonly writeFrom: >( + source: Entry, tag: Tag, payload: Event.PayloadWithTag ) => Effect.Effect["EncodingServices"]> + readonly forward: (entry: Entry) => Effect.Effect }) => Effect.Effect ): Layer.Layer["DecodingServices"] | Registry> => Layer.effectDiscard( @@ -616,70 +697,197 @@ export const groupCompaction = ( const isEventTag = (tag: string): tag is Event.Tag => Object.hasOwn(group.events, tag) const decodePayload = >(tag: Tag, payload: Uint8Array) => Schema.decodeUnknownEffect(group.events[tag].payloadSchemaBinary)(payload).pipe( - Effect.updateContext((input) => Context.merge(services, input)), - Effect.orDie - ) as unknown as Effect.Effect> - const writePayload = Effect.fnUntraced(function*>( - timestamp: number, + Effect.updateContext((input) => Context.merge(services, input)) + ) as unknown as Effect.Effect, Schema.SchemaError> + const writeFrom = Effect.fnUntraced(function*>( + source: Entry, tag: Tag, payload: Event.PayloadWithTag ): Effect.fn.Return["EncodingServices"]> { const event = Object.hasOwn(group.events, tag) ? group.events[tag] : undefined! const entry = new Entry({ - id: makeEntryIdUnsafe({ msecs: timestamp }), + id: makeEntryIdUnsafe({ msecs: source.createdAtMillis }), event: tag, payload: yield* Schema.encodeUnknownEffect(event.payloadSchemaBinary)(payload).pipe( Effect.orDie ) as any, primaryKey: event.primaryKey(payload) }, { disableChecks: true }) - yield* write(entry) + yield* write(entry, source) }) + const forward = (entry: Entry) => write(entry) - const byPrimaryKey = new Map< - string, - { - readonly entries: Array - readonly taggedPayloads: Array> - } - >() + const batch: Array<{ + readonly entry: Entry + readonly event: Event.TaggedPayload | undefined + }> = [] for (const entry of entries) { if (!isEventTag(entry.event)) { continue } - const payload = yield* decodePayload(entry.event, entry.payload) - const record = byPrimaryKey.get(entry.primaryKey) - const taggedPayload = { _tag: entry.event, payload } as unknown as Event.TaggedPayload - if (record) { - record.entries.push(entry) - record.taggedPayloads.push(taggedPayload) - } else { - byPrimaryKey.set(entry.primaryKey, { - entries: [entry], - taggedPayloads: [taggedPayload] - }) - } + const payload = yield* Effect.result(decodePayload(entry.event, entry.payload)) + batch.push({ + entry, + event: Result.isFailure(payload) + ? undefined + : { _tag: entry.event, payload: payload.success } as unknown as Event.TaggedPayload + }) } - for (const [primaryKey, { entries, taggedPayloads }] of byPrimaryKey) { - yield* Effect.orDie( - effect({ - primaryKey, - entries, - events: taggedPayloads, - write(tag, payload) { - return Effect.orDie(writePayload(entries[0].createdAtMillis, tag, payload)) - } - }).pipe( - Effect.updateContext((input) => Context.merge(services, input)) - ) - ) as any - } + yield* Effect.orDie( + effect({ entries: batch, writeFrom, forward }).pipe( + Effect.updateContext((input) => Context.merge(services, input)) + ) + ) as any }) }) }) ) +/** + * Registers a batch compaction handler for an event group. + * + * **When to use** + * + * Use when an event under one primary key affects entries under other keys. + * + * **Details** + * + * The compaction effect receives every entry for the group in batch order, with + * its decoded event or `event: undefined` if the payload cannot be decoded. + * + * Only entries that the effect outputs are kept: + * + * - `forward(entry)` keeps an input entry unchanged, with the same id, payload, + * and primary key. + * - `write(tag, payload)` encodes a replacement entry anchored to the first + * entry in the batch, with a fresh id in that entry's millisecond. + * + * Entries that are neither forwarded nor replaced are dropped, including entries + * that could not be decoded. During remote replay, kept entries are replayed in + * the id order of their anchors, then in write order, and a replacement's + * conflicts are the stored entries newer than its anchor. + * + * **Gotchas** + * + * Compaction runs before conflicts are read, so it never sees stored entries. + * Collapsing several entries into one is only safe when no stored entry for + * the same event and primary key has an id between theirs. Otherwise that + * stored entry should have applied between the collapsed ones, and replicas + * that received the entries in separate batches end up different. + * + * @see {@link groupCompaction} for compaction grouped by primary key + * + * @stability unstable + * @category compaction + * @since 4.0.0 + */ +export const groupBatchCompaction = ( + group: EventGroup.EventGroup, + effect: (options: { + readonly entries: ReadonlyArray<{ + readonly entry: Entry + readonly event: Event.TaggedPayload | undefined + }> + readonly write: >( + tag: Tag, + payload: Event.PayloadWithTag + ) => Effect.Effect["EncodingServices"]> + readonly forward: (entry: Entry) => Effect.Effect + }) => Effect.Effect +): Layer.Layer["DecodingServices"] | Registry> => + registerBatchCompaction(group, ({ entries, writeFrom, forward }) => + effect({ + entries, + write: (tag, payload) => writeFrom(entries[0].entry, tag, payload), + forward + })) + +/** + * Registers a compaction handler for an event group. + * + * **Details** + * + * During remote replay, decoded entries are grouped by primary key and passed to + * the compaction effect, which outputs the entries to keep: + * + * - `forward(entry)` keeps an input entry unchanged, with the same id, payload, + * and primary key. + * - `write(tag, payload)` encodes a replacement entry anchored to the first + * decoded entry for the primary key, with a fresh id in that entry's + * millisecond. + * + * Entries that are neither forwarded nor replaced are dropped. Entries that + * cannot be decoded are not passed to the effect and are kept unchanged. + * During remote replay, kept entries are replayed in the id order of their + * anchors, then in write order, and a replacement's conflicts are the stored + * entries newer than its anchor. + * + * **Gotchas** + * + * Compaction runs before conflicts are read, so it never sees stored entries. + * Collapsing several entries into one is only safe when no stored entry for + * the same event and primary key has an id between theirs. Otherwise that + * stored entry should have applied between the collapsed ones, and replicas + * that received the entries in separate batches end up different. + * + * @see {@link groupBatchCompaction} for compaction across primary keys + * + * @stability unstable + * @category compaction + * @since 4.0.0 + */ +export const groupCompaction = ( + group: EventGroup.EventGroup, + effect: (options: { + readonly primaryKey: string + readonly entries: ReadonlyArray + readonly events: ReadonlyArray> + readonly write: >( + tag: Tag, + payload: Event.PayloadWithTag + ) => Effect.Effect["EncodingServices"]> + readonly forward: (entry: Entry) => Effect.Effect + }) => Effect.Effect +): Layer.Layer["DecodingServices"] | Registry> => + registerBatchCompaction( + group, + Effect.fnUntraced(function*({ entries, writeFrom, forward }) { + const byPrimaryKey = new Map< + string, + { + readonly entries: Array + readonly events: Array> + } + >() + for (const { entry, event } of entries) { + if (event === undefined) { + yield* forward(entry) + continue + } + const record = byPrimaryKey.get(entry.primaryKey) + if (record) { + record.entries.push(entry) + record.events.push(event) + } else { + byPrimaryKey.set(entry.primaryKey, { entries: [entry], events: [event] }) + } + } + + for (const [primaryKey, { entries, events }] of byPrimaryKey) { + yield* effect({ + primaryKey, + entries, + events, + write: (tag, payload) => writeFrom(entries[0], tag, payload), + forward + }) + } + }) + ) + +type ReactivityKeyName = [keyof ReactivityLayer.Keys] extends [never] ? string : keyof ReactivityLayer.Keys + /** * Registers reactivity keys to invalidate when events from a group are written or * replayed. @@ -695,8 +903,8 @@ export const groupCompaction = ( export const groupReactivity = ( group: EventGroup.EventGroup, keys: - | { readonly [Tag in Event.Tag]?: ReadonlyArray } - | ReadonlyArray + | { readonly [Tag in Event.Tag]?: ReadonlyArray } + | ReadonlyArray ): Layer.Layer => Effect.gen(function*() { const registry = yield* Registry @@ -719,8 +927,10 @@ export const groupReactivity = ( * **Details** * * The returned handler decodes the entry and conflicts with the registered event - * schema, runs the matching handler with the supplied identity and store id, logs - * failures, and invalidates configured reactivity keys. + * schema, runs the matching handler with the supplied identity and store id, and + * invalidates configured reactivity keys. Decode and handler failures are logged + * and fail the effect. Invalidations are deferred until the end of an enclosing + * `Reactivity.withBatch`. * * @stability unstable * @category handlers @@ -736,60 +946,128 @@ export const makeReplayFromRemote = (options: { readonly service: string readonly effect: string } -}) => - Effect.fnUntraced( - function*({ conflicts, entry }: { - readonly entry: Entry - readonly conflicts: ReadonlyArray - }): Effect.fn.Return { +}) => { + const replayBatch = makeReplayBatchFromRemote(options) + return (input: ReplayInput) => replayBatch([input]) +} + +type ReplayInput = { + readonly entry: Entry + readonly conflicts: ReadonlyArray +} + +type DecodedConflict = { entry: Entry; payload: unknown } + +type ReplayBatch = { + readonly context: Context.Context + readonly handlerContexts: Map, Context.Context> + readonly decodedConflicts: Map +} + +/** + * Replays a batch, merging each handler's context with the batch context once + * and decoding each conflict entry shared by several entries once. The batch + * state is created per execution, so transaction retries start fresh. + */ +const makeReplayBatchFromRemote = (options: Parameters[0]) => { + const replayEntry = makeReplayEntryFromRemote(options) + return (batch: ReadonlyArray) => + Effect.contextWith((context: Context.Context) => { + const state: ReplayBatch = { context, handlerContexts: new Map(), decodedConflicts: new Map() } + return Effect.forEach(batch, (input) => replayEntry(input, state), { discard: true }) + }) +} + +// An entry's context is the batch context plus its log annotations, so only +// the annotations need to be copied into the merged handler context. +const withEntryAnnotations = (context: Context.Context, entryContext: Context.Context) => + Context.add(context, CurrentLogAnnotations, Context.get(entryContext, CurrentLogAnnotations)) + +const makeReplayEntryFromRemote = (options: Parameters[0]) => { + // Decoders depend only on the event schema, so they are shared across batches. + const decoders = new WeakMap< + Handlers.Item, + (input: unknown) => Effect.Effect + >() + const decoderFor = (handler: Handlers.Item) => { + let decoder = decoders.get(handler) + if (decoder === undefined) { + decoder = Schema.decodeUnknownEffect(handler.event.payloadSchemaBinary) + decoders.set(handler, decoder) + } + return decoder + } + const contextFor = (batch: ReplayBatch, handler: Handlers.Item) => { + let context = batch.handlerContexts.get(handler) + if (context === undefined) { + context = Context.merge(handler.context, batch.context) + batch.handlerContexts.set(handler, context) + } + return context + } + + return Effect.fnUntraced( + function*( + { conflicts, entry }: ReplayInput, + batch: ReplayBatch + ): Effect.fn.Return { const handler = options.handlers.get(entry.event) as Handlers.Item | undefined if (!handler) { return yield* Effect.logDebug(`Event handler not found for: "${entry.event}"`) } - const decodePayload = Schema.decodeUnknownEffect(handler.event.payloadSchemaBinary) - const decodedConflicts: Array<{ entry: Entry; payload: unknown }> = new Array(conflicts.length) + const decodePayload = decoderFor(handler) + const context = contextFor(batch, handler) + const conflictPayloads: Array = new Array(conflicts.length) for (let i = 0; i < conflicts.length; i++) { - decodedConflicts[i] = { - entry: conflicts[i], - payload: yield* decodePayload(conflicts[i].payload).pipe( - Effect.updateContext((input) => Context.merge(handler.context, input)) - ) as any + const conflict = conflicts[i] + let decoded = batch.decodedConflicts.get(conflict) + if (!decoded) { + decoded = { + entry: conflict, + payload: yield* decodePayload(conflict.payload).pipe( + Effect.updateContext((input) => withEntryAnnotations(context, input)) + ) as any + } + batch.decodedConflicts.set(conflict, decoded) } + conflictPayloads[i] = decoded } + // Conflicts decode with the caller's Identity; the payload and handler use the log's. yield* decodePayload(entry.payload).pipe( Effect.flatMap((payload) => handler.handler({ storeId: options.storeId, payload, entry, - conflicts: decodedConflicts + conflicts: conflictPayloads }) ), - Effect.provideService(Identity, options.identity), - Effect.updateContext((input) => Context.merge(handler.context, input)), + Effect.updateContext((input) => Context.add(withEntryAnnotations(context, input), Identity, options.identity)), Effect.asVoid ) as any - const keys = Object.hasOwn(options.reactivityKeys, entry.event) - ? options.reactivityKeys[entry.event] - : undefined - if (keys) { - for (const key of keys) { - options.reactivity.invalidateUnsafe({ - [key]: [entry.primaryKey] - }) - } - } + yield* invalidateEntry(options.reactivity, options.reactivityKeys, entry) }, - Effect.catchCause(Effect.logError), + Effect.tapCauseIf((cause) => !Cause.hasInterruptsOnly(cause), Effect.logError), (effect, { entry }) => Effect.annotateLogs(effect, { ...options.logAnnotations, entryId: entry.idString }) ) +} + +const invalidateEntry = ( + reactivity: Reactivity, + reactivityKeys: Record>, + entry: Entry +): Effect.Effect => { + const keys = Object.hasOwn(reactivityKeys, entry.event) ? reactivityKeys[entry.event] : undefined + if (!keys) return Effect.void + return Effect.forEach(keys, (key) => reactivity.invalidate({ [key]: [entry.primaryKey] }), { discard: true }) +} const remoteRetrySchedule = Schedule.min([ Schedule.exponential(200, 1.5), @@ -803,7 +1081,8 @@ const make = Effect.gen(function*() { const registry = yield* Registry const reactivity = yield* Reactivity - const replayFromRemote = makeReplayFromRemote({ + const handlerTransaction = yield* HandlerTransaction + const replayFromRemote = makeReplayBatchFromRemote({ handlers: registry.handlers, storeId, identity, @@ -816,78 +1095,80 @@ const make = Effect.gen(function*() { }) const invalidateReactivityEntries = (entries: ReadonlyArray) => - Effect.sync(() => { - for (const entry of entries) { - const keys = Object.hasOwn(registry.reactivityKeys, entry.event) - ? registry.reactivityKeys[entry.event] - : undefined - if (!keys) { - continue - } - for (const key of keys) { - reactivity.invalidateUnsafe({ - [key]: [entry.primaryKey] - }) - } - } - }) + Effect.forEach(entries, (entry) => invalidateEntry(reactivity, registry.reactivityKeys, entry), { discard: true }) const runRemote = Effect.fnUntraced( function*(remote: EventLogRemote["Service"]) { - const startSequence = yield* journal.nextRemoteSequence(remote.id) - yield* Effect.gen(function*() { - const changes = yield* remote.changes({ identity, startSequence, storeId }) + const startSequence = yield* journal.nextRemoteSequence(remote.id) + const changes = yield* remote.changes({ + identity, + startSequence, + storeId, + knownEntries: journal.entriesById + }) while (true) { const entries = yield* Queue.takeAll(changes) yield* journal.writeFromRemote({ remoteId: remote.id, - entries: entries.flat(), + entries, compact: registry.compactors.size > 0 ? Effect.fnUntraced(function*(remoteEntries) { - const finalEntries: Array = [] + // The journal sorts the batch by anchor, so output order is irrelevant. + const output: Array = [] + const inputs = new Map() + const outputCounts = new Map() const compactable = new Map< (options: { readonly entries: ReadonlyArray - readonly write: (entry: Entry) => Effect.Effect + readonly write: (entry: Entry, source?: Entry) => Effect.Effect }) => Effect.Effect, Array >() - for (let i = 0; i < remoteEntries.length; i++) { - const remoteEntry = remoteEntries[i] - const entry = remoteEntry.entry + for (const { entry } of remoteEntries) { + inputs.set(entry.idString, entry) const compactor = registry.compactors.get(entry.event) if (!compactor) { - finalEntries.push(entry) + output.push(entry) continue } - let arr = compactable.get(compactor.effect) - if (!arr) { - arr = [] - compactable.set(compactor.effect, arr) + let group = compactable.get(compactor.effect) + if (!group) { + group = [] + compactable.set(compactor.effect, group) } - arr.push(entry) + group.push(entry) } for (const [compact, entries] of compactable) { yield* compact({ entries, - write(entry) { + write(entry, source) { return Effect.sync(() => { - finalEntries.push(entry) + // Each output is ordered and conflict-checked by its source, + // then by write order, not by its own fresh id. An output + // without a known source stands for the group's first entry. + const anchor = inputs.get((source ?? entry).idString) ?? entries[0] + const index = outputCounts.get(anchor) ?? 0 + outputCounts.set(anchor, index + 1) + setAnchor(entry, anchor, index) + output.push(entry) }) } }) } - return finalEntries.sort(Entry.Order) + return output }) : undefined, - effect: replayFromRemote + // A failed batch is not committed, so the remote redelivers it on retry. + effect: (batch) => handlerTransaction(replayFromRemote(batch)).pipe(Effect.orDie) }).pipe( Effect.tap(({ duplicateEntries }) => invalidateReactivityEntries(duplicateEntries)), - journal.withLock(storeId) + journal.withLock(storeId), + // Outside the lock, so queries re-run after the journal transaction commits. + reactivity.withBatch ) } }).pipe( @@ -901,7 +1182,19 @@ const make = Effect.gen(function*() { Effect.forkScoped ) - const write = journal.withRemoteUncommited(remote.id, (entries) => remote.write({ identity, entries, storeId })) + const write = journal.withRemoteUncommited( + remote.id, + (entries) => + remote.write({ identity, entries, storeId }).pipe( + Effect.flatMap((acks) => + acks === undefined || acks.length === 0 + ? Effect.void + : journal.writeRemoteAcks({ remoteId: remote.id, acks, firstSequence: remote.firstSequence }).pipe( + journal.withLock(storeId) + ) + ) + ) + ) const writeUntilSuccess = write.pipe( Effect.tapCause(Effect.logDebug), Effect.retry(remoteRetrySchedule) @@ -926,60 +1219,49 @@ const make = Effect.gen(function*() { Effect.orDie ) - const writeHandler = Effect.fnUntraced(function*(handler: Handlers.Item, options: { - readonly schema: EventLogSchema - readonly event: string - readonly payload: unknown - }) { - const payload = yield* Schema.encodeUnknownEffect(handler.event.payloadSchemaBinary)(options.payload).pipe( - Effect.orDie - ) - return yield* journal.withLock(storeId)(journal.write({ - event: options.event, - primaryKey: handler.event.primaryKey(options.payload as never), - payload, - effect: (entry) => - handler.handler({ - storeId, - payload: options.payload, - entry, - conflicts: [] - }).pipe( - Effect.updateContext((input) => Context.merge(handler.context, input)), - Effect.provideService(Identity, identity), - Effect.tap(() => { - const keys = Object.hasOwn(registry.reactivityKeys, entry.event) - ? registry.reactivityKeys[entry.event] - : undefined - if (keys) { - for (const key of keys) { - reactivity.invalidateUnsafe({ - [key]: [entry.primaryKey] - }) - } - } - return Effect.void - }) - ) - })) - }) - - const eventLogWrite = (options: { - readonly schema: EventLogSchema - readonly event: string - readonly payload: unknown - }) => { - const handler = registry.handlers.get(options.event) as Handlers.Item | undefined - if (handler === undefined) { - return Effect.die(`Event handler not found for: "${options.event}"`) - } - return writeHandler(handler, options) - } + const writeEvents = ( + events: Arr.NonEmptyReadonlyArray<{ readonly event: string; readonly payload: unknown }> + ): Effect.Effect, unknown> => + Effect.forEach(events, ({ event, payload }) => { + const handler = registry.handlers.get(event) as Handlers.Item | undefined + if (handler === undefined) { + return Effect.die(`Event handler not found for: "${event}"`) + } + return Schema.encodeUnknownEffect(handler.event.payloadSchemaBinary)(payload).pipe( + Effect.orDie, + Effect.map((encoded) => ({ + handler, + payload, + entry: { event, primaryKey: handler.event.primaryKey(payload as never), payload: encoded as Uint8Array } + })) + ) + }).pipe( + Effect.flatMap((writes) => + journal.withLock(storeId)(journal.writeMany({ + entries: Arr.map(writes, (write) => write.entry), + effect: (written) => + handlerTransaction(Effect.forEach(written, (entry, i) => { + const { handler, payload } = writes[i] + return handler.handler({ storeId, payload, entry, conflicts: [] }).pipe( + Effect.updateContext((input) => Context.merge(handler.context, input)), + Effect.provideService(Identity, identity), + Effect.tap(() => invalidateEntry(reactivity, registry.reactivityKeys, entry)) + ) + })) + })) + ), + // Outside the lock, so queries re-run after the handler transaction and + // the journal commit. + reactivity.withBatch + ) as Effect.Effect, unknown> yield* registry.handleRemote(runRemote) return EventLog.of({ - write: eventLogWrite as EventLog["Service"]["write"], + write: ((options: { readonly event: string; readonly payload: unknown }) => + Effect.map(writeEvents([options]), Arr.headNonEmpty)) as EventLog["Service"]["write"], + writeMany: ((options: { readonly events: Arr.NonEmptyReadonlyArray }) => + writeEvents(options.events)) as EventLog["Service"]["writeMany"], entries: journal.entries, destroy: journal.destroy }) @@ -1049,8 +1331,27 @@ export const layer = ( * * **Details** * - * The returned function delegates to the `EventLog` service and preserves each - * event's success and error types. + * The returned function takes one `{ event, payload }` object, typed so the + * payload matches the event, as `writeMany` items are. It delegates to the + * `EventLog` service and preserves each event's success and error types. + * + * **Example** (Writing events with a client) + * + * ```ts + * import { Effect, Schema } from "effect" + * import { EventGroup, EventLog } from "effect/eventlog" + * + * const Users = EventGroup.empty.add({ + * tag: "UserCreated", + * primaryKey: (payload) => payload.id, + * payload: Schema.Struct({ id: Schema.String }) + * }) + * + * export const createUser = Effect.gen(function*() { + * const client = yield* EventLog.makeClient(EventLog.schema(Users)) + * yield* client({ event: "UserCreated", payload: { id: "user-1" } }) + * }) + * ``` * * @stability unstable * @category constructors @@ -1059,24 +1360,23 @@ export const layer = ( export const makeClient = ( schema: EventLogSchema ): Effect.Effect< - (>>( - event: Tag, - payload: Event.PayloadWithTag, Tag> - ) => Effect.Effect< - Event.SuccessWithTag, Tag>, - Event.ErrorWithTag, Tag> | EventJournalError - >), + < + // Pairs each event tag with its own payload. + Item extends { + readonly [Tag in Event.Tag>]: { + readonly event: Tag + readonly payload: Event.PayloadWithTag, Tag> + } + }[Event.Tag>] + >(item: Item) => Effect.Effect< + Event.SuccessWithTag, Item["event"]>, + Event.ErrorWithTag, Item["event"]> | EventJournalError + >, never, EventLog > => Effect.gen(function*() { const log = yield* EventLog - - return >>( - event: Tag, - payload: Event.PayloadWithTag, Tag> - ): Effect.Effect< - Event.SuccessWithTag, Tag>, - Event.ErrorWithTag, Tag> | EventJournalError - > => log.write({ schema, event, payload }) - }) + return (item: { readonly event: string; readonly payload: unknown }) => + log.write({ schema, event: item.event, payload: item.payload } as any) + }) as any diff --git a/repos/effect/packages/effect/src/eventlog/EventLogEncryption.ts b/repos/effect/packages/effect/src/eventlog/EventLogEncryption.ts index fd5b6573db..bfc20cf17a 100644 --- a/repos/effect/packages/effect/src/eventlog/EventLogEncryption.ts +++ b/repos/effect/packages/effect/src/eventlog/EventLogEncryption.ts @@ -9,62 +9,131 @@ * @stability unstable * @since 4.0.0 */ +import type { NonEmptyArray } from "../Array.ts" import * as Context from "../Context.ts" import * as Effect from "../Effect.ts" +import * as SchemaBinary from "../encoding/SchemaBinary.ts" import * as Layer from "../Layer.ts" import * as Redacted from "../Redacted.ts" import * as Schema from "../Schema.ts" import * as Transferable from "../workers/Transferable.ts" -import { Entry, EntryId, RemoteEntry } from "./EventJournal.ts" +import { type Entry, EntryFields, entryFromFields, EntryId, RemoteEntry } from "./EventJournal.ts" import type { Identity } from "./EventLog.ts" import { makeGetIdentityRootSecretMaterial } from "./internal/identityRootSecretDerivation.ts" /** - * Schema for an encrypted journal entry paired with its initialization vector - * and the id of the original entry. + * Schema for a batch of journal entries encrypted as one AES-GCM message. + * + * **Details** + * + * `entryIds` lists the ids of the encrypted entries in order. It is sent in + * plaintext so the server can deduplicate batches, and it is authenticated + * together with the store id, so a server cannot alter it or move the batch to + * another store. * * @stability unstable * @category models * @since 4.0.0 */ -export const EncryptedEntry = Schema.Struct({ - entryId: EntryId, +export const EncryptedBatch = Schema.Struct({ + entryIds: Schema.NonEmptyArray(EntryId), iv: Transferable.Uint8Array, - encryptedEntry: Transferable.Uint8Array + encryptedBatch: Transferable.Uint8Array }) /** - * Type of an encrypted remote entry, including its remote sequence number, - * initialization vector, entry id, and encrypted entry bytes. + * Type of an encrypted batch. * * @stability unstable * @category models * @since 4.0.0 */ -export interface EncryptedRemoteEntry extends Schema.Schema.Type {} +export interface EncryptedBatch extends Schema.Schema.Type {} /** - * Schema for encrypted entries exchanged with a remote event-log server. + * Type of an encrypted batch stored by a remote server, including the remote + * sequence of its first entry. * * @stability unstable * @category models * @since 4.0.0 */ -export const EncryptedRemoteEntry = Schema.Struct({ +export interface EncryptedRemoteBatch extends Schema.Schema.Type {} + +/** + * Schema for encrypted batches exchanged with a remote event-log server. + * + * **Details** + * + * The entries of a batch have consecutive remote sequences starting at + * `sequence`. + * + * @stability unstable + * @category models + * @since 4.0.0 + */ +export const EncryptedRemoteBatch = Schema.Struct({ sequence: Schema.Natural, - iv: Transferable.Uint8Array, - entryId: EntryId, - encryptedEntry: Transferable.Uint8Array + ...EncryptedBatch.fields }) +const maxBatchEntries = 100 +const maxBatchPayloadBytes = 256 * 1024 + +// A batch holds at most `maxBatchEntries` entries or `maxBatchPayloadBytes` of +// payload, whichever comes first. A single larger entry is a batch on its own. +const toBatches = (entries: ReadonlyArray): Array> => { + const batches: Array> = [] + let current: Array = [] + let bytes = 0 + for (const entry of entries) { + const size = entry.payload.byteLength + if (current.length > 0 && (current.length === maxBatchEntries || bytes + size > maxBatchPayloadBytes)) { + batches.push(current as NonEmptyArray) + current = [] + bytes = 0 + } + current.push(entry) + bytes += size + } + if (current.length > 0) batches.push(current as NonEmptyArray) + return batches +} + +const textEncoder = new TextEncoder() + +// The store id is length-prefixed, followed by the 16-byte entry ids. +const additionalData = (storeId: string, entryIds: ReadonlyArray): Uint8Array => { + const store = textEncoder.encode(storeId) + const data = new Uint8Array(4 + store.byteLength + entryIds.length * 16) + new DataView(data.buffer).setUint32(0, store.byteLength) + data.set(store, 4) + let offset = 4 + store.byteLength + for (const id of entryIds) { + data.set(id, offset) + offset += 16 + } + return data +} + +const equalBytes = (a: Uint8Array, b: Uint8Array): boolean => { + if (a.byteLength !== b.byteLength) return false + for (let i = 0; i < a.byteLength; i++) { + if (a[i] !== b[i]) return false + } + return true +} + +const BatchFromSchemaBinary = SchemaBinary.toCodec(Schema.NonEmptyArray(EntryFields)) +const encodeBatch = Schema.encodeEffect(BatchFromSchemaBinary) +const decodeBatch = Schema.decodeEffect(BatchFromSchemaBinary) + const toArrayBuffer = (data: Uint8Array): ArrayBuffer => { const buffer = new ArrayBuffer(data.byteLength) new Uint8Array(buffer).set(data) return buffer } -const toBufferSource = (data: Uint8Array): ArrayBufferView => new Uint8Array(toArrayBuffer(data)) - /** * Service that provides identity generation, entry * encryption and decryption, and SHA-256 hashing for event-log replication. @@ -81,16 +150,13 @@ const toBufferSource = (data: Uint8Array): ArrayBufferView => new U export class EventLogEncryption extends Context.Service - ) => Effect.Effect< - ReadonlyArray<{ - readonly iv: Uint8Array - readonly encryptedEntry: Uint8Array - }> - > + ) => Effect.Effect> readonly decrypt: ( identity: Identity["Service"], - entries: ReadonlyArray + storeId: string, + batches: ReadonlyArray ) => Effect.Effect> readonly sha256String: (data: Uint8Array) => Effect.Effect readonly sha256: (data: Uint8Array) => Effect.Effect @@ -110,35 +176,54 @@ export const makeEncryptionSubtle = (crypto: Crypto): Effect.Effect encodeBatch(batch))) return yield* Effect.promise(() => - Promise.all( - data.map((entry) => { - const iv = crypto.getRandomValues(new Uint8Array(12)) - return crypto.subtle.encrypt( - { name: "AES-GCM", iv: toBufferSource(iv), tagLength: 128 }, - key, - toBufferSource(entry) - ).then((encryptedEntry) => ({ iv, encryptedEntry: new Uint8Array(encryptedEntry) })) - }) - ) + Promise.all(batches.map((batch, i) => { + const entryIds = batch.map((entry) => entry.id) as NonEmptyArray + const iv = crypto.getRandomValues(new Uint8Array(12)) + return crypto.subtle.encrypt( + { name: "AES-GCM", iv, additionalData: additionalData(storeId, entryIds), tagLength: 128 }, + key, + data[i] + ).then((encryptedBatch) => ({ entryIds, iv, encryptedBatch: new Uint8Array(encryptedBatch) })) + })) ) }), - decrypt: Effect.fnUntraced(function*(identity, entries) { + decrypt: Effect.fnUntraced(function*(identity, storeId, batches) { const key = (yield* getIdentityRootSecretMaterial(identity)).encryptionKey - const decryptedData = (yield* Effect.promise(() => - Promise.all(entries.map((data) => + const decrypted = yield* Effect.promise(() => + Promise.all(batches.map((batch) => crypto.subtle.decrypt( - { name: "AES-GCM", iv: toBufferSource(data.iv), tagLength: 128 }, + { + name: "AES-GCM", + iv: batch.iv, + additionalData: additionalData(storeId, batch.entryIds), + tagLength: 128 + }, key, - toBufferSource(data.encryptedEntry) + batch.encryptedBatch ) )) - )).map((buffer) => new Uint8Array(buffer)) - const decoded = yield* Effect.orDie(Entry.decodeArray(decryptedData)) - return decoded.map((entry, index) => new RemoteEntry({ remoteSequence: entries[index].sequence, entry })) + ) + const result: Array = [] + for (let i = 0; i < batches.length; i++) { + const batch = batches[i] + const fields = yield* Effect.orDie(decodeBatch(new Uint8Array(decrypted[i]))) + const entries = fields.map(entryFromFields) + if ( + entries.length !== batch.entryIds.length || + entries.some((entry, j) => !equalBytes(entry.id, batch.entryIds[j])) + ) { + return yield* Effect.die(new Error("Encrypted batch entries do not match its entry ids")) + } + for (let j = 0; j < entries.length; j++) { + result.push(new RemoteEntry({ remoteSequence: batch.sequence + j, entry: entries[j] })) + } + } + return result }), sha256: (data) => Effect.promise(() => crypto.subtle.digest("SHA-256", toArrayBuffer(data))).pipe( diff --git a/repos/effect/packages/effect/src/eventlog/EventLogMessage.ts b/repos/effect/packages/effect/src/eventlog/EventLogMessage.ts index ceea16fd63..92011259a7 100644 --- a/repos/effect/packages/effect/src/eventlog/EventLogMessage.ts +++ b/repos/effect/packages/effect/src/eventlog/EventLogMessage.ts @@ -11,16 +11,18 @@ */ import type { NonEmptyArray, NonEmptyReadonlyArray } from "../Array.ts" import type { Brand } from "../Brand.ts" +import * as Effect from "../Effect.ts" import * as SchemaBinary from "../encoding/SchemaBinary.ts" import * as Predicate from "../Predicate.ts" import * as Rpc from "../rpc/Rpc.ts" import * as RpcGroup from "../rpc/RpcGroup.ts" import * as RpcMiddleware from "../rpc/RpcMiddleware.ts" import * as Schema from "../Schema.ts" +import type * as SchemaAST from "../SchemaAST.ts" import * as Transferable from "../workers/Transferable.ts" -import { Entry, RemoteEntry, RemoteId } from "./EventJournal.ts" +import { Entry, EntryFields, entryFromFields, EntryId, RemoteEntry, RemoteId } from "./EventJournal.ts" import type { Identity } from "./EventLog.ts" -import { EncryptedEntry, EncryptedRemoteEntry } from "./EventLogEncryption.ts" +import { EncryptedBatch, EncryptedRemoteBatch } from "./EventLogEncryption.ts" /** * Type-level identifier used to brand event-log store ids. @@ -109,8 +111,8 @@ export class EventLogAuthentication extends RpcMiddleware.Service("effect/eventlog/EventLogRemote/HelloResponse")({ remoteId: RemoteId, - challenge: Transferable.Uint8Array + challenge: Transferable.Uint8Array, + firstSequence: Schema.optional(Schema.Natural) }) {} /** @@ -271,15 +274,45 @@ export class ChunkedMessage } } +/** + * Schema for a written entry's id and a sequence at which the server stores + * it. Entries the server already held keep their stored sequence. When a write + * stores an entry again, as part of a partly known batch, it is acknowledged at + * its new position. + * + * @stability unstable + * @category protocols + * @since 4.0.6 + */ +export const WriteAck = Schema.Struct({ + entryId: EntryId, + sequence: Schema.Natural +}) + +/** + * Acknowledgement of one written entry. + * + * @stability unstable + * @category protocols + * @since 4.0.6 + */ +export type WriteAck = typeof WriteAck.Type + /** * RPC used to send one chunk of a large encoded write payload. * + * **Details** + * + * The completing chunk returns acknowledgements in request order; earlier + * chunks return an empty array. + * * @stability unstable * @category protocols * @since 4.0.0 */ export class WriteChunkedRpc extends Rpc.make("EventLog.WriteChunked", { payload: ChunkedMessage, + success: Schema.Array(WriteAck), error: EventLogProtocolError }).middleware(EventLogAuthentication) {} @@ -288,8 +321,8 @@ export class WriteChunkedRpc extends Rpc.make("EventLog.WriteChunked", { * * **Details** * - * It includes the client public key, target store id, and encrypted entries - * with their AES-GCM initialization vectors. + * It includes the client public key, target store id, and the entries + * encrypted as batches. * * @stability unstable * @category protocols @@ -298,7 +331,7 @@ export class WriteChunkedRpc extends Rpc.make("EventLog.WriteChunked", { export class WriteEntries extends Schema.Class("effect/eventlog/EventLogRemote/WriteEntries")({ publicKey: Schema.String, storeId: StoreId, - encryptedEntries: Schema.Array(EncryptedEntry) + batches: Schema.Array(EncryptedBatch) }) { static FromSchemaBinary = SchemaBinary.toCodec(WriteEntries) static encode = Schema.encodeEffect(this.FromSchemaBinary) @@ -324,15 +357,32 @@ export class WriteEntriesUnencrypted { static FromSchemaBinary = SchemaBinary.toCodec(WriteEntriesUnencrypted) static encode = Schema.encodeEffect(this.FromSchemaBinary) - static decode = Schema.decodeEffect(this.FromSchemaBinary) + static decode = ( + data: Uint8Array, + options?: SchemaAST.ParseOptions + ): Effect.Effect => + Effect.map( + decodeWriteEntriesUnencryptedFields(data, options), + (fields) => + Schema.makeClassTrusted(WriteEntriesUnencrypted, { ...fields, entries: fields.entries.map(entryFromFields) }) + ) get encoded() { return WriteEntriesUnencrypted.encode(this) } } +const decodeWriteEntriesUnencryptedFields = Schema.decodeEffect(SchemaBinary.toCodec(Schema.Struct({ + ...WriteEntriesUnencrypted.fields, + entries: Schema.Array(EntryFields) +}))) + /** * RPC used to send an encoded write payload that fits in one message. * + * **Details** + * + * The response acknowledges every written entry, in request order. + * * @stability unstable * @category protocols * @since 4.0.0 @@ -341,6 +391,7 @@ export class WriteSingleRpc extends Rpc.make("EventLog.WriteSingle", { payload: { data: Transferable.Uint8Array }, + success: Schema.Array(WriteAck), error: EventLogProtocolError }).middleware(EventLogAuthentication) {} @@ -351,7 +402,7 @@ export class WriteSingleRpc extends Rpc.make("EventLog.WriteSingle", { * **Details** * * Responses are encoded as either `SingleMessage` values or `ChunkedMessage` - * parts. + * parts. Entries written on this connection are acknowledged, not echoed. * * @stability unstable * @category protocols @@ -367,7 +418,7 @@ export class ChangesRpc extends Rpc.make("EventLog.Changes", { error: EventLogProtocolError, stream: true }).middleware(EventLogAuthentication) { - static EncryptedFromSchemaBinary = SchemaBinary.toCodec(Schema.NonEmptyArray(EncryptedRemoteEntry)) + static EncryptedFromSchemaBinary = SchemaBinary.toCodec(Schema.NonEmptyArray(EncryptedRemoteBatch)) static UnencryptedFromSchemaBinary = SchemaBinary.toCodec(Schema.NonEmptyArray(RemoteEntry)) static encodeEncrypted = Schema.encodeEffect(ChangesRpc.EncryptedFromSchemaBinary) static decodeEncrypted = Schema.decodeEffect(ChangesRpc.EncryptedFromSchemaBinary) diff --git a/repos/effect/packages/effect/src/eventlog/EventLogRemote.ts b/repos/effect/packages/effect/src/eventlog/EventLogRemote.ts index 9ecb392552..7beb1c258f 100644 --- a/repos/effect/packages/effect/src/eventlog/EventLogRemote.ts +++ b/repos/effect/packages/effect/src/eventlog/EventLogRemote.ts @@ -24,7 +24,7 @@ import type { RpcClientError } from "../rpc/RpcClientError.ts" import type * as RpcGroup from "../rpc/RpcGroup.ts" import type * as Schema from "../Schema.ts" import type * as Scope from "../Scope.ts" -import type { Entry, RemoteEntry, RemoteId } from "./EventJournal.ts" +import { type Entry, type EventJournal, type EventJournalError, RemoteEntry, type RemoteId } from "./EventJournal.ts" import { type Identity, Registry } from "./EventLog.ts" import { EventLogEncryption, layerSubtle } from "./EventLogEncryption.ts" import { @@ -35,6 +35,7 @@ import { EventLogRemoteRpcs, type HelloResponse, type StoreId, + type WriteAck, WriteEntries, WriteEntriesUnencrypted } from "./EventLogMessage.ts" @@ -54,22 +55,35 @@ import { makeGetIdentityRootSecretMaterial } from "./internal/identityRootSecret * It can write local entries to the remote, stream remote changes from a sequence * number, and run effects only after the supplied identity has authenticated. * + * Built-in remotes acknowledge writes instead of echoing them through `changes`. + * Custom remotes may return `void` from `write` to confirm uploads through + * `changes` instead. + * * @stability unstable * @category services * @since 4.0.0 */ export class EventLogRemote extends Context.Service Effect.Effect, never, Scope.Scope> readonly write: (options: { readonly identity: Identity["Service"] readonly storeId: StoreId readonly entries: ReadonlyArray - }) => Effect.Effect + }) => Effect.Effect | void, EventLogRemoteError> readonly whenAuthenticated: ( effect: Effect.Effect ) => Effect.Effect @@ -172,8 +186,10 @@ export const makeWith = Effect.fnUntraced(function*({ encodeWrite, decodeChanges }) => Effect.Effect, Schema.SchemaError> readonly decodeChanges: ( identity: Identity["Service"], - data: Uint8Array - ) => Effect.Effect, Schema.SchemaError> + storeId: StoreId, + data: Uint8Array, + knownEntries?: EventJournal["Service"]["entriesById"] + ) => Effect.Effect, Schema.SchemaError | EventJournalError> }): Effect.fn.Return { const client = yield* EventLogRemoteClient const registry = yield* Registry @@ -236,6 +252,7 @@ export const makeWith = Effect.fnUntraced(function*({ encodeWrite, decodeChanges const remote = EventLogRemote.of({ id: hello.remoteId, + firstSequence: hello.firstSequence, write: Effect.fnUntraced( function*(options) { yield* ensureAuthenticated(options.identity) @@ -243,9 +260,13 @@ export const makeWith = Effect.fnUntraced(function*({ encodeWrite, decodeChanges if (encoded.byteLength <= ChunkedMessage.chunkSize) { return yield* client["EventLog.WriteSingle"]({ data: encoded }) } - for (const part of ChunkedMessage.split(chunkedIdCounter++, encoded)) { - yield* client["EventLog.WriteChunked"](part) - } + // Only the part the server receives last carries acknowledgements. + const acks = yield* Effect.forEach( + ChunkedMessage.split(chunkedIdCounter++, encoded), + (part) => client["EventLog.WriteChunked"](part), + { concurrency: 4 } + ) + return acks.flat() }, retryForbidden, Effect.mapError((cause) => new EventLogRemoteError({ method: "write", cause })) @@ -263,20 +284,30 @@ export const makeWith = Effect.fnUntraced(function*({ encodeWrite, decodeChanges startSequence: options.startSequence }, { asQueue: true }) + // A batch can start before `startSequence`; its earlier entries are dropped. + const offer = Effect.fnUntraced(function*(data: Uint8Array) { + const entries = yield* decodeChanges(options.identity, options.storeId, data, options.knownEntries) + const first = entries.findIndex((entry) => entry.remoteSequence >= options.startSequence) + if (first === -1) return + yield* Queue.offerAll(outgoing, entries.slice(first)) + }) + while (true) { const parts = yield* Queue.takeAll(incoming) for (let i = 0; i < parts.length; i++) { const part = parts[i] if (part._tag === "Single") { - yield* Queue.offerAll(outgoing, yield* decodeChanges(options.identity, part.data)) + yield* offer(part.data) continue } const data = ChunkedMessage.join(chunkedState, part) if (!data) continue - yield* Queue.offerAll(outgoing, yield* decodeChanges(options.identity, data)) + yield* offer(data) } } }).pipe( + // Forbidden retries resubscribe with the same `startSequence`. Other failures + // propagate to EventLog, which reconnects from the journal's committed sequence. (effect) => retryForbidden(effect, options), Effect.mapError((cause) => { if (cause._tag === "EventLogRemoteError") { @@ -322,25 +353,50 @@ export const makeEncrypted = Effect.gen(function*(): Effect.fn.Return< > { const encryption = yield* EventLogEncryption + // Skip decryption only when every entry in a batch is already stored. + const decodeChanges = Effect.fnUntraced(function*( + identity: Identity["Service"], + storeId: StoreId, + data: Uint8Array, + knownEntries?: EventJournal["Service"]["entriesById"] + ) { + const batches = yield* ChangesRpc.decodeEncrypted(data) + if (knownEntries === undefined) return yield* encryption.decrypt(identity, storeId, batches) + const stored = yield* knownEntries(batches.flatMap((batch) => batch.entryIds)) + let offset = 0 + const knownBatches = batches.map((batch) => { + const entries = stored.slice(offset, offset + batch.entryIds.length) + offset += batch.entryIds.length + return entries.every((entry): entry is Entry => entry !== undefined) ? entries : undefined + }) + const decrypted = yield* encryption.decrypt( + identity, + storeId, + batches.filter((_, i) => knownBatches[i] === undefined) + ) + let next = 0 + return batches.flatMap((batch, i) => { + const entries = knownBatches[i] + return entries === undefined + ? batch.entryIds.map(() => decrypted[next++]) + : entries.map((entry, j) => + new RemoteEntry({ remoteSequence: batch.sequence + j, entry }, { disableChecks: true }) + ) + }) + }) + return yield* makeWith({ encodeWrite: (options) => - encryption.encrypt(options.identity, options.entries).pipe( - Effect.flatMap((encryptedEntries) => + encryption.encrypt(options.identity, options.storeId, options.entries).pipe( + Effect.flatMap((batches) => new WriteEntries({ publicKey: options.identity.publicKey, storeId: options.storeId, - encryptedEntries: encryptedEntries.map((entry, i) => ({ - entryId: options.entries[i].id, - iv: entry.iv, - encryptedEntry: entry.encryptedEntry - })) + batches }).encoded ) ), - decodeChanges: (identity, data) => - ChangesRpc.decodeEncrypted(data).pipe( - Effect.flatMap((entries) => encryption.decrypt(identity, entries)) - ) + decodeChanges }) }) @@ -362,7 +418,7 @@ export const makeUnencrypted: Effect.Effect< storeId: options.storeId, entries: options.entries }).encoded, - decodeChanges: (_identity, data) => ChangesRpc.decodeUnencrypted(data) + decodeChanges: (_identity, _storeId, data) => ChangesRpc.decodeUnencrypted(data) }) /** diff --git a/repos/effect/packages/effect/src/eventlog/EventLogServer.ts b/repos/effect/packages/effect/src/eventlog/EventLogServer.ts index 369ed9ec08..a7690e9098 100644 --- a/repos/effect/packages/effect/src/eventlog/EventLogServer.ts +++ b/repos/effect/packages/effect/src/eventlog/EventLogServer.ts @@ -14,16 +14,19 @@ import * as Arr from "../Array.ts" import * as Cache from "../Cache.ts" import * as Context from "../Context.ts" import * as Data from "../Data.ts" +import * as Duration from "../Duration.ts" import * as Effect from "../Effect.ts" import * as Equal from "../Equal.ts" +import * as Exit from "../Exit.ts" import * as Hash from "../Hash.ts" +import * as Uuid from "../internal/uuid.ts" import * as Layer from "../Layer.ts" import * as Option from "../Option.ts" import * as Redacted from "../Redacted.ts" import type * as Rpc from "../rpc/Rpc.ts" import type * as RpcGroup from "../rpc/RpcGroup.ts" import * as Stream from "../Stream.ts" -import type { RemoteId } from "./EventJournal.ts" +import type { EntryId, RemoteId } from "./EventJournal.ts" import * as EventLog from "./EventLog.ts" import { ChunkedMessage, @@ -32,7 +35,8 @@ import { EventLogRemoteRpcs, HelloResponse, SingleMessage, - type StoreId + type StoreId, + type WriteAck } from "./EventLogMessage.ts" import * as EventLogSessionAuth from "./EventLogSessionAuth.ts" @@ -72,12 +76,16 @@ export const layerAuthMiddleware: Layer.Layer< * chunked writes, delegates write and change handling to the supplied callbacks, * and frames large change payloads into chunks. * + * Use the connection's `ConnectionWrites.track` in `onWrite` and + * `ConnectionWrites.filter` in `changes` to suppress its own write echoes. + * * @stability unstable * @category layers * @since 4.0.0 */ export const layerRpcHandlers = (options: { readonly remoteId: RemoteId + readonly firstSequence?: number | undefined readonly authorizeIdentity?: (publicKey: string) => Effect.Effect readonly getOrCreateSessionAuthBinding: ( publicKey: string, @@ -85,12 +93,14 @@ export const layerRpcHandlers = (options: { ) => Effect.Effect> readonly onWrite: ( data: Uint8Array, - authenticatedPublicKeys: ReadonlySet - ) => Effect.Effect + authenticatedPublicKeys: ReadonlySet, + writes: ConnectionWrites + ) => Effect.Effect, EventLogProtocolError> readonly changes: (options: { readonly publicKey: string readonly storeId: StoreId readonly startSequence: number + readonly writes: ConnectionWrites }) => Stream.Stream, unknown> }): Layer.Layer< Rpc.ToHandler> | EventLogAuthentication @@ -103,8 +113,8 @@ export const layerRpcHandlers = (options: { }) let chunkedIdCounter = 0 - const persistedSigningPublicKeys = yield* Cache.make({ - lookup: (key: SessionAuthCacheKey) => + const persistedSigningPublicKeys = yield* Cache.makeWith( + (key: SessionAuthCacheKey) => options.getOrCreateSessionAuthBinding(key.publicKey, key.signingPublicKey).pipe( Effect.catchCause((_) => Effect.fail( @@ -117,15 +127,19 @@ export const layerRpcHandlers = (options: { ) ) ), - capacity: 4096 - }) + { + capacity: 4096, + timeToLive: (exit) => Exit.isSuccess(exit) ? Duration.infinity : Duration.zero + } + ) return EventLogRemoteRpcs.of({ "EventLog.Hello": Effect.fnUntraced(function*(_, { client }) { const challenge = yield* Cache.get(clientChallenges, client.id) return new HelloResponse({ remoteId: options.remoteId, - challenge + challenge, + firstSequence: options.firstSequence }) }), "EventLog.Authenticate": Effect.fnUntraced(function*(request, { client }) { @@ -181,21 +195,22 @@ export const layerRpcHandlers = (options: { .annotate(AuthenticatedIdentities, authenticatedIdentities) .annotate(ChunkedMessageState, new Map()) }), - "EventLog.WriteSingle": Effect.fnUntraced(function*(request, { client }) { - yield* options.onWrite( + "EventLog.WriteSingle": (request, { client }) => + options.onWrite( request.data, - Context.getOrUndefined(client.annotations, AuthenticatedIdentities) ?? new Set() - ) - }), - "EventLog.WriteChunked": Effect.fnUntraced(function*(request, { client }) { + Context.getOrUndefined(client.annotations, AuthenticatedIdentities) ?? new Set(), + getConnectionWrites(client) + ), + "EventLog.WriteChunked": (request, { client }) => { const state = Context.get(client.annotations, ChunkedMessageState) const data = ChunkedMessage.join(state, request) - if (!data) return - yield* options.onWrite( + if (!data) return Effect.succeed([]) + return options.onWrite( data, - Context.getOrUndefined(client.annotations, AuthenticatedIdentities) ?? new Set() + Context.getOrUndefined(client.annotations, AuthenticatedIdentities) ?? new Set(), + getConnectionWrites(client) ) - }), + }, "EventLog.Changes": (request, { client }) => { const authenticatedIdentities = Context.getOrUndefined(client.annotations, AuthenticatedIdentities) if (!authenticatedIdentities?.has(request.publicKey)) { @@ -211,7 +226,8 @@ export const layerRpcHandlers = (options: { return options.changes({ publicKey: request.publicKey, storeId: request.storeId, - startSequence: request.startSequence + startSequence: request.startSequence, + writes: getConnectionWrites(client) }).pipe( Stream.mapArray(Arr.flatMap((data): NonEmptyReadonlyArray => { if (data.byteLength <= ChunkedMessage.chunkSize) { @@ -259,6 +275,132 @@ export class ChunkedMessageState extends Context.Reference< defaultValue: () => new Map() }) {} +/** + * Tracks writes to suppress their echoes on the same RPC connection. + * + * **Details** + * + * Only tracks entries for stores with an open change stream. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export class ConnectionWrites { + private readonly streams = new Map>() + + /** + * Runs a storage write while suppressing its entries on this connection's streams. + * + * @stability unstable + * @since 4.0.6 + */ + track( + options: { + readonly publicKey: string + readonly storeId: StoreId + readonly entryIds: ReadonlyArray + }, + write: Effect.Effect, E, R> + ): Effect.Effect, E, R> { + const open = this.streams.get(makeStreamKey(options.publicKey, options.storeId)) + if (open === undefined || options.entryIds.length === 0) return write + const streams = Array.from(open) + const started = streams.map((stream) => stream.position) + const ids = options.entryIds.map(Uuid.stringify) + // Track before writing: storage can publish an entry before the write + // returns. Each occurrence is counted, as an id can be stored more than once. + for (const stream of streams) { + for (const id of ids) stream.ids.set(id, (stream.ids.get(id) ?? 0) + 1) + } + return Effect.onExit(write, (exit) => + Effect.sync(() => { + for (let i = 0; i < streams.length; i++) { + const stream = streams[i] + if (exit._tag === "Failure") { + for (const id of ids) release(stream.ids, id) + continue + } + // Release occurrences the stream had already passed when the write began. + for (const ack of exit.value) { + if (ack.sequence <= started[i]) release(stream.ids, Uuid.stringify(ack.entryId)) + } + } + })) + } + + /** + * Filters this connection's writes from a store's change stream. An item can + * hold several entries; `sequence` returns its highest one. + * + * @stability unstable + * @since 4.0.6 + */ + filter( + options: { + readonly publicKey: string + readonly storeId: StoreId + readonly entryIds: (item: A) => ReadonlyArray + readonly sequence: (item: A) => number + }, + stream: Stream.Stream + ): Stream.Stream { + return Stream.suspend(() => { + const key = makeStreamKey(options.publicKey, options.storeId) + const state: ConnectionStream = { ids: new Map(), position: -1 } + let open = this.streams.get(key) + if (open === undefined) { + open = new Set() + this.streams.set(key, open) + } + open.add(state) + return stream.pipe( + Stream.filter((item) => { + const sequence = options.sequence(item) + if (sequence > state.position) state.position = sequence + if (state.ids.size === 0) return true + // An item is written by one request, so it is either all or none of ours. + const ids = options.entryIds(item).map(Uuid.stringify) + if (!ids.every((id) => state.ids.has(id))) return true + for (const id of ids) release(state.ids, id) + return false + }), + Stream.ensuring(Effect.sync(() => { + open.delete(state) + if (open.size === 0) this.streams.delete(key) + })) + ) + }) + } +} + +interface ConnectionStream { + // Outstanding occurrences of this connection's writes, by entry id. + readonly ids: Map + position: number +} + +const release = (ids: Map, id: string): void => { + const count = ids.get(id) + if (count === undefined) return + if (count <= 1) ids.delete(id) + else ids.set(id, count - 1) +} + +const makeStreamKey = (publicKey: string, storeId: StoreId): string => `${publicKey}/${storeId}` + +class ConnectionWritesState extends Context.Service()( + "effect/eventlog/EventLogServer/ConnectionWrites" +) {} + +const getConnectionWrites = (client: Rpc.ServerClient): ConnectionWrites => { + const existing = Context.getOrUndefined(client.annotations, ConnectionWritesState) + if (existing !== undefined) return existing + const writes = new ConnectionWrites() + client.annotate(ConnectionWritesState, writes) + return writes +} + /** * Annotation containing the public keys authenticated on an RPC connection. * diff --git a/repos/effect/packages/effect/src/eventlog/EventLogServerEncrypted.ts b/repos/effect/packages/effect/src/eventlog/EventLogServerEncrypted.ts index d79205efd1..bef13abd54 100644 --- a/repos/effect/packages/effect/src/eventlog/EventLogServerEncrypted.ts +++ b/repos/effect/packages/effect/src/eventlog/EventLogServerEncrypted.ts @@ -14,18 +14,24 @@ import * as Arr from "../Array.ts" import * as Context from "../Context.ts" import * as Effect from "../Effect.ts" +import * as Equal from "../Equal.ts" import * as Uuid from "../internal/uuid.ts" import * as Layer from "../Layer.ts" import * as PubSub from "../PubSub.ts" import * as RcMap from "../RcMap.ts" import * as RpcServer from "../rpc/RpcServer.ts" -import * as Schema from "../Schema.ts" import type * as Scope from "../Scope.ts" import * as Stream from "../Stream.ts" -import * as Transferable from "../workers/Transferable.ts" -import { EntryId, makeRemoteIdUnsafe, type RemoteId } from "./EventJournal.ts" -import type { EncryptedRemoteEntry } from "./EventLogEncryption.ts" -import { ChangesRpc, EventLogProtocolError, EventLogRemoteRpcs, type StoreId, WriteEntries } from "./EventLogMessage.ts" +import { type EntryId, makeRemoteIdUnsafe, type RemoteId } from "./EventJournal.ts" +import type { EncryptedBatch, EncryptedRemoteBatch } from "./EventLogEncryption.ts" +import { + ChangesRpc, + EventLogProtocolError, + EventLogRemoteRpcs, + type StoreId, + type WriteAck, + WriteEntries +} from "./EventLogMessage.ts" import * as EventLogServer from "./EventLogServer.ts" /** @@ -33,9 +39,9 @@ import * as EventLogServer from "./EventLogServer.ts" * * **Details** * - * Incoming encrypted write payloads are decoded and persisted through `Storage`; - * change streams read encrypted entries from storage and encode them for the - * remote protocol. + * Decodes and persists writes through `Storage`, acknowledging their stored + * sequences. Change streams encode stored batches, excluding this connection's + * writes. * * @stability unstable * @category layers @@ -47,9 +53,10 @@ export const layerRpcHandlers = Layer.unwrap(Effect.gen(function*() { return EventLogServer.layerRpcHandlers({ remoteId, + firstSequence: storage.firstSequence, getOrCreateSessionAuthBinding: (publicKey, signingPublicKey) => storage.getOrCreateSessionAuthBinding(publicKey, signingPublicKey), - onWrite: Effect.fnUntraced(function*(data, authenticatedPublicKeys) { + onWrite: Effect.fnUntraced(function*(data, authenticatedPublicKeys, writes) { const request = yield* WriteEntries.decode(data).pipe( Effect.mapError((_) => new EventLogProtocolError({ @@ -68,15 +75,11 @@ export const layerRpcHandlers = Layer.unwrap(Effect.gen(function*() { message: "Identity is not authenticated" }) } - if (request.encryptedEntries.length === 0) return - const entries = request.encryptedEntries.map(({ encryptedEntry, entryId, iv }) => - new PersistedEntry({ - entryId, - iv, - encryptedEntry - }) - ) - return yield* storage.write(request.publicKey, request.storeId, entries).pipe( + if (request.batches.length === 0) return [] + const entryIds = request.batches.flatMap((batch) => batch.entryIds) + return yield* storage.write(request.publicKey, request.storeId, request.batches).pipe( + Effect.flatMap((stored) => toWriteAcks(storage, request.publicKey, request.storeId, request.batches, stored)), + (write) => writes.track({ publicKey: request.publicKey, storeId: request.storeId, entryIds }, write), Effect.catchCause((_) => Effect.fail( new EventLogProtocolError({ @@ -89,13 +92,53 @@ export const layerRpcHandlers = Layer.unwrap(Effect.gen(function*() { ) ) }), - changes: ({ publicKey, storeId, startSequence }) => - storage.changes(publicKey, storeId, startSequence).pipe( - Stream.mapArrayEffect((entries) => Effect.map(ChangesRpc.encodeEncrypted(entries), Arr.of)) + changes: ({ publicKey, startSequence, storeId, writes }) => + writes.filter( + { + publicKey, + storeId, + entryIds: (batch) => batch.entryIds, + sequence: (batch) => batch.sequence + batch.entryIds.length - 1 + }, + storage.changes(publicKey, storeId, startSequence) + ).pipe( + Stream.mapArrayEffect((batches) => Effect.map(ChangesRpc.encodeEncrypted(batches), Arr.of)) ) }) })) +// Acknowledges each entry occurrence in request order: at its position in the +// batch this write stored, or, for a skipped batch, where it was first stored. +const toWriteAcks = Effect.fnUntraced(function*( + storage: Storage["Service"], + publicKey: string, + storeId: StoreId, + batches: ReadonlyArray, + stored: ReadonlyArray +) { + // Stored batches keep request order, and each batch has its own random iv. + let next = 0 + const storedAt = batches.map((batch) => { + if (stored[next] === undefined || !Equal.equals(stored[next].iv, batch.iv)) return undefined + return stored[next++].sequence + }) + const skipped = batches.flatMap((batch, i) => storedAt[i] === undefined ? batch.entryIds : []) + const firstStored = new Map() + if (skipped.length > 0) { + for (const ack of yield* storage.entrySequences(publicKey, storeId, skipped)) { + firstStored.set(Uuid.stringify(ack.entryId), ack.sequence) + } + } + const acks: Array = [] + batches.forEach((batch, b) => + batch.entryIds.forEach((entryId, i) => { + const sequence = storedAt[b] === undefined ? firstStored.get(Uuid.stringify(entryId)) : storedAt[b] + i + if (sequence !== undefined) acks.push({ entryId, sequence }) + }) + ) + return acks +}) + /** * Provides an encrypted event-log RPC server using `EventLogRemoteRpcs` and the * encrypted server RPC handlers. @@ -108,7 +151,7 @@ export const layerRpcHandlers = Layer.unwrap(Effect.gen(function*() { * **Details** * * This layer installs `EventLogRemoteRpcs` on the provided RPC server protocol - * and wires those RPCs to `layerRpcHandlers`. Encrypted entries, session + * and wires those RPCs to `layerRpcHandlers`. Encrypted batches, session * authentication bindings, remote ids, and change streams are delegated to * `Storage`. * @@ -124,31 +167,6 @@ export const layer: Layer.Layer = Rp Layer.provide(layerRpcHandlers) ) -/** - * Schema for encrypted entries persisted by the encrypted event-log server. - * - * @stability unstable - * @category models - * @since 4.0.0 - */ -export class PersistedEntry extends Schema.Class( - "effect/eventlog/EventLogServerEncrypted/PersistedEntry" -)({ - entryId: EntryId, - iv: Transferable.Uint8Array, - encryptedEntry: Transferable.Uint8Array -}) { - /** - * String representation of the encrypted entry id. - * - * @stability unstable - * @since 4.0.0 - */ - get entryIdString(): string { - return Uuid.stringify(this.entryId) - } -} - /** * Defines the backing store service used by the encrypted event-log server. * @@ -160,15 +178,27 @@ export class PersistedEntry extends Schema.Class( * **Details** * * It provides the server remote id, stores session authentication bindings, - * persists encrypted entries, and streams encrypted changes for a public key and + * persists encrypted batches, and streams encrypted changes for a public key and * store id. * + * A batch owns one remote sequence per entry, starting at its `sequence`. + * `write` stores a batch unless every one of its entry ids is already stored, + * which makes retried writes idempotent, and returns the batches it stored. + * `entrySequences` returns the sequence each entry was first stored at, so + * skipped batches can still be acknowledged. `changes` streams every batch + * whose last sequence is at or after `startSequence`, followed by newly stored + * batches. + * * @stability unstable * @category services * @since 4.0.0 */ export class Storage extends Context.Service + /** + * Lowest assigned sequence (default 0). Overestimating skips entries. + */ + readonly firstSequence?: number | undefined readonly getOrCreateSessionAuthBinding: ( publicKey: string, signingPublicKey: Uint8Array @@ -176,13 +206,18 @@ export class Storage extends Context.Service - ) => Effect.Effect> + batches: ReadonlyArray + ) => Effect.Effect> + readonly entrySequences: ( + publicKey: string, + storeId: StoreId, + entryIds: ReadonlyArray + ) => Effect.Effect> readonly changes: ( publicKey: string, storeId: StoreId, startSequence: number - ) => Stream.Stream + ) => Stream.Stream }>()("effect/eventlog/EventLogServer/Storage") {} /** @@ -198,28 +233,25 @@ export class Storage extends Context.Service = Effect.gen(function*() { - const knownIds = new Map>() - const journals = new Map>() + const journals = new Map + readonly batches: Array + nextSequence: number + }>() const sessionAuthBindings = new Map>() const remoteId = makeRemoteIdUnsafe() - const ensureKnownIds = (scopeKey: string): Map => { - let storeKnownIds = knownIds.get(scopeKey) - if (storeKnownIds) return storeKnownIds - storeKnownIds = new Map() - knownIds.set(scopeKey, storeKnownIds) - return storeKnownIds - } const ensureJournal = (scopeKey: string) => { let journal = journals.get(scopeKey) if (journal) return journal - journal = [] + journal = { knownIds: new Map(), batches: [], nextSequence: 0 } journals.set(scopeKey, journal) return journal } const pubsubs = yield* RcMap.make({ lookup: (_scopeKey: string) => Effect.acquireRelease( - PubSub.unbounded(), + PubSub.unbounded(), PubSub.shutdown ), idleTimeToLive: 60000 @@ -234,33 +266,42 @@ export const makeStorageMemory: Effect.Effect = [] - for (const entry of entries) { - const idString = entry.entryIdString - if (storeKnownIds.has(idString)) continue - const encrypted: EncryptedRemoteEntry = { - sequence: journal.length, - entryId: entry.entryId, - iv: entry.iv, - encryptedEntry: entry.encryptedEntry - } - encryptedEntries.push(encrypted) - storeKnownIds.set(idString, encrypted.sequence) - journal.push(encrypted) - PubSub.publishUnsafe(pubsub, encrypted) + const stored: Array = [] + for (const batch of batches) { + const ids = batch.entryIds.map(Uuid.stringify) + if (ids.every((id) => journal.knownIds.has(id))) continue + ids.forEach((id, i) => { + if (!journal.knownIds.has(id)) journal.knownIds.set(id, journal.nextSequence + i) + }) + const remoteBatch: EncryptedRemoteBatch = { sequence: journal.nextSequence, ...batch } + journal.nextSequence += ids.length + journal.batches.push(remoteBatch) + stored.push(remoteBatch) + PubSub.publishUnsafe(pubsub, remoteBatch) } - return encryptedEntries + return stored }, Effect.scoped), + entrySequences: (publicKey, storeId, entryIds) => + Effect.sync(() => { + const knownIds = ensureJournal(makeEncryptedScopeKey({ publicKey, storeId })).knownIds + const acks: Array = [] + for (const entryId of entryIds) { + const sequence = knownIds.get(Uuid.stringify(entryId)) + if (sequence !== undefined) acks.push({ entryId, sequence }) + } + return acks + }), changes: Effect.fnUntraced(function*(publicKey, storeId, startSequence) { const scopeKey = makeEncryptedScopeKey({ publicKey, storeId }) const pubsub = yield* RcMap.get(pubsubs, scopeKey) const subscription = yield* PubSub.subscribe(pubsub) - return Stream.fromArray(ensureJournal(scopeKey).slice(startSequence)).pipe( + const batches = ensureJournal(scopeKey).batches + const first = batches.findIndex((batch) => batch.sequence + batch.entryIds.length > startSequence) + return Stream.fromArray(first === -1 ? [] : batches.slice(first)).pipe( Stream.concat(Stream.fromSubscription(subscription)) ) }, Stream.unwrap) diff --git a/repos/effect/packages/effect/src/eventlog/EventLogServerUnencrypted.ts b/repos/effect/packages/effect/src/eventlog/EventLogServerUnencrypted.ts index 1736172193..37d956cd75 100644 --- a/repos/effect/packages/effect/src/eventlog/EventLogServerUnencrypted.ts +++ b/repos/effect/packages/effect/src/eventlog/EventLogServerUnencrypted.ts @@ -39,6 +39,7 @@ import { EventLogProtocolError, EventLogRemoteRpcs, type StoreId, + type WriteAck, WriteEntriesUnencrypted } from "./EventLogMessage.ts" import * as EventLogServer from "./EventLogServer.ts" @@ -104,8 +105,9 @@ export const makeWrite = ( * **Details** * * Incoming plaintext entries are authorized, mapped to a server store, checked - * for conflicts, run through registered handlers, and persisted; change streams - * include compacted backlog entries when compactors are registered. + * for conflicts, run through registered handlers, and persisted. Acknowledgements + * include duplicates with their original sequences. Change streams apply registered + * compactors to backlog entries and exclude this connection's writes. * * @stability unstable * @category layers @@ -129,12 +131,12 @@ export const layerRpcHandlers: Layer.Layer< readonly entries: Arr.NonEmptyReadonlyArray }) { const entries = Arr.sort(options.entries, Entry.Order) - let history = yield* storage.entriesAfter(options.storeId, entries[0]) + const stored = yield* storage.entriesAfter(options.storeId, entries[0], entries) + const index = new HistoryIndex(stored.map((remoteEntry) => remoteEntry.entry)) const persistedEntries = Arr.empty() for (const entry of entries) { - const [duplicate, conflicts, newHistory] = toConflicts(history, entry) - if (duplicate) continue - history = newHistory + if (index.has(entry)) continue + const conflicts = index.conflicts(entry) yield* handler({ publicKey: options.publicKey, storeId: options.storeId, @@ -143,11 +145,26 @@ export const layerRpcHandlers: Layer.Layer< }) persistedEntries.push(entry) } - yield* storage.write(options.storeId, persistedEntries) + const committed = yield* storage.write(options.storeId, persistedEntries) + + const sequences = new Map() + for (const remoteEntry of stored) { + sequences.set(remoteEntry.entry.idString, remoteEntry.remoteSequence) + } + for (const remoteEntry of committed) { + sequences.set(remoteEntry.entry.idString, remoteEntry.remoteSequence) + } + const acks: Array = [] + for (const entry of options.entries) { + const sequence = sequences.get(entry.idString) + if (sequence !== undefined) acks.push({ entryId: entry.id, sequence }) + } + return acks }, storage.withTransaction) return EventLogServer.layerRpcHandlers({ remoteId, + firstSequence: storage.firstSequence, authorizeIdentity: (publicKey) => auth.authorizeIdentity({ publicKey }).pipe( Effect.mapError((error) => @@ -161,7 +178,7 @@ export const layerRpcHandlers: Layer.Layer< ), getOrCreateSessionAuthBinding: (publicKey, signingPublicKey) => storage.getOrCreateSessionAuthBinding(publicKey, signingPublicKey), - onWrite: Effect.fnUntraced(function*(data, authenticatedPublicKeys) { + onWrite: Effect.fnUntraced(function*(data, authenticatedPublicKeys, writes) { const request = yield* WriteEntriesUnencrypted.decode(data).pipe( Effect.mapError((_) => new EventLogProtocolError({ @@ -180,7 +197,7 @@ export const layerRpcHandlers: Layer.Layer< message: "Identity is not authenticated" }) } - if (!Arr.isReadonlyArrayNonEmpty(request.entries)) return + if (!Arr.isReadonlyArrayNonEmpty(request.entries)) return [] const resolvedStoreId = yield* mapping.resolve({ publicKey: request.publicKey, @@ -211,11 +228,17 @@ export const layerRpcHandlers: Layer.Layer< }) ) ) - yield* processEntries({ + return yield* processEntries({ publicKey: request.publicKey, storeId: resolvedStoreId, entries: request.entries }).pipe( + (write) => + writes.track({ + publicKey: request.publicKey, + storeId: request.storeId, + entryIds: request.entries.map((entry) => entry.id) + }, write), Effect.catchCause((_) => Effect.fail( new EventLogProtocolError({ @@ -238,11 +261,19 @@ export const layerRpcHandlers: Layer.Layer< publicKey: request.publicKey, storeId }) - return storage.changes({ - storeId, - startSequence: request.startSequence, - compactors: registry.compactors - }).pipe( + return request.writes.filter( + { + publicKey: request.publicKey, + storeId: request.storeId, + entryIds: (remoteEntry) => [remoteEntry.entry.id], + sequence: (remoteEntry) => remoteEntry.remoteSequence + }, + storage.changes({ + storeId, + startSequence: request.startSequence, + compactors: registry.compactors + }) + ).pipe( Stream.mapArrayEffect((entries) => Effect.map(ChangesRpc.encodeUnencrypted(entries), Arr.of)) ) }, Stream.unwrap) @@ -378,7 +409,8 @@ export const layerStoreMappingStatic = (options: { * * It provides the server remote id, stores session authentication bindings, * allocates remote sequence numbers, persists entries, streams changes, and - * exposes a transaction boundary. + * exposes a transaction boundary. `entriesAfter` returns stored entries with + * their sequences, so that duplicate writes can be acknowledged. * * @stability unstable * @category services @@ -386,11 +418,26 @@ export const layerStoreMappingStatic = (options: { */ export class Storage extends Context.Service + /** + * Lowest assigned sequence (default 0). Overestimating skips entries. + */ + readonly firstSequence?: number | undefined readonly getOrCreateSessionAuthBinding: ( publicKey: string, signingPublicKey: Uint8Array ) => Effect.Effect> - readonly entriesAfter: (storeId: StoreId, entry: Entry) => Effect.Effect> + /** + * Returns the stored entries at or after `entry`, in sequence order. + * + * The server also passes the incoming `batch`, sorted and starting with + * `entry`. Implementations may use it to return only the entries that share + * an id, or an event and primary key, with a batch entry. + */ + readonly entriesAfter: ( + storeId: StoreId, + entry: Entry, + batch?: Arr.NonEmptyReadonlyArray + ) => Effect.Effect> readonly write: ( storeId: StoreId, entries: ReadonlyArray @@ -411,43 +458,83 @@ const constEmptyPrivateKey = Redacted.make(new Uint8Array(32)) const makeServerWriteIdentityPublicKey = (storeId: StoreId): string => `effect-eventlog-server-write:${storeId}` -const entriesAfter = (journal: Array, startSequence: number): ReadonlyArray => - journal.filter((entry) => entry.remoteSequence >= startSequence) +// The memory journal is append-only with strictly increasing sequences. +const entriesAfter = (journal: Array, startSequence: number): ReadonlyArray => { + let low = 0 + let high = journal.length + while (low < high) { + const mid = Math.floor((low + high) / 2) + if (journal[mid].remoteSequence >= startSequence) { + high = mid + } else { + low = mid + 1 + } + } + return journal.slice(low) +} -const toConflicts = ( - history: ReadonlyArray, - originEntry: Entry -): [duplicate: boolean, conflicts: Array, newHistory: Array] => { - let duplicate = false +class KeyIndex { + private readonly byEvent = new Map>>() - for (let i = 0; i < history.length; i++) { - const entry = history[i] - if (entry.createdAtMillis < originEntry.createdAtMillis) { - continue - } else if (entry.idString === originEntry.idString) { - duplicate = true - continue + add(event: string, primaryKey: string, value: A): void { + let byKey = this.byEvent.get(event) + if (byKey === undefined) { + byKey = new Map() + this.byEvent.set(event, byKey) + } + const values = byKey.get(primaryKey) + if (values === undefined) { + byKey.set(primaryKey, [value]) + } else { + values.push(value) } + } - const newHistory = history.slice(i) - let conflicts: Array = [] - for (let j = 0; j < newHistory.length; j++) { - const scannedEntry = newHistory[j]! - if (scannedEntry.event === originEntry.event && scannedEntry.primaryKey === originEntry.primaryKey) { - conflicts.push(scannedEntry) - } + get(event: string, primaryKey: string): ReadonlyArray | undefined { + return this.byEvent.get(event)?.get(primaryKey) + } +} + +/** + * Indexes the history returned by `Storage.entriesAfter` for one batch. A + * conflict is a stored entry with the same event and primary key and a + * strictly greater id, reported in id order. Batch entries are never added, so + * they do not conflict with each other. + */ +class HistoryIndex { + private readonly ids = new Set() + private readonly byKey = new KeyIndex() + + constructor(history: ReadonlyArray) { + // Storage returns history in sequence order; conflicts are reported in id order. + for (const entry of Arr.sort(history, Entry.Order)) { + this.ids.add(entry.idString) + this.byKey.add(entry.event, entry.primaryKey, entry) } - return [duplicate, conflicts, newHistory] } - return [duplicate, [], []] + has(entry: Entry): boolean { + return this.ids.has(entry.idString) + } + + conflicts(entry: Entry): Array { + const stored = this.byKey.get(entry.event, entry.primaryKey) + if (stored === undefined) return [] + return stored.filter((candidate) => Entry.Order(candidate, entry) > 0) + } +} + +type MemoryStore = { + readonly journal: Array + readonly byId: Map + readonly byKey: KeyIndex } type RegisteredCompactor = { readonly events: ReadonlySet readonly effect: (options: { readonly entries: ReadonlyArray - readonly write: (entry: Entry) => Effect.Effect + readonly write: (entry: Entry, source?: Entry) => Effect.Effect }) => Effect.Effect } @@ -513,6 +600,12 @@ const toCompactedRemoteEntries = (options: { * entries when the replacement count can be mapped back to increasing remote * sequence numbers; otherwise the original entries are kept. * + * Outputs are ordered by the position of `source ?? entry` within the run, as in + * client replay. Unknown positions go to the start; ties keep write order. + * + * Client replay groups the whole batch per compactor; the server calls each + * compactor per contiguous run, so batch boundaries can differ. + * * @stability unstable * @category compaction * @since 4.0.0 @@ -539,7 +632,6 @@ export const compactBacklog = Effect.fnUntraced(function*(options: { const entries: Array = [remoteEntry.entry] const remoteGroup: Array = [remoteEntry] - const compacted: Array = [] index++ while (index < options.remoteEntries.length) { @@ -553,25 +645,29 @@ export const compactBacklog = Effect.fnUntraced(function*(options: { index++ } + const positions = new Map() + const compacted: Array> = [] + for (let i = 0; i < entries.length; i++) { + positions.set(entries[i].idString, i) + compacted.push([]) + } yield* compactor.effect({ entries, - write(entry) { + write(entry, source) { return Effect.sync(() => { - compacted.push(entry) + compacted[positions.get((source ?? entry).idString) ?? 0].push(entry) }) } }).pipe(Effect.orDie) const projected = toCompactedRemoteEntries({ - compacted, + compacted: compacted.flat(), remoteEntries: remoteGroup }) - if (projected === undefined) { - compactedRemoteEntries.push(...remoteGroup) - continue + for (const entry of projected ?? remoteGroup) { + compactedRemoteEntries.push(entry) } - compactedRemoteEntries.push(...projected) } return compactedRemoteEntries @@ -591,25 +687,16 @@ export const compactBacklog = Effect.fnUntraced(function*(options: { * @since 4.0.0 */ export const makeStorageMemory: Effect.Effect = Effect.gen(function*() { - const knownIds = new Map>() - const journals = new Map>() + const stores = new Map() const sessionAuthBindings = new Map>() const remoteId = makeRemoteIdUnsafe() - const ensureKnownIds = (storeId: StoreId): Map => { - let storeKnownIds = knownIds.get(storeId) - if (storeKnownIds) return storeKnownIds - storeKnownIds = new Map() - knownIds.set(storeId, storeKnownIds) - return storeKnownIds - } - - const ensureJournal = (storeId: StoreId): Array => { - let journal = journals.get(storeId) - if (journal) return journal - journal = [] - journals.set(storeId, journal) - return journal + const ensureStore = (storeId: StoreId): MemoryStore => { + let store = stores.get(storeId) + if (store) return store + store = { journal: [], byId: new Map(), byKey: new KeyIndex() } + stores.set(storeId, store) + return store } const pubsubs = yield* RcMap.make({ @@ -622,15 +709,13 @@ export const makeStorageMemory: Effect.Effect) { - const sequenceNumbers: Array = [] const committed: Array = [] - const storeKnownIds = ensureKnownIds(storeId) - const journal = ensureJournal(storeId) - let lastSequenceNumber = Arr.last(journal).pipe( + const store = ensureStore(storeId) + let lastSequenceNumber = Arr.last(store.journal).pipe( Option.map((entry) => entry.remoteSequence), Option.getOrElse(() => 0) ) - if (entries.some((entry) => storeKnownIds.has(entry.idString))) { + if (entries.some((entry) => store.byId.has(entry.idString))) { return yield* Effect.die("Duplicate entries") } @@ -640,10 +725,10 @@ export const makeStorageMemory: Effect.Effect Effect.sync(() => { const existing = sessionAuthBindings.get(publicKey) @@ -663,10 +749,27 @@ export const makeStorageMemory: Effect.Effect + entriesAfter: (storeId, oldest, batch) => Effect.sync(() => { - const journal = ensureJournal(storeId) - return journal.filter((e) => Entry.Order(e.entry, entry) >= 0).map((e) => e.entry) + const store = ensureStore(storeId) + if (batch === undefined) { + return store.journal + .filter((remoteEntry) => Entry.Order(remoteEntry.entry, oldest) >= 0) + } + const found = new Set() + const scanned = new Set>() + for (const entry of batch) { + const known = store.byId.get(entry.idString) + if (known !== undefined) found.add(known) + const sameKey = store.byKey.get(entry.event, entry.primaryKey) + if (sameKey === undefined || scanned.has(sameKey)) continue + scanned.add(sameKey) + for (const remoteEntry of sameKey) { + if (Entry.Order(remoteEntry.entry, oldest) >= 0) found.add(remoteEntry) + } + } + return Array.from(found) + .sort((a, b) => a.remoteSequence - b.remoteSequence) }), write, changes: Effect.fnUntraced(function*({ storeId, startSequence, compactors }) { @@ -674,7 +777,7 @@ export const makeStorageMemory: Effect.Effect 0 ? backlog[backlog.length - 1].remoteSequence : startSequence - 1 diff --git a/repos/effect/packages/effect/src/eventlog/EventRow.ts b/repos/effect/packages/effect/src/eventlog/EventRow.ts new file mode 100644 index 0000000000..9d09e93686 --- /dev/null +++ b/repos/effect/packages/effect/src/eventlog/EventRow.ts @@ -0,0 +1,706 @@ +/** + * Mutable rows that converge across event-log replicas. + * + * Each row type has one event with `set`, `patch` and `remove` payloads, keyed + * by one or more of its fields. Pass the result of `make` to `EventGroup.add`. + * Its `set`, `patch` and `remove` constructors return `{ event, payload }`, + * which spreads into `EventLog.write` and serves as an item of + * `EventLog.writeMany` or an argument of an `EventLog.makeClient` client. + * + * `handler` resolves conflicts per top-level field and calls at most one + * storage callback. Replicas converge to the result of applying entries in + * id order. `compaction` drops entries that newer ones shadow. + * + * **Details** + * + * Replay decodes entries and stored conflicts with the current row schema, + * which must accept every encoding previously written: + * + * - New fields must accept absence, for example with + * `Schema.withDecodingDefaultKey`. Nullable fields alone reject old `set` + * entries. Patches never apply defaults. + * - Removing a field is safe. Old entries decode without it. + * - To change a field's type, keep its encoded type and change its decoded + * type with a transformation, such as `Schema.NumberFromString` for a field + * that used to be a string. + * - Making a field required, renaming it, narrowing its type, changing its + * encoded type or changing the key breaks old entries. That includes + * widening a field to a union of its old and new types, because union + * encodings are tagged. A rename keeps old entries readable only if the + * field has an explicit `SchemaBinary.fieldId`. + * - Key-field schemas must decode written values to ones with the same + * primary key, since replicas derive the key from decoded values. + * + * Conflicts are scoped to one row; parent removal does not resolve concurrent + * child writes. For cascading deletion, retain a removed-parent marker and + * delete its children in the parent `remove` callback. The `remove` callback + * writes the marker even when the parent was never stored, since compaction + * or a newer `remove` can skip its `set`, and the marker must not depend on + * the old row. A parent `patch` treats a marker as a missing row. Child `set` + * and `patch` callbacks must skip marked parents but allow parents not yet + * received. Make the parent id part of the child key, as in + * `fields: ["parentId", "id"]`, so a child cannot move between parents. This + * converges only if parent deletion is final (no key reuse). Queries must + * exclude marked parents. Retain markers unless the app defines a safe cutoff + * for removing them. + * + * **Example** (Syncing grocery rows) + * + * ```ts + * import { Effect, Schema } from "effect" + * import { EventGroup, EventLog, EventRow } from "effect/eventlog" + * + * const Grocery = EventRow.make("grocery", { + * key: "id", + * schema: Schema.Struct({ + * id: Schema.String, + * name: Schema.String, + * completed: Schema.Boolean + * }) + * }) + * + * const Groceries = EventGroup.empty.add(Grocery) + * + * const store = new Map() + * + * export const GroceriesLive = EventLog.group(Groceries, (handlers) => + * handlers.handle( + * "grocery", + * EventRow.handler(Grocery, { + * set: (row) => Effect.sync(() => store.set(row.id, row)), + * patch: (id, fields) => + * Effect.sync(() => { + * const row = store.get(id) + * if (row !== undefined) store.set(id, { ...row, ...fields }) + * }), + * remove: (id) => Effect.sync(() => store.delete(id)) + * }) + * )) + * + * export const complete = Effect.fnUntraced(function*(id: string) { + * const log = yield* EventLog.EventLog + * yield* log.write({ + * schema: EventLog.schema(Groceries), + * ...Grocery.patch(id, { completed: true }) + * }) + * }) + * ``` + * + * **Example** (Evolving a row schema) + * + * ```ts import.meta.vitest + * import { Effect, Schema } from "effect" + * import { EventGroup, EventRow } from "effect/eventlog" + * + * const ItemV1 = EventRow.make("item", { + * key: "id", + * schema: Schema.Struct({ id: Schema.String, quantity: Schema.String }) + * }) + * + * // Adds a nullable field and reads the old string quantity as a number. + * const ItemV2 = EventRow.make("item", { + * key: "id", + * schema: Schema.Struct({ + * id: Schema.String, + * quantity: Schema.NumberFromString, + * note: Schema.NullOr(Schema.String).pipe( + * Schema.withDecodingDefaultKey(Effect.succeed(null)) + * ) + * }) + * }) + * + * const v1 = EventGroup.empty.add(ItemV1).events.item.payloadSchemaBinary + * const v2 = EventGroup.empty.add(ItemV2).events.item.payloadSchemaBinary + * + * const written = Schema.encodeUnknownSync(v1)(ItemV1.set({ id: "a", quantity: "3" }).payload) + * const decoded = Schema.decodeUnknownSync(v2)(written) + * decoded // => { _tag: "set", row: { id: "a", quantity: 3, note: null } } + * ``` + * + * @stability unstable + * @since 4.0.6 + */ +import * as Context from "../Context.ts" +import * as Effect from "../Effect.ts" +import * as SchemaBinary from "../encoding/SchemaBinary.ts" +import * as Layer from "../Layer.ts" +import * as Result from "../Result.ts" +import * as Schema from "../Schema.ts" +import * as SchemaGetter from "../SchemaGetter.ts" +import * as SchemaIssue from "../SchemaIssue.ts" +import * as SchemaParser from "../SchemaParser.ts" +import * as Struct from "../Struct.ts" +import type * as Event from "./Event.ts" +import type * as EventGroup from "./EventGroup.ts" +import { Entry } from "./EventJournal.ts" +import * as EventLog from "./EventLog.ts" +import type { StoreId } from "./EventLogMessage.ts" + +/** + * Required fields that can be part of a declared row key. + * + * @stability unstable + * @category utility types + * @since 4.0.6 + */ +export type RequiredField = + & { + [K in keyof Fields]: Fields[K] extends { readonly "~type.optionality": "optional" } ? never : K + }[keyof Fields] + & string + +/** + * Values of the declared key fields of a row. + * + * @stability unstable + * @category utility types + * @since 4.0.6 + */ +export type KeyValues = { + readonly [K in Names]: Schema.Schema.Type +} + +/** + * Required fields whose values are all strings, all numbers or all + * `Uint8Array`s, so their JSON encodings form keys without `derive`. + * + * @stability unstable + * @category utility types + * @since 4.0.6 + */ +export type DefaultKeyField = { + [K in RequiredField]: [Schema.Schema.Type] extends [string] ? K + : [Schema.Schema.Type] extends [number] ? K + : [Schema.Schema.Type] extends [Uint8Array] ? K + : never +}[RequiredField] + +/** + * A row key declared from one or more fields. + * + * **Details** + * + * The primary key is the result of `derive`, which must be deterministic and + * unique per row. Without it, the primary key is the JSON tuple of the + * fields' JSON encodings, in declaration order, which needs every field to be + * a {@link DefaultKeyField}. Other fields can collide in JSON, such as `null` + * and `undefined`, so they require `derive`. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export type KeyDeclaration> = + & { readonly fields: readonly [Names, ...Array] } + & ([Names] extends [DefaultKeyField] ? + { readonly derive?: ((fields: KeyValues) => string) | undefined } + : { readonly derive: (fields: KeyValues) => string }) + +/** + * Primary key type of a row: the key value for a string-typed single-field + * key, and `string` otherwise. + * + * @stability unstable + * @category utility types + * @since 4.0.6 + */ +export type PrimaryKey = Schema.Schema.Type extends string + ? Schema.Schema.Type + : string + +/** + * Optional non-key fields of a `patch` payload. + * + * @stability unstable + * @category utility types + * @since 4.0.6 + */ +export type PatchFields = { + readonly [K in Exclude]: Schema.optionalKey +} + +/** + * Schema of a row event's payload, derived from the row schema. + * + * **Details** + * + * - `set` carries the full row, validated by the row schema. + * - `patch` carries the row key and the fields to write. An absent field is + * not written. The key itself cannot be patched. + * - `remove` carries the row key. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export type PayloadSchema< + Fields extends Schema.Struct.Fields, + Key extends keyof Fields, + KeySchema extends Schema.Constraint = Fields[Key] +> = Schema.Union< + readonly [ + Schema.TaggedStruct<"set", { readonly row: Schema.Struct }>, + Schema.TaggedStruct<"patch", { + readonly key: KeySchema + readonly fields: Schema.Struct> + }>, + Schema.TaggedStruct<"remove", { readonly key: KeySchema }> + ] +> + +/** + * Decoded payload of a row event. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export type Payload< + Fields extends Schema.Struct.Fields, + Key extends keyof Fields, + KeySchema extends Schema.Constraint = Fields[Key] +> = Schema.Schema.Type> + +/** + * A row event to write, as returned by the `set`, `patch` and `remove` + * constructors of a row. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export type Write< + Tag extends string, + Fields extends Schema.Struct.Fields, + Key extends keyof Fields, + KeySchema extends Schema.Constraint = Fields[Key] +> = { + readonly event: Tag + readonly payload: Payload +} + +/** + * A row type that syncs through an `EventLog`. + * + * **Details** + * + * Includes `EventGroup.add` options, the row schema, its key fields and + * constructors for the events to write. The row key is the event primary key; + * success is `void`. + * + * `Key` is the union of the key fields. `KeySchema` is the schema of the key + * in `patch` and `remove` payloads and callbacks: the field's schema for a + * single-field key, and a struct of the key fields for a declared key. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export interface EventRow< + out Tag extends string, + in out Fields extends Schema.Struct.Fields, + in out Key extends keyof Fields & string, + in out Error extends Schema.Top = typeof Schema.Never, + in out KeySchema extends Schema.Constraint = Fields[Key] +> { + readonly tag: Tag + readonly keyFields: ReadonlyArray + readonly schema: Schema.Struct + readonly primaryKey: (payload: Payload) => string + readonly payload: PayloadSchema + readonly success: typeof Schema.Void + readonly error: Error + + /** + * Computes the primary key of a row from its key fields. + */ + readonly keyOf: (fields: KeyValues) => PrimaryKey + + /** + * Replaces the whole row, creating it if missing. + */ + readonly set: (row: Schema.Struct["Type"]) => Write + + /** + * Writes fields of an existing row. + * + * **Details** + * + * A patch on a missing row does nothing. To clear a value, use a nullable + * field and write `null`. + * + * Absent fields stay unwritten, even with decoding defaults. Values whose + * encoding omits the key, such as `Option.none()` for + * `Schema.OptionFromOptionalKey`, fail encoding: replicas cannot distinguish + * them from unwritten fields. + */ + readonly patch: ( + key: Schema.Schema.Type, + fields: Schema.Struct>["Type"] + ) => Write + + /** + * Deletes the row. + */ + readonly remove: (key: Schema.Schema.Type) => Write +} + +/** + * Defines a row type as a single event. + * + * **Details** + * + * `key` names a required field, or declares required fields with an + * optional `derive` function. `patch`, `remove` and the handler callbacks take + * the key as typed values: the field's value for a single field, and a struct + * of the fields for a declaration. Key fields cannot be patched. To move a + * row, remove it and set it again. + * + * A single-field key must be a {@link DefaultKeyField}. Declare other fields + * with a `derive` function. + * + * `keyOf` computes the primary key that conflicts are grouped by: the value + * of a string single-field key, the result of `derive`, or else the JSON + * tuple of the key fields' JSON encodings. `-0` and `0` are the same key. + * + * Last-writer-wins applies to top-level fields. Nested values are atomic, so + * concurrent updates within one nested value do not merge. + * + * `error` is the error schema of the event. Handler callbacks can fail with it. + * + * **Gotchas** + * + * Do not compact row events with `EventLog.groupCompaction` or + * `EventLog.groupBatchCompaction`. Compaction runs before stored conflicts + * are read, so merging row ops loses or misapplies writes. Use + * {@link compaction}, which only drops entries that newer ones shadow. + * + * For multi-row operations, use `EventLog.writeMany` with one op per row. + * + * @stability unstable + * @category constructors + * @since 4.0.6 + */ +export const make = < + const Tag extends string, + Fields extends Schema.Struct.Fields, + const Key extends DefaultKeyField = never, + const Declared extends RequiredField = never, + Error extends Schema.Top = typeof Schema.Never +>(tag: Tag, options: { + readonly key: Key | KeyDeclaration + readonly schema: Schema.Struct + readonly error?: Error | undefined +}): EventRow< + Tag, + Fields, + // A declaration also infers `Key`, as its constraint, so `Declared` decides. + // `NoInfer` keeps a contextual return type from inferring `Declared`. + [NoInfer] extends [never] ? NoInfer : NoInfer, + Error, + [NoInfer] extends [never] ? Fields[NoInfer] + : Schema.Struct<{ readonly [K in NoInfer]: Fields[K] }> +> => { + const fields = options.schema.fields + const key = typeof options.key === "string" + ? scalarKey(options.key, fields[options.key]) + : declaredKey(options.key as KeyDeclaration, fields) + const patchFields: Record = {} + for (const name of Object.keys(fields)) { + if (!key.fields.includes(name)) patchFields[name] = patchField(fields[name]) + } + const payload = Schema.Union([ + Schema.TaggedStruct("set", { row: options.schema }), + Schema.TaggedStruct("patch", { key: key.schema, fields: Schema.Struct(patchFields) }), + Schema.TaggedStruct("remove", { key: key.schema }) + ]) + return { + tag, + keyFields: key.fields, + schema: options.schema, + keyOf: key.primaryKey, + [keyFromId]: key.fromRow, + primaryKey: (payload: Op) => key.primaryKey(payload._tag === "set" ? payload.row : key.toValues(payload.key)), + payload, + success: Schema.Void, + error: options.error ?? Schema.Never, + set: (row: unknown) => ({ event: tag, payload: { _tag: "set", row } }), + patch: (key: unknown, fields: unknown) => ({ event: tag, payload: { _tag: "patch", key, fields } }), + remove: (key: unknown) => ({ event: tag, payload: { _tag: "remove", key } }) + } as any +} + +/** + * Builds the handler for a row event, to pass to `handlers.handle`. + * + * **Details** + * + * Resolves newer conflicts in id order and calls at most one callback with + * typed values and key, never conflicts. + * + * Newer writes take precedence per top-level field. Empty patches and entries + * with nothing left to write skip the callback but are still journaled and synced. + * A late `set` retains newer patches, including when the row is missing. + * + * Decodes stored entry bytes so local writes and replay pass the same values + * to callbacks, stripping keys outside the schema at any depth. Requires the + * payload schema's decoding services, plus any services the callbacks require. + * Callbacks may fail with the event's error type; success is always `void`. + * + * **Gotchas** + * + * Convergence requires this callback contract: + * + * - `set` replaces the whole row, creating it if it is missing. + * - `patch` updates an existing row and does nothing if the row is missing. + * Upserting can resurrect removed rows on only some replicas. + * - `remove` is idempotent. + * + * @stability unstable + * @category handlers + * @since 4.0.6 + */ +export const handler = < + Tag extends string, + Fields extends Schema.Struct.Fields, + Key extends keyof Fields & string, + Error extends Schema.Top, + KeySchema extends Schema.Constraint, + RSet = never, + RPatch = never, + RRemove = never +>( + row: EventRow, + callbacks: { + readonly set: ( + row: Schema.Struct["Type"], + key: Schema.Schema.Type + ) => Effect.Effect + readonly patch: ( + key: Schema.Schema.Type, + fields: Schema.Struct>["Type"] + ) => Effect.Effect + readonly remove: (key: Schema.Schema.Type) => Effect.Effect + } +): (options: { + readonly storeId: StoreId + readonly payload: Payload + readonly entry: Entry + readonly conflicts: ReadonlyArray<{ + readonly entry: Entry + readonly payload: Payload + }> +}) => Effect.Effect> => { + const decode = Schema.decodeUnknownEffect(SchemaBinary.toCodec(row.payload)) + const keyFrom: RowKey["fromRow"] = (row as any)[keyFromId] + return (options) => + Effect.flatMap( + Effect.orDie(decode(options.entry.payload)), + (payload): Effect.Effect => { + const op = resolve(payload as Op, options.conflicts as ReadonlyArray<{ readonly payload: Op }>) + if (op === undefined) return Effect.void + switch (op._tag) { + case "set": + return callbacks.set(op.row as any, keyFrom(op.row) as any) + case "patch": + return Object.keys(op.fields).length === 0 ? Effect.void : callbacks.patch(op.key as any, op.fields as any) + case "remove": + return callbacks.remove(op.key as any) + } + } + ).pipe(Effect.asVoid) +} + +/** + * Registers a compactor for one row event of a group. + * + * **Details** + * + * Drops entries superseded within the same remote batch and row by a newer + * `set` or `remove`. A `patch` is also dropped when newer patches together + * overwrite all its fields. Kept entries retain their ids and stored conflicts. + * + * A `set` or `remove` followed only by patches is kept: without it, the + * patches would apply to a different row, or to none. Entries that fail to + * decode are kept. + * + * Leaves other events' compactors unchanged. Register one compactor per row + * event. + * + * @stability unstable + * @category compaction + * @since 4.0.6 + */ +export const compaction = < + Events extends Event.Any, + Tag extends Event.Tag, + Fields extends Schema.Struct.Fields, + Key extends keyof Fields & string, + Error extends Schema.Top, + KeySchema extends Schema.Constraint +>( + group: EventGroup.EventGroup, + row: EventRow +): Layer.Layer> => + Layer.effectDiscard(Effect.gen(function*() { + const registry = yield* EventLog.Registry + const services = yield* Effect.context>() + const decode = Schema.decodeUnknownEffect(group.events[row.tag].payloadSchemaBinary) + yield* registry.registerCompaction({ + events: [row.tag], + effect: Effect.fnUntraced(function*({ entries, write }) { + const ops = new Map() + for (const entry of entries) { + const payload = yield* Effect.result( + decode(entry.payload).pipe(Effect.updateContext((input) => Context.merge(services, input))) + ) as Effect.Effect> + if (Result.isSuccess(payload)) ops.set(entry, payload.success) + } + const shadowed = shadowedEntries(entries, ops) + for (const entry of entries) { + if (!shadowed.has(entry)) yield* write(entry) + } + }) + }) + })) + +const shadowedEntries = (entries: ReadonlyArray, ops: ReadonlyMap): Set => { + const shadowed = new Set() + const byKey = new Map>() + for (const entry of entries) { + const row = byKey.get(entry.primaryKey) + if (row === undefined) byKey.set(entry.primaryKey, [entry]) + else row.push(entry) + } + for (const row of byKey.values()) { + row.sort(Entry.Order) + let replaced = false + let patched = false + const written = new Set() + for (let i = row.length - 1; i >= 0; i--) { + const op = ops.get(row[i]) + if (op === undefined) continue + if (op._tag !== "patch") { + if (replaced) shadowed.add(row[i]) + replaced = true + continue + } + const names = Object.keys(op.fields) + if (replaced || (patched && names.every((name) => written.has(name)))) shadowed.add(row[i]) + patched = true + for (const name of names) written.add(name) + } + } + return shadowed +} + +const keyFromId = "~effect/eventlog/EventRow/keyFrom" + +type Row = Readonly> + +// How a row's key is carried and derived, for a single key field or a declaration. +interface RowKey { + readonly fields: ReadonlyArray + // Schema of the key in `patch` and `remove` payloads. + readonly schema: Schema.Constraint + // The key as payloads and callbacks carry it. + readonly fromRow: (row: Row) => unknown + // The key field values a carried key stands for. + readonly toValues: (key: unknown) => Row + readonly primaryKey: (values: Row) => string +} + +const scalarKey = (name: string, field: Schema.Constraint): RowKey => ({ + fields: [name], + schema: field, + fromRow: (row) => row[name], + toValues: (key) => ({ [name]: key }), + primaryKey: jsonKey([[name, field]], true) +}) + +const declaredKey = (declaration: KeyDeclaration, fields: Schema.Struct.Fields): RowKey => { + const names = declaration.fields + const derive = declaration.derive + return { + fields: names, + schema: Schema.Struct(Struct.pick(fields, names) as Schema.Struct.Fields), + fromRow: (row) => Struct.pick(row, names), + toValues: (key) => key as Row, + primaryKey: derive !== undefined + ? (row) => derive(Struct.pick(row, names) as any) + : jsonKey(names.map((name) => [name, fields[name]]), false) + } +} + +// A string single-field key is its own primary key. Otherwise the key is the +// JSON tuple of the fields' JSON encodings, distinct for distinct values of +// `DefaultKeyField`s. A field holds one kind of value, so a raw string never +// meets a tuple from the same field. +const jsonKey = (fields: ReadonlyArray, single: boolean) => { + let encoders: ReadonlyArray unknown]> | undefined + return (row: Row): string => { + if (single && typeof row[fields[0][0]] === "string") return row[fields[0][0]] as string + encoders ??= fields.map(([name, field]) => + [name, Schema.encodeUnknownSync(Schema.toCodecJson(field as Schema.Codec))] as const + ) + return JSON.stringify(encoders.map(([name, encode]) => encode(row[name]))) + } +} + +// Run the field codec only for present keys, bypassing missing-key defaults. +const patchField = (field: Schema.Constraint): Schema.Constraint => { + const decode = SchemaParser.decodeUnknownEffect(field) + // The struct wrapper detects encodings that drop a written key. + const encode = SchemaParser.encodeUnknownEffect(Schema.Struct({ value: field })) + return Schema.optionalKey(Schema.toEncoded(field)).pipe(Schema.decodeTo(Schema.optionalKey(Schema.toType(field)), { + decode: SchemaGetter.transformEffect(decode), + encode: SchemaGetter.transformEffect((value, options) => + Effect.flatMap(encode({ value }, options), (encoded) => + Object.hasOwn(encoded, "value") + ? Effect.succeed(encoded.value) + : Effect.fail( + new SchemaIssue.Forbidden({ message: "A patch cannot write a value whose encoding omits the key" }, value) + )) + ) + })) +} + +type DecodingServices< + Fields extends Schema.Struct.Fields, + Key extends keyof Fields, + KeySchema extends Schema.Constraint +> = PayloadSchema["DecodingServices"] + +type Op = + | { readonly _tag: "set"; readonly row: Row } + | { readonly _tag: "patch"; readonly key: unknown; readonly fields: Row } + | { readonly _tag: "remove"; readonly key: unknown } + +const compose = (x: Op, y: Op): Op => { + if (y._tag !== "patch") return y + switch (x._tag) { + case "set": + return { _tag: "set", row: { ...x.row, ...y.fields } } + case "patch": + return { _tag: "patch", key: y.key, fields: { ...x.fields, ...y.fields } } + case "remove": + return x + } +} + +const resolve = (entry: Op, conflicts: ReadonlyArray<{ readonly payload: Op }>): Op | undefined => { + if (conflicts.length === 0) return entry + let newer = conflicts[0].payload + for (let i = 1; i < conflicts.length; i++) { + newer = compose(newer, conflicts[i].payload) + } + if (newer._tag !== "patch") return undefined + switch (entry._tag) { + case "set": + return { _tag: "set", row: { ...entry.row, ...newer.fields } } + case "patch": + return { _tag: "patch", key: entry.key, fields: Struct.omit(entry.fields, Object.keys(newer.fields)) } + case "remove": + return entry + } +} diff --git a/repos/effect/packages/effect/src/eventlog/SqlEventJournal.ts b/repos/effect/packages/effect/src/eventlog/SqlEventJournal.ts index 1c9c4437d5..20b00ba7a0 100644 --- a/repos/effect/packages/effect/src/eventlog/SqlEventJournal.ts +++ b/repos/effect/packages/effect/src/eventlog/SqlEventJournal.ts @@ -1,15 +1,11 @@ /** - * SQL-backed persistence for the unstable event-log journal. - * - * This module implements `EventJournal` on top of a `SqlClient`. It stores event - * entries as encoded bytes and stores per-remote sequence metadata in separate - * tables, giving event-log programs a durable journal that can be replayed after - * restart and synchronized with remote journals. + * SQL-backed event journal with per-remote upload checkpoints. * * @stability unstable * @since 4.0.0 */ import * as Arr from "../Array.ts" +import * as Clock from "../Clock.ts" import * as Effect from "../Effect.ts" import * as Uuid from "../internal/uuid.ts" import * as Layer from "../Layer.ts" @@ -20,6 +16,7 @@ import * as SqlClient from "../sql/SqlClient.ts" import * as SqlError from "../sql/SqlError.ts" import * as SqlSchema from "../sql/SqlSchema.ts" import * as EventJournal from "./EventJournal.ts" +import { ConflictIndex } from "./internal/conflicts.ts" type WriteFromRemoteOptions = Parameters[0] @@ -28,8 +25,20 @@ type WriteFromRemoteOptions = Parameters - sql` - CREATE TABLE IF NOT EXISTS ${entryTableSql} ( - id BYTEA PRIMARY KEY, - event TEXT NOT NULL, - primary_key TEXT NOT NULL, - payload BYTEA NOT NULL, - timestamp BIGINT NOT NULL - )`, - mysql: () => - sql` - CREATE TABLE IF NOT EXISTS ${entryTableSql} ( - id BINARY(16) PRIMARY KEY, - event TEXT NOT NULL, - primary_key TEXT NOT NULL, - payload BLOB NOT NULL, - timestamp BIGINT NOT NULL - )`, - mssql: () => - sql` - CREATE TABLE IF NOT EXISTS ${entryTableSql} ( - id UNIQUEIDENTIFIER PRIMARY KEY, - event NVARCHAR(MAX) NOT NULL, - primary_key NVARCHAR(MAX) NOT NULL, - payload VARBINARY(MAX) NOT NULL, - timestamp BIGINT NOT NULL - )`, - orElse: () => - sql` - CREATE TABLE IF NOT EXISTS ${entryTableSql} ( - id BLOB PRIMARY KEY, - event TEXT NOT NULL, - primary_key TEXT NOT NULL, - payload BLOB NOT NULL, - timestamp INTEGER NOT NULL - )` - }).pipe(withTracerDisabled) + // to_regclass folds unquoted names to lowercase, so pass the quoted identifier. + const quoted = (table: string) => sql`${sql(table)}`.compile(true)[0] + + const tableExists = (table: string) => + sql.onDialectOrElse({ + // Reading the catalog refreshes cached names after concurrent DDL. + pg: () => sql`SELECT 1 AS found FROM pg_class WHERE oid = to_regclass(${quoted(table)})`, + mysql: () => + sql`SELECT 1 AS found FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = ${table}`, + mssql: () => sql`SELECT 1 AS found WHERE OBJECT_ID(${table}, 'U') IS NOT NULL`, + orElse: () => sql`SELECT 1 AS found FROM sqlite_master WHERE type = 'table' AND name = ${table}` + }).pipe(Effect.map((rows) => rows.length > 0)) + + const createSchema = Effect.gen(function*() { + const eventIndex = sql(`${entryTable}_event_idx`) + const timestampIndex = sql(`${entryTable}_timestamp_idx`) + + yield* sql.onDialectOrElse({ + pg: () => + sql` + CREATE TABLE IF NOT EXISTS ${entryTableSql} ( + id BYTEA PRIMARY KEY, + event TEXT NOT NULL, + primary_key TEXT NOT NULL, + payload BYTEA NOT NULL, + timestamp BIGINT NOT NULL + )`, + mysql: () => + sql` + CREATE TABLE IF NOT EXISTS ${entryTableSql} ( + id BINARY(16) PRIMARY KEY, + event TEXT NOT NULL, + primary_key TEXT NOT NULL, + payload BLOB NOT NULL, + timestamp BIGINT NOT NULL + )`, + mssql: () => + sql` + CREATE TABLE IF NOT EXISTS ${entryTableSql} ( + id UNIQUEIDENTIFIER PRIMARY KEY, + event NVARCHAR(MAX) NOT NULL, + primary_key NVARCHAR(MAX) NOT NULL, + payload VARBINARY(MAX) NOT NULL, + timestamp BIGINT NOT NULL + )`, + orElse: () => + sql` + CREATE TABLE IF NOT EXISTS ${entryTableSql} ( + id BLOB PRIMARY KEY, + event TEXT NOT NULL, + primary_key TEXT NOT NULL, + payload BLOB NOT NULL, + timestamp INTEGER NOT NULL + )` + }) + + // Insertion sequences let uploads include older entries imported later. + yield* sql.onDialectOrElse({ + pg: () => + sql` + CREATE TABLE IF NOT EXISTS ${sequenceTableSql} ( + sequence BIGSERIAL PRIMARY KEY, + entry_id BYTEA NOT NULL UNIQUE + )`, + mysql: () => + sql` + CREATE TABLE IF NOT EXISTS ${sequenceTableSql} ( + sequence BIGINT AUTO_INCREMENT PRIMARY KEY, + entry_id BINARY(16) NOT NULL UNIQUE + )`, + mssql: () => + sql` + CREATE TABLE ${sequenceTableSql} ( + sequence BIGINT IDENTITY(1,1) PRIMARY KEY, + entry_id UNIQUEIDENTIFIER NOT NULL UNIQUE + )`, + orElse: () => + sql` + CREATE TABLE IF NOT EXISTS ${sequenceTableSql} ( + sequence INTEGER PRIMARY KEY, + entry_id BLOB NOT NULL UNIQUE + )` + }) + + yield* sql.onDialectOrElse({ + pg: () => + sql` + CREATE TABLE IF NOT EXISTS ${checkpointsTableSql} ( + remote_id BYTEA PRIMARY KEY, + sequence BIGINT NOT NULL + )`, + mysql: () => + sql` + CREATE TABLE IF NOT EXISTS ${checkpointsTableSql} ( + remote_id BINARY(16) PRIMARY KEY, + sequence BIGINT NOT NULL + )`, + mssql: () => + sql` + CREATE TABLE ${checkpointsTableSql} ( + remote_id UNIQUEIDENTIFIER PRIMARY KEY, + sequence BIGINT NOT NULL + )`, + orElse: () => + sql` + CREATE TABLE IF NOT EXISTS ${checkpointsTableSql} ( + remote_id BLOB PRIMARY KEY, + sequence INTEGER NOT NULL + )` + }) + + yield* sql.onDialectOrElse({ + pg: () => + sql` + CREATE TABLE IF NOT EXISTS ${remoteCursorsTableSql} ( + remote_id BYTEA PRIMARY KEY, + next_sequence INT NOT NULL + )`, + mysql: () => + sql` + CREATE TABLE IF NOT EXISTS ${remoteCursorsTableSql} ( + remote_id BINARY(16) PRIMARY KEY, + next_sequence INT NOT NULL + )`, + mssql: () => + sql` + CREATE TABLE ${remoteCursorsTableSql} ( + remote_id UNIQUEIDENTIFIER PRIMARY KEY, + next_sequence INT NOT NULL + )`, + orElse: () => + sql` + CREATE TABLE IF NOT EXISTS ${remoteCursorsTableSql} ( + remote_id BLOB PRIMARY KEY, + next_sequence INT NOT NULL + )` + }) + + // The remote each imported entry came from, so uploads skip it. + yield* sql.onDialectOrElse({ + pg: () => + sql` + CREATE TABLE IF NOT EXISTS ${sourcesTableSql} ( + entry_id BYTEA PRIMARY KEY, + remote_id BYTEA NOT NULL + )`, + mysql: () => + sql` + CREATE TABLE IF NOT EXISTS ${sourcesTableSql} ( + entry_id BINARY(16) PRIMARY KEY, + remote_id BINARY(16) NOT NULL + )`, + mssql: () => + sql` + CREATE TABLE ${sourcesTableSql} ( + entry_id UNIQUEIDENTIFIER PRIMARY KEY, + remote_id UNIQUEIDENTIFIER NOT NULL + )`, + orElse: () => + sql` + CREATE TABLE IF NOT EXISTS ${sourcesTableSql} ( + entry_id BLOB PRIMARY KEY, + remote_id BLOB NOT NULL + )` + }) + + yield* sql.onDialectOrElse({ + // MySQL indexes TEXT columns by prefix only. + mysql: () => + Effect.andThen( + sql`CREATE INDEX ${eventIndex} ON ${entryTableSql} (event(191), primary_key(191), timestamp)`, + sql`CREATE INDEX ${timestampIndex} ON ${entryTableSql} (timestamp)` + ), + // B-tree entries are limited to about 2.7 KB, so index hashes of the + // unbounded columns. + pg: () => + Effect.andThen( + sql`CREATE INDEX IF NOT EXISTS ${eventIndex} ON ${entryTableSql} (md5(event), md5(primary_key), timestamp)`, + sql`CREATE INDEX IF NOT EXISTS ${timestampIndex} ON ${entryTableSql} (timestamp)` + ), + // NVARCHAR(MAX) columns cannot be index keys. + mssql: () => sql`CREATE INDEX ${timestampIndex} ON ${entryTableSql} (timestamp)`, + orElse: () => + Effect.andThen( + sql`CREATE INDEX IF NOT EXISTS ${eventIndex} ON ${entryTableSql} (event, primary_key, timestamp)`, + sql`CREATE INDEX IF NOT EXISTS ${timestampIndex} ON ${entryTableSql} (timestamp)` + ) + }) + + // Number existing entries in creation order. `WHERE true` keeps SQLite + // from parsing ON CONFLICT as a join constraint. + yield* sql` + INSERT INTO ${sequenceTableSql} (entry_id) + SELECT id FROM ${entryTableSql} WHERE true ORDER BY timestamp ASC, id ASC + ON CONFLICT DO NOTHING + ` + }) + + // Earlier releases kept a remotes row per entry per remote. An entry known + // to several remotes keeps one of them as its source, and a remote without + // a cursor resumes after its highest downloaded sequence. + const importRemotes = Effect.gen(function*() { + yield* sql` + INSERT INTO ${sourcesTableSql} (entry_id, remote_id) + SELECT entry_id, remote_id FROM ${remotesTableSql} WHERE true + ON CONFLICT DO NOTHING + ` + yield* sql` + INSERT INTO ${remoteCursorsTableSql} (remote_id, next_sequence) + SELECT remote_id, MAX(sequence) + 1 FROM ${remotesTableSql} GROUP BY remote_id + ON CONFLICT DO NOTHING + ` + yield* sql`DROP TABLE ${remotesTableSql}` + }) - yield* sql.onDialectOrElse({ - pg: () => - sql` - CREATE TABLE IF NOT EXISTS ${remotesTableSql} ( - remote_id BYTEA NOT NULL, - entry_id BYTEA NOT NULL, - sequence INT NOT NULL, - PRIMARY KEY (remote_id, entry_id) - )`, - mysql: () => - sql` - CREATE TABLE IF NOT EXISTS ${remotesTableSql} ( - remote_id BINARY(16) NOT NULL, - entry_id BINARY(16) NOT NULL, - sequence INT NOT NULL, - PRIMARY KEY (remote_id, entry_id) - )`, - mssql: () => - sql` - CREATE TABLE IF NOT EXISTS ${remotesTableSql} ( - remote_id UNIQUEIDENTIFIER NOT NULL, - entry_id UNIQUEIDENTIFIER NOT NULL, - sequence INT NOT NULL, - PRIMARY KEY (remote_id, entry_id) - )`, - orElse: () => - sql` - CREATE TABLE IF NOT EXISTS ${remotesTableSql} ( - remote_id BLOB NOT NULL, - entry_id BLOB NOT NULL, - sequence INT NOT NULL, - PRIMARY KEY (remote_id, entry_id) - )` + // Both steps run in one transaction, so a complete schema has the sources + // table and no remotes table. Checking that first lets complete databases + // open read-only; PostgreSQL also locks and rechecks so concurrent opens + // migrate once. + const schemaState = Effect.all({ sources: tableExists(sourcesTable), remotes: tableExists(remotesTable) }) + const isComplete = (state: { readonly sources: boolean; readonly remotes: boolean }) => + state.sources && !state.remotes + yield* Effect.gen(function*() { + if (isComplete(yield* schemaState)) return + yield* Effect.gen(function*() { + yield* sql.onDialectOrElse({ + pg: () => sql`SELECT pg_advisory_xact_lock(hashtext(${quoted(entryTable)}))`, + orElse: () => Effect.void + }) + const state = yield* schemaState + if (!state.sources) yield* createSchema + if (state.remotes) yield* importRemotes + }).pipe(sql.withTransaction) }).pipe(withTracerDisabled) const decodeEntryRows = Schema.decodeUnknownEffect(EntryRowArray) const decodeEntryIdRows = Schema.decodeUnknownEffect(EntryIdRowArray) const toEntries = (rows: ReadonlyArray): ReadonlyArray => rows.map(toEntry) + // Sort timestamp ties by id bytes consistently across dialects. + const toOrderedEntries = (rows: ReadonlyArray): ReadonlyArray => + rows.map(toEntry).sort(EventJournal.Entry.Order) - const insertEntry = SqlSchema.void({ - Request: EntryRow, - execute: (entry) => sql`INSERT INTO ${entryTableSql} ${sql.insert(entry)} ON CONFLICT DO NOTHING` - }) const insertEntries = SqlSchema.void({ Request: EntryRowArray, execute: (entries) => sql`INSERT INTO ${entryTableSql} ${sql.insert(entries)} ON CONFLICT DO NOTHING` }) - const insertRemotes = SqlSchema.void({ - Request: RemoteRowArray, - execute: (entries) => sql`INSERT INTO ${remotesTableSql} ${sql.insert(entries)} ON CONFLICT DO NOTHING` + const insertSources = (entries: ReadonlyArray, remoteId: EventJournal.RemoteId) => + sql`INSERT INTO ${sourcesTableSql} ${ + sql.insert(entries.map((entry) => ({ entry_id: entry.id, remote_id: remoteId }))) + } ON CONFLICT DO NOTHING` + // Serialize writers so checkpoints cannot skip out-of-order commits. + // Lock before callbacks in both paths to avoid projection-row deadlocks. + const lockWriters = sql.onDialectOrElse({ + pg: () => Effect.asVoid(sql`LOCK TABLE ${sequenceTableSql} IN SHARE ROW EXCLUSIVE MODE`), + orElse: () => Effect.void }) + const insertSequence = (entries: ReadonlyArray) => + sql`INSERT INTO ${sequenceTableSql} ${ + sql.insert(entries.map((entry) => ({ entry_id: entry.id }))) + } ON CONFLICT DO NOTHING` + + const decodeCursorRows = Schema.decodeUnknownEffect(CursorRowArray) + + const readCursor = (remoteId: EventJournal.RemoteId) => + sql`SELECT next_sequence FROM ${remoteCursorsTableSql} WHERE remote_id = ${remoteId}`.pipe( + Effect.flatMap(decodeCursorRows), + Effect.map((rows) => rows.length > 0 ? rows[0].next_sequence : 0) + ) + + // Cursors only move forward. + const writeCursor = (remoteId: EventJournal.RemoteId, next: number) => + sql` + INSERT INTO ${remoteCursorsTableSql} ${sql.insert({ remote_id: remoteId, next_sequence: next })} + ON CONFLICT (remote_id) DO UPDATE SET next_sequence = excluded.next_sequence + WHERE ${remoteCursorsTableSql}.next_sequence < excluded.next_sequence + ` const pubsub = yield* PubSub.unbounded() + const clock = yield* Clock.Clock const writeFromRemote = Effect.fnUntraced(function*(options: WriteFromRemoteOptions): Effect.fn.Return< { @@ -152,11 +358,8 @@ export const make = (options?: { EventJournal.EventJournalError | Schema.SchemaError | SqlError.SqlError > { const entries = options.entries.map((remoteEntry) => remoteEntry.entry) - const remoteRows = options.entries.map((remoteEntry) => ({ - remote_id: options.remoteId, - entry_id: remoteEntry.entry.id, - sequence: remoteEntry.remoteSequence - })) + + yield* lockWriters const existingIds = new Set() if (entries.length > 0) { @@ -176,14 +379,14 @@ export const make = (options?: { ) ) } - if (entries.length > 0) { - yield* insertEntries(entries.map(toEntryRow)) - } - if (remoteRows.length > 0) { - yield* insertRemotes(remoteRows) - } - - const uncommitted = options.entries.filter((entry) => !existingIds.has(entry.entry.idString)) + // Overlapping batches can repeat an entry; the first copy wins. + const incomingIds = new Set() + const uncommitted = options.entries.filter((entry) => { + const idString = entry.entry.idString + if (existingIds.has(idString) || incomingIds.has(idString)) return false + incomingIds.add(idString) + return true + }) const duplicateEntries = options.entries .filter((entry) => existingIds.has(entry.entry.idString)) .map((entry) => entry.entry) @@ -191,56 +394,111 @@ export const make = (options?: { ? yield* options.compact(uncommitted) : uncommitted.map((remoteEntry) => remoteEntry.entry) - for (const entry of compacted) { - const conflicts = yield* sql` - SELECT * - FROM ${entryTableSql} - WHERE event = ${entry.event} AND - primary_key = ${entry.primaryKey} AND - timestamp >= ${entry.createdAtMillis} - ORDER BY timestamp ASC - `.pipe( + if (Arr.isReadonlyArrayNonEmpty(compacted)) { + const index = new ConflictIndex(compacted) + const primaryKeys = Array.from(new Set(compacted.map((entry) => entry.primaryKey))) + // Match the hash index; exact predicates below rule out hash collisions. + const hashed = sql.onDialectOrElse({ + pg: () => [ + sql`md5(event) IN (${sql.csv(index.events.map((event) => sql`md5(${event})`))})`, + sql`md5(primary_key) IN (${sql.csv(primaryKeys.map((primaryKey) => sql`md5(${primaryKey})`))})` + ], + orElse: () => [] + }) + const conflicts = sql.and([ + ...hashed, + sql.in("event", index.events), + sql.in("primary_key", primaryKeys), + // A coarse filter: the index compares ids exactly. + sql`timestamp >= ${index.oldestMillis}` + ]) + const stored = yield* sql`SELECT * FROM ${entryTableSql} WHERE ${conflicts}`.pipe( Effect.flatMap(decodeEntryRows), Effect.map(toEntries) ) - yield* options.effect({ entry, conflicts }) + for (const entry of stored) { + index.add(entry) + } + yield* options.effect(index.toBatch()) + } + + // Keep the incoming batch out of conflict queries and persist only after + // compaction and every callback have succeeded. + if (uncommitted.length > 0) { + const imported = uncommitted.map((remoteEntry) => remoteEntry.entry) + yield* insertEntries(imported.map(toEntryRow)) + yield* insertSequence(imported) + yield* insertSources(imported, options.remoteId) + } + // The change stream is ordered, so every earlier sequence is now known. + if (options.entries.length > 0) { + yield* writeCursor(options.remoteId, EventJournal.nextSequenceAfter(options.entries)) } + yield* EventJournal.warnIfAhead(clock, uncommitted.map((remoteEntry) => remoteEntry.entry.id)) return { duplicateEntries } }) + // Other processes share the database but not the entry id sequence. Ids in + // the latest millisecond are few, and comparing them here avoids ordering + // id bytes in SQL. + const latestEntryIds = sql` + SELECT id FROM ${entryTableSql} WHERE timestamp = (SELECT MAX(timestamp) FROM ${entryTableSql}) + `.pipe( + Effect.flatMap(decodeEntryIdRows), + Effect.map((rows) => rows.map((row) => row.id)) + ) + + const writeMany: EventJournal.EventJournal["Service"]["writeMany"] = ({ effect, entries }) => + Effect.uninterruptibleMask((restore) => + Effect.gen(function*() { + yield* lockWriters + // Rows visible here include an enclosing transaction's uncommitted + // ones, which local ids must also sort after. + const committed = yield* EventJournal.makeEntries(entries, yield* latestEntryIds) + const result = yield* restore(effect(committed)).pipe( + Effect.mapError((error) => new CallbackFailure(error)) + ) + yield* insertEntries(committed.map(toEntryRow)) + yield* insertSequence(committed) + return { result, committed } + }).pipe( + sql.withTransaction, + Effect.mapError((error) => + error instanceof CallbackFailure + ? error.error + : error instanceof EventJournal.EventJournalError + ? error + : new EventJournal.EventJournalError({ cause: error, method: "writeMany" }) + ), + Effect.flatMap(({ committed, result }) => Effect.as(PubSub.publishAll(pubsub, committed), result)) + ) + ).pipe(withTracerDisabled) + return EventJournal.EventJournal.of({ entries: sql`SELECT * FROM ${entryTableSql} ORDER BY timestamp ASC`.pipe( withTracerDisabled, Effect.flatMap(decodeEntryRows), - Effect.map(toEntries), + Effect.map(toOrderedEntries), Effect.mapError((cause) => new EventJournal.EventJournalError({ cause, method: "entries" })) ), - write: Effect.fnUntraced( - function*({ effect, event, payload, primaryKey }) { - const entry = new EventJournal.Entry({ - id: EventJournal.makeEntryIdUnsafe(), - event, - primaryKey, - payload - }, { disableChecks: true }) - return yield* Effect.uninterruptibleMask((restore) => - restore(effect(entry)).pipe( - Effect.tap( - insertEntry(toEntryRow(entry)).pipe( - Effect.mapError((cause) => new EventJournal.EventJournalError({ cause, method: "write" })) - ) - ), - Effect.tap(PubSub.publish(pubsub, entry)) - ) - ) - }, - withTracerDisabled - ), + entriesById: (ids) => + ids.length === 0 ? Effect.succeed([]) : sql`SELECT * FROM ${entryTableSql} WHERE ${sql.in("id", ids)}`.pipe( + withTracerDisabled, + Effect.flatMap(decodeEntryRows), + Effect.map((rows) => { + const byId = new Map(toEntries(rows).map((entry) => [entry.idString, entry])) + return ids.map((id) => byId.get(Uuid.stringify(id))) + }), + Effect.mapError((cause) => new EventJournal.EventJournalError({ cause, method: "entriesById" })) + ), + write: ({ effect, ...entry }) => writeMany({ entries: [entry], effect: (entries) => effect(entries[0]) }), + writeMany, writeFromRemote: (options) => writeFromRemote(options).pipe( + sql.withTransaction, withTracerDisabled, Effect.catchIf( (e) => e._tag !== "EventJournalError", @@ -249,36 +507,66 @@ export const make = (options?: { ), withRemoteUncommited: Effect.fnUntraced( function*(remoteId, f) { + const mapError = Effect.mapError((cause: unknown) => + new EventJournal.EventJournalError({ cause, method: "withRemoteUncommited" }) + ) + // Capture an upper bound so entries committed during upload remain pending. + const [{ checkpoint, latest }] = yield* sql` + SELECT + COALESCE((SELECT sequence FROM ${checkpointsTableSql} WHERE remote_id = ${remoteId}), 0) AS checkpoint, + COALESCE((SELECT MAX(sequence) FROM ${sequenceTableSql}), 0) AS latest + `.pipe(Effect.flatMap(decodeUploadBounds), mapError) + if (latest <= checkpoint) return yield* Effect.succeedNone const entries = yield* sql` - SELECT * - FROM ${entryTableSql} - WHERE id NOT IN (SELECT entry_id FROM ${remotesTableSql} WHERE remote_id = ${remoteId}) - ORDER BY timestamp ASC + SELECT e.* + FROM ${sequenceTableSql} s + JOIN ${entryTableSql} e ON e.id = s.entry_id + WHERE s.sequence > ${checkpoint} AND + s.sequence <= ${latest} AND + NOT EXISTS ( + SELECT 1 FROM ${sourcesTableSql} src WHERE src.entry_id = s.entry_id AND src.remote_id = ${remoteId} + ) `.pipe( Effect.flatMap(decodeEntryRows), - Effect.map(toEntries), - Effect.mapError((cause) => new EventJournal.EventJournalError({ cause, method: "withRemoteUncommited" })) + Effect.map(toOrderedEntries), + mapError + ) + // Uploads can complete out of order; the checkpoint only moves forward. + const saveCheckpoint = sql` + INSERT INTO ${checkpointsTableSql} ${sql.insert({ remote_id: remoteId, sequence: latest })} + ON CONFLICT (remote_id) DO UPDATE SET sequence = excluded.sequence + WHERE ${checkpointsTableSql}.sequence < excluded.sequence + `.pipe(mapError) + return yield* Effect.uninterruptibleMask((restore) => + restore(Arr.isReadonlyArrayNonEmpty(entries) ? Effect.asSome(f(entries)) : Effect.succeedNone).pipe( + Effect.tap(saveCheckpoint) + ) ) - if (!Arr.isReadonlyArrayNonEmpty(entries)) return yield* Effect.succeedNone - return yield* Effect.asSome(f(entries)) }, withTracerDisabled ), + writeRemoteAcks: Effect.fnUntraced( + function*({ acks, firstSequence, remoteId }) { + if (acks.length === 0) return + const next = yield* readCursor(remoteId) + yield* writeCursor(remoteId, EventJournal.advanceSequence(next, acks, firstSequence)) + }, + (_) => sql.withTransaction(_), + (_) => withTracerDisabled(_), + Effect.mapError((cause) => new EventJournal.EventJournalError({ cause, method: "writeRemoteAcks" })) + ), nextRemoteSequence: (remoteId) => - sql<{ max: number | null }>`SELECT MAX(sequence) AS max FROM ${remotesTableSql} WHERE remote_id = ${remoteId}` - .pipe( - Effect.map((rows) => { - const value = rows[0]?.max - if (value === null || value === undefined) return 0 - return Number(value) + 1 - }), - withTracerDisabled, - Effect.mapError((cause) => new EventJournal.EventJournalError({ cause, method: "nextRemoteSequence" })) - ), + readCursor(remoteId).pipe( + withTracerDisabled, + Effect.mapError((cause) => new EventJournal.EventJournalError({ cause, method: "nextRemoteSequence" })) + ), changes: PubSub.subscribe(pubsub), destroy: Effect.gen(function*() { yield* sql`DROP TABLE ${entryTableSql}` - yield* sql`DROP TABLE ${remotesTableSql}` + yield* sql`DROP TABLE ${sequenceTableSql}` + yield* sql`DROP TABLE ${sourcesTableSql}` + yield* sql`DROP TABLE ${checkpointsTableSql}` + yield* sql`DROP TABLE ${remoteCursorsTableSql}` }).pipe( withTracerDisabled, Effect.mapError((cause) => new EventJournal.EventJournalError({ cause, method: "destroy" })) @@ -293,25 +581,11 @@ export const make = (options?: { }) /** - * Provides `EventJournal` using the SQL-backed implementation created by - * `make`. - * - * **When to use** - * - * Use when composing a Layer graph that should provide a persistent SQL-backed - * `EventJournal` from an existing `SqlClient` service. - * - * **Details** - * - * The layer delegates to `make(options)`, so the same optional `entryTable` and - * `remotesTable` settings are used and construction requires `SqlClient` and - * may fail with `SqlError`. + * Provides a SQL-backed `EventJournal` from an existing `SqlClient` service. * * **Gotchas** * - * Layer construction performs the same minimal `CREATE TABLE IF NOT EXISTS` - * setup as `make`; manage indexes and schema migrations outside this layer when - * your SQL schema needs more than the built-in tables. + * Creating or migrating the schema requires a writable connection. * * @see {@link make} for constructing the SQL-backed service directly * @see {@link EventJournal.layerMemory} for an in-memory `EventJournal` layer @@ -324,6 +598,7 @@ export const make = (options?: { export const layer = (options?: { readonly entryTable?: string readonly remotesTable?: string + readonly remoteCursorsTable?: string }): Layer.Layer => Layer.effect(EventJournal.EventJournal)(make(options)) @@ -366,6 +641,9 @@ const EntryRow = Schema.Struct({ }) const EntryRowArray = Schema.Array(EntryRow) +const decodeUploadBounds = Schema.decodeUnknownEffect(Schema.Tuple([ + Schema.Struct({ checkpoint: SqlNatural, latest: SqlNatural }) +])) const EntryIdRowArray = Schema.Array(Schema.Struct({ id: SqlEntryId })) type EntryRow = Schema.Schema.Type @@ -386,12 +664,14 @@ const toEntryRow = (entry: EventJournal.Entry): EntryRow => ({ timestamp: EventJournal.entryIdMillis(entry.id) }) -const RemoteRow = Schema.Struct({ - remote_id: EventJournal.RemoteId, - entry_id: EventJournal.EntryId, - sequence: Schema.Natural -}) - -const RemoteRowArray = Schema.Array(RemoteRow) +const CursorRowArray = Schema.Array(Schema.Struct({ next_sequence: SqlNatural })) const withTracerDisabled = Effect.withTracerEnabled(false) + +// Preserve callback errors without wrapping them as journal SQL errors. +class CallbackFailure { + readonly error: E + constructor(error: E) { + this.error = error + } +} diff --git a/repos/effect/packages/effect/src/eventlog/SqlEventLogServerEncrypted.ts b/repos/effect/packages/effect/src/eventlog/SqlEventLogServerEncrypted.ts index a686f4b788..6efa588d17 100644 --- a/repos/effect/packages/effect/src/eventlog/SqlEventLogServerEncrypted.ts +++ b/repos/effect/packages/effect/src/eventlog/SqlEventLogServerEncrypted.ts @@ -4,24 +4,29 @@ * This module provides the durable `Storage` implementation used by * `EventLogServerEncrypted` when entries should be stored without exposing * plaintext event data to the database. It persists the server remote id, - * session authentication bindings, and encrypted entry tables, assigns stable + * session authentication bindings, and encrypted batch tables, assigns stable * sequence numbers, and streams changes. Clients remain responsible for * encrypting writes and decrypting reads. * * @stability unstable * @since 4.0.0 */ +import * as Arr from "../Array.ts" +import type { NonEmptyArray } from "../Array.ts" import * as Effect from "../Effect.ts" import * as Layer from "../Layer.ts" import * as PubSub from "../PubSub.ts" import * as RcMap from "../RcMap.ts" import * as Schema from "../Schema.ts" import type * as Scope from "../Scope.ts" +import * as Semaphore from "../Semaphore.ts" import * as SqlClient from "../sql/SqlClient.ts" import type * as SqlError from "../sql/SqlError.ts" +import type * as Statement from "../sql/Statement.ts" import * as Stream from "../Stream.ts" -import { EntryId, makeRemoteIdUnsafe, type RemoteId } from "./EventJournal.ts" +import { type EntryId, makeRemoteIdUnsafe, type RemoteId } from "./EventJournal.ts" import * as EventLogEncryption from "./EventLogEncryption.ts" +import type { WriteAck } from "./EventLogMessage.ts" import * as EventLogServerEncrypted from "./EventLogServerEncrypted.ts" /** @@ -30,8 +35,16 @@ import * as EventLogServerEncrypted from "./EventLogServerEncrypted.ts" * **Details** * * It persists the server remote id, session authentication bindings, and encrypted - * entries in dialect-specific tables, creating per-identity/store entry tables as - * needed. + * batches in dialect-specific tables, creating per-identity/store batch and entry + * id tables as needed. + * + * **Gotchas** + * + * Storage does not retry sequence conflicts between competing instances. + * `EventLog.runRemote` retries failed writes; direct `EventLogRemote.write` + * callers must retry failures other than Forbidden themselves. Live notifications + * are local to each storage instance; clients on other instances must reconnect + * to read those writes. * * @stability unstable * @category constructors @@ -40,7 +53,6 @@ import * as EventLogServerEncrypted from "./EventLogServerEncrypted.ts" export const makeStorage = (options?: { readonly entryTablePrefix?: string readonly remoteIdTable?: string - readonly insertBatchSize?: number }): Effect.Effect< EventLogServerEncrypted.Storage["Service"], SqlError.SqlError, @@ -53,7 +65,6 @@ export const makeStorage = (options?: { const tablePrefix = options?.entryTablePrefix ?? "effect_events" const remoteIdTable = options?.remoteIdTable ?? "effect_remote_id" const sessionAuthBindingsTable = `${tablePrefix}_session_auth_bindings` - const insertBatchSize = options?.insertBatchSize ?? 200 const remoteIdTableSql = sql(remoteIdTable) const sessionAuthBindingsTableSql = sql(sessionAuthBindingsTable) @@ -71,7 +82,8 @@ export const makeStorage = (options?: { )`, mssql: () => sql` - CREATE TABLE IF NOT EXISTS ${remoteIdTableSql} ( + IF OBJECT_ID(${remoteIdTable}, N'U') IS NULL + CREATE TABLE ${remoteIdTableSql} ( remote_id VARBINARY(16) PRIMARY KEY )`, orElse: () => @@ -96,7 +108,8 @@ export const makeStorage = (options?: { )`, mssql: () => sql` - CREATE TABLE IF NOT EXISTS ${sessionAuthBindingsTableSql} ( + IF OBJECT_ID(${sessionAuthBindingsTable}, N'U') IS NULL + CREATE TABLE ${sessionAuthBindingsTableSql} ( public_key NVARCHAR(191) PRIMARY KEY, signing_public_key VARBINARY(32) NOT NULL )`, @@ -126,51 +139,115 @@ export const makeStorage = (options?: { const scopeHash = (yield* encryptions.sha256String(new TextEncoder().encode(scopeKey))).slice(0, 16) const table = `${tablePrefix}_${scopeHash}` const tableSql = sql(table) + const idsTableSql = sql(`${table}_ids`) + const indexSql = sql(`${table}_last_sequence`) yield* sql.onDialectOrElse({ pg: () => sql` CREATE TABLE IF NOT EXISTS ${tableSql} ( - sequence SERIAL PRIMARY KEY, + sequence INTEGER PRIMARY KEY, + last_sequence INTEGER NOT NULL, iv BYTEA NOT NULL, - entry_id BYTEA UNIQUE NOT NULL, - encrypted_entry BYTEA NOT NULL - )`, + entry_ids BYTEA NOT NULL, + encrypted_batch BYTEA NOT NULL + )`.pipe( + Effect.andThen(sql`CREATE INDEX IF NOT EXISTS ${indexSql} ON ${tableSql} (last_sequence)`), + Effect.andThen(sql` + CREATE TABLE IF NOT EXISTS ${idsTableSql} ( + entry_id BYTEA PRIMARY KEY, + sequence INTEGER NOT NULL + )`) + ), mysql: () => sql` CREATE TABLE IF NOT EXISTS ${tableSql} ( - sequence INT AUTO_INCREMENT PRIMARY KEY, + sequence INT PRIMARY KEY, + last_sequence INT NOT NULL, iv BINARY(12) NOT NULL, - entry_id BINARY(16) UNIQUE NOT NULL, - encrypted_entry BLOB NOT NULL - )`, + entry_ids LONGBLOB NOT NULL, + encrypted_batch LONGBLOB NOT NULL, + INDEX (last_sequence) + )`.pipe( + Effect.andThen(sql` + CREATE TABLE IF NOT EXISTS ${idsTableSql} ( + entry_id BINARY(16) PRIMARY KEY, + sequence INT NOT NULL + )`) + ), mssql: () => sql` - CREATE TABLE IF NOT EXISTS ${tableSql} ( - sequence INT IDENTITY(1,1) PRIMARY KEY, + IF OBJECT_ID(${table}, N'U') IS NULL + CREATE TABLE ${tableSql} ( + sequence INT PRIMARY KEY, + last_sequence INT NOT NULL, iv VARBINARY(12) NOT NULL, - entry_id VARBINARY(16) UNIQUE NOT NULL, - encrypted_entry VARBINARY(MAX) NOT NULL - )`, + entry_ids VARBINARY(MAX) NOT NULL, + encrypted_batch VARBINARY(MAX) NOT NULL, + INDEX ${indexSql} NONCLUSTERED (last_sequence) + )`.pipe( + Effect.andThen(sql` + IF OBJECT_ID(${`${table}_ids`}, N'U') IS NULL + CREATE TABLE ${idsTableSql} ( + entry_id VARBINARY(16) PRIMARY KEY, + sequence INT NOT NULL + )`) + ), orElse: () => sql` CREATE TABLE IF NOT EXISTS ${tableSql} ( - sequence INTEGER PRIMARY KEY AUTOINCREMENT, + sequence INTEGER PRIMARY KEY, + last_sequence INTEGER NOT NULL, iv BLOB NOT NULL, - entry_id BLOB UNIQUE NOT NULL, - encrypted_entry BLOB NOT NULL - )` + entry_ids BLOB NOT NULL, + encrypted_batch BLOB NOT NULL + )`.pipe( + Effect.andThen(sql`CREATE INDEX IF NOT EXISTS ${indexSql} ON ${tableSql} (last_sequence)`), + Effect.andThen(sql` + CREATE TABLE IF NOT EXISTS ${idsTableSql} ( + entry_id BLOB PRIMARY KEY, + sequence INTEGER NOT NULL + )`) + ) }) const pubsub = yield* Effect.acquireRelease( - PubSub.unbounded(), + PubSub.unbounded(), PubSub.shutdown ) - return { pubsub, table } as const + // Serializes writes within this storage instance. + const semaphore = yield* Semaphore.make(1) + return { pubsub, semaphore, tableSql, idsTableSql } as const }, withTracerDisabled), idleTimeToLive: "5 minutes" }) + // Returns how many of the ids were not stored yet. Stored ids keep their + // first sequence. + const insertNewIds = (idsTableSql: Statement.Identifier, rows: ReadonlyArray) => + sql.onDialectOrElse({ + mysql: () => + sql`INSERT IGNORE INTO ${idsTableSql} ${sql.insert(rows)}`.raw.pipe( + Effect.map((result) => (result as { affectedRows: number }).affectedRows) + ), + mssql: () => + sql` + MERGE ${idsTableSql} WITH (HOLDLOCK) AS target + USING (VALUES ${ + sql.join(",", false)(rows.map((row) => sql`(${row.entry_id}, ${row.sequence})`)) + }) AS source (entry_id, sequence) + ON target.entry_id = source.entry_id + WHEN NOT MATCHED THEN + INSERT (entry_id, sequence) VALUES (source.entry_id, source.sequence) + OUTPUT inserted.entry_id; + `.pipe(Effect.map((inserted) => inserted.length)), + orElse: () => + sql` + INSERT INTO ${idsTableSql} ${sql.insert(rows)} + ON CONFLICT DO NOTHING RETURNING entry_id + `.pipe(Effect.map((inserted) => inserted.length)) + }) + const getSessionAuthBinding = (publicKey: string) => sql` SELECT public_key, signing_public_key @@ -187,6 +264,8 @@ export const makeStorage = (options?: { return EventLogServerEncrypted.Storage.of({ getId: Effect.succeed(remoteId), + // Auto-increment sequences start at 1. + firstSequence: 1, getOrCreateSessionAuthBinding: Effect.fnUntraced( function*(publicKey, signingPublicKey) { const existing = yield* getSessionAuthBinding(publicKey) @@ -205,47 +284,67 @@ export const makeStorage = (options?: { Effect.orDie ), write: Effect.fnUntraced( - function*(publicKey, storeId, entries) { - if (entries.length === 0) return [] + function*(publicKey, storeId, batches) { + if (batches.length === 0) return [] const scopeKey = makeEncryptedScopeKey(publicKey, storeId) - const { pubsub, table } = yield* RcMap.get(resources, scopeKey) - const forInsert: Array<{ - readonly ids: Array - readonly entries: Array<{ - iv: Uint8Array - entry_id: Uint8Array - encrypted_entry: Uint8Array - }> - }> = [{ ids: [], entries: [] }] - let currentBatch = forInsert[0] - for (const entry of entries) { - currentBatch.ids.push(entry.entryId) - currentBatch.entries.push({ - iv: entry.iv, - entry_id: entry.entryId, - encrypted_entry: entry.encryptedEntry - }) - if (currentBatch.entries.length === insertBatchSize) { - currentBatch = { ids: [], entries: [] } - forInsert.push(currentBatch) + const { idsTableSql, pubsub, semaphore, tableSql } = yield* RcMap.get(resources, scopeKey) + + const writeBatches = Effect.gen(function*() { + const stored: Array = [] + // Read before inserting ids, which record the sequence they are stored at. + const [last] = yield* sql<{ last_sequence: number | string | null }>` + SELECT MAX(last_sequence) AS last_sequence FROM ${tableSql} + ` + let nextSequence = Number(last?.last_sequence ?? 0) + 1 + for (const batch of batches) { + // Inserting the ids decides whether the batch is new. SQL Server's + // MERGE also requires unique source ids. + const ids = new Map() + for (let i = 0; i < batch.entryIds.length; i++) { + const entry_id = batch.entryIds[i] + const key = entry_id.join(",") + if (!ids.has(key)) ids.set(key, { entry_id, sequence: nextSequence + i }) + } + let inserted = 0 + for (const chunk of Arr.chunksOf(ids.values(), 200)) { + inserted += yield* insertNewIds(idsTableSql, chunk) + } + if (inserted === 0) continue + const row: BatchRow = { + sequence: nextSequence, + last_sequence: nextSequence + batch.entryIds.length - 1, + iv: batch.iv, + entry_ids: concatEntryIds(batch.entryIds), + encrypted_batch: batch.encryptedBatch + } + yield* sql`INSERT INTO ${tableSql} ${sql.insert(row)}` + stored.push({ sequence: nextSequence, ...batch }) + nextSequence += batch.entryIds.length } - } + return stored + }) - const allEntries: Array = [] - for (const batch of forInsert) { - if (batch.entries.length === 0) continue - const encryptedEntries = yield* sql` - INSERT INTO ${sql(table)} ${sql.insert(batch.entries)} ON CONFLICT DO NOTHING - `.pipe( - Effect.andThen( - sql`SELECT * FROM ${sql(table)} WHERE ${sql.in("entry_id", batch.ids)} ORDER BY sequence ASC` - ), - Effect.flatMap(decodeEntries) - ) - yield* PubSub.publishAll(pubsub, encryptedEntries) - allEntries.push(...encryptedEntries) + return yield* semaphore.withPermits(1)(Effect.gen(function*() { + const stored = yield* sql.withTransaction(writeBatches) + yield* PubSub.publishAll(pubsub, stored) + return stored + })) + }, + Effect.orDie, + Effect.scoped, + withTracerDisabled + ), + entrySequences: Effect.fnUntraced( + function*(publicKey, storeId, entryIds) { + const { idsTableSql } = yield* RcMap.get(resources, makeEncryptedScopeKey(publicKey, storeId)) + const acks: Array = [] + for (const chunk of Arr.chunksOf(entryIds, 200)) { + const rows = yield* sql` + SELECT entry_id, sequence FROM ${idsTableSql} WHERE ${sql.in("entry_id", chunk)} + `.pipe(Effect.flatMap(decodeIdRows)) + for (const row of rows) acks.push({ entryId: row.entry_id as EntryId, sequence: row.sequence }) } - return allEntries + return acks }, Effect.orDie, Effect.scoped, @@ -254,12 +353,14 @@ export const makeStorage = (options?: { changes: Effect.fnUntraced( function*(publicKey, storeId, startSequence) { const scopeKey = makeEncryptedScopeKey(publicKey, storeId) - const { pubsub, table } = yield* RcMap.get(resources, scopeKey) + const { pubsub, tableSql } = yield* RcMap.get(resources, scopeKey) const subscription = yield* PubSub.subscribe(pubsub) const initial = yield* sql` - SELECT * FROM ${sql(table)} WHERE sequence >= ${startSequence} ORDER BY sequence ASC + SELECT sequence, iv, entry_ids, encrypted_batch FROM ${tableSql} + WHERE last_sequence >= ${startSequence} + ORDER BY sequence ASC `.pipe( - Effect.flatMap(decodeEntries) + Effect.flatMap(decodeBatches) ) return Stream.fromArray(initial).pipe(Stream.concat(Stream.fromSubscription(subscription))) }, @@ -270,14 +371,38 @@ export const makeStorage = (options?: { }) }).pipe(withTracerDisabled) -const EncryptedRemoteEntrySql = Schema.Struct({ +type IdRow = { + readonly entry_id: Uint8Array + readonly sequence: number +} + +const decodeIdRows = Schema.decodeUnknownEffect(Schema.Array(Schema.Struct({ + entry_id: Schema.Uint8Array, + sequence: Schema.Number +}))) + +type BatchRow = { + readonly sequence: number + readonly last_sequence: number + readonly iv: Uint8Array + readonly entry_ids: Uint8Array + readonly encrypted_batch: Uint8Array +} + +const concatEntryIds = (entryIds: ReadonlyArray): Uint8Array => { + const data = new Uint8Array(entryIds.length * 16) + for (let i = 0; i < entryIds.length; i++) data.set(entryIds[i], i * 16) + return data +} + +const EncryptedRemoteBatchSql = Schema.Struct({ sequence: Schema.Natural, iv: Schema.Uint8Array, - entry_id: EntryId, - encrypted_entry: Schema.Uint8Array + entry_ids: Schema.Uint8Array, + encrypted_batch: Schema.Uint8Array }) -type EncryptedRemoteEntrySql = Schema.Schema.Type +type EncryptedRemoteBatchSql = Schema.Schema.Type const SessionAuthBindingSql = Schema.Struct({ public_key: Schema.String, @@ -286,20 +411,26 @@ const SessionAuthBindingSql = Schema.Struct({ type SessionAuthBindingSql = Schema.Schema.Type -const decodeEntryRows = Schema.decodeUnknownEffect(Schema.Array(EncryptedRemoteEntrySql)) +const decodeBatchRows = Schema.decodeUnknownEffect(Schema.Array(EncryptedRemoteBatchSql)) const decodeSessionAuthBindingRows = Schema.decodeUnknownEffect(Schema.Array(SessionAuthBindingSql)) -const toEncryptedRemoteEntry = (row: EncryptedRemoteEntrySql): EventLogEncryption.EncryptedRemoteEntry => ({ - sequence: row.sequence, - iv: row.iv as Uint8Array, - entryId: row.entry_id, - encryptedEntry: row.encrypted_entry as Uint8Array -}) +const toEncryptedRemoteBatch = (row: EncryptedRemoteBatchSql): EventLogEncryption.EncryptedRemoteBatch => { + const entryIds: Array = [] + for (let offset = 0; offset < row.entry_ids.byteLength; offset += 16) { + entryIds.push(row.entry_ids.subarray(offset, offset + 16) as EntryId) + } + return { + sequence: row.sequence, + entryIds: entryIds as NonEmptyArray, + iv: row.iv as Uint8Array, + encryptedBatch: row.encrypted_batch as Uint8Array + } +} -const decodeEntries = ( +const decodeBatches = ( rows: unknown -): Effect.Effect, Schema.SchemaError> => - decodeEntryRows(rows).pipe(Effect.map((entries) => entries.map(toEncryptedRemoteEntry))) +): Effect.Effect, Schema.SchemaError> => + decodeBatchRows(rows).pipe(Effect.map((batches) => batches.map(toEncryptedRemoteBatch))) const decodeSessionAuthBindings = ( rows: unknown @@ -315,7 +446,6 @@ const decodeSessionAuthBindings = ( export const layerStorage = (options?: { readonly entryTablePrefix?: string readonly remoteIdTable?: string - readonly insertBatchSize?: number }): Layer.Layer< EventLogServerEncrypted.Storage, SqlError.SqlError, @@ -333,7 +463,6 @@ export const layerStorage = (options?: { export const layerStorageSubtle = (options?: { readonly entryTablePrefix?: string readonly remoteIdTable?: string - readonly insertBatchSize?: number }): Layer.Layer => layerStorage(options).pipe( Layer.provide(EventLogEncryption.layerSubtle) diff --git a/repos/effect/packages/effect/src/eventlog/SqlEventLogServerUnencrypted.ts b/repos/effect/packages/effect/src/eventlog/SqlEventLogServerUnencrypted.ts index 35a59f9d74..10047a53e3 100644 --- a/repos/effect/packages/effect/src/eventlog/SqlEventLogServerUnencrypted.ts +++ b/repos/effect/packages/effect/src/eventlog/SqlEventLogServerUnencrypted.ts @@ -11,7 +11,6 @@ * @since 4.0.0 */ import * as Arr from "../Array.ts" -import * as Context from "../Context.ts" import * as Effect from "../Effect.ts" import * as Layer from "../Layer.ts" import * as PubSub from "../PubSub.ts" @@ -26,8 +25,9 @@ import * as Stream from "../Stream.ts" import { Entry, EntryId, makeRemoteIdUnsafe, RemoteEntry, type RemoteId } from "./EventJournal.ts" import * as EventLogServerUnencrypted from "./EventLogServerUnencrypted.ts" -let storageIdCounter = 0 const withTracerDisabled = Effect.withTracerEnabled(false) +// Keeps each history lookup within dialect parameter limits. +const lookupBatchSize = 200 type StorageOptions = { readonly entryTablePrefix?: string @@ -42,7 +42,13 @@ type StorageOptions = { * * The implementation creates tables for the server remote id, store sequences, * entries, and session authentication bindings, then persists and streams - * plaintext remote entries. + * plaintext remote entries. Writes reach `changes` subscribers once the + * outermost SQL transaction commits, including caller-owned transactions, and are + * dropped if it rolls back. + * + * **Gotchas** + * + * Concurrent caller-owned transactions can publish writes out of sequence order. * * @stability unstable * @category constructors @@ -144,6 +150,40 @@ export const makeStorage = Effect.fnUntraced(function*(options?: StorageOptions) )` }) + // Serves the batch lookup in `entriesAfter`. Postgres indexes hashes to stay + // under its B-tree row limit, MySQL indexes prefixes, and MSSQL has no + // bounded index yet. + const eventKeyIndex = `${entriesTable}_event_key_idx` + yield* sql.onDialectOrElse({ + pg: () => + sql` + CREATE INDEX IF NOT EXISTS ${sql(eventKeyIndex)} + ON ${entriesTableSql} (store_id, hashtextextended(event, 0), hashtextextended(primary_key, 0), entry_id) + `, + mysql: () => { + const indexExists = sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM information_schema.statistics + WHERE table_schema = DATABASE() + AND table_name = ${entriesTable} + AND index_name = ${eventKeyIndex} + `.pipe( + Effect.map((rows) => Number(rows[0].count) > 0) + ) + return sql` + CREATE INDEX ${sql(eventKeyIndex)} + ON ${entriesTableSql} (store_id, event(191), primary_key(191), entry_id) + `.pipe( + Effect.catch((error) => Effect.flatMap(indexExists, (exists) => exists ? Effect.void : Effect.fail(error))) + ) + }, + mssql: () => Effect.void, + orElse: () => + sql` + CREATE INDEX IF NOT EXISTS ${sql(eventKeyIndex)} + ON ${entriesTableSql} (store_id, event, primary_key, entry_id) + ` + }) + yield* sql.onDialectOrElse({ pg: () => sql` @@ -237,49 +277,22 @@ export const makeStorage = Effect.fnUntraced(function*(options?: StorageOptions) }) const transactionSemaphore = yield* Semaphore.make(1) - type PendingWrites = Array]> - const pendingWrites = Context.Service( - `effect/eventlog/SqlEventLogServerUnencrypted/PendingWrites/${storageIdCounter++}` - ) - const publish = Effect.fnUntraced(function*([storeId, entries]: PendingWrites[number]) { - const pubsub = yield* RcMap.get(pubsubs, storeId) - return yield* PubSub.publishAll(pubsub, entries) - }, Effect.scoped) - // Merge nested buffers on success; publish only after the outer commit. - // Hold the outer permit through publication to prevent later commits from overtaking it. - const withTransaction = Effect.fnUntraced(function*( - effect: Effect.Effect - ): Effect.fn.Return> { - const parent = yield* Effect.serviceOption(pendingWrites) - // Allow body interruption, but mask commit through publication. - const transaction = Effect.uninterruptibleMask(Effect.fnUntraced(function*(restore) { - const writes: PendingWrites = [] - const result = yield* sql.withTransaction( - Effect.provideService(restore(effect), pendingWrites, writes) - ) - if (parent._tag === "Some") { - parent.value.push(...writes) - } else { - const stores = new Map>() - for (const [storeId, entries] of writes) { - const existing = stores.get(storeId) - if (existing === undefined) { - stores.set(storeId, [...entries]) - } else { - existing.push(...entries) - } - } - // Sibling buffers may merge out of sequence order. - for (const entries of stores.values()) { - entries.sort((a, b) => a.remoteSequence - b.remoteSequence) - } - yield* Effect.forEach(stores, publish, { discard: true }) - } - return result - })) - // Wait interruptibly; nested transactions reuse the outer permit. - return yield* parent._tag === "Some" ? transaction : transactionSemaphore.withPermit(transaction) - }) + const publish = (storeId: string, entries: ReadonlyArray) => + RcMap.get(pubsubs, storeId).pipe( + Effect.flatMap((pubsub) => PubSub.publishAll(pubsub, entries)), + Effect.asVoid, + Effect.scoped + ) + // Hold the permit through commit and publication to preserve commit order + // for storage-owned transactions. + const withTransaction = (effect: Effect.Effect): Effect.Effect => + Effect.flatMap( + Effect.serviceOption(sql.transactionService), + (transaction) => + transaction._tag === "Some" + ? sql.withTransaction(effect) + : transactionSemaphore.withPermit(sql.withTransaction(effect)) + ) const ensureStore = (storeId: string) => sql.onDialectOrElse({ @@ -362,6 +375,77 @@ export const makeStorage = Effect.fnUntraced(function*(options?: StorageOptions) Effect.flatMap(decodeRemoteEntries) ) + // Postgres matches on the indexed hashes, then on the values to rule out + // collisions. + const matchesKey = sql.onDialectOrElse({ + pg: () => (event: string, primaryKeys: ReadonlyArray) => + sql` + hashtextextended(event, 0) = hashtextextended(${event}, 0) AND event = ${event} + AND hashtextextended(primary_key, 0) IN (${ + sql.csv(primaryKeys.map((primaryKey) => sql`hashtextextended(${primaryKey}, 0)`)) + }) + AND ${sql.in("primary_key", primaryKeys)} + `, + orElse: () => (event: string, primaryKeys: ReadonlyArray) => + sql`event = ${event} AND ${sql.in("primary_key", primaryKeys)}` + }) + + // One SELECT per index lookup, joined with UNION ALL: MySQL scans the whole + // store for the equivalent OR. + const selectChunkHistory = (storeId: string, oldest: Entry, entries: ReadonlyArray) => { + const primaryKeysByEvent = new Map>() + for (const entry of entries) { + const primaryKeys = primaryKeysByEvent.get(entry.event) + if (primaryKeys === undefined) { + primaryKeysByEvent.set(entry.event, new Set([entry.primaryKey])) + } else { + primaryKeys.add(entry.primaryKey) + } + } + let statement = sql` + SELECT sequence, entry_id, event, primary_key, payload + FROM ${entriesTableSql} + WHERE store_id = ${storeId} AND ${sql.in("entry_id", entries.map((entry) => entry.id))} + ` + for (const [event, primaryKeys] of primaryKeysByEvent) { + statement = sql` + ${statement} + UNION ALL + SELECT sequence, entry_id, event, primary_key, payload + FROM ${entriesTableSql} + WHERE store_id = ${storeId} + AND ${matchesKey(event, Array.from(primaryKeys))} + AND entry_id >= ${oldest.id} + ` + } + return Effect.flatMap(statement, decodeRemoteEntries) + } + + // Chunks and the UNION ALL branches can return the same row more than once. + const selectBatchHistory = (storeId: string, oldest: Entry, batch: Arr.NonEmptyReadonlyArray) => + Effect.forEach( + Arr.chunksOf(batch, lookupBatchSize), + (chunk) => selectChunkHistory(storeId, oldest, chunk) + ).pipe( + Effect.map((chunks) => { + const bySequence = new Map() + for (const remoteEntry of chunks.flat()) { + bySequence.set(remoteEntry.remoteSequence, remoteEntry) + } + return Array.from(bySequence.values()).sort((a, b) => a.remoteSequence - b.remoteSequence) + }) + ) + + const selectEntriesFrom = (storeId: string, entry: Entry) => + sql` + SELECT sequence, entry_id, event, primary_key, payload + FROM ${entriesTableSql} + WHERE store_id = ${storeId} AND entry_id >= ${entry.id} + ORDER BY sequence ASC + `.pipe( + Effect.flatMap(decodeRemoteEntries) + ) + const getSessionAuthBinding = (publicKey: string) => sql` SELECT public_key, signing_public_key @@ -378,6 +462,7 @@ export const makeStorage = Effect.fnUntraced(function*(options?: StorageOptions) return EventLogServerUnencrypted.Storage.of({ getId: Effect.succeed(remoteId), + firstSequence: 1, getOrCreateSessionAuthBinding: Effect.fnUntraced( function*(publicKey, signingPublicKey) { const existing = yield* getSessionAuthBinding(publicKey) @@ -395,15 +480,8 @@ export const makeStorage = Effect.fnUntraced(function*(options?: StorageOptions) Effect.orDie, withTracerDisabled ), - entriesAfter: (storeId, entry) => - sql` - SELECT sequence, entry_id, event, primary_key, payload - FROM ${entriesTableSql} - WHERE store_id = ${storeId} AND entry_id >= ${entry.id} - ORDER BY sequence ASC - `.pipe( - Effect.flatMap(decodeRemoteEntries), - Effect.map(Arr.map((r) => r.entry)), + entriesAfter: (storeId, entry, batch) => + (batch === undefined ? selectEntriesFrom(storeId, entry) : selectBatchHistory(storeId, entry, batch)).pipe( Effect.orDie, withTracerDisabled ), @@ -454,8 +532,7 @@ export const makeStorage = Effect.fnUntraced(function*(options?: StorageOptions) const nextSequence = currentNextSequence + entries.length yield* setNextSequence(storeId, nextSequence) - const writes = yield* pendingWrites - writes.push([storeId, committed]) + yield* sql.afterCommit(publish(storeId, committed)) return committed }, @@ -467,11 +544,13 @@ export const makeStorage = Effect.fnUntraced(function*(options?: StorageOptions) function*({ storeId, startSequence, compactors }) { const pubsub = yield* RcMap.get(pubsubs, storeId) const subscription = yield* PubSub.subscribe(pubsub) - const backlog = yield* EventLogServerUnencrypted.compactBacklog({ - compactors, - remoteEntries: yield* selectEntriesAfter(storeId, startSequence) - }) - let watermark = backlog.length > 0 ? backlog[backlog.length - 1]!.remoteSequence : startSequence - 1 + const remoteEntries = yield* selectEntriesAfter(storeId, startSequence) + const backlog = yield* EventLogServerUnencrypted.compactBacklog({ compactors, remoteEntries }) + // Compaction can drop trailing entries, so skip live entries up to the + // last uncompacted one. + const watermark = remoteEntries.length > 0 + ? remoteEntries[remoteEntries.length - 1]!.remoteSequence + : startSequence - 1 return Stream.fromArray(backlog).pipe( Stream.concat( diff --git a/repos/effect/packages/effect/src/eventlog/index.ts b/repos/effect/packages/effect/src/eventlog/index.ts index ce45972844..bf4a1de7b1 100644 --- a/repos/effect/packages/effect/src/eventlog/index.ts +++ b/repos/effect/packages/effect/src/eventlog/index.ts @@ -71,6 +71,12 @@ export * as EventLogServerUnencrypted from "./EventLogServerUnencrypted.ts" */ export * as EventLogSessionAuth from "./EventLogSessionAuth.ts" +/** + * @stability unstable + * @since 4.0.6 + */ +export * as EventRow from "./EventRow.ts" + /** * @stability unstable * @since 4.0.0 diff --git a/repos/effect/packages/effect/src/eventlog/internal/conflicts.ts b/repos/effect/packages/effect/src/eventlog/internal/conflicts.ts new file mode 100644 index 0000000000..fd6baed6fd --- /dev/null +++ b/repos/effect/packages/effect/src/eventlog/internal/conflicts.ts @@ -0,0 +1,123 @@ +import type * as Arr from "../../Array.ts" +import * as Order from "../../Order.ts" +import type { Entry } from "../EventJournal.ts" + +/** Orders entry ids by their raw UUID bytes. */ +export const entryIdOrder = Order.make((a, b) => { + for (let i = 0; i < 16; i++) { + if (a[i] !== b[i]) { + return a[i] < b[i] ? -1 : 1 + } + } + return 0 +}) + +/** Orders entries by id. */ +const entryOrder = Order.make((a, b) => entryIdOrder(a.id, b.id)) + +/** + * The source entry a compaction output stands for, and the output's position + * among the outputs written for that source. + */ +interface Anchor { + readonly entry: Entry + readonly index: number +} + +// Set during client replay only, so it never leaves the process. +const anchors = new WeakMap() + +/** + * Records that `entry` was written by a compactor for `source`, as its + * `index`th output for that source. Untagged entries are their own anchor. + */ +export const setAnchor = (entry: Entry, source: Entry, index: number): void => { + anchors.set(entry, { entry: source, index }) +} + +const anchorEntry = (entry: Entry): Entry => anchors.get(entry)?.entry ?? entry +const anchorIndex = (entry: Entry): number => anchors.get(entry)?.index ?? 0 + +// Sorts by anchor id, then by write order among one anchor's outputs. +const compareAnchors = (a: Entry, b: Entry): number => + entryOrder(anchorEntry(a), anchorEntry(b)) || anchorIndex(a) - anchorIndex(b) + +/** + * Pairs a batch of incoming entries with the stored entries that conflict with + * them: same event and primary key, and an id strictly greater than the + * incoming entry's anchor. + * + * The batch is sorted by anchor id, then by write order, so it is applied in + * the same order on every replica and a compaction output takes its source's + * place and precedence. Add the stored entries that may conflict, then call + * `toBatch` to sort and filter them. + */ +export class ConflictIndex { + /** The incoming entries in anchor order. */ + readonly batch: ReadonlyArray + /** The anchor of the first incoming entry. Only stored entries with a greater id can conflict. */ + readonly oldest: Entry + /** Creation time of `oldest`, for coarse filters. */ + readonly oldestMillis: number + private readonly stored = new Map>() + + constructor(batch: Arr.NonEmptyReadonlyArray) { + this.batch = batch.slice().sort(compareAnchors) + this.oldest = anchorEntry(this.batch[0]) + this.oldestMillis = this.oldest.createdAtMillis + for (const entry of this.batch) { + let byKey = this.stored.get(entry.event) + if (byKey === undefined) { + byKey = new Map() + this.stored.set(entry.event, byKey) + } + if (!byKey.has(entry.primaryKey)) { + byKey.set(entry.primaryKey, { + event: entry.event, + primaryKey: entry.primaryKey, + oldest: anchorEntry(entry), + entries: [] + }) + } + } + } + + get events(): Array { + return Array.from(this.stored.keys()) + } + + /** The `(event, primaryKey)` groups in the batch. */ + *groups(): Iterable { + for (const byKey of this.stored.values()) { + yield* byKey.values() + } + } + + add(entry: Entry): void { + this.stored.get(entry.event)?.get(entry.primaryKey)?.entries.push(entry) + } + + toBatch(): Array<{ readonly entry: Entry; readonly conflicts: ReadonlyArray }> { + for (const group of this.groups()) { + group.entries.sort(entryOrder) + } + return this.batch.map((entry) => { + const anchor = anchorEntry(entry) + return { + entry, + conflicts: this.stored.get(entry.event)!.get(entry.primaryKey)!.entries.filter((stored) => + entryOrder(stored, anchor) > 0 + ) + } + }) + } +} + +/** An event and primary key in the batch, with the stored entries added for it. */ +export interface ConflictGroup { + readonly event: string + readonly primaryKey: string + /** The lowest anchor among the group's incoming entries. Only stored entries with a greater id can conflict. */ + readonly oldest: Entry + readonly entries: Array +} diff --git a/repos/effect/packages/effect/src/graphql/GraphQLClientError.ts b/repos/effect/packages/effect/src/graphql/GraphQLClientError.ts index 8935736a7a..9976cefb07 100644 --- a/repos/effect/packages/effect/src/graphql/GraphQLClientError.ts +++ b/repos/effect/packages/effect/src/graphql/GraphQLClientError.ts @@ -5,11 +5,12 @@ * @stability experimental * @since 4.0.0 */ -import * as Duration from "../Duration.ts" -import type * as Headers from "../http/Headers.ts" +import type * as Duration from "../Duration.ts" +import * as Headers from "../http/Headers.ts" import type * as HttpClientError from "../http/HttpClientError.ts" import { HeadersSchema } from "../internal/graphqlHeaders.ts" import { hasProperty } from "../Predicate.ts" +import * as Result from "../Result.ts" import * as Schema from "../Schema.ts" const TypeId = "~effect/graphql/GraphQLClientError" @@ -226,16 +227,8 @@ const isRetryableStatus = (status: number): boolean => status === 429 || (status */ const parseRetryAfter = (value: string | undefined, now: number): Duration.Duration | undefined => { if (value === undefined) return undefined - const trimmed = value.trim() - if (/^\d+$/.test(trimmed)) { - return Duration.seconds(Number(trimmed)) - } - // Every HTTP date names a day or month; `Date.parse` alone also accepts - // bare numbers such as `1.5` or `-1`. - if (!/[a-z]/i.test(trimmed)) return undefined - const date = Date.parse(trimmed) - if (Number.isNaN(date)) return undefined - return Duration.millis(Math.max(0, date - now)) + const retryAfter = Headers.parseRetryAfter(value) + return Result.isSuccess(retryAfter) ? Headers.retryAfterDelay(retryAfter.success, now) : undefined } /** diff --git a/repos/effect/packages/effect/src/graphql/GraphQLProtocol.ts b/repos/effect/packages/effect/src/graphql/GraphQLProtocol.ts index 2985d62db7..6f4d446beb 100644 --- a/repos/effect/packages/effect/src/graphql/GraphQLProtocol.ts +++ b/repos/effect/packages/effect/src/graphql/GraphQLProtocol.ts @@ -511,9 +511,10 @@ export const makeHttp: { * * **Gotchas** * - * `GraphQLResolver` resolves each lookup's URL once, before middleware run and - * before the batched request exists, then sends each batch to the URL it was - * grouped by. A single call resolves it in the transport, after middleware. + * `GraphQLResolver` resolves each lookup's URL once, in the calling fiber, + * before middleware run and before the batched request exists, then sends + * each batch to the URL it was grouped by. A single call resolves it in the + * transport, after middleware. * So the URL must depend only on the context and the operation, and * middleware must not change `kind` or `operationName` or provide services * the URL reads. diff --git a/repos/effect/packages/effect/src/graphql/GraphQLResolver.ts b/repos/effect/packages/effect/src/graphql/GraphQLResolver.ts index 07699bc951..863e4080e5 100644 --- a/repos/effect/packages/effect/src/graphql/GraphQLResolver.ts +++ b/repos/effect/packages/effect/src/graphql/GraphQLResolver.ts @@ -9,7 +9,8 @@ * in separate batches. Compose `RequestResolver.setDelay` and * `RequestResolver.batchN` with `.pipe` to control batching. * - * Lookups with different protocol URLs are batched separately. See + * Lookups with different protocol URLs are batched separately, and + * `RequestResolver.batchN` limits each URL's batches. See * `GraphQLProtocol.layerHttp` for middleware and caching constraints. * * @stability experimental @@ -27,6 +28,7 @@ import * as internalGraphQL from "../internal/graphql.ts" import * as MutableHashMap from "../MutableHashMap.ts" import * as Request from "../Request.ts" import * as RequestResolver from "../RequestResolver.ts" +import * as Tracer from "../Tracer.ts" import type * as Types from "../Types.ts" import type * as GraphQL from "./GraphQL.ts" import type { OperationMethod, PartialResult } from "./GraphQLClient.ts" @@ -65,10 +67,13 @@ type BatchKey = [GraphQL.Services] extends [never] ? interface KeyRequest extends Request.Request { readonly key: K + /** The protocol URL {@link request} resolved for this lookup, if it could. */ + readonly url: string | undefined } -// Requests are structural, so two requests with equal keys are one request. -const makeRequest = (key: K): KeyRequest => Request.of>()({ key }) +// Requests are structural, so two requests with equal keys and URLs are one request. +const makeRequest = (key: K, url: string | undefined): KeyRequest => + Request.of>()({ key, url }) // ----------------------------------------------------------------------------- // Running @@ -129,8 +134,13 @@ export const request: { (key: K, resolver: RequestResolver.RequestResolver>): Effect.Effect } = dual( 2, - (key: K, resolver: RequestResolver.RequestResolver>): Effect.Effect => - Effect.request(makeRequest(key), resolver) + (key: K, resolver: RequestResolver.RequestResolver>): Effect.Effect => { + // Resolved in the caller's fiber, so the URL is part of the pending batch's key. + const resolveUrl = urlResolvers.get(resolver.batchKey) + return resolveUrl === undefined + ? queue(makeRequest(key, undefined), resolver) + : Effect.flatMap(resolveUrl, (url) => queue(makeRequest(key, url), resolver)) + } ) // ----------------------------------------------------------------------------- @@ -609,41 +619,90 @@ interface Slice { const constNoSuchElement = Exit.fail(new Cause.NoSuchElementError()) /** - * Resolves each entry's URL in its own context, then splits the entries by URL - * and `batchKey`. Each batch runs in its first entry's context, carrying the - * URL it was grouped by, so the transport doesn't resolve it again. + * URL resolvers by resolver `batchKey`, which `RequestResolver.batchN` and + * `setDelay` keep, so {@link request} can resolve a lookup's URL before it is + * queued. + */ +const urlResolvers = new WeakMap>() + +/** + * Groups pending entries by URL and `batchKey`, so collection limits apply to + * each group. A URL that `request` couldn't resolve, for example after + * `RequestResolver.grouped` replaced the key, is resolved here in the entry's + * context, and the batch is split again. Each batch runs in its first entry's + * context, under any span opened around the batch, carrying the URL it was + * grouped by, so the transport doesn't resolve it again. */ -const makeResolver = ( +const makeResolver = >( method: OperationMethod, batchKey: ((context: Context.Context) => unknown) | undefined, runAll: (entries: Arr.NonEmptyArray>) => Effect.Effect, any> ): RequestResolver.RequestResolver => { const resolve = internalGraphQL.rawExecutors.get(method)?.url - const operation = { kind: method.operation.kind, operationName: method.operation.name } + const resolveUrl = resolve?.({ kind: method.operation.kind, operationName: method.operation.name }) + // Arrays are compared structurally, so equal keys share a batch. + const keyOf = (entry: Request.Entry, url: string | undefined) => [url, batchKey?.(entry.context)] as const const urlOf = (entry: Request.Entry) => - resolve === undefined ? Effect.succeed(undefined) : Effect.provideContext(resolve(operation), entry.context) - return RequestResolver.make((entries) => - Effect.flatMap(Effect.forEach(entries, urlOf, { concurrency: "unbounded" }), (urls) => { - // Arrays are compared structurally, so equal keys share a batch. - const batches = MutableHashMap.empty< - readonly [string | undefined, unknown], - Arr.NonEmptyArray> - >() - entries.forEach((entry, i) => { - const key = [urls[i], batchKey?.(entry.context)] as const - const batch = MutableHashMap.get(batches, key) - if (batch._tag === "Some") batch.value.push(entry) - else MutableHashMap.set(batches, key, [entry]) + entry.request.url !== undefined || resolveUrl === undefined + ? Effect.succeed(entry.request.url) + : runIn(resolveUrl, entry.context) + const resolver = RequestResolver.makeGrouped({ + key: (entry) => keyOf(entry, entry.request.url), + resolver: (entries) => + Effect.contextWith((current: Context.Context) => { + const span = batchSpan(current) + return Effect.flatMap(Effect.forEach(entries, urlOf, { concurrency: "unbounded" }), (urls) => { + const batches = MutableHashMap.empty< + readonly [string | undefined, unknown], + Arr.NonEmptyArray> + >() + entries.forEach((entry, i) => { + const key = keyOf(entry, urls[i]) + const batch = MutableHashMap.get(batches, key) + if (batch._tag === "Some") batch.value.push(entry) + else MutableHashMap.set(batches, key, [entry]) + }) + return Effect.forEach(batches, ([[url], entries]) => { + let context: Context.Context = entries[0].context + if (url !== undefined) context = Context.add(context, internalGraphQL.ResolvedUrl, { resolve, url }) + if (span !== undefined) context = Context.add(context, Tracer.ParentSpan, span) + return runIn(runAll(entries), context) + }, { concurrency: "unbounded", discard: true }) + }) }) - return Effect.forEach(batches, ([[url], entries]) => { - const context: Context.Context = entries[0].context - return Effect.provideContext( - runAll(entries), - url === undefined ? context : Context.add(context, internalGraphQL.ResolvedUrl, { resolve, url }) - ) - }, { concurrency: "unbounded", discard: true }) - }) + }) + if (resolveUrl !== undefined) urlResolvers.set(resolver.batchKey, resolveUrl) + return resolver +} + +// Replaces, rather than merges into, the batch fiber's context, which may +// belong to a lookup with other services or `Context.Reference` values. +const runIn = (effect: Effect.Effect, context: Context.Context): Effect.Effect => + Effect.updateContext(effect, (_: Context.Context) => context) + +// The caller's span when it queued a lookup. A batch fiber inherits it from the +// caller that started or filled the batch, even if that caller was interrupted. +const QueuedSpan = Context.Reference("effect/graphql/GraphQLResolver/QueuedSpan", { + defaultValue: () => undefined +}) + +const queue = ( + request: KeyRequest, + resolver: RequestResolver.RequestResolver> +): Effect.Effect => + Effect.contextWith((context: Context.Context) => + Effect.provideService( + Effect.request(request, resolver), + QueuedSpan, + Context.getOrUndefined(context, Tracer.ParentSpan) + ) ) + +// A span opened around the batch, such as `RequestResolver.withSpan`'s, rather +// than the one the batch fiber inherited from a caller. +const batchSpan = (context: Context.Context) => { + const span = Context.getOrUndefined(context, Tracer.ParentSpan) + return span === Context.get(context, QueuedSpan) ? undefined : span } /** diff --git a/repos/effect/packages/effect/src/http-api/HttpApi.ts b/repos/effect/packages/effect/src/http-api/HttpApi.ts index 6f9ab62b84..ec8bf46049 100644 --- a/repos/effect/packages/effect/src/http-api/HttpApi.ts +++ b/repos/effect/packages/effect/src/http-api/HttpApi.ts @@ -322,7 +322,7 @@ const extractResponseContent = ( function add(schema: Schema.Top) { const body = HttpApiSchema.isWithHeaders(schema) ? schema.schema : schema - if (HttpApiSchema.isStreamSchema(body)) return + if (HttpApiSchema.isStreamSchema(body) || HttpApiSchema.isUpgrade(body)) return const status = getStatus(schema) const schemas = map.get(status) if (schemas === undefined) { diff --git a/repos/effect/packages/effect/src/http-api/HttpApiBuilder.ts b/repos/effect/packages/effect/src/http-api/HttpApiBuilder.ts index 21a0339197..24a8145aa8 100644 --- a/repos/effect/packages/effect/src/http-api/HttpApiBuilder.ts +++ b/repos/effect/packages/effect/src/http-api/HttpApiBuilder.ts @@ -10,10 +10,12 @@ * @since 4.0.0 */ import type { NonEmptyReadonlyArray } from "../Array.ts" +import * as ByteSize from "../ByteSize.ts" import type * as Cause from "../Cause.ts" import * as Context from "../Context.ts" import * as Effect from "../Effect.ts" import * as Base64 from "../encoding/Base64.ts" +import * as Ndjson from "../encoding/Ndjson.ts" import * as Sse from "../encoding/Sse.ts" import * as Fiber from "../Fiber.ts" import type { FileSystem } from "../FileSystem.ts" @@ -26,6 +28,7 @@ import type { HttpPlatform } from "../http/HttpPlatform.ts" import * as HttpRouter from "../http/HttpRouter.ts" import * as Request from "../http/HttpServerRequest.ts" import { HttpServerRequest } from "../http/HttpServerRequest.ts" +import * as Respondable from "../http/HttpServerRespondable.ts" import * as Response from "../http/HttpServerResponse.ts" import type { HttpServerResponse } from "../http/HttpServerResponse.ts" import * as Multipart from "../http/Multipart.ts" @@ -47,7 +50,7 @@ import type { NoInfer } from "../Types.ts" import * as UndefinedOr from "../UndefinedOr.ts" import * as HttpApi from "./HttpApi.ts" import * as HttpApiEndpoint from "./HttpApiEndpoint.ts" -import { HttpApiSchemaError } from "./HttpApiError.ts" +import { HttpApiPayloadTooLarge, HttpApiSchemaError } from "./HttpApiError.ts" import type * as HttpApiGroup from "./HttpApiGroup.ts" import * as HttpApiMiddleware from "./HttpApiMiddleware.ts" import * as HttpApiSchema from "./HttpApiSchema.ts" @@ -739,6 +742,11 @@ type PayloadDecoder = readonly nullOnEmpty: boolean readonly decode: (input: unknown) => Effect.Effect } + | { + readonly _tag: "Stream" + readonly maxBytes: number | undefined + readonly decode: (request: HttpServerRequest) => Stream.Stream + } function buildPayloadDecoders( payloadMap: HttpApiEndpoint.PayloadMap, @@ -746,6 +754,15 @@ function buildPayloadDecoders( ): Map { const result = new Map() payloadMap.forEach(({ encoding, schemas }, contentType) => { + if (encoding._tag === "Stream") { + const schema = schemas[0] as HttpApiSchema.StreamUint8Array | HttpApiSchema.StreamNdjson + result.set(contentType, { + _tag: "Stream", + maxBytes: schema.maxBytes === undefined ? undefined : ByteSize.toNumberUnsafe(schema.maxBytes), + decode: makePayloadStreamDecoder(schema, options) + }) + return + } const schema = Schema.Union(schemas) const decode = Schema.decodeUnknownEffect( encoding._tag === "FormUrlEncoded" ? Schema.toCodecArrayFromSingle(schema) : schema, @@ -777,6 +794,17 @@ function decodePayload( if (!existing) { return Response.text(`Unsupported content-type: ${contentType}`, { status: 415 }) } + if (existing._tag === "Stream") { + if (existing.maxBytes !== undefined && Number(httpRequest.headers["content-length"]) > existing.maxBytes) { + return payloadTooLargeResponse + } + // The stream can be pulled after the handler returns, inside a streaming + // response, so it carries the element schema's services with it. + return Effect.map( + Effect.context(), + (context) => Stream.provideContext(existing.decode(httpRequest), context) + ) + } const { _tag, decode } = existing switch (_tag) { case "Multipart": { @@ -824,6 +852,69 @@ function decodePayload( } } +const payloadTooLargeResponse = Response.empty({ status: 413, headers: { connection: "close" } }) + +// The handler pulls the body itself, so nothing is read ahead of it. Body read +// failures stay defects, as they are for buffered payloads. +function makePayloadStreamDecoder( + schema: HttpApiSchema.StreamUint8Array | HttpApiSchema.StreamNdjson, + options: SchemaAST.ParseOptions | undefined +): (request: HttpServerRequest) => Stream.Stream { + const maxBytes = schema.maxBytes + const readBody = (request: HttpServerRequest) => { + const body = Stream.orDie(request.stream) + return maxBytes === undefined ? body : limitBytes(body, maxBytes) + } + if (HttpApiSchema.isStreamUint8Array(schema)) { + return readBody + } + const decodeElement = Schema.decodeUnknownEffect(Schema.toCodecJson(schema.schema), options) + const maxElementBytes = schema.maxElementBytes + const elementTooLarge = new HttpApiPayloadTooLarge({ limit: "maxElementBytes", size: maxElementBytes }) + const maxLineBytes = ByteSize.toNumberUnsafe(maxElementBytes) + return (request) => + readBody(request).pipe( + Stream.pipeThroughChannel(Ndjson.decode({ ignoreEmptyLines: true, maxLineBytes })), + Stream.mapEffect((json) => decodeElement(json)), + Stream.mapError((error) => { + if (HttpApiPayloadTooLarge.is(error)) return error + if (!Ndjson.isNdjsonError(error)) return new HttpApiSchemaError({ kind: "Payload", cause: error }) + if (error.reason === "LineTooLarge") return elementTooLarge + return new HttpApiSchemaError({ kind: "Payload", cause: ndjsonSchemaError(error) }) + }) + ) +} + +// `JSON.stringify` found no JSON representation for an encoded element +const isNdjsonPackError = (error: unknown): error is Ndjson.NdjsonError => + Ndjson.isNdjsonError(error) && error.kind === "Pack" + +const ndjsonSchemaError = (error: Ndjson.NdjsonError): Schema.SchemaError => + new Schema.SchemaError( + new SchemaIssue.InvalidValue({ + message: error.reason === "InvalidUtf8" + ? "Expected valid UTF-8" + : error.unterminated + ? "Expected a valid JSON line, got an unterminated final line" + : "Expected a valid JSON line" + }) + ) + +function limitBytes( + self: Stream.Stream, + maxBytes: ByteSize.ByteSize +): Stream.Stream { + const limit = ByteSize.toNumberUnsafe(maxBytes) + const error = new HttpApiPayloadTooLarge({ limit: "maxBytes", size: maxBytes }) + return Stream.suspend(() => { + let total = 0 + return Stream.mapArrayEffect(self, (chunks) => { + for (const chunk of chunks) total += chunk.length + return total > limit ? Effect.fail(error) : Effect.succeed(chunks) + }) + }) +} + function handlerToHttpEffect( api: Pick, group: HttpApiGroup.Top, @@ -847,6 +938,7 @@ function handlerToHttpEffect( ) const encodeStream = makeStreamEncoder(endpoint, options.success) const encodeWithHeaders = makeWithHeadersEncoder(endpoint, options.success, options.headers) + const upgrade = getUpgradeSuccessSchema(endpoint) const shouldParsePayload = endpoint.payload.size > 0 && !isRaw const payloadBy = shouldParsePayload ? buildPayloadDecoders(endpoint.payload, options.payload) : undefined @@ -884,10 +976,20 @@ function handlerToHttpEffect( request.payload = yield* HttpApiSchemaError.wrap("Payload", result) } } + if (upgrade !== undefined) { + if (!isUpgradeRequest(httpRequest, upgrade.protocol)) { + return upgradeRequired(upgrade.protocol) + } + request.socket = yield* Effect.orDie(httpRequest.upgrade) + } let response = yield* handler(request) if (Response.isHttpServerResponse(response)) { return response } + if (upgrade !== undefined) { + // The server replaces this with the 101 once the connection is upgraded. + return Response.empty() + } let responseHeaders: unknown | undefined if (encodeWithHeaders !== undefined && HttpApiSchema.isWithHeadersValue(response)) { responseHeaders = response.headers @@ -908,8 +1010,11 @@ function handlerToHttpEffect( ).pipe( Effect.withErrorReporting, Effect.catch((error) => { - if (HttpApiSchemaError.is(error)) return Effect.die(error) - return Effect.orDie(encodeError(error)) + if (HttpApiSchemaError.is(error) || HttpApiPayloadTooLarge.is(error)) return Effect.die(error) + return Effect.catch(encodeError(error), (schemaError) => + Respondable.isRespondable(error) + ? Respondable.toResponse(error) + : Effect.die(schemaError)) }), Effect.provideContext(context) ) @@ -1071,25 +1176,7 @@ function makeStreamEncoder( const status = HttpApiSchema.getStatusSuccessSchema(successSchema.schema) const contentType = streamSchema.contentType - if (HttpApiSchema.isStreamUint8Array(streamSchema)) { - return (response, context) => { - if (!Stream.isStream(response)) { - return hasBuffered ? undefined : new Schema.SchemaError( - new SchemaIssue.InvalidValue({ message: "Expected a streaming response" }) - ) - } - - return Effect.succeed(Response.stream( - Stream.provideContext( - response as Stream.Stream, - context as Context.Context - ), - { status, contentType } - )) - } - } - - const sseEncoder = makeSseEncoder(streamSchema, options) + const encodeBody = makeStreamBodyEncoder(streamSchema, options) return (response, context) => { if (!Stream.isStream(response)) { @@ -1100,7 +1187,7 @@ function makeStreamEncoder( return Effect.succeed(Response.stream( Stream.provideContext( - encodeSseStream(response, sseEncoder), + encodeBody(response), context as Context.Context ), { status, contentType } @@ -1108,6 +1195,47 @@ function makeStreamEncoder( } } +function makeStreamBodyEncoder( + streamSchema: HttpApiSchema.StreamSchema, + options: SchemaAST.ParseOptions | undefined +): (stream: Stream.Stream) => Stream.Stream { + if (HttpApiSchema.isStreamUint8Array(streamSchema)) { + return (stream) => stream as Stream.Stream + } + if (HttpApiSchema.isStreamNdjson(streamSchema)) { + const encodeElements = Schema.encodeUnknownEffect( + Schema.NonEmptyArray(Schema.toCodecJson(streamSchema.schema)), + options + ) + // There is no error schema, so a bad element aborts the response + return (stream) => + stream.pipe( + Stream.mapArrayEffect((elements) => Effect.orDie(encodeElements(elements))), + Stream.pipeThroughChannel(Ndjson.encode()), + Stream.catchIf(isNdjsonPackError, Stream.die) + ) + } + const sseEncoder = makeSseEncoder(streamSchema, options) + return (stream) => encodeSseStream(stream, sseEncoder) +} + +function getUpgradeSuccessSchema(endpoint: HttpApiEndpoint.Top): HttpApiSchema.Upgrade | undefined { + for (const schema of endpoint.success) { + if (HttpApiSchema.isUpgrade(schema)) return schema + } +} + +// RFC 9110 §7.8: `Upgrade` only applies with an `upgrade` connection option. +const isUpgradeRequest = (request: HttpServerRequest, protocol: string): boolean => + request.headers.upgrade?.toLowerCase() === protocol.toLowerCase() && + /(?:^|,)\s*upgrade\s*(?:,|$)/i.test(request.headers.connection ?? "") + +const upgradeRequired = (protocol: string) => + Response.empty({ + status: 426, + headers: { connection: "Upgrade", upgrade: protocol } + }) + function getStreamSuccessSchema(endpoint: HttpApiEndpoint.Top) { for (const schema of endpoint.success) { const body = HttpApiSchema.isWithHeaders(schema) ? schema.schema : schema diff --git a/repos/effect/packages/effect/src/http-api/HttpApiClient.ts b/repos/effect/packages/effect/src/http-api/HttpApiClient.ts index e5e2287a33..47721e683a 100644 --- a/repos/effect/packages/effect/src/http-api/HttpApiClient.ts +++ b/repos/effect/packages/effect/src/http-api/HttpApiClient.ts @@ -12,11 +12,14 @@ * @since 4.0.0 */ import * as Arr from "../Array.ts" +import * as ByteSize from "../ByteSize.ts" import * as Cause from "../Cause.ts" import type * as Context from "../Context.ts" import * as Effect from "../Effect.ts" +import * as Ndjson from "../encoding/Ndjson.ts" import * as Sse from "../encoding/Sse.ts" import { identity } from "../Function.ts" +import type * as Headers from "../http/Headers.ts" import * as HttpBody from "../http/HttpBody.ts" import * as HttpClient from "../http/HttpClient.ts" import * as HttpClientError from "../http/HttpClientError.ts" @@ -31,6 +34,7 @@ import * as Schema from "../Schema.ts" import * as SchemaAST from "../SchemaAST.ts" import * as SchemaIssue from "../SchemaIssue.ts" import * as SchemaTransformation from "../SchemaTransformation.ts" +import type * as Socket from "../socket/Socket.ts" import * as Stream from "../Stream.ts" import type { Simplify } from "../Types.ts" import * as UndefinedOr from "../UndefinedOr.ts" @@ -96,9 +100,31 @@ export type SuccessType = S extends HttpApiSchema.WithHeaders< never > : S extends HttpApiSchema.StreamUint8Array ? Stream.Stream + : S extends HttpApiSchema.StreamNdjson ? + Stream.Stream<_S["Type"], HttpClientError.HttpClientError | Schema.SchemaError, never> + : S extends HttpApiSchema.Upgrade ? Upgraded : S extends Schema.Constraint ? S["Type"] : never +/** + * The value a client method returns for an endpoint declared with + * `HttpApiSchema.Upgrade`: the headers of the `101 Switching Protocols` + * response and the upgraded connection. + * + * **Details** + * + * See `HttpClientResponse.upgrade` for how the connection behaves and when it + * closes. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export interface Upgraded { + readonly headers: Headers.Headers + readonly socket: Socket.Socket +} + type SuccessDecodingServices = S extends HttpApiSchema.StreamSse< infer _Events, infer _Error, @@ -107,6 +133,7 @@ type SuccessDecodingServices = S extends HttpApiSchema.StreamSse< | _Events["DecodingServices"] | _Error["DecodingServices"] : S extends HttpApiSchema.StreamUint8Array ? never + : S extends HttpApiSchema.StreamNdjson ? _S["DecodingServices"] : S extends Schema.Constraint ? S["DecodingServices"] : never @@ -161,7 +188,8 @@ export declare namespace Client { Endpoint extends HttpApiEndpoint.ConstraintRequest, E, R, - Mode extends ResponseMode + Mode extends ResponseMode, + PayloadR > = Effect.Effect< Response, Mode>, | HttpApiMiddleware.Error @@ -170,6 +198,7 @@ export declare namespace Client { | HttpClientError.HttpClientError | ([Mode] extends ["response-only"] ? never : Endpoint["~Error"]["Type"] | Schema.SchemaError), | R + | PayloadR | Endpoint["~Params"]["EncodingServices"] | Endpoint["~Query"]["EncodingServices"] | Endpoint["~Payload"]["EncodingServices"] @@ -186,6 +215,11 @@ export declare namespace Client { * error, and service channels reflect the endpoint schemas, middleware, and * selected response mode. * + * **Details** + * + * The services of a streaming `payload` are inferred and added to the + * returned effect. + * * @stability unstable * @category models * @since 4.0.0 @@ -194,17 +228,18 @@ export declare namespace Client { Endpoint extends HttpApiEndpoint.ConstraintRequest, E, R - > = ( + > = ( request: Simplify< HttpApiEndpoint.ClientRequest< Endpoint["~Params"], Endpoint["~Query"], Endpoint["~Payload"], Endpoint["~Headers"], - Mode + Mode, + PayloadR > > - ) => MethodReturn + ) => MethodReturn /** * Extracts client methods for endpoints in top-level groups so they can be exposed @@ -412,13 +447,30 @@ export const makeClient = + Effect.map(response.upgrade, (socket): Upgraded => ({ headers: response.headers, socket })) + } const encodeParams = UndefinedOr.map( endpoint.params, (schema) => Schema.encodeUnknownEffect(schema, parseOptions.params) ) - const payloadSchemas = HttpApiEndpoint.getPayloadSchemas(endpoint) + const payloadSchemas: Array = [] + let streamPayload: StreamPayloadSchema | undefined + for (const schema of HttpApiEndpoint.getPayloadSchemas(endpoint)) { + if (HttpApiSchema.isStreamUint8Array(schema) || HttpApiSchema.isStreamNdjson(schema)) { + streamPayload = schema + } else { + payloadSchemas.push(schema) + } + } + const streamPayloadEncoder = UndefinedOr.map(streamPayload, (schema) => ({ + contentType: schema.contentType, + encode: makeStreamPayloadEncoder(schema, parseOptions.payload) + })) const encodePayload = Arr.isArrayNonEmpty(payloadSchemas) ? Schema.encodeUnknownEffect( HttpMethod.hasBody(endpoint.method) @@ -459,7 +511,19 @@ export const makeClient = () + httpRequest = HttpClientRequest.setBody( + httpRequest, + HttpBody.stream( + Stream.provideContext( + streamPayloadEncoder.encode(request.payload), + context as Context.Context + ), + streamPayloadEncoder.contentType + ) + ) + } else if (encodePayload !== undefined) { if (HttpMethod.hasBody(endpoint.method)) { if (request.payload instanceof FormData) { httpRequest = HttpClientRequest.bodyFormData(httpRequest, request.payload) @@ -471,6 +535,10 @@ export const makeClient = httpRequest = HttpClientRequest.appendUrlParams(httpRequest, urlParams) } + } else if (streamPayloadEncoder !== undefined) { + return yield* new Schema.SchemaError( + new SchemaIssue.InvalidValue({ message: "Expected a Stream payload" }, request.payload) + ) } // headers @@ -486,6 +554,10 @@ export const makeClient = (api: Api, options?: { // ---------------------------------------------------------------------------- -const paramsRegExp = /(\/?):(\w+)(\?)?/g - const compilePath = (path: string, schema: Schema.Top | undefined) => { if (schema === undefined || !path.includes(":")) { return (_: any) => path } const paramNames = HttpApiPath.getParamNames(schema) - return (params: Record) => { - return path.replace(paramsRegExp, (match, slash: string, key: string, optional: string | undefined) => { - if (paramNames !== undefined && !paramNames.has(key)) { - return match - } + return (params: Record) => + HttpApiPath.replaceParams(path, paramNames, (key, optional) => { const value = params[key] - if (value === undefined) { - if (optional !== undefined) { - return "" - } - throw new Error(`Missing path parameter: ${key}`) - } - return `${slash}${encodeURIComponent(value)}` + if (value !== undefined) return encodeURIComponent(value) + if (optional) return undefined + throw new Error(`Missing path parameter: ${key}`) }) - } } function schemasToResponse( @@ -932,6 +994,12 @@ const isWithHeadersStreamSuccess = ( schema: StreamSuccessSchema ): schema is HttpApiSchema.WithHeaders => HttpApiSchema.isWithHeaders(schema) +function getUpgradeSuccessSchema(endpoint: HttpApiEndpoint.Top): HttpApiSchema.Upgrade | undefined { + for (const schema of endpoint.success) { + if (HttpApiSchema.isUpgrade(schema)) return schema + } +} + function getStreamSuccessSchemas(endpoint: HttpApiEndpoint.Top): Array { const schemas: Array = [] for (const schema of endpoint.success) { @@ -950,18 +1018,11 @@ function streamToResponse( ) { const isWithHeaders = isWithHeadersStreamSuccess(successSchema) const streamSchema = isWithHeaders ? successSchema.schema : successSchema - const sse = HttpApiSchema.isStreamUint8Array(streamSchema) - ? undefined - : { - declaration: streamSchema, - decoder: makeSseDecoder(streamSchema, options) - } + const decodeBody = makeStreamBodyDecoder(streamSchema, options) const toStream: ResponseDecoder = (response, sseOptions) => Effect.map(Effect.context(), (context) => Stream.provideContext( - sse === undefined ? - response.stream : - decodeSseStream(response.stream, sse.declaration, sse.decoder(sseOptions)), + decodeBody(response, sseOptions), context as Context.Context )) if (!isWithHeaders) return toStream @@ -974,6 +1035,55 @@ function streamToResponse( ) } +function makeStreamBodyDecoder( + streamSchema: HttpApiSchema.StreamSchema, + options: SchemaAST.ParseOptions | undefined +): ( + response: HttpClientResponse.HttpClientResponse, + sseOptions: Sse.DecodeOptions | undefined +) => Stream.Stream { + if (HttpApiSchema.isStreamUint8Array(streamSchema)) { + return (response) => response.stream + } + if (HttpApiSchema.isStreamNdjson(streamSchema)) { + const decodeElement = Schema.decodeUnknownEffect(Schema.toCodecJson(streamSchema.schema), options) + const maxElementBytes = ByteSize.toNumberUnsafe(streamSchema.maxElementBytes) + return (response) => + response.stream.pipe( + Stream.pipeThroughChannel(Ndjson.decode({ ignoreEmptyLines: true, maxLineBytes: maxElementBytes })), + Stream.mapError((error) => { + if (!Ndjson.isNdjsonError(error)) return error + if (error.reason !== "LineTooLarge") return ndjsonSchemaError(error) + return new HttpClientError.HttpClientError({ + reason: new HttpClientError.DecodeError({ + request: response.request, + response, + description: `NDJSON line exceeds maxElementBytes of ${maxElementBytes} bytes` + }) + }) + }), + Stream.mapEffect((json) => decodeElement(json)) + ) + } + const decoder = makeSseDecoder(streamSchema, options) + return (response, sseOptions) => decodeSseStream(response.stream, streamSchema, decoder(sseOptions)) +} + +// `JSON.stringify` found no JSON representation for an encoded element +const isNdjsonPackError = (error: unknown): error is Ndjson.NdjsonError => + Ndjson.isNdjsonError(error) && error.kind === "Pack" + +const ndjsonSchemaError = (error: Ndjson.NdjsonError): Schema.SchemaError => + new Schema.SchemaError( + new SchemaIssue.InvalidValue({ + message: error.reason === "InvalidUtf8" + ? "Expected valid UTF-8" + : error.unterminated + ? "Expected a valid JSON line, got an unterminated final line" + : "Expected a valid JSON line" + }) + ) + function makeSseDecoder( declaration: HttpApiSchema.StreamSse, parseOptions: SchemaAST.ParseOptions | undefined @@ -1138,6 +1248,24 @@ const statusOrElse = (response: HttpClientResponse.HttpClientResponse) => const $HttpBody = Schema.declare(HttpBody.isHttpBody) +type StreamPayloadSchema = HttpApiSchema.StreamUint8Array | HttpApiSchema.StreamNdjson + +function makeStreamPayloadEncoder( + schema: StreamPayloadSchema, + options: SchemaAST.ParseOptions | undefined +): (stream: Stream.Stream) => Stream.Stream { + if (HttpApiSchema.isStreamUint8Array(schema)) { + return (stream) => stream as Stream.Stream + } + const encodeElements = Schema.encodeUnknownEffect(Schema.NonEmptyArray(Schema.toCodecJson(schema.schema)), options) + return (stream) => + stream.pipe( + Stream.mapArrayEffect((elements) => encodeElements(elements)), + Stream.pipeThroughChannel(Ndjson.encode()), + Stream.catchIf(isNdjsonPackError, Stream.die) + ) +} + function getEncodePayloadSchema( schemas: readonly [Schema.Constraint, ...Array], method: HttpMethod.HttpMethod diff --git a/repos/effect/packages/effect/src/http-api/HttpApiEndpoint.ts b/repos/effect/packages/effect/src/http-api/HttpApiEndpoint.ts index 7149265468..db3a91a885 100644 --- a/repos/effect/packages/effect/src/http-api/HttpApiEndpoint.ts +++ b/repos/effect/packages/effect/src/http-api/HttpApiEndpoint.ts @@ -18,7 +18,7 @@ import * as Context from "../Context.ts" import type { Effect } from "../Effect.ts" import type * as Sse from "../encoding/Sse.ts" import { identity, memoize } from "../Function.ts" -import type { HttpMethod } from "../http/HttpMethod.ts" +import { hasBody, type HttpMethod } from "../http/HttpMethod.ts" import * as HttpRouter from "../http/HttpRouter.ts" import type { HttpServerRequest } from "../http/HttpServerRequest.ts" import type { HttpServerResponse } from "../http/HttpServerResponse.ts" @@ -27,9 +27,11 @@ import { type Pipeable, pipeArguments } from "../Pipeable.ts" import * as Predicate from "../Predicate.ts" import * as Schema from "../Schema.ts" import * as AST from "../SchemaAST.ts" +import type * as Socket from "../socket/Socket.ts" import type * as Stream from "../Stream.ts" import type { Simplify } from "../Struct.ts" import type * as Types from "../Types.ts" +import type * as HttpApiError from "./HttpApiError.ts" import type * as HttpApiGroup from "./HttpApiGroup.ts" import type * as HttpApiMiddleware from "./HttpApiMiddleware.ts" import * as HttpApiSchema from "./HttpApiSchema.ts" @@ -57,6 +59,8 @@ type SuccessType = S extends HttpApiSchema.WithHeaders< infer _Value > ? Stream.Stream<_Value, _Error["Type"], never> : S extends HttpApiSchema.StreamUint8Array ? Stream.Stream + : S extends HttpApiSchema.StreamNdjson ? Stream.Stream<_S["Type"], unknown, never> + : S extends HttpApiSchema.Upgrade ? void : S extends Schema.Constraint ? S["Type"] : never @@ -66,6 +70,7 @@ type SuccessEncodingServices = S extends HttpApiSchema.StreamSse< infer _Value > ? _Events["EncodingServices"] | _Error["EncodingServices"] : S extends HttpApiSchema.StreamUint8Array ? never + : S extends HttpApiSchema.StreamNdjson ? _S["EncodingServices"] : S extends Schema.Constraint ? S["EncodingServices"] : never @@ -75,6 +80,7 @@ type SuccessDecodingServices = S extends HttpApiSchema.StreamSse< infer _Value > ? _Events["DecodingServices"] | _Error["DecodingServices"] : S extends HttpApiSchema.StreamUint8Array ? never + : S extends HttpApiSchema.StreamNdjson ? _S["DecodingServices"] : S extends Schema.Constraint ? S["DecodingServices"] : never @@ -82,13 +88,18 @@ type UnwrapReadonlyArray = S extends ReadonlyArray ? A : S type ExtractBufferedSuccess = Exclude< Extract, Schema.Top>, - HttpApiSchema.StreamSchema | HttpApiSchema.WithHeaders + HttpApiSchema.StreamSchema | HttpApiSchema.Upgrade | HttpApiSchema.WithHeaders > type ExtractStreamSuccess = UnwrapReadonlyArray extends infer Success ? Success extends HttpApiSchema.StreamSchema ? Success : never : never +type ExtractUpgradeSuccess = Extract, HttpApiSchema.Upgrade> + +type UpgradeSocket = [Extract] extends [never] ? {} + : { readonly socket: Socket.Socket } + type ExtractWithHeadersSuccess = UnwrapReadonlyArray extends infer Success ? Success extends HttpApiSchema.WithHeaders ? HttpApiSchema.WithHeaders< _Inner extends HttpApiSchema.StreamSchema ? _Inner : Schema.toCodecJson<_Inner>, @@ -98,8 +109,12 @@ type ExtractWithHeadersSuccess = UnwrapReadonlyArra : never type ToSuccessCodec = [ExtractBufferedSuccess] extends [never] ? - ExtractStreamSuccess | ExtractWithHeadersSuccess - : Schema.toCodecJson> | ExtractStreamSuccess | ExtractWithHeadersSuccess + ExtractStreamSuccess | ExtractUpgradeSuccess | ExtractWithHeadersSuccess + : + | Schema.toCodecJson> + | ExtractStreamSuccess + | ExtractUpgradeSuccess + | ExtractWithHeadersSuccess type ToJsonCodec = [S] extends [never] ? never : [S] extends [Schema.Constraint] ? Schema.toCodecJson @@ -110,13 +125,33 @@ type ToStringTreeCodec = [S] extends [never] ? never : [S] extends [Schema.Constraint] ? Schema.toCodecStringTree : never -type RequestFromParts = +type ToPayloadCodec = [Exclude] extends [never] ? + Extract + : ToJsonCodec> | Extract + +// Stream payloads are keyed on the schema, because a stream schema's `Type` +// describes the response direction. +type RequestPayload = Payload extends HttpApiSchema.StreamUint8Array ? + Stream.Stream + : Payload extends HttpApiSchema.StreamNdjson ? + Stream.Stream<_S["Type"], HttpApiError.HttpApiSchemaError | HttpApiError.HttpApiPayloadTooLarge> + : Payload extends Schema.Top ? BufferedRequestPayload + : never + +type BufferedRequestPayload = PayloadType extends Brand ? + Stream.Stream + : Simplify + +type PayloadError = Payload extends HttpApiSchema.StreamUint8Array ? HttpApiError.HttpApiPayloadTooLarge + : Payload extends HttpApiSchema.StreamNdjson ? + HttpApiError.HttpApiSchemaError | HttpApiError.HttpApiPayloadTooLarge + : never + +type RequestFromParts = + & UpgradeSocket & ([ParamsType] extends [never] ? {} : { readonly params: Simplify }) & ([QueryType] extends [never] ? {} : { readonly query: Simplify }) - & ([PayloadType] extends [never] ? {} - : PayloadType extends Brand ? - { readonly payload: Stream.Stream } - : { readonly payload: Simplify }) + & ([Payload] extends [never] ? {} : { readonly payload: RequestPayload }) & ([HeadersType] extends [never] ? {} : { readonly headers: Simplify }) & { readonly request: HttpServerRequest @@ -124,7 +159,8 @@ type RequestFromParts readonly group: HttpApiGroup.Top } -type RequestRawFromParts = +type RequestRawFromParts = + & UpgradeSocket & ([ParamsType] extends [never] ? {} : { readonly params: Simplify }) & ([QueryType] extends [never] ? {} : { readonly query: Simplify }) & ([HeadersType] extends [never] ? {} : { readonly headers: Simplify }) @@ -138,14 +174,26 @@ type RequestRawFromParts = * Maps normalized media types to a payload encoding strategy and one or more * schemas. Each schema retains its declared content type in its encoding annotation. * + * **Details** + * + * A streaming payload (`HttpApiSchema.StreamUint8Array` or + * `HttpApiSchema.StreamNdjson`) has a `Stream` encoding and is the only schema + * for its media type. + * * @stability unstable * @category models * @since 4.0.0 */ -export type PayloadMap = ReadonlyMap] -}> +export type PayloadMap = ReadonlyMap< + string, + { + readonly encoding: HttpApiSchema.PayloadEncoding + readonly schemas: readonly [Schema.Top, ...Array] + } | { + readonly encoding: { readonly _tag: "Stream"; readonly contentType: string } + readonly schemas: readonly [HttpApiSchema.StreamUint8Array | HttpApiSchema.StreamNdjson] + } +> /** * Represents an API endpoint. An API endpoint is mapped to a single route on @@ -178,8 +226,15 @@ export interface HttpApiEndpoint< readonly "~Error": Error readonly "~Middleware": Middleware readonly "~MiddlewareServices": MiddlewareServices - readonly "~Request": RequestFromParts - readonly "~RequestRaw": RequestRawFromParts + readonly "~Request": RequestFromParts< + this, + Params["Type"], + Query["Type"], + Payload, + Headers["Type"], + Success + > + readonly "~RequestRaw": RequestRawFromParts readonly identifier: Identifier readonly path: Path @@ -518,7 +573,7 @@ export type RequestRaw = Endpoint extends ConstraintRequest ? Endpoint * Builds the request object accepted by a generated client method, including the * params, query, headers, and payload required by the endpoint, plus optional * response mode and SSE decoding options. Multipart payloads are supplied as - * `FormData`. + * `FormData`, and streaming payloads as a `Stream` that requires `PayloadR`. * * @stability unstable * @category utility types @@ -529,7 +584,8 @@ export type ClientRequest< Query extends Schema.Constraint, Payload extends Schema.Constraint, Headers extends Schema.Constraint, - ResponseMode extends ClientResponseMode + ResponseMode extends ClientResponseMode, + PayloadR = never > = ( & ([Params["Type"]] extends [never] ? {} : { readonly params: Params["Type"] }) & ([Query["Type"]] extends [never] ? {} : { readonly query: Query["Type"] }) @@ -538,8 +594,8 @@ export type ClientRequest< : Payload["Type"] extends infer P ? P extends Brand | Brand ? { readonly payload: FormData } - : { readonly payload: Payload["Type"] } - : { readonly payload: Payload["Type"] }) + : { readonly payload: ClientPayload } + : { readonly payload: ClientPayload }) ) extends infer Req ? keyof Req extends never ? (void | { readonly responseMode?: ResponseMode readonly sseOptions?: Sse.DecodeOptions | undefined @@ -550,6 +606,11 @@ export type ClientRequest< } : void +type ClientPayload = Payload extends HttpApiSchema.StreamUint8Array + ? Stream.Stream + : Payload extends HttpApiSchema.StreamNdjson ? Stream.Stream + : Payload["Type"] + /** * Controls what a generated client method returns: the decoded success value, * the decoded value paired with the raw response, or only the raw response. @@ -628,7 +689,11 @@ export type ErrorServicesDecode = Endpoint extends ConstraintRequest ? */ export type Handler = ( request: Simplify -) => Effect | HttpServerResponse, Endpoint["~Error"]["Type"] | E, R> +) => Effect< + SuccessType | HttpServerResponse, + Endpoint["~Error"]["Type"] | PayloadError> | E, + R +> /** * The raw server handler for an endpoint, receiving a request shape without a @@ -981,7 +1046,9 @@ export type QueryConstraint = * encode to `string | ReadonlyArray | undefined` and OpenAPI can expand * it into `in: query` parameters * - for body methods, payload may be any `Schema.Top` (or content-type keyed - * schemas) and OpenAPI uses `requestBody` instead of `parameters` + * schemas) and OpenAPI uses `requestBody` instead of `parameters`. This + * includes one `HttpApiSchema.StreamUint8Array` or `HttpApiSchema.StreamNdjson`, + * but not `HttpApiSchema.StreamSse`, which is response-only * * @stability unstable * @category utility types @@ -991,7 +1058,10 @@ export type PayloadConstraint = Method extends HttpMe string, Schema.Encoder | undefined, unknown> > : - Schema.Top | ReadonlyArray + PayloadSchema | ReadonlyArray + +// `StreamSse` is response-only, so it is the one stream schema excluded here. +type PayloadSchema = Schema.Top & { readonly sseMode?: never } /** * Payload constraint used when automatic codecs are enabled: no-body methods @@ -1004,7 +1074,7 @@ export type PayloadConstraint = Method extends HttpMe */ export type PayloadConstraintCodecs = Method extends HttpMethod.NoBody ? Record : - Schema.Top | ReadonlyArray + PayloadSchema | ReadonlyArray /** * Constraint for success response schemas, allowing either a single schema or a @@ -1031,7 +1101,7 @@ type ErrorSchema = S extends HttpApiSchema.WithHeaders = [ Extract< ErrorSchema ? S[number] : S>, - HttpApiSchema.StreamSchema + HttpApiSchema.StreamSchema | HttpApiSchema.Upgrade > ] extends [never] ? S : never @@ -1073,7 +1143,7 @@ export const make = (method: Method): { Path, ToStringTreeCodec, ToStringTreeCodec, - Method extends HttpMethod.WithBody ? ToJsonCodec> + Method extends HttpMethod.WithBody ? ToPayloadCodec> : ToStringTreeCodec>, ToStringTreeCodec, ToSuccessCodec, @@ -1156,7 +1226,7 @@ export const make = (method: Method): { query: ensureStruct(options?.query, transformStringTree), headers: ensureStruct(options?.headers, transformStringTree), payload: getPayload(options?.payload, method, disableCodecs), - success: getSuccessResponse(options?.success, method, disableCodecs), + success: getSuccessResponse(options?.success, method, options?.payload !== undefined, disableCodecs), error: getErrorResponse(options?.error, disableCodecs), annotations: Context.empty(), middlewares: new Set(), @@ -1185,21 +1255,46 @@ function getPayload( ): PayloadMap { const result: Map< string, - { encoding: HttpApiSchema.PayloadEncoding; schemas: [Schema.Top, ...Array] } + { + encoding: HttpApiSchema.PayloadEncoding | { readonly _tag: "Stream"; readonly contentType: string } + schemas: [Schema.Top, ...Array] + } > = new Map() - if (payload === undefined) return result + if (payload === undefined) return result as PayloadMap const schemas: Array = Array.isArray(payload) ? payload : Schema.isSchema(payload) ? [payload] : [(Schema.Struct(payload as any)).pipe(HttpApiSchema.asFormUrlEncoded())] const transform = disableCodecs ? identity : transformPayload + let hasStream = false for (const schema of schemas) { + if (HttpApiSchema.isStreamSchema(schema)) { + if (HttpApiSchema.isStreamSse(schema)) { + throw new Error("StreamSse is response-only and cannot be used as a payload") + } + if (!hasBody(method)) { + throw new Error(`${method} endpoints cannot declare a streaming payload`) + } + if (hasStream) { + throw new Error("Multiple streaming payloads are not supported") + } + hasStream = true + const contentType = MediaType.normalize(schema.contentType) + if (result.has(contentType)) { + throw new Error(`Cannot combine buffered and streaming payloads for content-type: ${schema.contentType}`) + } + result.set(contentType, { encoding: { _tag: "Stream", contentType: schema.contentType }, schemas: [schema] }) + continue + } const encoding = HttpApiSchema.getPayloadEncoding(schema.ast, method) const contentType = MediaType.normalize(encoding.contentType) const existing = result.get(contentType) if (existing) { + if (existing.encoding._tag === "Stream") { + throw new Error(`Cannot combine buffered and streaming payloads for content-type: ${encoding.contentType}`) + } if (existing.encoding._tag !== encoding._tag) { throw new Error(`Multiple payload encodings for content-type: ${encoding.contentType}`) } @@ -1211,7 +1306,7 @@ function getPayload( result.set(contentType, { encoding, schemas: [transform(schema, method)] }) } } - return result + return result as PayloadMap } const reservedStreamFailureEvent = "effect/http-api/stream/failure" @@ -1219,16 +1314,17 @@ const reservedStreamFailureEvent = "effect/http-api/stream/failure" function getSuccessResponse( success: Schema.Top | ReadonlyArray | undefined, method: HttpMethod, + hasPayload: boolean, disableCodecs: boolean ): Set { if (success === undefined) return new Set() const schemas = Arr.ensure(success) - validateSuccessResponse(schemas, method) + validateSuccessResponse(schemas, method, hasPayload) return new Set(disableCodecs ? schemas : schemas.map(transformResponseSchema)) } const transformResponseSchema = memoize((schema: Schema.Top): Schema.Top => { - if (HttpApiSchema.isStreamSchema(schema)) return schema + if (HttpApiSchema.isStreamSchema(schema) || HttpApiSchema.isUpgrade(schema)) return schema if (HttpApiSchema.isWithHeaders(schema)) { const inner = HttpApiSchema.isStreamSchema(schema.schema) ? schema.schema @@ -1249,12 +1345,27 @@ function getErrorResponse( if (HttpApiSchema.isStreamSchema(body)) { throw new Error("Streaming schemas are not supported in error responses") } + if (HttpApiSchema.isUpgrade(body)) { + throw new Error("Upgrade schemas are not supported in error responses") + } } validateResponseExclusivity(schemas, HttpApiSchema.getStatusErrorSchema) return new Set(disableCodecs ? schemas : schemas.map(transformResponseSchema)) } -function validateSuccessResponse(schemas: ReadonlyArray, method: HttpMethod) { +function validateSuccessResponse(schemas: ReadonlyArray, method: HttpMethod, hasPayload: boolean) { + if (schemas.some(HttpApiSchema.isUpgrade)) { + if (schemas.length > 1) { + throw new Error("Cannot combine an upgrade with other success responses") + } + if (method === "HEAD") { + throw new Error("HEAD endpoints cannot declare an upgrade") + } + if (hasPayload) { + throw new Error("Upgrade endpoints cannot declare a payload") + } + return + } let hasStream = false const statuses = new Map { + readonly [HttpApiPayloadTooLargeTypeId]: HttpApiPayloadTooLargeTypeId = HttpApiPayloadTooLargeTypeId + readonly [ErrorReporter.ignore] = true + + static is(u: unknown): u is HttpApiPayloadTooLarge { + return hasProperty(u, HttpApiPayloadTooLargeTypeId) + } + + readonly name = "HttpApiPayloadTooLarge" + readonly message = `Payload exceeds ${this.limit} of ${this.size} bytes`; + + [HttpServerRespondable.symbol]() { + return Effect.succeed(payloadTooLargeResponse) + } +} diff --git a/repos/effect/packages/effect/src/http-api/HttpApiScalar.ts b/repos/effect/packages/effect/src/http-api/HttpApiScalar.ts index eb585fa801..a8666cd4b2 100644 --- a/repos/effect/packages/effect/src/http-api/HttpApiScalar.ts +++ b/repos/effect/packages/effect/src/http-api/HttpApiScalar.ts @@ -17,7 +17,6 @@ import type * as Layer from "../Layer.ts" import type * as HttpApi from "./HttpApi.ts" import type * as HttpApiGroup from "./HttpApiGroup.ts" import * as Html from "./internal/html.ts" -import * as internal from "./internal/httpApiScalar.ts" import * as OpenApi from "./OpenApi.ts" /** @@ -212,7 +211,8 @@ const makeHandler = ( * **Details** * * The route serves the OpenAPI specification generated from the API at the - * configured path, defaulting to `/docs`. + * configured path, defaulting to `/docs`. The bundled script is loaded when + * the layer is built, not when this module is imported. * * @stability unstable * @category layers @@ -226,11 +226,12 @@ export const layer = } | undefined ): Layer.Layer => HttpRouter.use(Effect.fnUntraced(function*(router) { + const { javascript } = yield* Effect.promise(() => import("./internal/httpApiScalar.ts")) const handler = makeHandler({ api, source: { _tag: "Inline", - source: internal.javascript + source: javascript }, scalar: options?.scalar }) diff --git a/repos/effect/packages/effect/src/http-api/HttpApiSchema.ts b/repos/effect/packages/effect/src/http-api/HttpApiSchema.ts index 617a816094..129c8ee712 100644 --- a/repos/effect/packages/effect/src/http-api/HttpApiSchema.ts +++ b/repos/effect/packages/effect/src/http-api/HttpApiSchema.ts @@ -10,6 +10,7 @@ * @stability unstable * @since 4.0.0 */ +import * as ByteSize from "../ByteSize.ts" import * as Sse from "../encoding/Sse.ts" import { constVoid, type LazyArg } from "../Function.ts" import { hasBody, type HttpMethod } from "../http/HttpMethod.ts" @@ -19,6 +20,7 @@ import * as Predicate from "../Predicate.ts" import * as Schema from "../Schema.ts" import * as SchemaAST from "../SchemaAST.ts" import * as SchemaTransformation from "../SchemaTransformation.ts" +import * as Socket from "../socket/Socket.ts" import * as Stream from "../Stream.ts" declare module "../Schema.ts" { @@ -120,8 +122,12 @@ export type StatusLiteral = HttpStatus.Literal * * **Details** * - * This is equivalent to calling `.annotate({ httpApiStatus: code })` on the - * schema. You can pass either a numeric status code (for example, `201`) or a + * The status is stored with `.annotateKey({ httpApiStatus: code })`, so a named + * schema reused with different statuses still emits a single OpenAPI component. + * If the schema has checks or already has an `httpApiStatus` annotation, it is + * stored with `.annotate({ httpApiStatus: code })` instead. A later + * `.annotate({ httpApiStatus })` takes precedence, and a later check drops the + * status. You can pass either a numeric status code (for example, `201`) or a * common literal name (for example, `"Created"`). * * @stability unstable @@ -142,7 +148,10 @@ export function status(code: StatusLiteral): { */ export function status(code: number | StatusLiteral) { const statusCode = typeof code === "string" ? HttpStatus.fromLiteral(code) : code - return (self: S): S["Rebuild"] => self.annotate({ httpApiStatus: statusCode }) + return (self: S): S["Rebuild"] => + self.ast.checks !== undefined || hasStatusAnnotation(self.ast) + ? self.annotate({ httpApiStatus: statusCode }) + : self.annotateKey({ httpApiStatus: statusCode }) } /** @@ -251,7 +260,7 @@ export function asNoContent(options: { } } -type StreamMode = "sse" | "uint8array" +type StreamMode = "sse" | "uint8array" | "ndjson" /** * Mode describing whether an SSE stream emits full events or raw data values. @@ -282,6 +291,9 @@ export type StreamSseMode = "events" | "data" * application event. Endpoint construction rejects event schemas that declare * the reserved name statically. * + * `StreamSse` is response-only. The `payload` slot's type excludes it, and + * endpoint construction rejects it. + * * @stability unstable * @category models * @since 4.0.0 @@ -339,13 +351,14 @@ export interface SseEventFromData extends {} /** - * Schema for a streaming `Uint8Array` success response. + * Schema for a streaming `Uint8Array` body. * * **Details** * - * This declaration stores the response content type for later endpoint, - * server, client, and OpenAPI integrations. It is intentionally separate from - * the buffered `asUint8Array` response encoding. + * It works as a success response or as a request payload. It is intentionally + * separate from the buffered `asUint8Array` encoding. As a payload, the handler + * receives the body as it is read, and `maxBytes` caps the bytes read. It has + * no default and does not fall back to `MaxBodySize`. * * @stability unstable * @category models @@ -366,22 +379,91 @@ export interface StreamUint8Array extends readonly _tag: "StreamUint8Array" readonly mode: "uint8array" readonly contentType: string + readonly maxBytes: ByteSize.ByteSize | undefined +} + +/** + * Schema for a stream of newline-delimited JSON elements. + * + * **Details** + * + * Each element is encoded as one line of JSON followed by `\n`, including the + * last. When decoding, `\n`, `\r\n` and a bare `\r` end a line, empty lines are + * skipped, a leading BOM is stripped and invalid UTF-8 fails. The first + * malformed line or element that fails to decode ends the stream, after every + * element before it has been delivered. A missing final newline is accepted if + * the last line parses. + * + * `maxElementBytes` caps the UTF-8 bytes of one line, without its terminator. + * `maxBytes` caps the whole body and has no default. + * + * As a request payload, the handler receives the elements as the body is + * read. Malformed lines and elements that fail to decode are an + * `HttpApiSchemaError`, and limit breaches are an `HttpApiPayloadTooLarge`. + * + * **Gotchas** + * + * There is no `error` schema. A failure mid-stream aborts the connection, so + * the reader sees a body read error, never a clean end. Typed failures + * mid-stream are a {@link StreamSse} feature. A body cut cleanly at a line + * boundary can't be told apart from a complete one. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export interface StreamNdjson extends + Schema.BottomLazy< + SchemaAST.Declaration, + StreamNdjson + > +{ + readonly "Type": Stream.Stream + readonly "Encoded": Stream.Stream + readonly "DecodingServices": S["DecodingServices"] + readonly "EncodingServices": S["EncodingServices"] + readonly "Rebuild": StreamNdjson + readonly "~type.make.in": Stream.Stream + readonly "~type.make": Stream.Stream + readonly "Iso": Stream.Stream + readonly [StreamSchemaTypeId]: typeof StreamSchemaTypeId + readonly _tag: "StreamNdjson" + readonly mode: "ndjson" + readonly contentType: string + readonly schema: S + readonly maxBytes: ByteSize.ByteSize | undefined + readonly maxElementBytes: ByteSize.ByteSize } /** - * Schema for a streaming HTTP API success response. + * Schema for a streaming HTTP API body. + * + * **Details** + * + * {@link StreamUint8Array} and {@link StreamNdjson} describe the body in either + * direction, with the endpoint slot deciding which. {@link StreamSse} is for + * success responses only. * * @stability unstable * @category models * @since 4.0.0 */ -export type StreamSchema = StreamSse | StreamUint8Array +export type StreamSchema = + | StreamSse + | StreamUint8Array + | StreamNdjson const streamSchema = Schema.declare(Stream.isStream) +const defaultMaxElementBytes = ByteSize.mebibytes(1) + /** * Creates a Server-Sent Events streaming success response schema. * + * **Details** + * + * `StreamSse` is response-only. Endpoint construction rejects it in `payload`. + * * @stability unstable * @category constructors * @since 4.0.0 @@ -422,7 +504,12 @@ export const StreamSse: { } /** - * Creates a streaming `Uint8Array` success response schema. + * Creates a streaming `Uint8Array` body schema. + * + * **Details** + * + * `contentType` defaults to `application/octet-stream`. `maxBytes` only applies + * to request payloads and has no default. * * @stability unstable * @category constructors @@ -430,12 +517,64 @@ export const StreamSse: { */ export const StreamUint8Array = (options?: { readonly contentType?: string | undefined + readonly maxBytes?: ByteSize.Input | undefined }): StreamUint8Array => Schema.make(streamSchema.ast, { [StreamSchemaTypeId]: StreamSchemaTypeId, _tag: "StreamUint8Array", mode: "uint8array", - contentType: options?.contentType ?? defaultStreamContentType("uint8array") + contentType: options?.contentType ?? defaultStreamContentType("uint8array"), + maxBytes: options?.maxBytes === undefined ? undefined : ByteSize.fromInputUnsafe(options.maxBytes) + }) + +/** + * Creates a schema for a stream of newline-delimited JSON elements. + * + * **Details** + * + * `contentType` defaults to `application/x-ndjson` and `maxElementBytes` to 1 + * MiB. `maxBytes` has no default. + * + * **Example** (Declaring an NDJSON request payload) + * + * ```ts import.meta.vitest + * import { Schema } from "effect" + * import { HttpApiEndpoint, HttpApiSchema } from "effect/http-api" + * + * const Event = Schema.Struct({ id: Schema.Number }) + * const payload = HttpApiSchema.StreamNdjson({ schema: Event, maxBytes: "100 MiB" }) + * + * // The handler receives a Stream of Events + * const importEvents = HttpApiEndpoint.post("importEvents", "/events", { + * payload, + * success: Schema.Number + * }) + * + * importEvents.path // => "/events" + * payload.contentType // => "application/x-ndjson" + * payload.maxElementBytes // => 1048576n + * ``` + * + * @stability unstable + * @category constructors + * @since 4.0.6 + */ +export const StreamNdjson = (options: { + readonly schema: S + readonly contentType?: string | undefined + readonly maxBytes?: ByteSize.Input | undefined + readonly maxElementBytes?: ByteSize.Input | undefined +}): StreamNdjson => + Schema.make>(streamSchema.ast, { + [StreamSchemaTypeId]: StreamSchemaTypeId, + _tag: "StreamNdjson", + mode: "ndjson", + contentType: options.contentType ?? defaultStreamContentType("ndjson"), + schema: options.schema, + maxBytes: options.maxBytes === undefined ? undefined : ByteSize.fromInputUnsafe(options.maxBytes), + maxElementBytes: options.maxElementBytes === undefined + ? defaultMaxElementBytes + : ByteSize.fromInputUnsafe(options.maxElementBytes) }) /** @@ -456,12 +595,109 @@ export const isStreamSse = (u: unknown): u is StreamSse isStreamSchema(u) && u._tag === "StreamUint8Array" +/** + * @internal + */ +export const isStreamNdjson = (u: unknown): u is StreamNdjson => + isStreamSchema(u) && u._tag === "StreamNdjson" + +const UpgradeTypeId = "~effect/http-api/HttpApiSchema/Upgrade" + +/** + * Schema for a success response that switches the connection to another + * protocol with an HTTP/1.1 upgrade. + * + * **Details** + * + * Created with {@link Upgrade}. The response is `101 Switching Protocols`, and + * the connection then carries raw bytes in both directions as a `Socket`. + * + * @stability unstable + * @category models + * @since 4.0.6 + */ +export interface Upgrade extends + Schema.Bottom< + Socket.Socket, + Socket.Socket, + never, + never, + SchemaAST.Declaration, + Upgrade + > +{ + readonly "Rebuild": Upgrade + readonly [UpgradeTypeId]: typeof UpgradeTypeId + readonly protocol: string +} + +const upgradeSchema = Schema.declare(Socket.isSocket).pipe(status(101)) + +/** + * Declares an endpoint success that upgrades the connection to `protocol`, + * such as Docker's `tcp` for container attach. + * + * **When to use** + * + * Use as the only success schema of an endpoint that hands over its + * connection as a raw byte stream after `101 Switching Protocols`. For + * WebSocket, upgrade the request with `HttpServerRequest.upgrade` and + * connect with `Socket.makeWebSocket` instead. + * + * **Details** + * + * The handler receives the connection as `socket`. The `101` response is + * sent when the handler acquires the socket's reader, so a handler can still + * fail with a declared error before then. Requests that do not ask for + * `protocol` get `426 Upgrade Required`. + * + * The client sends `Connection: Upgrade` and `Upgrade: protocol`, and returns + * the response headers with the connection. Other responses are decoded as + * the endpoint's errors. + * + * The endpoint cannot declare a payload or other success schemas. + * + * **Example** (Declaring an attach endpoint) + * + * ```ts + * import { Schema } from "effect" + * import { HttpApiEndpoint, HttpApiSchema } from "effect/http-api" + * + * const attach = HttpApiEndpoint.post("attach", "/containers/:id/attach", { + * params: { id: Schema.String }, + * success: HttpApiSchema.Upgrade({ protocol: "tcp" }) + * }) + * ``` + * + * @stability unstable + * @category constructors + * @since 4.0.6 + */ +export const Upgrade = (options: { + readonly protocol: string +}): Upgrade => { + if (options.protocol.toLowerCase() === "websocket") { + throw new Error("Upgrade does not support WebSocket") + } + return Schema.make(upgradeSchema.ast, { + [UpgradeTypeId]: UpgradeTypeId, + protocol: options.protocol + }) +} + +/** + * @internal + */ +export const isUpgrade = (u: unknown): u is Upgrade => Schema.isSchema(u) && Predicate.hasProperty(u, UpgradeTypeId) + function defaultStreamContentType(mode: StreamMode): string { switch (mode) { case "sse": return "text/event-stream" case "uint8array": return "application/octet-stream" + case "ndjson": + return "application/x-ndjson" } } @@ -627,6 +863,9 @@ export function WithHeaders( if (isWithHeaders(schema)) { throw new Error("WithHeaders schemas cannot be nested") } + if (isUpgrade(schema)) { + throw new Error("WithHeaders cannot wrap an upgrade") + } return Schema.make>(withHeadersValueSchema.ast, { [WithHeadersTypeId]: WithHeadersTypeId, schema, @@ -791,6 +1030,9 @@ export function encodeToWithHeaders< }) { return (self: S): encodeToWithHeaders => { const body = options.body + if (isUpgrade(body)) { + throw new Error("encodeToWithHeaders cannot wrap an upgrade") + } const headers = Schema.Struct(options.headers) const status = resolveHttpApiStatus(body.ast) const encoding = resolveHttpApiEncoding(body.ast) @@ -1031,7 +1273,19 @@ const resolveHttpApiEncoding = SchemaAST.resolveAt("~httpApiEncoding") */ export const getWithHeadersAnnotation = SchemaAST.resolveAt("~httpApiWithHeaders") -const resolveHttpApiStatus = SchemaAST.resolveAt("httpApiStatus") +const resolveHttpApiStatusAnnotation = SchemaAST.resolveAt("httpApiStatus") + +function hasStatusAnnotation(ast: SchemaAST.AST): boolean { + const annotations = SchemaAST.resolve(ast) + return annotations !== undefined && "httpApiStatus" in annotations +} + +// An `httpApiStatus` annotation, even `undefined`, overrides the key context set by +// `status`, which a later check drops, matching annotations +function resolveHttpApiStatus(ast: SchemaAST.AST): number | undefined { + if (hasStatusAnnotation(ast)) return resolveHttpApiStatusAnnotation(ast) + return ast.checks === undefined ? ast.context?.annotations?.httpApiStatus : undefined +} const defaultJsonEncoding: Encoding = { _tag: "Json", diff --git a/repos/effect/packages/effect/src/http-api/HttpApiSwagger.ts b/repos/effect/packages/effect/src/http-api/HttpApiSwagger.ts index d25913a8c4..6a694e4b90 100644 --- a/repos/effect/packages/effect/src/http-api/HttpApiSwagger.ts +++ b/repos/effect/packages/effect/src/http-api/HttpApiSwagger.ts @@ -17,11 +17,12 @@ import type * as Layer from "../Layer.ts" import type * as HttpApi from "./HttpApi.ts" import type * as HttpApiGroup from "./HttpApiGroup.ts" import * as Html from "./internal/html.ts" -import * as internal from "./internal/httpApiSwagger.ts" import * as OpenApi from "./OpenApi.ts" const makeHandler = (options: { readonly api: HttpApi.HttpApi + readonly javascript: string + readonly css: string }) => { const makeResponse = Function.memoize((api: HttpApi.HttpApi) => { const spec = OpenApi.fromApi(api) @@ -31,7 +32,7 @@ const makeHandler = ( ${Html.escape(spec.info.title)} Documentation - +
@@ -39,7 +40,7 @@ const makeHandler = ( ${Html.escapeJson(spec)}