feat: Add a Dart harness

.github/workflows/compatibility.yml

@@ -82,6 +82,11 @@ jobs:
             ref: feat/sdk-spec
             language: kotlin
             tags: ''
+          - name: flutter
+            repository: Flagsmith/flagsmith-flutter-client
+            ref: feat/sdk-spec
+            language: dart
+            tags: ''
     uses: ./.github/workflows/run-spec.yml
     with:
       spec-repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}

HARNESS.md

@@ -6,6 +6,7 @@ A harness runs an SDK's steps file for the spec runner. There is one per languag
 | Language | Harness                            |
 | -------- | ---------------------------------- |
 | .NET     | [`harness/dotnet`](harness/dotnet) |
+| Dart     | [`harness/dart`](harness/dart)     |
 | Elixir   | [`harness/elixir`](harness/elixir) |
 | Go       | [`harness/go`](harness/go)         |
 | Java     | [`harness/java`](harness/java)     |

harness/dart/README.md

@@ -0,0 +1,84 @@
+# Dart harness
+
+Runs a Dart or Flutter SDK's [steps file](../../README.md#onboarding-an-sdk) for the spec runner, over the
+[stdio protocol](../../HARNESS.md#protocol).
+
+Dart cannot call a function by name at runtime, so the harness generates an entrypoint for each steps
+file. `build.dart` reads `sdk-spec/steps.dart`, finds each step-marker comment and the first top-level
+function after it, and writes an entrypoint that passes those functions to `harness.dart`, keyed by
+step text. At runtime, `harness.dart` calls them with `Function.apply`.
+
+## Steps file
+
+- Step functions are public, top-level functions in `sdk-spec/steps.dart`, each below its step marker
+  (`// Create the SDK client`, `// When …`). Import the SDK by its package name.
+- `Create the SDK client` returns the client. `When` steps take it first.
+- Arguments arrive as decoded JSON: `String`, `int`, `double`, `bool`, `null`, and data tables as
+  `Map<String, dynamic>`. Declare them with those types, e.g. `Map<String, Object?> traits`.
+- Return a JSON-encodable value, or a `Future` of one.
+- Let SDK errors propagate. They are reported as `"<type>: <message>"`, with the stack trace on stderr.
+- `print` output goes to stderr, so the SDK may log freely.
+
+```dart
+// When the application gets the environment flags
+Future<List<Map<String, Object?>>> getEnvironmentFlags(FlagsmithClient flagsmith) async =>
+    asSpecFlags(await flagsmith.getFeatureFlags());
+```
+
+## Dockerfile
+
+Generate the entrypoint inside the SDK's package, so that the steps file's `package:` imports resolve,
+then compile it.
+
+For a **Flutter** package, which needs `dart:ui`, build a kernel with `flutter build bundle` and run
+it on `flutter_tester`, the headless engine `flutter test` uses:
+
+```dockerfile
+FROM ghcr.io/cirruslabs/flutter:3.44.0
+WORKDIR /sdk
+COPY . .
+COPY --from=harness . /harness
+RUN flutter pub get \
+    && dart /harness/build.dart -o sdk-spec/harness_main.dart sdk-spec/steps.dart \
+    && flutter build bundle --debug --target sdk-spec/harness_main.dart \
+    && ln -s "$(find "$(dirname "$(readlink -f "$(command -v flutter)")")/cache/artifacts/engine" \
+        -path '*/linux-*/flutter_tester' -print -quit)" /usr/local/bin/flutter_tester
+ENTRYPOINT ["flutter_tester", "--non-interactive", "--disable-vm-service", "build/flutter_assets/kernel_blob.bin"]
+```
+
+For a **Dart** package, compile the entrypoint to an executable:
+
+```dockerfile
+FROM docker.io/library/dart:stable AS build
+WORKDIR /sdk
+COPY . .
+COPY --from=harness . /harness
+RUN dart pub get \
+    && dart /harness/build.dart -o sdk-spec/harness_main.dart sdk-spec/steps.dart \
+    && mkdir /out \
+    && dart compile exe sdk-spec/harness_main.dart -o /out/harness
+
+FROM scratch
+# The dart image provides /runtime/: the minimal files an AOT-compiled executable needs.
+COPY --from=build /runtime/ /
+COPY --from=build /out/harness /harness
+ENTRYPOINT ["/harness"]
+```
+
+`build.dart` prints the steps it found to stderr, and fails if a marked function is private.
+
+## Running locally
+
+Generate and compile the entrypoint in the SDK checkout, then point the runner at it:
+
+```sh
+cd <sdk checkout>
+dart <spec>/harness/dart/build.dart -o sdk-spec/harness_main.dart sdk-spec/steps.dart
+flutter build bundle --debug --target sdk-spec/harness_main.dart
+
+cd <spec>/runner
+SDK_COMMAND="$(dirname "$(readlink -f "$(command -v flutter)")")/cache/artifacts/engine/darwin-x64/flutter_tester \
+  --non-interactive --disable-vm-service <sdk checkout>/build/flutter_assets/kernel_blob.bin" uv run pytest -v
+```
+
+For a Dart package, use `SDK_COMMAND="dart run <sdk checkout>/sdk-spec/harness_main.dart"` instead.

harness/dart/build.dart

@@ -0,0 +1,71 @@
+// Generates the Dart harness entrypoint for a steps file. See README.md.
+//
+// Usage: dart build.dart -o path/to/harness_main.dart path/to/steps.dart
+//
+// Dart cannot call a function by name at runtime, so this finds the marked functions in the steps
+// file and generates an entrypoint that passes them to harness.dart. Compile or run the entrypoint
+// in the SDK's package, so that the steps file's `package:` imports resolve.
+import 'dart:convert';
+import 'dart:io';
+
+final _marker = RegExp(r'^\s*//+\s*(When .+?|Create the SDK client)\s*$');
+// A top-level function declaration: an optional return type, then the name and its parameters.
+final _function = RegExp(
+  r'^(?:[\w$][\w$<>?,.\[\] ]*\s)?([A-Za-z_$][\w$]*)\s*(?:<[^()]*>)?\s*\(',
+);
+final _notFunction = RegExp(
+  r'^(?:import|export|part|library|class|mixin|enum|extension|typedef|final|const|var|late)\b',
+);
+
+void main(List<String> arguments) {
+  if (arguments.length != 3 || arguments[0] != '-o') {
+    stderr.writeln('usage: dart build.dart -o <entrypoint> <steps file>');
+    exit(64);
+  }
+  final entrypoint = File(arguments[1]).absolute;
+  final stepsFile = File(arguments[2]).absolute;
+
+  final steps = findSteps(stepsFile);
+  final harness = Platform.script.resolve('harness.dart');
+  entrypoint.writeAsStringSync('''
+// Generated by build.dart from ${stepsFile.path}. Do not edit.
+import ${_quote('$harness')} as harness;
+import ${_quote('${stepsFile.uri}')} as steps;
+
+void main() => harness.run({
+${steps.entries.map((s) => '  ${_quote(s.key)}: steps.${s.value},').join('\n')}
+});
+''');
+  for (final MapEntry(key: text, value: function) in steps.entries) {
+    stderr.writeln('$text -> $function');
+  }
+}
+
+/// Returns the steps marked in the file, in order: a marker comment marks the first top-level
+/// function after it.
+Map<String, String> findSteps(File file) {
+  final steps = <String, String>{};
+  String? pending;
+  final lines = file.readAsLinesSync();
+  for (final (index, line) in lines.indexed) {
+    if (_marker.firstMatch(line) case final marker?) {
+      pending = marker[1];
+    } else if (pending != null && !_notFunction.hasMatch(line)) {
+      if (_function.firstMatch(line) case final function?) {
+        final name = function[1]!;
+        if (name.startsWith('_')) {
+          stderr.writeln(
+            '${file.path}:${index + 1}: step "$pending" must be a public function, '
+            'not $name',
+          );
+          exit(1);
+        }
+        steps[pending] = name;
+        pending = null;
+      }
+    }
+  }
+  return steps;
+}
+
+String _quote(String value) => jsonEncode(value).replaceAll(r'$', r'\$');

harness/dart/harness.dart

@@ -0,0 +1,93 @@
+// Dart harness for the Flagsmith SDK behavioural spec. See HARNESS.md.
+//
+// Dart cannot look up a function by name at runtime, so build.dart generates an entrypoint that
+// passes the steps file's functions to `run`, keyed by step text in file order.
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+Future<void> run(Map<String, Function> steps) async {
+  // Keep stdout for responses: anything the SDK prints goes to stderr.
+  await runZoned(
+    () => _serve(steps),
+    zoneSpecification: ZoneSpecification(
+      print: (self, parent, zone, line) => stderr.writeln(line),
+    ),
+  );
+  // The SDK may leave timers running, so exit rather than wait for them.
+  exit(0);
+}
+
+Future<void> _serve(Map<String, Function> steps) async {
+  final clients = <String, Object?>{};
+  final lines = stdin.transform(utf8.decoder).transform(const LineSplitter());
+  await for (final line in lines) {
+    if (line.trim().isEmpty) {
+      continue;
+    }
+    final request = jsonDecode(line) as Map<String, dynamic>;
+    stdout.writeln(jsonEncode(await _handle(request, steps, clients)));
+    await stdout.flush();
+  }
+}
+
+Future<Map<String, Object?>> _handle(
+  Map<String, dynamic> request,
+  Map<String, Function> steps,
+  Map<String, Object?> clients,
+) async {
+  final response = <String, Object?>{'id': request['id']};
+  if (request['command'] == 'hello') {
+    response['steps'] = steps.keys.toList();
+    return response;
+  }
+  final step = steps[request['step']];
+  if (step == null) {
+    response['missing'] = true;
+    return response;
+  }
+
+  try {
+    final args = [
+      if (request['client'] case final String client)
+        clients.containsKey(client)
+            ? clients[client]
+            : throw HarnessException('no client $client'),
+      ...request['args'] as List<dynamic>,
+    ];
+    var result = Function.apply(step, args);
+    if (result is Future) {
+      result = await result;
+    }
+    if (request['returnsClient'] == true) {
+      final handle = '${clients.length + 1}';
+      clients[handle] = result;
+      response['client'] = handle;
+    } else {
+      // Encode here, so a result that is not JSON is reported as the step's error.
+      response['result'] = jsonDecode(jsonEncode(result));
+    }
+  } catch (e, stackTrace) {
+    stderr.writeln('harness: ${request['step']}: $e\n$stackTrace');
+    response['error'] = _describe(e);
+  }
+  return response;
+}
+
+/// Formats an error as "<type>: <message>". Many Dart exceptions already start their
+/// `toString()` with the type, e.g. "FormatException: …", so the type is not repeated.
+String _describe(Object error) {
+  final type = '${error.runtimeType}';
+  final message = '$error';
+  return message.startsWith(type) ? message : '$type: $message';
+}
+
+/// Reports a step the harness could not call.
+class HarnessException implements Exception {
+  HarnessException(this.message);
+
+  final String message;
+
+  @override
+  String toString() => 'HarnessException: $message';
+}