8.3.1: stream application/jsonl, and refuse the deprecated application/stream+json

README.md

@@ -30,7 +30,7 @@ Maven and Gradle
   - [Usage](#usage)
   - [Calling an API from your service (callMode)](#calling-an-api-from-your-service-callmode)
   - [Object-typed query parameters and multipart bodies](#object-typed-query-parameters-and-multipart-bodies)
-  - [Streaming endpoints (application/x-ndjson)](#streaming-endpoints-applicationx-ndjson)
+  - [Streaming endpoints (NDJSON and JSON Lines)](#streaming-endpoints-ndjson-and-json-lines)
   - [Camel case Java names (useCamelCaseNames)](#camel-case-java-names-usecamelcasenames)
   - [Unknown enum values (useUnknownEnumValue)](#unknown-enum-values-useunknownenumvalue)
 - [Property Validation](#property-validation)
@@ -1093,12 +1093,14 @@ own: every generated API takes one argument per part (`MultipartFile file,
 String comment`). A multipart body that references a component schema keeps
 taking that model.
 
-### Streaming endpoints (application/x-ndjson)
+### Streaming endpoints (NDJSON and JSON Lines)
 
 An operation whose successful response, or whose request body, is
-`application/x-ndjson` (or the older `application/stream+json`) streams one JSON
+`application/x-ndjson` or `application/jsonl` (the same format) streams one JSON
 document per line. Its schema is the type of each item, or an array of it, and
-an OpenAPI 3.2 `itemSchema` is read the same way:
+an OpenAPI 3.2 `itemSchema` is read the same way. The deprecated
+`application/stream+json`, which Spring Framework 7 no longer streams, fails the
+generation:
 
 ```yaml
 responses:
@@ -1148,6 +1150,34 @@ for it.
   or a `Mono<ResponseEntity<Flux<ItemDTO>>>` from `...WithHttpInfo`, and send a
   streamed body from a `Flux<ItemDTO>`. **Reactive `@HttpExchange` interfaces**
   return a `Mono<ResponseEntity<Flux<ItemDTO>>>`.
+- Spring's codecs stream `application/x-ndjson` only. The generated
+  `ApiWebClient` registers `application/jsonl` too, but a WebFlux server or a
+  reactive `@HttpExchange` interface uses the application's codecs, which need
+  it registered for JSON Lines:
+
+  ```java
+  @Bean
+  CodecCustomizer jsonLinesCodecCustomizer(final JsonMapper mapper) {
+    final MediaType jsonLines = new MediaType("application", "jsonl");
+    return configurer -> {
+      final JacksonJsonEncoder encoder = new JacksonJsonEncoder(mapper,
+          MediaType.APPLICATION_JSON, MediaType.APPLICATION_NDJSON, jsonLines);
+      encoder.setStreamingMediaTypes(
+          List.of(MediaType.APPLICATION_NDJSON, jsonLines));
+      configurer.defaultCodecs().jacksonJsonEncoder(encoder);
+      configurer.defaultCodecs().jacksonJsonDecoder(new JacksonJsonDecoder(
+          mapper, MediaType.APPLICATION_JSON, MediaType.APPLICATION_NDJSON,
+          jsonLines));
+    };
+  }
+  ```
+
+  This is Spring Boot 4 (`org.springframework.boot.http.codec.CodecCustomizer`).
+  On Spring Boot 2 and 3 it is
+  `org.springframework.boot.web.codec.CodecCustomizer`, with an `ObjectMapper`
+  and `Jackson2JsonEncoder`/`Jackson2JsonDecoder` set through
+  `jackson2JsonEncoder`/`jackson2JsonDecoder`. The customizer applies to the
+  WebFlux server and to the WebClients built from Boot's `WebClient.Builder`.
 - The **blocking clients** (RestClient, and `@HttpExchange` without `reactive`)
   read a response whole, so they cannot hand the items over as they arrive, and
   the generation fails for a streaming operation: generate that client with

multiapi-engine/pom.xml

@@ -4,7 +4,7 @@
 
   <groupId>com.sngular</groupId>
   <artifactId>multiapi-engine</artifactId>
-  <version>8.3.0</version>
+  <version>8.3.1</version>
   <packaging>jar</packaging>
 
   <properties>

multiapi-engine/src/main/java/com/sngular/api/generator/plugin/openapi/utils/MapperPathUtil.java

@@ -61,10 +61,13 @@ public class MapperPathUtil {
   public static final String SCHEMA = "schema";
 
   /**
-   * The media types that stream a sequence of JSON documents, one per line, which Spring reads and writes item by item. Their
-   * schema is the type of each item, or an array of it.
+   * The media types that stream a sequence of JSON documents, one per line: NDJSON and JSON Lines, which are the same format.
+   * Their schema is the type of each item, or an array of it.
    */
-  private static final Set<String> STREAMING_MEDIA_TYPES = Set.of("application/x-ndjson", "application/stream+json");
+  private static final Set<String> STREAMING_MEDIA_TYPES = Set.of("application/x-ndjson", "application/jsonl");
+
+  /** Deprecated for application/x-ndjson, and no longer streamed by Spring Framework 7. */
+  private static final String STREAM_JSON_MEDIA_TYPE = "application/stream+json";
 
   private static final String JSON_MEDIA_TYPE = "application/json";
 
@@ -190,6 +193,10 @@ public static boolean isStreamingMediaType(final String mediaType) {
    * content.
    */
   private static void applyStreaming(final OperationObject operation, final SpecFile specFile) {
+    if (operation.getProduces().contains(STREAM_JSON_MEDIA_TYPE) || operation.getConsumes().contains(STREAM_JSON_MEDIA_TYPE)) {
+      throw new CodeGenerationException("Operation '" + operation.getOperationId() + "' declares " + STREAM_JSON_MEDIA_TYPE + ", which is deprecated and "
+                                        + "which Spring Framework 7 no longer streams: declare the stream as application/x-ndjson or application/jsonl");
+    }
     final Optional<ResponseObject> streamedResponse = operation.getResponseObjects().stream()
                                                               .filter(response -> StringUtils.startsWith(response.getResponseName(), "2"))
                                                               .filter(response -> response.getContentObjects().stream().anyMatch(ContentObject::isStreaming))
@@ -209,7 +216,7 @@ private static void applyStreaming(final OperationObject operation, final SpecFi
     }
     final boolean streams = operation.isStreamingResponse() || operation.getRequestObjects().stream().anyMatch(RequestObject::isStreaming);
     if (streams && specFile.isCallMode() && !specFile.isReactive()) {
-      throw new CodeGenerationException("Operation '" + operation.getOperationId() + "' streams application/x-ndjson, which the blocking clients cannot "
+      throw new CodeGenerationException("Operation '" + operation.getOperationId() + "' streams its items, which the blocking clients cannot "
                                         + "consume: generate its client with reactive=true (a WebClient client or a reactive @HttpExchange interface)");
     }
   }

multiapi-engine/src/main/resources/templates/openapi/template.ftlh

@@ -74,7 +74,7 @@
     produces = {"${operation.streamingMediaType}"}${consumesStream}
   )
   default ResponseEntity<StreamingResponseBody> ${operation.operationId}Ndjson(${streamingEndpointArgs(operation, path, true, true)})${throwsIo} {
-    return NdjsonSupport.stream(${streamingMethodCall(operation, path)}, servletRequest, servletResponse);
+    return NdjsonSupport.stream(${streamingMethodCall(operation, path)}, "${operation.streamingMediaType}", servletRequest, servletResponse);
   }
 
   <#if operation.streamingAlsoJson>

multiapi-engine/src/main/resources/templates/openapi/templateNdjsonSupport.ftlh

@@ -25,31 +25,29 @@ import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBo
 import org.springframework.web.servlet.support.RequestContextUtils;
 
 /**
- * Sends and reads application/x-ndjson, one JSON document per line, for the API interfaces of this package. Streamed
+ * Sends and reads application/x-ndjson and application/jsonl, one JSON document per line, for the API interfaces of this package. Streamed
  * responses are written by Spring MVC on its asynchronous executor, each item sent to the client as soon as it is produced,
  * and the stream the operation returns is closed once it is sent, also when sending fails. Items are serialized with the
  * application's ObjectMapper.
  */
 public final class NdjsonSupport {
 
-  private static final MediaType APPLICATION_NDJSON = MediaType.parseMediaType("application/x-ndjson");
-
   private NdjsonSupport() {
   }
 
   /**
-   * The response streaming the items of the given one, with its status and headers. A response without items, such as the
-   * NOT_IMPLEMENTED of an operation that is not implemented, is sent without a body.
+   * The response streaming the items of the given one as the given media type, with its status and headers. A response
+   * without items, such as the NOT_IMPLEMENTED of an operation that is not implemented, is sent without a body.
    */
-  public static <T> ResponseEntity<StreamingResponseBody> stream(final ResponseEntity<Stream<T>> response, final HttpServletRequest servletRequest,
-      final HttpServletResponse servletResponse) {
+  public static <T> ResponseEntity<StreamingResponseBody> stream(final ResponseEntity<Stream<T>> response, final String mediaType,
+      final HttpServletRequest servletRequest, final HttpServletResponse servletResponse) {
     final ResponseEntity.BodyBuilder builder = ResponseEntity.status(response.getStatusCode()).headers(response.getHeaders());
     final Stream<T> items = response.getBody();
     if (items == null) {
       return builder.build();
     }
     final ObjectMapper mapper = objectMapper(servletRequest);
-    return builder.contentType(APPLICATION_NDJSON).body(output -> write(items, mapper, output, servletResponse));
+    return builder.contentType(MediaType.parseMediaType(mediaType)).body(output -> write(items, mapper, output, servletResponse));
   }
 
   /** The response sending the items of the given one as a JSON array, with its status and headers. */

multiapi-engine/src/main/resources/templates/openapi/templateWebClient.ftlh

@@ -6,6 +6,7 @@ import java.text.ParsePosition;
 import java.time.OffsetDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
@@ -158,17 +159,22 @@ public class ApiWebClient {
   }
 
   private static WebClient buildWebClient(final ${mapperType} mapper) {
-    // application/x-ndjson is read and written with the same mapper, one item per line.
+    // application/x-ndjson and application/jsonl are read and written with the same mapper, one item per line.
     final MediaType ndjson = new MediaType("application", "x-ndjson");
+    final MediaType jsonLines = new MediaType("application", "jsonl");
     ExchangeStrategies strategies = ExchangeStrategies
       .builder()
       .codecs(clientDefaultCodecsConfigurer -> {
 <#if isJackson3>
-        clientDefaultCodecsConfigurer.defaultCodecs().jacksonJsonEncoder(new JacksonJsonEncoder(mapper, MediaType.APPLICATION_JSON, ndjson));
-        clientDefaultCodecsConfigurer.defaultCodecs().jacksonJsonDecoder(new JacksonJsonDecoder(mapper, MediaType.APPLICATION_JSON, ndjson));
+        final JacksonJsonEncoder encoder = new JacksonJsonEncoder(mapper, MediaType.APPLICATION_JSON, ndjson, jsonLines);
+        encoder.setStreamingMediaTypes(Arrays.asList(ndjson, jsonLines));
+        clientDefaultCodecsConfigurer.defaultCodecs().jacksonJsonEncoder(encoder);
+        clientDefaultCodecsConfigurer.defaultCodecs().jacksonJsonDecoder(new JacksonJsonDecoder(mapper, MediaType.APPLICATION_JSON, ndjson, jsonLines));
 <#else>
-        clientDefaultCodecsConfigurer.defaultCodecs().jackson2JsonEncoder(new Jackson2JsonEncoder(mapper, MediaType.APPLICATION_JSON, ndjson));
-        clientDefaultCodecsConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(mapper, MediaType.APPLICATION_JSON, ndjson));
+        final Jackson2JsonEncoder encoder = new Jackson2JsonEncoder(mapper, MediaType.APPLICATION_JSON, ndjson, jsonLines);
+        encoder.setStreamingMediaTypes(Arrays.asList(ndjson, jsonLines));
+        clientDefaultCodecsConfigurer.defaultCodecs().jackson2JsonEncoder(encoder);
+        clientDefaultCodecsConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(mapper, MediaType.APPLICATION_JSON, ndjson, jsonLines));
 </#if>
       }).build();
     WebClient.Builder webClientBuilder = WebClient.builder().exchangeStrategies(strategies);

multiapi-engine/src/test/java/com/sngular/api/generator/plugin/openapi/OpenApiGeneratorFixtures.java

@@ -367,6 +367,12 @@ public final class OpenApiGeneratorFixtures {
 					.modelPackage("com.sngular.multifileplugin.ndjsonmixedrequest.model")
 					.modelNameSuffix("DTO").useTagsGroup(true).build());
 
+	static final List<SpecFile> TEST_NDJSON_STREAMING_STREAM_JSON = List
+			.of(SpecFile.builder().filePath("openapigenerator/testNdjsonStreaming/api-stream-json.yml")
+					.apiPackage("com.sngular.multifileplugin.ndjsonstreamjson")
+					.modelPackage("com.sngular.multifileplugin.ndjsonstreamjson.model")
+					.modelNameSuffix("DTO").useTagsGroup(true).build());
+
 	static final List<SpecFile> TEST_HTTP_EXCHANGE_CLIENT = List
 			.of(SpecFile.builder().filePath("openapigenerator/testHttpExchangeClient/api-test.yml")
 					.apiPackage("com.sngular.multifileplugin.httpexchange")

multiapi-engine/src/test/java/com/sngular/api/generator/plugin/openapi/OpenApiGeneratorTest.java

@@ -275,6 +275,14 @@ void testNdjsonStreamingJsonAlternativeMustListTheItems() {
         .hasMessageContaining("must be the array of them");
   }
 
+  @Test
+  void testDeprecatedStreamJsonIsRefused() {
+    Assertions.assertThatThrownBy(() -> openApiGenerator.processFileSpec(OpenApiGeneratorFixtures.TEST_NDJSON_STREAMING_STREAM_JSON))
+        .isInstanceOf(CodeGenerationException.class)
+        .hasMessageContaining("listItems")
+        .hasMessageContaining("application/stream+json");
+  }
+
   @Test
   void testNdjsonStreamedBodyMustBeItsOnlyContent() {
     Assertions.assertThatThrownBy(() -> openApiGenerator.processFileSpec(OpenApiGeneratorFixtures.TEST_NDJSON_STREAMING_MIXED_REQUEST))

multiapi-engine/src/test/resources/openapigenerator/testNdjsonStreaming/api-stream-json.yml

@@ -0,0 +1,23 @@
+openapi: "3.0.3"
+info:
+  title: A stream declared as the deprecated application/stream+json
+  version: 1.0.0
+paths:
+  /items:
+    get:
+      tags: [items]
+      operationId: listItems
+      responses:
+        '200':
+          description: One item per line
+          content:
+            application/stream+json:
+              schema:
+                $ref: "#/components/schemas/Item"
+components:
+  schemas:
+    Item:
+      type: object
+      properties:
+        id:
+          type: string

multiapi-engine/src/test/resources/openapigenerator/testNdjsonStreaming/api-test.yml

@@ -26,6 +26,33 @@ paths:
             application/json:
               schema:
                 $ref: "#/components/schemas/Error"
+  /items/lines:
+    get:
+      tags: [items]
+      operationId: listItemLines
+      responses:
+        '200':
+          description: One item per line, as JSON Lines
+          content:
+            application/jsonl:
+              schema:
+                $ref: "#/components/schemas/Item"
+    post:
+      tags: [items]
+      operationId: importItemLines
+      requestBody:
+        required: true
+        content:
+          application/jsonl:
+            schema:
+              $ref: "#/components/schemas/Item"
+      responses:
+        '200':
+          description: Import summary
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ImportResult"
   /items/export:
     get:
       tags: [items]

multiapi-engine/src/test/resources/openapigenerator/testNdjsonStreaming/assets/httpexchange/ItemsApi.java

@@ -42,6 +42,21 @@ public interface ItemsApi {
   @GetExchange(url = "/items", accept = {"application/x-ndjson"})
   Mono<ResponseEntity<Flux<ItemDTO>>> listItems(@RequestParam(name = "filter", required = false) String filter);
 
+  /**
+   * GET /items/lines
+   * @return One item per line, as JSON Lines (status code 200);
+   */
+  @GetExchange(url = "/items/lines", accept = {"application/jsonl"})
+  Mono<ResponseEntity<Flux<ItemDTO>>> listItemLines();
+
+  /**
+   * POST /items/lines
+   * @param itemDTO  (required)
+   * @return Import summary (status code 200);
+   */
+  @PostExchange(url = "/items/lines", accept = {"application/json"}, contentType = "application/jsonl")
+  Mono<ResponseEntity<ImportResultDTO>> importItemLines(@RequestBody(required = true) Flux<ItemDTO> itemDTO);
+
   /**
    * GET /items/export
    * @return The items, as a JSON array or one per line (status code 200);

multiapi-engine/src/test/resources/openapigenerator/testNdjsonStreaming/assets/imperative-boot3/EventsApi.java

@@ -40,7 +40,7 @@ public interface EventsApi {
     produces = {"application/x-ndjson"}
   )
   default ResponseEntity<StreamingResponseBody> streamEventsNdjson(@Parameter(hidden = true) final HttpServletRequest servletRequest, @Parameter(hidden = true) final HttpServletResponse servletResponse) {
-    return NdjsonSupport.stream(streamEvents(), servletRequest, servletResponse);
+    return NdjsonSupport.stream(streamEvents(), "application/x-ndjson", servletRequest, servletResponse);
   }
 
   /**

multiapi-engine/src/test/resources/openapigenerator/testNdjsonStreaming/assets/imperative-boot3/ItemsApi.java

@@ -45,7 +45,7 @@ public interface ItemsApi {
     produces = {"application/x-ndjson"}
   )
   default ResponseEntity<StreamingResponseBody> listItemsNdjson(@Parameter(name = "filter", required = false, schema = @Schema(description = "")) @RequestParam(name = "filter", required = false) String filter, @Parameter(hidden = true) final HttpServletRequest servletRequest, @Parameter(hidden = true) final HttpServletResponse servletResponse) {
-    return NdjsonSupport.stream(listItems(filter), servletRequest, servletResponse);
+    return NdjsonSupport.stream(listItems(filter), "application/x-ndjson", servletRequest, servletResponse);
   }
 
   /**
@@ -55,6 +55,59 @@ default ResponseEntity<Stream<ItemDTO>> listItems(final String filter) {
     return new ResponseEntity<>(HttpStatus.NOT_IMPLEMENTED);
   }
 
+  /**
+   * GET /items/lines, sending the items {@link #listItemLines} returns as application/jsonl, one JSON document per line.
+   */
+  @Operation(
+    operationId = "listItemLines",
+    tags = {"items"},
+    responses = {
+      @ApiResponse(responseCode = "200", description = "One item per line, as JSON Lines", content = @Content(mediaType = "application/jsonl", schema = @Schema(implementation = ItemDTO.class)))
+    }
+  )
+  @RequestMapping(
+    method = RequestMethod.GET,
+    value = "/items/lines",
+    produces = {"application/jsonl"}
+  )
+  default ResponseEntity<StreamingResponseBody> listItemLinesNdjson(@Parameter(hidden = true) final HttpServletRequest servletRequest, @Parameter(hidden = true) final HttpServletResponse servletResponse) {
+    return NdjsonSupport.stream(listItemLines(), "application/jsonl", servletRequest, servletResponse);
+  }
+
+  /**
+   * GET /items/lines. Implement this method rather than the endpoint above: the items are sent as they are produced, and the stream is closed once they are sent.
+   */
+  default ResponseEntity<Stream<ItemDTO>> listItemLines() {
+    return new ResponseEntity<>(HttpStatus.NOT_IMPLEMENTED);
+  }
+
+  /**
+   * POST /items/lines, reading the application/jsonl body, one JSON document per line, as the items {@link #importItemLines} receives.
+   */
+  @Operation(
+    operationId = "importItemLines",
+    tags = {"items"},
+    responses = {
+      @ApiResponse(responseCode = "200", description = "Import summary", content = @Content(mediaType = "application/json", schema = @Schema(implementation = ImportResultDTO.class)))
+    }
+  )
+  @RequestMapping(
+    method = RequestMethod.POST,
+    value = "/items/lines",
+    produces = {"application/json"},
+    consumes = {"application/jsonl"}
+  )
+  default ResponseEntity<ImportResultDTO> importItemLinesNdjson(@Parameter(hidden = true) final HttpServletRequest servletRequest) throws IOException {
+    return importItemLines(NdjsonSupport.read(servletRequest, new TypeReference<ItemDTO>() {}));
+  }
+
+  /**
+   * POST /items/lines. Implement this method rather than the endpoint above: the body items are read as the stream is consumed.
+   */
+  default ResponseEntity<ImportResultDTO> importItemLines(final Stream<ItemDTO> itemDTO) {
+    return new ResponseEntity<>(HttpStatus.NOT_IMPLEMENTED);
+  }
+
   /**
    * GET /items/export, sending the items {@link #exportItems} returns as application/x-ndjson, one JSON document per line.
    */
@@ -71,7 +124,7 @@ default ResponseEntity<Stream<ItemDTO>> listItems(final String filter) {
     produces = {"application/x-ndjson"}
   )
   default ResponseEntity<StreamingResponseBody> exportItemsNdjson(@Parameter(hidden = true) final HttpServletRequest servletRequest, @Parameter(hidden = true) final HttpServletResponse servletResponse) {
-    return NdjsonSupport.stream(exportItems(), servletRequest, servletResponse);
+    return NdjsonSupport.stream(exportItems(), "application/x-ndjson", servletRequest, servletResponse);
   }
 
   /**