From 078e6648916e82ec811b4974a0c90e55111da635 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 11 Aug 2026 17:05:21 +0200 Subject: [PATCH 1/2] fix(client): page explicit LIMITs above index.max_result_window through scroll (#224) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A SELECT with an explicit LIMIT above the index's max_result_window (ES default 10,000) failed outright, while the same query with NO LIMIT succeeded and returned every row: the one-shot search path issued a single search with size = LIMIT, which Elasticsearch rejects whenever from + size exceeds the window. The failure reached the caller as an opaque error naming neither LIMIT nor max_result_window. Routing (defect 1): extend #209's scroll routing in SearchApi.search / searchAsync — a row-shaped query whose LIMIT window (offset + limit) exceeds the ES default window (SearchApi.DefaultMaxResultWindow, 10,000) now pages through scroll bounded by maxDocuments = offset + limit, with the first offset rows dropped client-side and the statement's LIMIT stripped before translation (scroll contexts reject "from"). The per-index setting is deliberately not probed: an index tuned higher just pages (still correct, and fast post-#197); aggregation-shaped queries are never routed (their result is the aggregation itself). The licensed cap is unaffected — CoreDqlExtension rule (1) still 402-rejects an explicit LIMIT above quota before execution, so the bounded scroll can never exceed a vetted limit. Error opacity (defect 2), for the residual one-shot rejections (index tuned BELOW the threshold; UNION legs): - core: translate a max_result_window rejection into an actionable message naming LIMIT/OFFSET and the remedies, scanning message, cause chain and suppressed exceptions (ES 6/7 RHLC nests the per-shard root cause as suppressed on "all shards failed"); - core: singleSearchAsync / multiSearchAsync now recover a FAILED future into an ElasticFailure — previously the raw Throwable propagated to consumers, which flattened it into a generic error; - es8/es9: extract rootCause / causedBy reasons from the typed ErrorCause tree (invisible outside the module), and recover the async search paths through the same extraction. New LimitCompletenessSpec (testkit + 5 client subclasses): 12,000-doc 3-shard index asserting the asymmetry directly — LIMIT 11000 and no-LIMIT both complete, ORDER BY + OFFSET stays exact through the routing, LIMIT 10000 stays one-shot (boundary), and a window-lowered index yields an error naming max_result_window. Green on real ES 6.8 (rest + jest) / 7.17 / 8.18 / 9.0; sibling completeness guards (#197/#207/#209) and the 737 core unit tests stay green. Closes #224 Co-Authored-By: Claude Fable 5 --- .../elastic/client/SearchApi.scala | 364 ++++++++++++------ .../JestClientLimitCompletenessSpec.scala | 19 + ...HighLevelClientLimitCompletenessSpec.scala | 19 + ...HighLevelClientLimitCompletenessSpec.scala | 19 + .../elastic/client/java/JavaClientApi.scala | 14 +- .../client/java/JavaClientHelpers.scala | 76 +++- .../JavaClientLimitCompletenessSpec.scala | 19 + .../elastic/client/java/JavaClientApi.scala | 14 +- .../client/java/JavaClientHelpers.scala | 76 +++- .../JavaClientLimitCompletenessSpec.scala | 19 + .../client/LimitCompletenessSpec.scala | 225 +++++++++++ 11 files changed, 716 insertions(+), 148 deletions(-) create mode 100644 es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientLimitCompletenessSpec.scala create mode 100644 es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientLimitCompletenessSpec.scala create mode 100644 es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientLimitCompletenessSpec.scala create mode 100644 es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientLimitCompletenessSpec.scala create mode 100644 es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientLimitCompletenessSpec.scala create mode 100644 testkit/src/main/scala/app/softnetwork/elastic/client/LimitCompletenessSpec.scala diff --git a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala index 0dd85e71..82a07741 100644 --- a/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala +++ b/core/src/main/scala/app/softnetwork/elastic/client/SearchApi.scala @@ -29,6 +29,7 @@ import app.softnetwork.elastic.sql.PainlessContextType import app.softnetwork.elastic.sql.function.aggregate.{PercentileAgg, RankingWindow} import app.softnetwork.elastic.sql.macros.SQLQueryMacros import app.softnetwork.elastic.sql.query.{ + Limit, MultiSearch, SQLAggregation, SearchStatement, @@ -114,11 +115,11 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { explodeNested = single.explodeNested ) this match { - case scrollApi: ScrollApi if single.limit.isEmpty && single.returnsRows => + case scrollApi: ScrollApi if single.returnsRows && requiresScrollPaging(single.limit) => // A row query is data-bound, not time-bound: every page request below // carries its own timeout, so the stream always terminates. Await.result( - scrollAllRows(scrollApi, single, elasticQuery), + scrollRows(scrollApi, single, elasticQuery), Duration.Inf ) case _ => @@ -285,7 +286,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { .getOrElse(query)}\nin indices '$indices' -> ${error.message}" ) ElasticResult.failure( - error.copy( + enrichMaxResultWindowError(error).copy( operation = Some("search"), index = Some(elasticQuery.indices.mkString(",")) ) @@ -390,7 +391,7 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { s"❌ Failed to execute multi-search for query \n$elasticQueries\n -> ${error.message}" ) ElasticResult.failure( - error.copy( + enrichMaxResultWindowError(error).copy( operation = Some("multiSearch") ) ) @@ -442,8 +443,8 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { collection.immutable.Seq(single.sources: _*) ) this match { - case scrollApi: ScrollApi if single.limit.isEmpty && single.returnsRows => - scrollAllRows(scrollApi, single, elasticQuery) + case scrollApi: ScrollApi if single.returnsRows && requiresScrollPaging(single.limit) => + scrollRows(scrollApi, single, elasticQuery) case _ => if (single.windowRowQuery) Future.successful(searchWithWindowEnrichment(single)) @@ -514,75 +515,97 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { val sql = elasticQuery.sql val query = elasticQuery.query val indices = elasticQuery.indices.mkString(",") - executeSingleSearchAsync(elasticQuery).flatMap { - case ElasticSuccess(Some(response)) => - logger.info( - s"✅ Successfully executed asynchronous search for query \n$elasticQuery\nin indices '$indices'" - ) - val aggs = toClientAggregations(aggregations) - ElasticResult.fromTry( - parseResponse( - response, - fieldAliases, - aggs, - fields, - nestedHits, - elasticQuery.explodeNested + executeSingleSearchAsync(elasticQuery) + .flatMap { + case ElasticSuccess(Some(response)) => + logger.info( + s"✅ Successfully executed asynchronous search for query \n$elasticQuery\nin indices '$indices'" ) - ) match { - case success @ ElasticSuccess(_) => - logger.info( - s"✅ Successfully parsed search results for query \n$elasticQuery\nin indices '$indices'" + val aggs = toClientAggregations(aggregations) + ElasticResult.fromTry( + parseResponse( + response, + fieldAliases, + aggs, + fields, + nestedHits, + elasticQuery.explodeNested ) - Future.successful( - ElasticResult.success( - ElasticResponse( - sql, - query, - success.value, - fieldAliases, - aggs + ) match { + case success @ ElasticSuccess(_) => + logger.info( + s"✅ Successfully parsed search results for query \n$elasticQuery\nin indices '$indices'" + ) + Future.successful( + ElasticResult.success( + ElasticResponse( + sql, + query, + success.value, + fieldAliases, + aggs + ) ) ) - ) - case ElasticFailure(error) => - logger.error( - s"❌ Failed to parse search results for query \n${sql - .getOrElse(query)}\nin indices '$indices' -> ${error.message}" - ) - Future.successful( - ElasticResult.failure( - error.copy( - operation = Some("searchAsync"), - index = Some(indices) + case ElasticFailure(error) => + logger.error( + s"❌ Failed to parse search results for query \n${sql + .getOrElse(query)}\nin indices '$indices' -> ${error.message}" + ) + Future.successful( + ElasticResult.failure( + error.copy( + operation = Some("searchAsync"), + index = Some(indices) + ) ) ) + } + case ElasticSuccess(_) => + val error = + ElasticError( + message = + s"Failed to execute asynchronous search for query \n$elasticQuery\nin indices '$indices'", + index = Some(elasticQuery.indices.mkString(",")), + operation = Some("searchAsync") + ) + logger.error(s"❌ ${error.message}") + Future.successful(ElasticResult.failure(error)) + case ElasticFailure(error) => + logger.error( + s"❌ Failed to execute asynchronous search for query \n${sql + .getOrElse(query)}\nin indices '$indices' -> ${error.message}" + ) + Future.successful( + ElasticResult.failure( + enrichMaxResultWindowError(error).copy( + operation = Some("searchAsync"), + index = Some(elasticQuery.indices.mkString(",")) + ) ) - } - case ElasticSuccess(_) => - val error = - ElasticError( - message = - s"Failed to execute asynchronous search for query \n$elasticQuery\nin indices '$indices'", - index = Some(elasticQuery.indices.mkString(",")), - operation = Some("searchAsync") ) - logger.error(s"❌ ${error.message}") - Future.successful(ElasticResult.failure(error)) - case ElasticFailure(error) => - logger.error( - s"❌ Failed to execute asynchronous search for query \n${sql - .getOrElse(query)}\nin indices '$indices' -> ${error.message}" - ) - Future.successful( + } + .recover { + // Issue #224 — some client implementations surface an execution failure as a FAILED future + // rather than an ElasticFailure; without this recover the raw Throwable propagates to the + // consumer, which typically flattens it into an opaque generic error. Honor the + // ElasticResult contract here (and translate a max_result_window rejection on the way). + case t: Throwable => + logger.error( + s"❌ Failed to execute asynchronous search for query \n${sql + .getOrElse(query)}\nin indices '$indices' -> ${t.getMessage}" + ) ElasticResult.failure( - error.copy( - operation = Some("searchAsync"), - index = Some(elasticQuery.indices.mkString(",")) + enrichMaxResultWindowError( + ElasticError( + message = s"Failed to execute search: ${t.getMessage}", + cause = Some(t), + operation = Some("searchAsync"), + index = Some(indices) + ) ) ) - ) - } + } } /** Asynchronous multi-search with Elasticsearch queries. @@ -611,69 +634,87 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { Option(elasticQueries.queries.flatMap(_.sql).mkString("\nUNION ALL\n")) ) - executeMultiSearchAsync(elasticQueries).flatMap { - case ElasticSuccess(Some(response)) => - logger.info( - s"✅ Successfully executed asynchronous multi-search for query \n$elasticQueries" - ) - val aggs = toClientAggregations(aggregations) - ElasticResult.fromTry( - parseResponse( - response, - fieldAliases, - aggs, - fields, - nestedHits, - elasticQueries.explodeNested + executeMultiSearchAsync(elasticQueries) + .flatMap { + case ElasticSuccess(Some(response)) => + logger.info( + s"✅ Successfully executed asynchronous multi-search for query \n$elasticQueries" ) - ) match { - case success @ ElasticSuccess(_) => - logger.info( - s"✅ Successfully parsed multi-search results for query '$elasticQueries'" + val aggs = toClientAggregations(aggregations) + ElasticResult.fromTry( + parseResponse( + response, + fieldAliases, + aggs, + fields, + nestedHits, + elasticQueries.explodeNested ) - Future.successful( - ElasticResult.success( - ElasticResponse( - sql, - query, - success.value, - fieldAliases, - aggs + ) match { + case success @ ElasticSuccess(_) => + logger.info( + s"✅ Successfully parsed multi-search results for query '$elasticQueries'" + ) + Future.successful( + ElasticResult.success( + ElasticResponse( + sql, + query, + success.value, + fieldAliases, + aggs + ) ) ) - ) - case ElasticFailure(error) => - logger.error( - s"❌ Failed to parse multi-search results for query \n$elasticQueries\n -> ${error.message}" - ) - Future.successful( - ElasticResult.failure( - error.copy( - operation = Some("multiSearchAsync") + case ElasticFailure(error) => + logger.error( + s"❌ Failed to parse multi-search results for query \n$elasticQueries\n -> ${error.message}" + ) + Future.successful( + ElasticResult.failure( + error.copy( + operation = Some("multiSearchAsync") + ) ) ) + } + case ElasticSuccess(_) => + val error = + ElasticError( + message = s"Failed to execute asynchronous multi-search for query \n$elasticQueries", + operation = Some("multiSearchAsync") ) - } - case ElasticSuccess(_) => - val error = - ElasticError( - message = s"Failed to execute asynchronous multi-search for query \n$elasticQueries", - operation = Some("multiSearchAsync") + logger.error(s"❌ ${error.message}") + Future.successful(ElasticResult.failure(error)) + case ElasticFailure(error) => + logger.error( + s"❌ Failed to execute asynchronous multi-search for query \n$elasticQueries\n -> ${error.message}" + ) + Future.successful( + ElasticResult.failure( + enrichMaxResultWindowError(error).copy( + operation = Some("multiSearchAsync") + ) + ) + ) + } + .recover { + // Issue #224 — same contract repair as singleSearchAsync: a client implementation may fail + // the future instead of returning an ElasticFailure. + case t: Throwable => + logger.error( + s"❌ Failed to execute asynchronous multi-search for query \n$elasticQueries\n -> ${t.getMessage}" ) - logger.error(s"❌ ${error.message}") - Future.successful(ElasticResult.failure(error)) - case ElasticFailure(error) => - logger.error( - s"❌ Failed to execute asynchronous multi-search for query \n$elasticQueries\n -> ${error.message}" - ) - Future.successful( ElasticResult.failure( - error.copy( - operation = Some("multiSearchAsync") + enrichMaxResultWindowError( + ElasticError( + message = s"Failed to execute multi-search: ${t.getMessage}", + cause = Some(t), + operation = Some("multiSearchAsync") + ) ) ) - ) - } + } } // ======================================================================== @@ -1667,19 +1708,73 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { } // ======================================================================== - // ROW COMPLETENESS — SCROLL ROUTING (issue #209) + // ROW COMPLETENESS — SCROLL ROUTING (issues #209 / #224) // ======================================================================== - /** Collect EVERY row of an un-LIMITed row query through the scroll path. + /** True when a row-shaped query must page through scroll instead of the one-shot search: + * + * - no `LIMIT` at all (#209): a one-shot search cannot honor "no LIMIT means every row" — with + * no `size` Elasticsearch returns its default 10 hits; + * - an explicit `LIMIT` whose window (`offset + limit`) exceeds + * [[SearchApi.DefaultMaxResultWindow]] (#224): Elasticsearch rejects a one-shot search + * whenever `from + size > index.max_result_window` (default 10,000), so the SAME query would + * fail with `LIMIT 20000` yet succeed with no LIMIT. The window is per-index and not probed + * (a multi-index query has no single window anyway); on an index tuned HIGHER the query + * pages unnecessarily but stays correct — and a >10k one-shot response is better paged + * regardless. + */ + private def requiresScrollPaging(limit: Option[Limit]): Boolean = + limit match { + case None => true + case Some(l) => + l.limit.toLong + l.offset.map(_.offset.toLong).getOrElse(0L) > + SearchApi.DefaultMaxResultWindow + } + + /** Issue #224 — a one-shot search whose `from + size` exceeds `index.max_result_window` is + * rejected by Elasticsearch with an `illegal_argument_exception` that names neither the SQL + * `LIMIT` that produced the `size` nor the remedy — and downstream consumers often flatten it + * further. An index tuned BELOW the routing threshold of [[SearchApi.DefaultMaxResultWindow]] + * can still surface the rejection despite the scroll routing, so translate it into an actionable + * message here; every other error passes through unchanged. + */ + private def enrichMaxResultWindowError(error: ElasticError): ElasticError = { + def mentionsWindow(message: String): Boolean = + message != null && + (message.contains("max_result_window") || message.contains("Result window is too large")) + // The REST high-level clients (ES 6/7) surface the per-shard root cause as SUPPRESSED + // exceptions on an "all shards failed" wrapper, so the scan walks both chains (bounded — + // exception graphs can be cyclic in theory). + def throwableMentionsWindow(t: Throwable, depth: Int = 10): Boolean = + t != null && depth > 0 && + (mentionsWindow(t.getMessage) || + t.getSuppressed.exists(s => throwableMentionsWindow(s, depth - 1)) || + throwableMentionsWindow(t.getCause, depth - 1)) + if (mentionsWindow(error.message) || error.cause.exists(t => throwableMentionsWindow(t))) + error.copy( + message = + "LIMIT/OFFSET exceeds the index's `index.max_result_window` for a single search: " + + "lower LIMIT/OFFSET so `offset + limit` fits within the window, raise " + + "`index.max_result_window` on the index, or drop the LIMIT to page through every row. " + + s"Elasticsearch said: ${error.message}", + statusCode = error.statusCode.orElse(Some(400)) + ) + else error + } + + /** Collect the rows of a row-shaped query through the scroll path. * - * A one-shot search cannot honor "no LIMIT means every row": with no `size` Elasticsearch - * returns its default 10 hits, and any explicit `size` is bounded by `index.max_result_window`. * The scroll path pages completely (PIT / search_after) and already handles window enrichment - * and script fields, so `search` / `searchAsync` route row-shaped queries with no LIMIT here. - * Aggregation-shaped queries must never be routed — their result is the aggregation itself, - * already bounded by an explicit `terms` size. + * and script fields, so `search` / `searchAsync` route here every row query that + * [[requiresScrollPaging]] flags: with no LIMIT the stream is unbounded (every matching row, + * #209); with an explicit LIMIT above the one-shot window (#224) the stream is bounded by + * `maxDocuments = offset + limit` (enforced by ScrollApi's `.take`) and the first `offset` rows + * are dropped client-side — scroll contexts reject `from`, and per-page `size` is the scroll + * batch size, so the statement's LIMIT is stripped before translation. Aggregation-shaped + * queries must never be routed — their result is the aggregation itself, already bounded by an + * explicit `terms` size. */ - private[client] def scrollAllRows( + private[client] def scrollRows( scrollApi: ScrollApi, single: SingleSearch, elasticQuery: ElasticQuery @@ -1687,12 +1782,17 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { implicit val system: ActorSystem = SearchApi.scrollRoutingSystem implicit val ec: ExecutionContext = system.dispatcher val sql = elasticQuery.sql.orElse(Option(single.sql)) + val offset = single.limit.flatMap(_.offset).map(_.offset.toLong).getOrElse(0L) + val maxDocuments = single.limit.map(l => offset + l.limit.toLong) + val statement = if (single.limit.isEmpty) single else single.copy(limit = None) logger.info( - s"▶ Row query without LIMIT — routing through scroll for row completeness:\n${sql.getOrElse(elasticQuery.query)}" + s"▶ Row query ${maxDocuments.fold("without LIMIT")(max => s"with LIMIT window $max above ${SearchApi.DefaultMaxResultWindow}")} — routing through scroll for row completeness:\n${sql + .getOrElse(elasticQuery.query)}" ) scrollApi - .scroll(single, ScrollConfig()) + .scroll(statement, ScrollConfig(maxDocuments = maxDocuments)) .map(_._1) + .drop(offset) .runWith(Sink.seq) .map { rows => logger.info(s"✅ Scroll-routed search returned ${rows.size} rows") @@ -1725,8 +1825,16 @@ trait SearchApi extends ElasticConversion with ElasticClientHelpers { object SearchApi { - /** JVM-shared materializer for [[SearchApi.scrollAllRows]]. Daemonic so an un-terminated system - * can never keep the JVM alive — clients don't own it, so no `close()` reaches it. + /** Elasticsearch's default `index.max_result_window`: the ceiling on `from + size` for a one-shot + * search, identical across ES 6/7/8/9. Row queries whose explicit LIMIT window exceeds it are + * routed through scroll (#224) — see [[SearchApi.requiresScrollPaging]]. The actual per-index + * setting is deliberately NOT probed; an index tuned higher just pages, an index tuned lower + * keeps its (translated) one-shot rejection below this threshold. + */ + val DefaultMaxResultWindow: Long = 10000L + + /** JVM-shared materializer for [[SearchApi.scrollRows]]. Daemonic so an un-terminated system can + * never keep the JVM alive — clients don't own it, so no `close()` reaches it. */ private[client] lazy val scrollRoutingSystem: ActorSystem = ActorSystem( diff --git a/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientLimitCompletenessSpec.scala b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientLimitCompletenessSpec.scala new file mode 100644 index 00000000..f1001146 --- /dev/null +++ b/es6/jest/src/test/scala/app/softnetwork/elastic/client/JestClientLimitCompletenessSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JestClientLimitCompletenessSpec extends LimitCompletenessSpec diff --git a/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientLimitCompletenessSpec.scala b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientLimitCompletenessSpec.scala new file mode 100644 index 00000000..87d991b3 --- /dev/null +++ b/es6/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientLimitCompletenessSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class RestHighLevelClientLimitCompletenessSpec extends LimitCompletenessSpec diff --git a/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientLimitCompletenessSpec.scala b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientLimitCompletenessSpec.scala new file mode 100644 index 00000000..87d991b3 --- /dev/null +++ b/es7/rest/src/test/scala/app/softnetwork/elastic/client/RestHighLevelClientLimitCompletenessSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class RestHighLevelClientLimitCompletenessSpec extends LimitCompletenessSpec diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index e6878e81..02a70a35 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -1071,8 +1071,10 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { classOf[JMap[String, Object]] ) ).map { response => - ElasticSuccess(Some(convertToJson(response))) - } + ElasticSuccess(Some(convertToJson(response))): ElasticResult[Option[String]] + }.recover( + asyncElasticFailure("singleSearch", Some(elasticQuery.indices.mkString(","))) + ) override private[client] def executeMultiSearchAsync( elasticQueries: ElasticQueries @@ -1089,8 +1091,14 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { async().msearch(request, classOf[JMap[String, Object]]) } .map { response => - ElasticSuccess(Some(convertToJson(response))) + ElasticSuccess(Some(convertToJson(response))): ElasticResult[Option[String]] } + .recover( + asyncElasticFailure( + "multiSearch", + Some(elasticQueries.queries.flatMap(_.indices).distinct.mkString(",")) + ) + ) } diff --git a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala index 32aab1b2..ea8d25ac 100644 --- a/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala +++ b/es8/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala @@ -19,6 +19,7 @@ package app.softnetwork.elastic.client.java import app.softnetwork.elastic.client.ElasticClientHelpers import app.softnetwork.elastic.client.result.{ElasticError, ElasticResult} +import scala.jdk.CollectionConverters._ import scala.util.{Failure, Success, Try} trait JavaClientHelpers extends ElasticClientHelpers with JavaClientConversion { @@ -28,6 +29,71 @@ trait JavaClientHelpers extends ElasticClientHelpers with JavaClientConversion { // GENERIC METHODS FOR EXECUTING JAVA CLIENT ACTIONS // ======================================================================== + /** Flatten an Elasticsearch failure into a message that names the ACTUAL cause. + * + * The top-level `reason` of a search failure is often just "all shards failed" + * (`search_phase_execution_exception`); the actionable detail — e.g. a `max_result_window` + * rejection (#224) — lives in the typed `rootCause` / `causedBy` tree of the ErrorCause, which + * is invisible outside this module (it is NOT part of any Throwable message or cause chain). + */ + private[client] def elasticsearchErrorMessage( + operation: String, + ex: co.elastic.clients.elasticsearch._types.ElasticsearchException + ): String = { + val error = Option(ex.error()) + val errorType = error.flatMap(e => Option(e.`type`())) + val reason = error.flatMap(e => Option(e.reason())) + val rootCauses = + error.toSeq.flatMap(_.rootCause().asScala).flatMap(rc => Option(rc.reason())) + val causedBy = { + // Walk the causedBy chain (bounded — defensive against cyclic metadata). + Iterator + .iterate(error.flatMap(e => Option(e.causedBy())))(_.flatMap(e => Option(e.causedBy()))) + .takeWhile(_.isDefined) + .take(10) + .flatten + .flatMap(cb => Option(cb.reason())) + .toSeq + } + val reasons = (reason.toSeq ++ rootCauses ++ causedBy).distinct + s"Elasticsearch error during $operation: ${errorType.getOrElse("unknown")} - ${if (reasons.nonEmpty) reasons.mkString("; ") + else ex.getMessage}" + } + + /** Convert a Throwable failing an asynchronous Java-client call into an ElasticFailure, + * unwrapping the CompletionException layer and extracting the full Elasticsearch error detail + * (see [[elasticsearchErrorMessage]]). Without this, an async search failure propagates as a raw + * failed future whose message hides the root cause (#224). + */ + private[client] def asyncElasticFailure[T]( + operation: String, + index: Option[String] + ): PartialFunction[Throwable, ElasticResult[T]] = { case t: Throwable => + val unwrapped = t match { + case ce: java.util.concurrent.CompletionException if ce.getCause != null => ce.getCause + case other => other + } + val error = unwrapped match { + case ex: co.elastic.clients.elasticsearch._types.ElasticsearchException => + ElasticError( + message = elasticsearchErrorMessage(operation, ex), + cause = Some(ex), + statusCode = Option(ex.status()).map(_.intValue()), + index = index, + operation = Some(operation) + ) + case other => + ElasticError( + message = s"Exception during $operation: ${other.getMessage}", + cause = Some(other), + index = index, + operation = Some(operation) + ) + } + logger.warn(s"${error.message}${index.map(i => s" on index '$i'").getOrElse("")}") + ElasticResult.failure(error) + } + //format:off /** Execute a Java Client action with a generic transformation of the result. * @@ -84,14 +150,10 @@ trait JavaClientHelpers extends ElasticClientHelpers with JavaClientConversion { case Success(result) => ElasticResult.success(result) case Failure(ex: co.elastic.clients.elasticsearch._types.ElasticsearchException) => - // Extract error details from Elasticsearch exception + // Extract error details from Elasticsearch exception — including the rootCause / + // causedBy detail, without which a search failure reads "all shards failed" (#224) val statusCode = Option(ex.status()).map(_.intValue()) - val errorType = Option(ex.error()).flatMap(e => Option(e.`type`())) - val reason = Option(ex.error()).flatMap(e => Option(e.reason())) - - val message = - s"Elasticsearch error during $operation: ${errorType.getOrElse("unknown")} - ${reason - .getOrElse(ex.getMessage)}" + val message = elasticsearchErrorMessage(operation, ex) logger.warn(s"$message$indexStr") ElasticResult.failure( diff --git a/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientLimitCompletenessSpec.scala b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientLimitCompletenessSpec.scala new file mode 100644 index 00000000..085b884e --- /dev/null +++ b/es8/java/src/test/scala/app/softnetwork/elastic/client/JavaClientLimitCompletenessSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JavaClientLimitCompletenessSpec extends LimitCompletenessSpec diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala index e639f4b8..fa116d2b 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientApi.scala @@ -1071,8 +1071,10 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { classOf[JMap[String, Object]] ) ).map { response => - ElasticSuccess(Some(convertToJson(response))) - } + ElasticSuccess(Some(convertToJson(response))): ElasticResult[Option[String]] + }.recover( + asyncElasticFailure("singleSearch", Some(elasticQuery.indices.mkString(","))) + ) override private[client] def executeMultiSearchAsync( elasticQueries: ElasticQueries @@ -1089,8 +1091,14 @@ trait JavaClientSearchApi extends SearchApi with JavaClientHelpers { async().msearch(request, classOf[JMap[String, Object]]) } .map { response => - ElasticSuccess(Some(convertToJson(response))) + ElasticSuccess(Some(convertToJson(response))): ElasticResult[Option[String]] } + .recover( + asyncElasticFailure( + "multiSearch", + Some(elasticQueries.queries.flatMap(_.indices).distinct.mkString(",")) + ) + ) } diff --git a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala index 32aab1b2..ea8d25ac 100644 --- a/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala +++ b/es9/java/src/main/scala/app/softnetwork/elastic/client/java/JavaClientHelpers.scala @@ -19,6 +19,7 @@ package app.softnetwork.elastic.client.java import app.softnetwork.elastic.client.ElasticClientHelpers import app.softnetwork.elastic.client.result.{ElasticError, ElasticResult} +import scala.jdk.CollectionConverters._ import scala.util.{Failure, Success, Try} trait JavaClientHelpers extends ElasticClientHelpers with JavaClientConversion { @@ -28,6 +29,71 @@ trait JavaClientHelpers extends ElasticClientHelpers with JavaClientConversion { // GENERIC METHODS FOR EXECUTING JAVA CLIENT ACTIONS // ======================================================================== + /** Flatten an Elasticsearch failure into a message that names the ACTUAL cause. + * + * The top-level `reason` of a search failure is often just "all shards failed" + * (`search_phase_execution_exception`); the actionable detail — e.g. a `max_result_window` + * rejection (#224) — lives in the typed `rootCause` / `causedBy` tree of the ErrorCause, which + * is invisible outside this module (it is NOT part of any Throwable message or cause chain). + */ + private[client] def elasticsearchErrorMessage( + operation: String, + ex: co.elastic.clients.elasticsearch._types.ElasticsearchException + ): String = { + val error = Option(ex.error()) + val errorType = error.flatMap(e => Option(e.`type`())) + val reason = error.flatMap(e => Option(e.reason())) + val rootCauses = + error.toSeq.flatMap(_.rootCause().asScala).flatMap(rc => Option(rc.reason())) + val causedBy = { + // Walk the causedBy chain (bounded — defensive against cyclic metadata). + Iterator + .iterate(error.flatMap(e => Option(e.causedBy())))(_.flatMap(e => Option(e.causedBy()))) + .takeWhile(_.isDefined) + .take(10) + .flatten + .flatMap(cb => Option(cb.reason())) + .toSeq + } + val reasons = (reason.toSeq ++ rootCauses ++ causedBy).distinct + s"Elasticsearch error during $operation: ${errorType.getOrElse("unknown")} - ${if (reasons.nonEmpty) reasons.mkString("; ") + else ex.getMessage}" + } + + /** Convert a Throwable failing an asynchronous Java-client call into an ElasticFailure, + * unwrapping the CompletionException layer and extracting the full Elasticsearch error detail + * (see [[elasticsearchErrorMessage]]). Without this, an async search failure propagates as a raw + * failed future whose message hides the root cause (#224). + */ + private[client] def asyncElasticFailure[T]( + operation: String, + index: Option[String] + ): PartialFunction[Throwable, ElasticResult[T]] = { case t: Throwable => + val unwrapped = t match { + case ce: java.util.concurrent.CompletionException if ce.getCause != null => ce.getCause + case other => other + } + val error = unwrapped match { + case ex: co.elastic.clients.elasticsearch._types.ElasticsearchException => + ElasticError( + message = elasticsearchErrorMessage(operation, ex), + cause = Some(ex), + statusCode = Option(ex.status()).map(_.intValue()), + index = index, + operation = Some(operation) + ) + case other => + ElasticError( + message = s"Exception during $operation: ${other.getMessage}", + cause = Some(other), + index = index, + operation = Some(operation) + ) + } + logger.warn(s"${error.message}${index.map(i => s" on index '$i'").getOrElse("")}") + ElasticResult.failure(error) + } + //format:off /** Execute a Java Client action with a generic transformation of the result. * @@ -84,14 +150,10 @@ trait JavaClientHelpers extends ElasticClientHelpers with JavaClientConversion { case Success(result) => ElasticResult.success(result) case Failure(ex: co.elastic.clients.elasticsearch._types.ElasticsearchException) => - // Extract error details from Elasticsearch exception + // Extract error details from Elasticsearch exception — including the rootCause / + // causedBy detail, without which a search failure reads "all shards failed" (#224) val statusCode = Option(ex.status()).map(_.intValue()) - val errorType = Option(ex.error()).flatMap(e => Option(e.`type`())) - val reason = Option(ex.error()).flatMap(e => Option(e.reason())) - - val message = - s"Elasticsearch error during $operation: ${errorType.getOrElse("unknown")} - ${reason - .getOrElse(ex.getMessage)}" + val message = elasticsearchErrorMessage(operation, ex) logger.warn(s"$message$indexStr") ElasticResult.failure( diff --git a/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientLimitCompletenessSpec.scala b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientLimitCompletenessSpec.scala new file mode 100644 index 00000000..085b884e --- /dev/null +++ b/es9/java/src/test/scala/app/softnetwork/elastic/client/JavaClientLimitCompletenessSpec.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +class JavaClientLimitCompletenessSpec extends LimitCompletenessSpec diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/LimitCompletenessSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/LimitCompletenessSpec.scala new file mode 100644 index 00000000..531e4114 --- /dev/null +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/LimitCompletenessSpec.scala @@ -0,0 +1,225 @@ +/* + * Copyright 2025 SOFTNETWORK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package app.softnetwork.elastic.client + +import akka.NotUsed +import akka.actor.ActorSystem +import akka.stream.scaladsl.Source +import app.softnetwork.elastic.client.bulk._ +import app.softnetwork.elastic.client.result.{ElasticFailure, ElasticSuccess} +import app.softnetwork.elastic.client.spi.ElasticClientFactory +import app.softnetwork.elastic.scalatest.ElasticDockerTestKit +import app.softnetwork.elastic.sql.query.SelectStatement +import app.softnetwork.persistence.generateUUID +import org.scalatest.flatspec.AnyFlatSpecLike +import org.scalatest.matchers.should.Matchers +import org.slf4j.{Logger, LoggerFactory} + +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, ExecutionContext} +import scala.language.implicitConversions + +case class LimitRow(id: String, amount: Int) + +/** Regression test for issue #224: a `SELECT` with an explicit `LIMIT` above the index's + * `index.max_result_window` (ES default 10,000) failed outright, while the SAME query with no + * `LIMIT` succeeded and returned every row — the one-shot search path issued a single search + * with `size = LIMIT`, which Elasticsearch rejects whenever `from + size` exceeds the window. + * + * Row queries whose LIMIT window (`offset + limit`) exceeds the ES default window are now routed + * through the scroll path bounded at `maxDocuments = offset + limit` (#209's routing, extended). + * This spec asserts the asymmetry directly on a multi-shard index: LIMIT above the window and no + * LIMIT both return their full expected row counts, small LIMITs keep their one-shot bound, and + * ORDER BY + OFFSET stay exact through the scroll routing. On an index tuned BELOW the routing + * threshold the one-shot rejection remains — the spec asserts it now names `max_result_window` + * instead of surfacing an opaque failure. + */ +trait LimitCompletenessSpec extends AnyFlatSpecLike with ElasticDockerTestKit with Matchers { + + lazy val log: Logger = LoggerFactory.getLogger(getClass.getName) + + implicit val system: ActorSystem = ActorSystem(generateUUID()) + + lazy val client: ElasticClientApi = ElasticClientFactory.create(elasticConfig) + + private val index = "limit_completeness" + + /** 12,000 docs — above the ES default `index.max_result_window` of 10,000, so an explicit + * `LIMIT 11000` can only be served by paging past the window. Zero-padded ids make ORDER BY + + * OFFSET content exact oracles. + */ + private val totalDocs = 12000 + + /** A second index tuned BELOW the scroll-routing threshold: `max_result_window = 100`, so a + * one-shot `LIMIT 250` (well under 10,000) is still rejected by Elasticsearch — the error must + * be actionable, naming `max_result_window`. + */ + private val loweredIndex = "limit_window_lowered" + + private val loweredDocs = 300 + + private def indexDocs(indexName: String, count: Int, settings: String): Unit = { + val mapping = + """{ + | "properties": { + | "id": { "type": "keyword" }, + | "amount": { "type": "integer" } + | } + |}""".stripMargin + + client.createIndex(indexName, settings = settings).get shouldBe true + client.setMapping(indexName, mapping).get shouldBe true + + val docs = (1 to count).map { i => + s"""{"id":"id_${"%05d".format(i)}","amount":$i}""" + }.toList + + implicit val bulkOptions: BulkOptions = BulkOptions( + defaultIndex = indexName, + logEvery = 10000 + ) + + implicit def listToSource[T](list: List[T]): Source[T, NotUsed] = + Source.fromIterator(() => list.iterator) + + client.bulk[String](docs, identity, idKey = Some(Set("id"))) match { + case ElasticSuccess(_) => // ok + case ElasticFailure(error) => + error.cause.foreach(_.printStackTrace()) + fail(s"Bulk indexing failed: ${error.message}") + } + + client.refresh(indexName) + } + + override def beforeAll(): Unit = { + super.beforeAll() + indexDocs(index, totalDocs, """{"number_of_shards": 3, "number_of_replicas": 0}""") + indexDocs( + loweredIndex, + loweredDocs, + """{"number_of_shards": 1, "number_of_replicas": 0, "index.max_result_window": 100}""" + ) + } + + override def afterAll(): Unit = { + client.deleteIndex(index) + client.deleteIndex(loweredIndex) + super.afterAll() + } + + "SELECT with LIMIT above index.max_result_window" should "return exactly LIMIT rows" in { + client.searchAs[LimitRow]( + "SELECT id, amount FROM limit_completeness LIMIT 11000" + ) match { + case ElasticSuccess(rows) => + rows should have size 11000 + rows.map(_.id).toSet should have size 11000 + + log.info(s"✓ ${rows.size} rows from $index with LIMIT 11000 (window 10000)") + + case ElasticFailure(error) => + fail(s"Query failed: ${error.message}") + } + } + + "the same SELECT without LIMIT" should "return every row (the #224 asymmetry pair)" in { + client.searchAs[LimitRow]( + "SELECT id, amount FROM limit_completeness" + ) match { + case ElasticSuccess(rows) => + rows should have size totalDocs.toLong + + case ElasticFailure(error) => + fail(s"Query failed: ${error.message}") + } + } + + "SELECT with LIMIT above the window through searchAsync" should "return exactly LIMIT rows" in { + implicit val ec: ExecutionContext = system.dispatcher + implicit val context: ConversionContext = NativeContext + Await.result( + client.searchAsync( + SelectStatement("SELECT id, amount FROM limit_completeness LIMIT 11000") + ), + Duration.Inf + ) match { + case ElasticSuccess(response) => + response.results should have size 11000 + + case ElasticFailure(error) => + fail(s"Query failed: ${error.message}") + } + } + + "ORDER BY + OFFSET with a LIMIT window above the ES window" should "stay exact" in { + // offset (1000) + limit (10500) = 11500 > 10000 → scroll-routed; ORDER BY id with + // zero-padded ids makes the returned slice an exact oracle. + client.searchAs[LimitRow]( + "SELECT id, amount FROM limit_completeness ORDER BY id ASC LIMIT 10500 OFFSET 1000" + ) match { + case ElasticSuccess(rows) => + rows should have size 10500 + rows.head.id shouldBe "id_01001" + rows.last.id shouldBe "id_11500" + rows.map(_.id) shouldBe sorted + + case ElasticFailure(error) => + fail(s"Query failed: ${error.message}") + } + } + + "SELECT with a small LIMIT" should "keep its one-shot bound" in { + client.searchAs[LimitRow]( + "SELECT id, amount FROM limit_completeness LIMIT 42" + ) match { + case ElasticSuccess(rows) => + rows should have size 42 + + case ElasticFailure(error) => + fail(s"Query failed: ${error.message}") + } + } + + "SELECT with LIMIT exactly at the window" should "succeed one-shot (boundary, not routed)" in { + client.searchAs[LimitRow]( + "SELECT id, amount FROM limit_completeness LIMIT 10000" + ) match { + case ElasticSuccess(rows) => + rows should have size 10000 + + case ElasticFailure(error) => + fail(s"Query failed: ${error.message}") + } + } + + "a one-shot rejection on an index tuned below the routing threshold" should + "name max_result_window" in { + client.searchAs[LimitRow]( + "SELECT id, amount FROM limit_window_lowered LIMIT 250" + ) match { + case ElasticSuccess(rows) => + fail(s"Expected a max_result_window rejection, got ${rows.size} rows") + + case ElasticFailure(error) => + error.message should include("max_result_window") + error.message should include("LIMIT") + + log.info(s"✓ Actionable rejection: ${error.message}") + } + } +} From 411eaf7365f1048b6d388d1a23a8826ae70b8a44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Manciot?= Date: Tue, 11 Aug 2026 17:15:12 +0200 Subject: [PATCH 2/2] fix(tests): improve regression test for LIMIT handling above index.max_result_window --- .../elastic/client/LimitCompletenessSpec.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/testkit/src/main/scala/app/softnetwork/elastic/client/LimitCompletenessSpec.scala b/testkit/src/main/scala/app/softnetwork/elastic/client/LimitCompletenessSpec.scala index 531e4114..a0d6447d 100644 --- a/testkit/src/main/scala/app/softnetwork/elastic/client/LimitCompletenessSpec.scala +++ b/testkit/src/main/scala/app/softnetwork/elastic/client/LimitCompletenessSpec.scala @@ -37,8 +37,8 @@ case class LimitRow(id: String, amount: Int) /** Regression test for issue #224: a `SELECT` with an explicit `LIMIT` above the index's * `index.max_result_window` (ES default 10,000) failed outright, while the SAME query with no - * `LIMIT` succeeded and returned every row — the one-shot search path issued a single search - * with `size = LIMIT`, which Elasticsearch rejects whenever `from + size` exceeds the window. + * `LIMIT` succeeded and returned every row — the one-shot search path issued a single search with + * `size = LIMIT`, which Elasticsearch rejects whenever `from + size` exceeds the window. * * Row queries whose LIMIT window (`offset + limit`) exceeds the ES default window are now routed * through the scroll path bounded at `maxDocuments = offset + limit` (#209's routing, extended). @@ -58,9 +58,9 @@ trait LimitCompletenessSpec extends AnyFlatSpecLike with ElasticDockerTestKit wi private val index = "limit_completeness" - /** 12,000 docs — above the ES default `index.max_result_window` of 10,000, so an explicit - * `LIMIT 11000` can only be served by paging past the window. Zero-padded ids make ORDER BY + - * OFFSET content exact oracles. + /** 12,000 docs — above the ES default `index.max_result_window` of 10,000, so an explicit `LIMIT + * 11000` can only be served by paging past the window. Zero-padded ids make ORDER BY + OFFSET + * content exact oracles. */ private val totalDocs = 12000