GITHUB#15239: Honor exhaustive ScoreMode in BatchScoreBulkScorer

lucene/CHANGES.txt

@@ -226,6 +226,11 @@ Bug Fixes
 * GITHUB#16565: IndexWriter#updateBinaryDocValue now rejects updating a field that is part of the
   index sort, matching IndexWriter#updateNumericDocValue. (Jim Ferenczi)
 
+* GITHUB#15239: BatchScoreBulkScorer and DenseConjunctionBulkScorer no longer skip hits from
+  setMinCompetitiveScore when the search ScoreMode is exhaustive. Nested TopScoreDocCollectors can
+  still track a competitive threshold without dropping matches from an outer COMPLETE collector.
+  (Shrey Narayan, NextBrick)
+
 * GITHUB#14049: Randomize KNN codec params in RandomCodec. Fixes scalar quantization div-by-zero
   when all values are identical. (Mike Sokolov)
 
@@ -483,6 +488,11 @@ Optimizations
 Bug Fixes
 ---------------------
 
+* GITHUB#15239: BatchScoreBulkScorer and DenseConjunctionBulkScorer no longer skip hits from
+  setMinCompetitiveScore when the search ScoreMode is exhaustive. Nested TopScoreDocCollectors can
+  still track a competitive threshold without dropping matches from an outer COMPLETE collector.
+  (Shrey Narayan, NextBrick)
+
 * GITHUB#16546: Prevent RangeBulkScorer from passing empty ranges to LeafCollector. (jxy)
 
 * GITHUB#16450: Fix DocValuesRangeIterator.docIDRunEnd() returning incorrect run boundaries,

lucene/core/src/java/org/apache/lucene/search/BatchScoreBulkScorer.java

@@ -28,9 +28,13 @@ class BatchScoreBulkScorer extends BulkScorer {
   private final SimpleScorable scorable = new SimpleScorable();
   private final DocAndFloatFeatureBuffer buffer = new DocAndFloatFeatureBuffer();
   private final Scorer scorer;
+  private final boolean applyMinCompetitiveScore;
 
-  BatchScoreBulkScorer(Scorer scorer) {
+  BatchScoreBulkScorer(Scorer scorer, ScoreMode scoreMode) {
     this.scorer = scorer;
+    // Exhaustive collection (COMPLETE*) must visit every match even if a nested
+    // collector calls setMinCompetitiveScore (GITHUB#15239).
+    this.applyMinCompetitiveScore = scoreMode.isExhaustive() == false;
   }
 
   @Override
@@ -40,7 +44,9 @@ public int score(LeafCollector collector, Bits acceptDocs, int min, int max) thr
     }
 
     collector.setScorer(scorable);
-    scorer.setMinCompetitiveScore(scorable.minCompetitiveScore);
+    if (applyMinCompetitiveScore) {
+      scorer.setMinCompetitiveScore(scorable.minCompetitiveScore);
+    }
 
     if (scorer.docID() < min) {
       scorer.iterator().advance(min);
@@ -51,11 +57,13 @@ public int score(LeafCollector collector, Bits acceptDocs, int min, int max) thr
         scorer.nextDocsAndScores(max, acceptDocs, buffer)) {
       for (int i = 0, size = buffer.size; i < size; i++) {
         float score = scorable.score = buffer.features[i];
-        if (score >= scorable.minCompetitiveScore) {
+        if (applyMinCompetitiveScore == false || score >= scorable.minCompetitiveScore) {
           collector.collect(buffer.docs[i]);
         }
       }
-      scorer.setMinCompetitiveScore(scorable.minCompetitiveScore);
+      if (applyMinCompetitiveScore) {
+        scorer.setMinCompetitiveScore(scorable.minCompetitiveScore);
+      }
     }
 
     return scorer.docID();

lucene/core/src/java/org/apache/lucene/search/BooleanScorerSupplier.java

@@ -361,7 +361,7 @@ BulkScorer filteredOptionalBulkScorer() throws IOException {
 
       if (maxDoc >= DenseConjunctionBulkScorer.WINDOW_SIZE
           && cost >= maxDoc / DenseConjunctionBulkScorer.DENSITY_THRESHOLD_INVERSE) {
-        return DenseConjunctionBulkScorer.of(filters, maxDoc, 0f);
+        return DenseConjunctionBulkScorer.of(filters, maxDoc, 0f, scoreMode);
       }
 
       Scorer scorer = new ConjunctionScorer(filters, Collections.emptyList());
@@ -434,7 +434,7 @@ private BulkScorer requiredBulkScorer() throws IOException {
       if (requiredScoring.isEmpty()
           && maxDoc >= DenseConjunctionBulkScorer.WINDOW_SIZE
           && leadCost >= maxDoc / DenseConjunctionBulkScorer.DENSITY_THRESHOLD_INVERSE) {
-        return DenseConjunctionBulkScorer.of(requiredNoScoring, maxDoc, 0f);
+        return DenseConjunctionBulkScorer.of(requiredNoScoring, maxDoc, 0f, scoreMode);
       } else if (requiredNoScoring.stream()
           .map(Scorer::twoPhaseIterator)
           .allMatch(Objects::isNull)) {

lucene/core/src/java/org/apache/lucene/search/CombinedFieldQuery.java

@@ -265,12 +265,14 @@ class CombinedFieldWeight extends Weight {
     private final IndexSearcher searcher;
     private final TermStates[] termStates;
     private final Similarity.SimScorer simWeight;
+    private final ScoreMode scoreMode;
 
     CombinedFieldWeight(Query query, IndexSearcher searcher, ScoreMode scoreMode, float boost)
         throws IOException {
       super(query);
       assert scoreMode.needsScores();
       this.searcher = searcher;
+      this.scoreMode = scoreMode;
       long docFreq = 0;
       long totalTermFreq = 0;
       termStates = new TermStates[fieldTerms.length];
@@ -394,7 +396,7 @@ public long cost() {
 
         @Override
         public BulkScorer bulkScorer() throws IOException {
-          return new BatchScoreBulkScorer(get(Long.MAX_VALUE));
+          return new BatchScoreBulkScorer(get(Long.MAX_VALUE), scoreMode);
         }
       };
     }

lucene/core/src/java/org/apache/lucene/search/ConstantScoreScorerSupplier.java

@@ -89,7 +89,7 @@ public final BulkScorer bulkScorer() throws IOException {
         iterators = Collections.emptyList();
         twoPhases = Collections.singletonList(twoPhase);
       }
-      return new DenseConjunctionBulkScorer(iterators, twoPhases, maxDoc, score);
+      return new DenseConjunctionBulkScorer(iterators, twoPhases, maxDoc, score, scoreMode);
     } else if (scoreMode.needsScores() == false) {
       // Collect window-by-window via intoBitSet. For a two-phase iterator this confirms matches in
       // its (possibly bulk) intoBitSet; the only overhead over a plain leap-frog is the reusable

lucene/core/src/java/org/apache/lucene/search/DenseConjunctionBulkScorer.java

@@ -72,6 +72,7 @@ void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOException {
   private final int maxDoc;
   private final List<DisiWrapper> iterators;
   private final SimpleScorable scorable;
+  private final boolean applyMinCompetitiveScore;
 
   private final FixedBitSet windowMatches = new FixedBitSet(WINDOW_SIZE);
   private final FixedBitSet clauseWindowMatches = new FixedBitSet(WINDOW_SIZE);
@@ -80,7 +81,8 @@ void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOException {
   private final List<DocIdSetIterator> windowApproximations = new ArrayList<>();
   private final List<TwoPhaseIterator> windowTwoPhases = new ArrayList<>();
 
-  static DenseConjunctionBulkScorer of(List<Scorer> filters, int maxDoc, float constantScore) {
+  static DenseConjunctionBulkScorer of(
+      List<Scorer> filters, int maxDoc, float constantScore, ScoreMode scoreMode) {
     List<DocIdSetIterator> iterators = new ArrayList<>();
     List<TwoPhaseIterator> twoPhases = new ArrayList<>();
     for (Scorer filter : filters) {
@@ -91,14 +93,15 @@ static DenseConjunctionBulkScorer of(List<Scorer> filters, int maxDoc, float con
         iterators.add(filter.iterator());
       }
     }
-    return new DenseConjunctionBulkScorer(iterators, twoPhases, maxDoc, constantScore);
+    return new DenseConjunctionBulkScorer(iterators, twoPhases, maxDoc, constantScore, scoreMode);
   }
 
   DenseConjunctionBulkScorer(
       List<DocIdSetIterator> iterators,
       List<TwoPhaseIterator> twoPhases,
       int maxDoc,
-      float constantScore) {
+      float constantScore,
+      ScoreMode scoreMode) {
     if (iterators.isEmpty() && twoPhases.isEmpty()) {
       throw new IllegalArgumentException("Expected one or more iterators, got 0");
     }
@@ -121,6 +124,9 @@ static DenseConjunctionBulkScorer of(List<Scorer> filters, int maxDoc, float con
             .thenComparingDouble(w -> w.twoPhase() == null ? 0 : w.twoPhase().matchCost()));
     this.scorable = new SimpleScorable();
     scorable.score = constantScore;
+    // Exhaustive collection must visit every match even if a nested collector calls
+    // setMinCompetitiveScore (GITHUB#15239).
+    this.applyMinCompetitiveScore = scoreMode.isExhaustive() == false;
   }
 
   @Override
@@ -145,7 +151,7 @@ public int score(LeafCollector collector, Bits acceptDocs, int min, int max) thr
     }
 
     while (min < max) {
-      if (scorable.minCompetitiveScore > scorable.score) {
+      if (applyMinCompetitiveScore && scorable.minCompetitiveScore > scorable.score) {
         return DocIdSetIterator.NO_MORE_DOCS;
       }
       min = scoreWindow(collector, acceptDocs, iterators, min, max);

lucene/core/src/java/org/apache/lucene/search/ScoreMode.java

@@ -19,7 +19,12 @@
 /** Different modes of search. */
 public enum ScoreMode {
 
-  /** Produced scorers will allow visiting all matches and get their score. */
+  /**
+   * Produced scorers will allow visiting all matches and get their score. This score mode is
+   * exhaustive: a call to {@link Scorable#setMinCompetitiveScore(float)} must never cause a match
+   * to be skipped, even if a nested collector makes one in violation of the contract documented on
+   * that method.
+   */
   COMPLETE(true, true),
 
   /** Produced scorers will allow visiting all matches but scores won't be available. */

lucene/core/src/java/org/apache/lucene/search/TermQuery.java

@@ -172,7 +172,7 @@ public BulkScorer bulkScorer() throws IOException {
             return ConstantScoreScorerSupplier.fromIterator(iterator, 0f, scoreMode, maxDoc)
                 .bulkScorer();
           }
-          return new BatchScoreBulkScorer(get(Long.MAX_VALUE));
+          return new BatchScoreBulkScorer(get(Long.MAX_VALUE), scoreMode);
         }
 
         @Override

lucene/core/src/test/org/apache/lucene/search/ReadAheadMatchAllDocsQuery.java

@@ -78,7 +78,7 @@ public BulkScorer bulkScorer() throws IOException {
             List<DocIdSetIterator> clauses =
                 Collections.singletonList(DocIdSetIterator.all(context.reader().maxDoc()));
             return new DenseConjunctionBulkScorer(
-                clauses, Collections.emptyList(), context.reader().maxDoc(), score());
+                clauses, Collections.emptyList(), context.reader().maxDoc(), score(), scoreMode);
           }
 
           @Override