diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 4121b695d7c0..39c8d26a90f4 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -484,6 +484,8 @@ Optimizations * GITHUB#16297: Use LongHashSet for sparse ordinal sets in DocValuesRangeIterator so memory scales with matching terms instead of segment cardinality. (Costin Leau) +* GITHUB#16520: Support two-phase filters in MaxScoreBulkScorer bitset path. (Ke Wei) + Bug Fixes --------------------- diff --git a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java index 8d517092bc8f..dd553b8dbef5 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java @@ -70,7 +70,7 @@ final class MaxScoreBulkScorer extends BulkScorer { essentialQueue = DisiPriorityQueue.ofMaxSize(allScorers.length); maxScoreSums = new double[allScorers.length]; - if (this.filter != null && this.filter.twoPhaseView == null && maxDoc >= INNER_WINDOW_SIZE) { + if (this.filter != null && maxDoc >= INNER_WINDOW_SIZE) { long minScorerCost = allScorers[0].cost; for (int j = 1; j < allScorers.length; j++) { minScorerCost = Math.min(minScorerCost, allScorers[j].cost); @@ -234,7 +234,11 @@ private void fillScoreBufferViaBitSet(DisiWrapper top, Bits acceptDocs, int inne filter.doc = filter.approximation.advance(innerWindowMin); } if (filter.doc < innerWindowMax) { - filter.approximation.intoBitSet(innerWindowMax, filterMatches, innerWindowMin); + if (filter.twoPhaseView != null) { + filter.twoPhaseView.intoBitSet(innerWindowMax, filterMatches, innerWindowMin); + } else { + filter.approximation.intoBitSet(innerWindowMax, filterMatches, innerWindowMin); + } filter.doc = filter.approximation.docID(); } } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java index 834da692c978..f553dfb1e529 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java @@ -22,6 +22,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; @@ -1111,7 +1112,7 @@ public void testSparseFilterUsesLeapFrog() throws IOException { Query filterQuery = new CountingFilterQuery( - new TermQuery(new Term("filter", "yes")), intoBitSetCalls, advanceCalls); + new TermQuery(new Term("filter", "yes")), intoBitSetCalls, null, advanceCalls); BooleanQuery innerOr = new BooleanQuery.Builder() @@ -1190,7 +1191,7 @@ public void testDenseScorersUseBitSet() throws IOException { Query filterQuery = new CountingFilterQuery( - new TermQuery(new Term("filter", "yes")), intoBitSetCalls, advanceCalls); + new TermQuery(new Term("filter", "yes")), intoBitSetCalls, null, advanceCalls); BooleanQuery innerOr = new BooleanQuery.Builder() @@ -1242,11 +1243,17 @@ public void collect(int doc) {} private static class CountingFilterQuery extends Query { private final Query delegate; private final int[] intoBitSetCalls; + private final int[] twoPhaseIntoIntoBitSetCalls; private final int[] advanceCalls; - CountingFilterQuery(Query delegate, int[] intoBitSetCalls, int[] advanceCalls) { + CountingFilterQuery( + Query delegate, + int[] intoBitSetCalls, + int[] twoPhaseIntoIntoBitSetCalls, + int[] advanceCalls) { this.delegate = delegate; this.intoBitSetCalls = intoBitSetCalls; + this.twoPhaseIntoIntoBitSetCalls = twoPhaseIntoIntoBitSetCalls; this.advanceCalls = advanceCalls; } @@ -1263,6 +1270,29 @@ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOExcepti @Override public Scorer get(long leadCost) throws IOException { Scorer innerScorer = innerSS.get(leadCost); + TwoPhaseIterator innerTwoPhase = innerScorer.twoPhaseIterator(); + if (innerTwoPhase != null) { + TwoPhaseIterator countingTwoPhase = + new TwoPhaseIterator(innerTwoPhase.approximation()) { + @Override + public boolean matches() throws IOException { + return innerTwoPhase.matches(); + } + + @Override + public float matchCost() { + return innerTwoPhase.matchCost(); + } + + @Override + public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) + throws IOException { + twoPhaseIntoIntoBitSetCalls[0]++; + innerTwoPhase.intoBitSet(upTo, bitSet, offset); + } + }; + return new ConstantScoreScorer(0f, scoreMode, countingTwoPhase); + } DocIdSetIterator innerIter = innerScorer.iterator(); DocIdSetIterator countingIter = new FilterDocIdSetIterator(innerIter) { @@ -1405,6 +1435,69 @@ public void collect(int doc) {} dir.close(); } + public void testTwoPhaseFilterUsesBitSet() throws IOException { + Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig()); + for (int i = 0; i < 10000; i++) { + Document doc = new Document(); + doc.add(new TextField("body", "dense1", Field.Store.NO)); + doc.add(new TextField("body", "dense2", Field.Store.NO)); + doc.add(SortedNumericDocValuesField.indexedField("filter", i % 20)); + w.addDocument(doc); + } + w.close(); + + DirectoryReader reader = DirectoryReader.open(dir); + IndexSearcher searcher = new IndexSearcher(reader); + searcher.setQueryCache(null); + + BooleanQuery innerOr = + new BooleanQuery.Builder() + .add(new TermQuery(new Term("body", "dense1")), Occur.SHOULD) + .add(new TermQuery(new Term("body", "dense2")), Occur.SHOULD) + .build(); + + int[] twoPhaseIntoIntoBitSetCalls = {0}; + int[] collectedDocs = {0}; + Query delegateFilterQuery = SortedNumericDocValuesField.newSlowRangeQuery("filter", 1, 1); + Query filterQuery = + new CountingFilterQuery(delegateFilterQuery, null, twoPhaseIntoIntoBitSetCalls, new int[1]); + BooleanQuery outerQuery = + new BooleanQuery.Builder().add(innerOr, Occur.MUST).add(filterQuery, Occur.FILTER).build(); + + Query rewritten = searcher.rewrite(outerQuery); + Weight weight = searcher.createWeight(rewritten, ScoreMode.TOP_SCORES, 1f); + for (LeafReaderContext ctx : reader.leaves()) { + ScorerSupplier ss = weight.scorerSupplier(ctx); + if (ss != null) { + BulkScorer bs = ss.bulkScorer(); + assertTrue( + "Expected MaxScoreBulkScorer but got " + bs.getClass().getSimpleName(), + bs instanceof MaxScoreBulkScorer); + bs.score( + new LeafCollector() { + @Override + public void setScorer(Scorable scorer) {} + + @Override + public void collect(int doc) { + collectedDocs[0]++; + } + }, + null, + 0, + DocIdSetIterator.NO_MORE_DOCS); + } + } + + assertEquals(500, collectedDocs[0]); + assertTrue( + "Expected twoPhaseIntoIntoBitSetCalls() to be called on the two-phase filter", + twoPhaseIntoIntoBitSetCalls[0] > 0); + reader.close(); + dir.close(); + } + /** * A query wrapper that counts the smallest target/upTo ever passed to advanceShallow()/ * getMaxScore() on its scorer. This lets us detect whether MaxScoreBulkScorer computed score