diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 8df9186b6387..a96997f42f3c 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -26,6 +26,11 @@ API Changes New Features --------------------- +* GITHUB#16418: Incremental doc-values updates. A set-only doc-values update (NUMERIC or BINARY) is written as a sparse + "delta" overlay holding just the updated documents and layered over the base column at read time, instead of + rewriting the whole column, turning per-update write amplification from O(column) into O(updated docs). Opt-in; + enable with IndexWriterConfig#setMaxDocValuesOverlays(n). (Jim Ferenczi) + * GITHUB#16243: Add StableTflSimilarity, a similarity that estimates term rarity from term length and document length instead of corpus statistics such as document frequency. It also supports k3 query-term frequency saturation. (Tianxiao Wei) @@ -135,6 +140,9 @@ Bug Fixes * GITHUB#16546: Prevent RangeBulkScorer from passing empty ranges to LeafCollector. (jxy) +* GITHUB#16565: IndexWriter#updateBinaryDocValue now rejects updating a field that is part of the + index sort, matching IndexWriter#updateNumericDocValue. (Jim Ferenczi) + * GITHUB#16450: Fix DocValuesRangeIterator.docIDRunEnd() returning incorrect run boundaries, causing bulk-scoring to skip or mis-score documents in range and ordinal set queries. (Parker Timmins) diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java index ffcb9f07c9b1..034816fa4236 100644 --- a/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java +++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java @@ -248,6 +248,10 @@ boolean hasValue() { protected final int maxDoc; protected PagedMutable docs; protected int size; + // true once any doc's value was reset (removed); such a buffer cannot take the sparse incremental + // path. Protected so subclasses that override reset() (e.g. NumericDocValuesFieldUpdates) can set + // it too. + protected boolean anyReset; protected DocValuesFieldUpdates(int maxDoc, long delGen, String field, DocValuesType type) { this.maxDoc = maxDoc; @@ -360,9 +364,15 @@ final synchronized int size() { * @param doc the doc to update */ synchronized void reset(int doc) { + anyReset = true; addInternal(doc, HAS_NO_VALUE_MASK); } + /** Whether this buffer removed any doc's value (vs. only setting values). */ + final synchronized boolean anyReset() { + return anyReset; + } + final synchronized int add(int doc) { return addInternal(doc, HAS_VALUE_MASK); } diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesOverlayMerger.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesOverlayMerger.java new file mode 100644 index 000000000000..4a98517dfb03 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesOverlayMerger.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.index; + +import java.io.IOException; +import org.apache.lucene.search.DocIdSetIterator; + +/** + * Merges the doc iterators of several doc-values layers (ordered newest first) into the union of + * their docs and tracks which layer supplies the current value: the newest layer positioned on the + * current doc. Backed by a binary min-heap keyed by (docID, layer index), so advancing only + * re-positions the layers sitting on the current doc (the rest keep their place) instead of + * scanning every layer per doc. Shared by {@link OverlayNumericDocValues} and {@link + * OverlayBinaryDocValues}; layers are advanced with {@code advance} only (never {@code + * advanceExact}) so iteration and random access can be interleaved on the same instance. + */ +final class DocValuesOverlayMerger { + + private final DocValuesIterator[] layers; // newest first + private final int[] heap; // 1-based; holds layer indices, ordered by (docID, index) + private final int size; + private int docID = -1; + private int winner = -1; + + DocValuesOverlayMerger(DocValuesIterator[] layers) { + this.layers = layers; + this.size = layers.length; + this.heap = new int[size + 1]; + // all layers start at docID -1, so indices in natural order are already a valid min-heap + for (int i = 0; i < size; i++) { + heap[i + 1] = i; + } + } + + int docID() { + return docID; + } + + /** Index of the layer supplying the current value, or -1 if the current doc has no value. */ + int valueLayer() { + return winner; + } + + int nextDoc() throws IOException { + return advance(docID + 1); + } + + int advance(int target) throws IOException { + while (layers[heap[1]].docID() < target) { + layers[heap[1]].advance(target); + siftDown(); + } + docID = layers[heap[1]].docID(); + winner = docID == DocIdSetIterator.NO_MORE_DOCS ? -1 : heap[1]; + return docID; + } + + boolean advanceExact(int target) throws IOException { + while (layers[heap[1]].docID() < target) { + layers[heap[1]].advance(target); + siftDown(); + } + docID = target; + winner = layers[heap[1]].docID() == target ? heap[1] : -1; + return winner != -1; + } + + long cost() { + // Over-estimates the union (a doc in several layers counts once per layer); cost() is only an + // upper-bound hint. + long cost = 0; + for (DocValuesIterator layer : layers) { + cost += layer.cost(); + } + return cost; + } + + /** Restore the heap after the root layer advanced (its docID only ever increases). */ + private void siftDown() { + int i = 1; + int node = heap[1]; + while (true) { + int left = i << 1; + if (left > size) { + break; + } + int right = left + 1; + int child = right <= size && less(heap[right], heap[left]) ? right : left; + if (less(heap[child], node) == false) { + break; + } + heap[i] = heap[child]; + i = child; + } + heap[i] = node; + } + + private boolean less(int a, int b) { + int da = layers[a].docID(); + int db = layers[b].docID(); + if (da != db) { + return da < db; + } + return a < b; // same doc: the newer layer (smaller index) wins + } +} diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java index 9a0a00b5eeb9..374cb483c8b5 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java @@ -1159,7 +1159,8 @@ public IndexWriter(Directory d, IndexWriterConfig conf) throws IOException { bufferedUpdatesStream::getCompletedDelGen, infoStream, conf.getSoftDeletesField(), - reader); + reader, + config); if (config.getReaderPooling()) { readerPool.enableReaderPooling(); } @@ -2009,6 +2010,15 @@ public long updateBinaryDocValue(Term term, String field, BytesRef value) throws if (value == null) { throw new IllegalArgumentException("cannot update a field to a null value: " + field); } + // Checked before the doc-values-type check below: an index sort field is never binary, so this + // would otherwise surface as a less clear doc-values-type mismatch. + if (config.getIndexSortFields().contains(field)) { + throw new IllegalArgumentException( + "cannot update docvalues field involved in the index sort, field=" + + field + + ", sort=" + + config.getIndexSort()); + } globalFieldNumberMap.verifyOrCreateDvOnlyField(field, DocValuesType.BINARY, true); try { return maybeProcessEvents( @@ -3660,6 +3670,13 @@ private SegmentCommitInfo copySegmentAsIs( newInfo.setFiles(info.info.files()); newInfoPerCommit.setFieldInfosFiles(info.getFieldInfosFiles()); newInfoPerCommit.setDocValuesUpdatesFiles(info.getDocValuesUpdatesFiles()); + // Carry over the incremental doc-values overlay generations (the copied update files keep their + // gen numbers). + for (Map.Entry e : info.getDocValuesOverlays().entrySet()) { + final long[] packed = e.getValue(); + newInfoPerCommit.setDocValuesOverlay( + e.getKey(), packed[0], ArrayUtil.copyOfSubArray(packed, 1, packed.length)); + } boolean success = false; diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java index e20a6371ef2c..eac4c056c5e1 100644 --- a/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java +++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java @@ -107,6 +107,14 @@ public enum OpenMode { */ public static final long DEFAULT_MAX_FULL_FLUSH_MERGE_WAIT_MILLIS = 500; + /** + * Default maximum number of sparse doc-values overlays a field keeps before they are folded into + * one: {@code 0}, which disables the feature (classic full-column rewrite). Set a positive value + * to enable it; higher values lower write amplification but keep more overlays (hence more files) + * live to merge at read time. + */ + public static final int DEFAULT_MAX_DOC_VALUES_OVERLAYS = 0; + // indicates whether this config instance is already attached to a writer. // not final so that it can be cloned properly. private SetOnce writer = new SetOnce<>(); @@ -324,6 +332,29 @@ public boolean getReaderPooling() { return readerPooling; } + /** + * Sets the maximum number of sparse doc-values overlays a field may accumulate before they are + * folded into a single overlay, which also enables or disables the feature. When enabled, a + * doc-values update that only sets values (no removals) is written as a sparse "delta" holding + * just the updated documents and overlaid on the existing column at read time, rather than + * rewriting the whole column. This trades some read cost for much lower write amplification on + * frequently updated fields. Pass {@code 0} to disable the feature and keep the classic + * full-column rewrite; any value {@code > 0} enables it, keeping up to that many overlays before + * a fold. Disabled by default (see {@link #DEFAULT_MAX_DOC_VALUES_OVERLAYS}). + * + *

Only takes effect when IndexWriter is first created. + * + * @lucene.experimental + */ + public IndexWriterConfig setMaxDocValuesOverlays(int maxDocValuesOverlays) { + if (maxDocValuesOverlays < 0) { + throw new IllegalArgumentException( + "maxDocValuesOverlays must be >= 0; got " + maxDocValuesOverlays); + } + this.maxDocValuesOverlays = maxDocValuesOverlays; + return this; + } + /** * Expert: Controls when segments are flushed to disk during indexing. The {@link FlushPolicy} * initialized during {@link IndexWriter} instantiation and once initialized the given instance is diff --git a/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java b/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java index c9db6d0c6f66..0cb6b235b0d8 100644 --- a/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java +++ b/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java @@ -89,6 +89,12 @@ public class LiveIndexWriterConfig { /** True if calls to {@link IndexWriter#close()} should first do a commit. */ protected boolean commitOnClose = IndexWriterConfig.DEFAULT_COMMIT_ON_CLOSE; + /** + * Maximum number of sparse doc-values overlays a field keeps before they are folded into one; + * {@code 0} disables the feature. See {@link IndexWriterConfig#setMaxDocValuesOverlays}. + */ + protected int maxDocValuesOverlays; + /** The sort order to use to write merged segments. */ protected Sort indexSort = null; @@ -136,6 +142,7 @@ public class LiveIndexWriterConfig { mergePolicy = new TieredMergePolicy(); flushPolicy = new FlushByRamOrCountsPolicy(); readerPooling = IndexWriterConfig.DEFAULT_READER_POOLING; + maxDocValuesOverlays = IndexWriterConfig.DEFAULT_MAX_DOC_VALUES_OVERLAYS; perThreadHardLimitMB = IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB; maxFullFlushMergeWaitMillis = IndexWriterConfig.DEFAULT_MAX_FULL_FLUSH_MERGE_WAIT_MILLIS; eventListener = IndexWriterEventListener.NO_OP_LISTENER; @@ -385,6 +392,16 @@ public boolean getUseCompoundFile() { return useCompoundFile; } + /** + * Returns the maximum number of sparse doc-values overlays a field keeps before they are folded + * into one, or {@code 0} if the feature is disabled. + * + * @lucene.experimental + */ + public int getMaxDocValuesOverlays() { + return maxDocValuesOverlays; + } + /** * Returns true if {@link IndexWriter#close()} should first commit before closing. */ @@ -487,6 +504,7 @@ public String toString() { sb.append("readerPooling=").append(getReaderPooling()).append("\n"); sb.append("perThreadHardLimitMB=").append(getRAMPerThreadHardLimitMB()).append("\n"); sb.append("useCompoundFile=").append(getUseCompoundFile()).append("\n"); + sb.append("maxDocValuesOverlays=").append(getMaxDocValuesOverlays()).append("\n"); sb.append("commitOnClose=").append(getCommitOnClose()).append("\n"); sb.append("indexSort=").append(getIndexSort()).append("\n"); sb.append("checkPendingFlushOnUpdate=").append(isCheckPendingFlushOnUpdate()).append("\n"); diff --git a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java index a3c14486fbda..41b642ddf599 100644 --- a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java +++ b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java @@ -168,6 +168,7 @@ void add(int doc, BytesRef value) { @Override synchronized void reset(int doc) { + anyReset = true; bitSet.set(doc); this.hasAtLeastOneValue = true; if (hasNoValue == null) { diff --git a/lucene/core/src/java/org/apache/lucene/index/OverlayBinaryDocValues.java b/lucene/core/src/java/org/apache/lucene/index/OverlayBinaryDocValues.java new file mode 100644 index 000000000000..cb6a1f92e0b0 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/index/OverlayBinaryDocValues.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.index; + +import java.io.IOException; +import org.apache.lucene.util.BytesRef; + +/** Binary counterpart of {@link OverlayNumericDocValues}; see it for the layering and invariant. */ +final class OverlayBinaryDocValues extends BinaryDocValues { + + private final BinaryDocValues[] layers; // newest first, base last + private final DocValuesOverlayMerger merger; + + OverlayBinaryDocValues(BinaryDocValues[] layers) { + assert layers != null && layers.length > 0; + this.layers = layers; + this.merger = new DocValuesOverlayMerger(layers); + } + + @Override + public BytesRef binaryValue() throws IOException { + return layers[merger.valueLayer()].binaryValue(); + } + + @Override + public boolean advanceExact(int target) throws IOException { + return merger.advanceExact(target); + } + + @Override + public int docID() { + return merger.docID(); + } + + @Override + public int nextDoc() throws IOException { + return merger.nextDoc(); + } + + @Override + public int advance(int target) throws IOException { + return merger.advance(target); + } + + @Override + public long cost() { + return merger.cost(); + } +} diff --git a/lucene/core/src/java/org/apache/lucene/index/OverlayNumericDocValues.java b/lucene/core/src/java/org/apache/lucene/index/OverlayNumericDocValues.java new file mode 100644 index 000000000000..af7eccf55830 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/index/OverlayNumericDocValues.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.index; + +import java.io.IOException; + +/** + * Merges several {@link NumericDocValues} layers (newest first, base last) into one view: a doc's + * value comes from the newest layer that has it, else the base. The delta layers are set-only + * (removals go through the dense rewrite), so a doc has a value iff some layer has it and the + * newest wins, which makes iteration a plain union-merge. + */ +final class OverlayNumericDocValues extends NumericDocValues { + + private final NumericDocValues[] layers; // newest first, base last + private final DocValuesOverlayMerger merger; + + OverlayNumericDocValues(NumericDocValues[] layers) { + assert layers != null && layers.length > 0; + this.layers = layers; + this.merger = new DocValuesOverlayMerger(layers); + } + + @Override + public long longValue() throws IOException { + return layers[merger.valueLayer()].longValue(); + } + + @Override + public boolean advanceExact(int target) throws IOException { + return merger.advanceExact(target); + } + + @Override + public int docID() { + return merger.docID(); + } + + @Override + public int nextDoc() throws IOException { + return merger.nextDoc(); + } + + @Override + public int advance(int target) throws IOException { + return merger.advance(target); + } + + @Override + public long cost() { + return merger.cost(); + } +} diff --git a/lucene/core/src/java/org/apache/lucene/index/ReaderPool.java b/lucene/core/src/java/org/apache/lucene/index/ReaderPool.java index 1bac886c7b4c..ed9c93ec5186 100644 --- a/lucene/core/src/java/org/apache/lucene/index/ReaderPool.java +++ b/lucene/core/src/java/org/apache/lucene/index/ReaderPool.java @@ -52,6 +52,7 @@ final class ReaderPool implements Closeable { private final InfoStream infoStream; private final SegmentInfos segmentInfos; private final String softDeletesField; + private final LiveIndexWriterConfig config; // This is a "write once" variable (like the organic dye // on a DVD-R that may or may not be heated by a laser and // then cooled to permanently record the event): it's @@ -75,7 +76,8 @@ final class ReaderPool implements Closeable { LongSupplier completedDelGenSupplier, InfoStream infoStream, String softDeletesField, - StandardDirectoryReader reader) + StandardDirectoryReader reader, + LiveIndexWriterConfig config) throws IOException { this.directory = directory; this.originalDirectory = originalDirectory; @@ -84,6 +86,7 @@ final class ReaderPool implements Closeable { this.completedDelGenSupplier = completedDelGenSupplier; this.infoStream = infoStream; this.softDeletesField = softDeletesField; + this.config = config; if (reader != null) { // Pre-enroll all segment readers into the reader pool; this is necessary so // any in-memory NRT live docs are correctly carried over, and so NRT readers @@ -106,7 +109,8 @@ final class ReaderPool implements Closeable { new ReadersAndUpdates( segmentInfos.getIndexCreatedVersionMajor(), newReader, - newPendingDeletes(newReader, newReader.getOriginalSegmentInfo()))); + newPendingDeletes(newReader, newReader.getOriginalSegmentInfo()), + config)); } } } @@ -404,7 +408,7 @@ synchronized ReadersAndUpdates get(SegmentCommitInfo info, boolean create) { } rld = new ReadersAndUpdates( - segmentInfos.getIndexCreatedVersionMajor(), info, newPendingDeletes(info)); + segmentInfos.getIndexCreatedVersionMajor(), info, newPendingDeletes(info), config); // Steal initial reference: readerMap.put(info, rld); } else { diff --git a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java index 761ff5e7d4fd..ecffedfa9a12 100644 --- a/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java +++ b/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Locale; @@ -32,11 +33,13 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.codecs.FieldInfosFormat; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FlushInfo; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.TrackingDirectoryWrapper; +import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; @@ -48,6 +51,13 @@ // searching or merging), plus pending deletes and updates, // for a given segment final class ReadersAndUpdates { + + private static final long[] EMPTY_GENS = new long[0]; + + // Fold the deltas back into a dense column once they cover at least this fraction of the segment, + // capping the read-time overlay depth. TODO: expose as a config option. + private static final double FOLD_TO_DENSE_COVERAGE_RATIO = 0.5; + // Not final because we replace (clone) when we need to // change it and it's been shared: final SegmentCommitInfo info; @@ -65,6 +75,8 @@ final class ReadersAndUpdates { // the major version this index was created with private final int indexCreatedVersionMajor; + private final LiveIndexWriterConfig config; + // Indicates whether this segment is currently being merged. While a segment // is merging, all field updates are also registered in the // mergingDVUpdates map. Also, calls to writeFieldUpdates merge the @@ -88,10 +100,14 @@ final class ReadersAndUpdates { final AtomicLong ramBytesUsed = new AtomicLong(); ReadersAndUpdates( - int indexCreatedVersionMajor, SegmentCommitInfo info, PendingDeletes pendingDeletes) { + int indexCreatedVersionMajor, + SegmentCommitInfo info, + PendingDeletes pendingDeletes, + LiveIndexWriterConfig config) { this.info = info; this.pendingDeletes = pendingDeletes; this.indexCreatedVersionMajor = indexCreatedVersionMajor; + this.config = config; } /** @@ -100,9 +116,12 @@ final class ReadersAndUpdates { *

NOTE: steals incoming ref from reader. */ ReadersAndUpdates( - int indexCreatedVersionMajor, SegmentReader reader, PendingDeletes pendingDeletes) + int indexCreatedVersionMajor, + SegmentReader reader, + PendingDeletes pendingDeletes, + LiveIndexWriterConfig config) throws IOException { - this(indexCreatedVersionMajor, reader.getOriginalSegmentInfo(), pendingDeletes); + this(indexCreatedVersionMajor, reader.getOriginalSegmentInfo(), pendingDeletes, config); this.reader = reader; pendingDeletes.onNewReader(reader, info); } @@ -287,6 +306,7 @@ private synchronized void handleDVUpdates( DocValuesFormat dvFormat, final SegmentReader reader, Map> fieldFiles, + Map newOverlays, long maxDelGen, InfoStream infoStream) throws IOException { @@ -325,6 +345,66 @@ private synchronized void handleDVUpdates( final IOContext updatesContext = IOContext.flush(new FlushInfo(info.info.maxDoc(), bytes)); final FieldInfo fieldInfo = infos.fieldInfo(field); assert fieldInfo != null; + // The field's current overlay {baseGen, deltas...} (or null). baseGen is the recorded base + // once the field has deltas, otherwise the field's current column (-1 = core, or an earlier + // dense generation). + final long[] existingOverlay = info.getDocValuesOverlay(fieldInfo.number); + final long baseGen = + existingOverlay != null ? existingOverlay[0] : fieldInfo.getDocValuesGen(); + final long[] priorGens = + existingOverlay != null + ? ArrayUtil.copyOfSubArray(existingOverlay, 1, existingOverlay.length) + : EMPTY_GENS; + boolean anyRemoval = false; + for (DocValuesFieldUpdates u : updatesToApply) { + if (u.anyReset()) { + anyRemoval = true; + break; + } + } + // Set-only updates become a sparse delta generation overlaid at read time; a removal falls + // back to the dense + // rewrite. (Skip-indexed fields can't reach here: IndexWriter rejects doc-values updates on + // them.) + final boolean sparseDelta = config.getMaxDocValuesOverlays() > 0 && anyRemoval == false; + // Past maxDocValuesOverlays, fold the prior deltas into this write as one sparse generation + // (base untouched) instead of appending another. + // TODO: folding all deltas each cycle rewrites the folded generation repeatedly; a + // size-tiered policy would help. + final boolean compact = sparseDelta && priorGens.length >= config.getMaxDocValuesOverlays(); + final List deltaProducers = new ArrayList<>(); + long deltaCoverage = 0; // sum of the delta generations' docs-with-value counts (>= distinct) + // Open the prior deltas to measure their coverage; a sparse fold keeps the base separate + // (the core column or an earlier dense generation) and merges only the deltas. + if (compact) { + try { + for (long gen : priorGens) { + FieldInfo fiGen = SegmentDocValuesProducer.withGen(fieldInfo, gen); + SegmentReadState srs = + new SegmentReadState( + info.info.dir, + info.info, + new FieldInfos(new FieldInfo[] {fiGen}), + IOContext.DEFAULT, + Long.toString(gen, Character.MAX_RADIX)); + DocValuesProducer p = dvFormat.fieldsProducer(srs); + deltaProducers.add(p); + deltaCoverage += + type == DocValuesType.BINARY + ? p.getBinary(fieldInfo).cost() + : p.getNumeric(fieldInfo).cost(); + } + } catch (Throwable t) { + // The try-with-resources below owns the normal close; close here if opening a delta fails + // first. + IOUtils.closeWhileHandlingException(deltaProducers); + throw t; + } + } + // Fold into one dense column (reclaiming the base) once the deltas cover at least + // FOLD_TO_DENSE_COVERAGE_RATIO of the segment. + final boolean foldToDense = + compact && deltaCoverage >= info.info.maxDoc() * FOLD_TO_DENSE_COVERAGE_RATIO; fieldInfo.setDocValuesGen(nextDocValuesGen); final FieldInfos fieldInfos = new FieldInfos(new FieldInfo[] {fieldInfo}); // separately also track which files were created for this gen @@ -354,9 +434,19 @@ private synchronized void handleDVUpdates( @Override public BinaryDocValues getBinary(FieldInfo fieldInfoIn) throws IOException { DocValuesFieldUpdates.Iterator iterator = updateSupplier.apply(fieldInfo); + if (sparseDelta && compact == false) { + // write only the updated docs; unchanged docs come from the base at read time + return DocValuesFieldUpdates.Iterator.asBinaryDocValues(iterator); + } + // sparse fold merges the prior deltas (stays sparse); a dense rewrite (removal or + // fold-to-dense) merges the base column + final BinaryDocValues onDisk = + compact && foldToDense == false + ? overlayBinary(deltaProducers, fieldInfo) + : reader.getBinaryDocValues(field); final MergedDocValues mergedDocValues = new MergedDocValues<>( - reader.getBinaryDocValues(field), + onDisk, DocValuesFieldUpdates.Iterator.asBinaryDocValues(iterator), iterator); // Merge sort of the original doc values with updated doc values: @@ -407,9 +497,19 @@ public long cost() { @Override public NumericDocValues getNumeric(FieldInfo fieldInfoIn) throws IOException { DocValuesFieldUpdates.Iterator iterator = updateSupplier.apply(fieldInfo); + if (sparseDelta && compact == false) { + // write only the updated docs; unchanged docs come from the base at read time + return DocValuesFieldUpdates.Iterator.asNumericDocValues(iterator); + } + // sparse fold merges the prior deltas (stays sparse); a dense rewrite (removal or + // fold-to-dense) merges the base column + final NumericDocValues onDisk = + compact && foldToDense == false + ? overlayNumeric(deltaProducers, fieldInfo) + : reader.getNumericDocValues(field); final MergedDocValues mergedDocValues = new MergedDocValues<>( - reader.getNumericDocValues(field), + onDisk, DocValuesFieldUpdates.Iterator.asNumericDocValues(iterator), iterator); // Merge sort of the original doc values with updated doc values: @@ -453,6 +553,26 @@ public long cost() { } }); } + } finally { + IOUtils.close(deltaProducers); + } + // Stage the field's overlay generations; writeFieldUpdates commits them to the commit info + // only if the whole batch succeeds, so a later field's failure (whose rollback deletes these + // gen files) never leaves info referencing them. Packed as {baseGen, deltas...}. + if (compact && foldToDense == false) { + // the single folded generation replaces the prior deltas (base kept) + newOverlays.put(fieldInfo.number, new long[] {baseGen, nextDocValuesGen}); + } else if (sparseDelta && compact == false) { + // prepend this generation to the field's overlay list (base unchanged; -1 = the core + // column) + final long[] packed = new long[priorGens.length + 2]; + packed[0] = baseGen; + packed[1] = nextDocValuesGen; + System.arraycopy(priorGens, 0, packed, 2, priorGens.length); + newOverlays.put(fieldInfo.number, packed); + } else { + // a dense rewrite (removal or fold-to-dense) is a single column again: clear the overlay + newOverlays.put(fieldInfo.number, new long[] {-1}); } info.advanceDocValuesGen(); assert !fieldFiles.containsKey(fieldInfo.number); @@ -460,6 +580,28 @@ public long cost() { } } + /** + * Fresh overlay (newest generation first) over the given delta producers, used to fold deltas + * during compaction. + */ + private static NumericDocValues overlayNumeric( + List producers, FieldInfo fieldInfo) throws IOException { + NumericDocValues[] layers = new NumericDocValues[producers.size()]; + for (int i = 0; i < layers.length; i++) { + layers[i] = producers.get(i).getNumeric(fieldInfo); + } + return new OverlayNumericDocValues(layers); + } + + private static BinaryDocValues overlayBinary( + List producers, FieldInfo fieldInfo) throws IOException { + BinaryDocValues[] layers = new BinaryDocValues[producers.size()]; + for (int i = 0; i < layers.length; i++) { + layers[i] = producers.get(i).getBinary(fieldInfo); + } + return new OverlayBinaryDocValues(layers); + } + /** * This class merges the current on-disk DV with an incoming update DV instance and merges the two * instances giving the incoming update precedence in terms of values, in other words the values @@ -617,6 +759,9 @@ public synchronized boolean writeFieldUpdates( throws IOException { long startTimeNS = System.nanoTime(); final Map> newDVFiles = new HashMap<>(); + // Overlay generations staged per field (committed to info only if the whole write succeeds); it + // also drives file retention: a field keeps the files of the generations it still references. + final Map newOverlays = new HashMap<>(); Set fieldInfosFiles = null; FieldInfos fieldInfos = null; boolean any = false; @@ -687,7 +832,14 @@ public synchronized boolean writeFieldUpdates( final DocValuesFormat docValuesFormat = codec.docValuesFormat(); handleDVUpdates( - fieldInfos, trackingDir, docValuesFormat, reader, newDVFiles, maxDelGen, infoStream); + fieldInfos, + trackingDir, + docValuesFormat, + reader, + newDVFiles, + newOverlays, + maxDelGen, + infoStream); fieldInfosFiles = writeFieldInfosGen(fieldInfos, trackingDir, codec.fieldInfosFormat()); } finally { @@ -742,18 +894,39 @@ public synchronized boolean writeFieldUpdates( assert fieldInfosFiles != null; info.setFieldInfosFiles(fieldInfosFiles); - // update the doc-values updates files. the files map each field to its set - // of files, hence we copy from the existing map all fields w/ updates that - // were not updated in this session, and add new mappings for fields that - // were updated now. + // A field's live update files are those of the generations its overlay still references (the + // core column, baseGen -1, lives in the segment's own files). Carry over untouched fields; for + // the rest keep this session's files plus prior files whose generation is still referenced. assert newDVFiles.isEmpty() == false; for (Entry> e : info.getDocValuesUpdatesFiles().entrySet()) { - if (newDVFiles.containsKey(e.getKey()) == false) { - newDVFiles.put(e.getKey(), e.getValue()); + final int field = e.getKey(); + if (newDVFiles.containsKey(field) == false) { + newDVFiles.put(field, e.getValue()); + continue; } + final long[] overlay = newOverlays.get(field); // {baseGen, deltas...}, or {-1} when cleared + final Set live = new HashSet<>(newDVFiles.get(field)); + for (String f : e.getValue()) { + final long gen = IndexFileNames.parseGeneration(f); + for (long liveGen : overlay) { + if (liveGen != -1 && liveGen == gen) { + live.add(f); + break; + } + } + } + newDVFiles.put(field, live); } info.setDocValuesUpdatesFiles(newDVFiles); + // Commit the staged overlay generations now that the write and file bookkeeping succeeded, and + // before reopening the reader below so it reflects them. + for (Entry e : newOverlays.entrySet()) { + final long[] packed = e.getValue(); + info.setDocValuesOverlay( + e.getKey(), packed[0], ArrayUtil.copyOfSubArray(packed, 1, packed.length)); + } + // if there is a reader open, reopen it to reflect the updates if (reader != null) { swapNewReaderWithLatestLiveDocs(); diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java b/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java index ed101ecda9e0..7a04ed6b99ea 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java +++ b/lucene/core/src/java/org/apache/lucene/index/SegmentCommitInfo.java @@ -71,6 +71,11 @@ public class SegmentCommitInfo { // Track the per-field DocValues update files private final Map> dvUpdatesFiles = new HashMap<>(); + // Per-field sparse doc-values overlay: field number -> {baseGen, deltaGenNewestFirst...}, where a + // baseGen of -1 is a sentinel for the core column. Absent for a field using the classic + // single-generation column. + private final Map dvOverlays = new HashMap<>(); + // TODO should we add .files() to FieldInfosFormat, like we have on // LiveDocsFormat? // track the fieldInfos update files @@ -134,6 +139,45 @@ public void setDocValuesUpdatesFiles(Map> dvUpdatesFiles) { } } + /** + * Per-field incremental doc-values overlay, as {@code field number -> {baseGen, + * deltaGenNewestFirst...}}. Empty for segments whose doc-values fields all use the classic + * single-generation column. + * + * @lucene.internal + */ + public Map getDocValuesOverlays() { + return Collections.unmodifiableMap(dvOverlays); + } + + /** + * The {@code {baseGen, deltaGenNewestFirst...}} overlay for a field, or {@code null} if it has + * none. + */ + long[] getDocValuesOverlay(int fieldNumber) { + return dvOverlays.get(fieldNumber); + } + + /** + * Records the sparse doc-values overlay generations for a field (newest delta first, over {@code + * baseGen}), or clears it when {@code deltaGensNewestFirst} is empty. + */ + void setDocValuesOverlay(int fieldNumber, long baseGen, long[] deltaGensNewestFirst) { + if (deltaGensNewestFirst.length == 0) { + dvOverlays.remove(fieldNumber); + return; + } + long[] packed = new long[deltaGensNewestFirst.length + 1]; + packed[0] = baseGen; + System.arraycopy(deltaGensNewestFirst, 0, packed, 1, deltaGensNewestFirst.length); + dvOverlays.put(fieldNumber, packed); + } + + /** True if any field in this commit carries a sparse doc-values overlay. */ + public boolean hasDocValuesOverlays() { + return dvOverlays.isEmpty() == false; + } + /** Returns the FieldInfos file names. */ public Set getFieldInfosFiles() { return Collections.unmodifiableSet(fieldInfosFiles); @@ -397,6 +441,10 @@ public SegmentCommitInfo clone() { other.dvUpdatesFiles.put(e.getKey(), new HashSet<>(e.getValue())); } + for (Entry e : dvOverlays.entrySet()) { + other.dvOverlays.put(e.getKey(), e.getValue().clone()); + } + other.fieldInfosFiles.addAll(fieldInfosFiles); return other; diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java index 0f4df818ddcb..16fb388bb7ea 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java +++ b/lucene/core/src/java/org/apache/lucene/index/SegmentDocValuesProducer.java @@ -18,11 +18,13 @@ import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.IdentityHashMap; import java.util.Set; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.internal.hppc.IntObjectHashMap; import org.apache.lucene.internal.hppc.LongArrayList; +import org.apache.lucene.internal.hppc.LongObjectHashMap; import org.apache.lucene.store.Directory; /** Encapsulates multiple producers when there are docvalues updates as one producer */ @@ -31,11 +33,21 @@ // producer? class SegmentDocValuesProducer extends DocValuesProducer { - final IntObjectHashMap dvProducersByField = new IntObjectHashMap<>(); + // one entry per field: the producers for that field, newest generation first, authoritative base + // last + final IntObjectHashMap dvProducersByField = new IntObjectHashMap<>(); final Set dvProducers = Collections.newSetFromMap(new IdentityHashMap()); final LongArrayList dvGens = new LongArrayList(); + private final SegmentCommitInfo si; + private final Directory dir; + private final FieldInfos coreInfos; + private final SegmentDocValues segDocValues; + // producers opened for this reader, keyed by generation, so each generation is opened (and + // ref-counted) once + private final LongObjectHashMap openedByGen = new LongObjectHashMap<>(); + /** * Creates a new producer that handles updated docvalues fields * @@ -52,30 +64,32 @@ class SegmentDocValuesProducer extends DocValuesProducer { FieldInfos allInfos, SegmentDocValues segDocValues) throws IOException { + this.si = si; + this.dir = dir; + this.coreInfos = coreInfos; + this.segDocValues = segDocValues; try { - DocValuesProducer baseProducer = null; for (FieldInfo fi : allInfos) { if (fi.getDocValuesType() == DocValuesType.NONE) { continue; } + long[] overlay = si.getDocValuesOverlay(fi.number); + if (overlay != null) { + // incremental update: overlay the sparse delta generations over the base + dvProducersByField.put(fi.number, openOverlay(fi, overlay)); + continue; + } long docValuesGen = fi.getDocValuesGen(); if (docValuesGen == -1) { - if (baseProducer == null) { - // the base producer gets the original fieldinfos it wrote - baseProducer = segDocValues.getDocValuesProducer(docValuesGen, si, dir, coreInfos); - dvGens.add(docValuesGen); - dvProducers.add(baseProducer); - } - dvProducersByField.put(fi.number, baseProducer); + // the base producer gets the original fieldinfos it wrote (shared across all base fields) + dvProducersByField.put(fi.number, new DocValuesProducer[] {getProducer(-1, coreInfos)}); } else { - assert !dvGens.contains(docValuesGen); // otherwise, producer sees only the one fieldinfo it wrote - final DocValuesProducer dvp = - segDocValues.getDocValuesProducer( - docValuesGen, si, dir, new FieldInfos(new FieldInfo[] {fi})); - dvGens.add(docValuesGen); - dvProducers.add(dvp); - dvProducersByField.put(fi.number, dvp); + dvProducersByField.put( + fi.number, + new DocValuesProducer[] { + getProducer(docValuesGen, new FieldInfos(new FieldInfo[] {fi})) + }); } } } catch (Throwable t) { @@ -88,46 +102,130 @@ class SegmentDocValuesProducer extends DocValuesProducer { } } + /** + * Opens the producer for one generation, opening (and ref-counting) each generation at most once + * per reader. + */ + private DocValuesProducer getProducer(long gen, FieldInfos infos) throws IOException { + DocValuesProducer p = openedByGen.get(gen); + if (p == null) { + p = segDocValues.getDocValuesProducer(gen, si, dir, infos); + openedByGen.put(gen, p); + dvGens.add(gen); + dvProducers.add(p); + } + return p; + } + + /** + * Builds the ordered producer stack (newest delta first, base last) for an overlay field from its + * {@code {baseGen, deltaGenNewestFirst...}} generations (see {@link + * SegmentCommitInfo#getDocValuesOverlay}). + */ + private DocValuesProducer[] openOverlay(FieldInfo fi, long[] overlay) throws IOException { + final long baseGen = overlay[0]; + final int numDeltas = overlay.length - 1; + final boolean hasCoreBase = baseGen == -1 && coreInfos.fieldInfo(fi.name) != null; + DocValuesProducer[] producers = + new DocValuesProducer[numDeltas + ((baseGen != -1 || hasCoreBase) ? 1 : 0)]; + int i = 0; + for (int d = 1; d <= numDeltas; d++) { + long gen = overlay[d]; + producers[i++] = getProducer(gen, new FieldInfos(new FieldInfo[] {withGen(fi, gen)})); + } + if (baseGen == -1) { + if (hasCoreBase) { + producers[i] = getProducer(-1, coreInfos); + } + } else { + producers[i] = getProducer(baseGen, new FieldInfos(new FieldInfo[] {withGen(fi, baseGen)})); + } + return producers; + } + + /** + * A copy of {@code fi} with its doc-values generation set to {@code gen}, so the codec reads the + * right gen files. + */ + static FieldInfo withGen(FieldInfo fi, long gen) { + FieldInfo copy = + new FieldInfo( + fi.name, + fi.number, + fi.hasTermVectors(), + fi.omitsNorms(), + fi.hasPayloads(), + fi.getIndexOptions(), + fi.getDocValuesType(), + fi.docValuesSkipIndexType(), + gen, + new HashMap<>(fi.attributes()), + fi.getPointDimensionCount(), + fi.getPointIndexDimensionCount(), + fi.getPointNumBytes(), + fi.getVectorDimension(), + fi.getVectorEncoding(), + fi.getVectorSimilarityFunction(), + fi.isSoftDeletesField(), + fi.isParentField()); + return copy; + } + @Override public NumericDocValues getNumeric(FieldInfo field) throws IOException { - DocValuesProducer dvProducer = dvProducersByField.get(field.number); - assert dvProducer != null; - return dvProducer.getNumeric(field); + DocValuesProducer[] producers = dvProducersByField.get(field.number); + assert producers != null; + if (producers.length == 1) { + return producers[0].getNumeric(field); + } + NumericDocValues[] layers = new NumericDocValues[producers.length]; + for (int i = 0; i < producers.length; i++) { + layers[i] = producers[i].getNumeric(field); + } + return new OverlayNumericDocValues(layers); } @Override public BinaryDocValues getBinary(FieldInfo field) throws IOException { - DocValuesProducer dvProducer = dvProducersByField.get(field.number); - assert dvProducer != null; - return dvProducer.getBinary(field); + DocValuesProducer[] producers = dvProducersByField.get(field.number); + assert producers != null; + if (producers.length == 1) { + return producers[0].getBinary(field); + } + BinaryDocValues[] layers = new BinaryDocValues[producers.length]; + for (int i = 0; i < producers.length; i++) { + layers[i] = producers[i].getBinary(field); + } + return new OverlayBinaryDocValues(layers); + } + + // sorted/sorted-set/sorted-numeric and skippers are never overlaid: only numeric/binary values + // can be updated in place, so these always have a single producer. + private DocValuesProducer single(FieldInfo field) { + DocValuesProducer[] producers = dvProducersByField.get(field.number); + assert producers != null && producers.length == 1 + : "field is not a single-generation field: " + field.name; + return producers[0]; } @Override public SortedDocValues getSorted(FieldInfo field) throws IOException { - DocValuesProducer dvProducer = dvProducersByField.get(field.number); - assert dvProducer != null; - return dvProducer.getSorted(field); + return single(field).getSorted(field); } @Override public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { - DocValuesProducer dvProducer = dvProducersByField.get(field.number); - assert dvProducer != null; - return dvProducer.getSortedNumeric(field); + return single(field).getSortedNumeric(field); } @Override public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException { - DocValuesProducer dvProducer = dvProducersByField.get(field.number); - assert dvProducer != null; - return dvProducer.getSortedSet(field); + return single(field).getSortedSet(field); } @Override public DocValuesSkipper getSkipper(FieldInfo field) throws IOException { - DocValuesProducer dvProducer = dvProducersByField.get(field.number); - assert dvProducer != null; - return dvProducer.getSkipper(field); + return single(field).getSkipper(field); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java index 5332e637bc3c..27bd1473cf3d 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java +++ b/lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java @@ -62,7 +62,8 @@ *

* * Data types: @@ -82,6 +83,10 @@ *
  • CommitUserData --> {@link DataOutput#writeMapOfStrings Map<String,String>} *
  • UpdatesFiles --> Map<{@link DataOutput#writeInt Int32}, {@link * DataOutput#writeSetOfStrings(Set) Set<String>}> + *
  • DocValuesOverlays --> OverlayCount, <FieldNumber, BaseGen, DeltaCount, + * DeltaGenDeltaCount>OverlayCount; OverlayCount, FieldNumber and + * DeltaCount are {@link DataOutput#writeInt Int32}, BaseGen and DeltaGen are {@link + * DataOutput#writeLong Int64} *
  • Footer --> {@link CodecUtil#writeFooter CodecFooter} * * @@ -106,6 +111,9 @@ * no updates to DocValues in that segment. Anything above zero means there are updates to * DocValues stored by {@link DocValuesFormat}. *
  • UpdatesFiles stores the set of files that were updated in that segment per field. + *
  • DocValuesOverlays records, per field that has incremental (sparse) doc-values updates, the + * base generation and the delta generations (newest first) overlaid on it at read time; empty + * for segments without overlays. * * * @lucene.experimental @@ -118,7 +126,10 @@ public final class SegmentInfos implements Cloneable, Iterable= VERSION_10_6) { + final int numOverlayFields = CodecUtil.readBEInt(input); + for (int i = 0; i < numOverlayFields; i++) { + final int fieldNumber = CodecUtil.readBEInt(input); + final long baseGen = CodecUtil.readBELong(input); + final int numDeltas = CodecUtil.readBEInt(input); + final long[] deltaGens = new long[numDeltas]; + for (int g = 0; g < numDeltas; g++) { + deltaGens[g] = CodecUtil.readBELong(input); + } + siPerCommit.setDocValuesOverlay(fieldNumber, baseGen, deltaGens); + } + } infos.add(siPerCommit); Version segmentVersion = info.getVersion(); @@ -699,6 +723,20 @@ public void write(IndexOutput out) throws IOException { CodecUtil.writeBEInt(out, e.getKey()); out.writeSetOfStrings(e.getValue()); } + // Doc-values overlays, part of the format since VERSION_10_6 (which VERSION_CURRENT always + // is). + // field -> {baseGen, deltaGenNewestFirst...}; empty for segments without overlays. + final Map overlays = siPerCommit.getDocValuesOverlays(); + CodecUtil.writeBEInt(out, overlays.size()); + for (Entry e : overlays.entrySet()) { + final long[] packed = e.getValue(); + CodecUtil.writeBEInt(out, e.getKey()); + CodecUtil.writeBELong(out, packed[0]); // baseGen + CodecUtil.writeBEInt(out, packed.length - 1); // number of delta generations + for (int g = 1; g < packed.length; g++) { + CodecUtil.writeBELong(out, packed[g]); + } + } } out.writeMapOfStrings(userData); CodecUtil.writeFooter(out); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java index e014f69275ab..d28dfe1f7020 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java @@ -1326,7 +1326,11 @@ public void testAddIndexes() throws Exception { public void testDeleteUnusedUpdatesFiles() throws Exception { Directory dir = newDirectory(); - IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); + // Asserts the dense-rewrite behavior of superseding a field's prior generation, so it disables + // the sparse incremental path, which intentionally retains prior overlays to layer them at read + // time. + IndexWriterConfig conf = + newIndexWriterConfig(new MockAnalyzer(random())).setMaxDocValuesOverlays(0); IndexWriter writer = new IndexWriter(dir, conf); Document doc = new Document(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIncrementalDocValuesUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestIncrementalDocValuesUpdates.java new file mode 100644 index 000000000000..de5fa719dc60 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/index/TestIncrementalDocValuesUpdates.java @@ -0,0 +1,622 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.document.BinaryDocValuesField; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.TestUtil; +import org.apache.lucene.util.BytesRef; + +/** + * Tests the incremental doc-values update path ({@link IndexWriterConfig#setMaxDocValuesOverlays}), + * where a set-only update is stored as a sparse delta overlaid on the base column at read time and + * the overlays are folded once they exceed the configured maximum. + */ +public class TestIncrementalDocValuesUpdates extends LuceneTestCase { + + private IndexWriterConfig incrementalConfig() { + return new IndexWriterConfig(new MockAnalyzer(random())) + .setMaxDocValuesOverlays(TestUtil.nextInt(random(), 1, 6)); + } + + private static void assertNumeric(IndexReader reader, String id, long expected) + throws IOException { + assertNumericField(reader, id, "val", expected); + } + + private static void assertNumericField(IndexReader reader, String id, String field, long expected) + throws IOException { + for (LeafReaderContext ctx : reader.leaves()) { + TermsEnum te = ctx.reader().terms("id").iterator(); + if (te.seekExact(new BytesRef(id))) { + PostingsEnum pe = te.postings(null); + int doc = pe.nextDoc(); + NumericDocValues dv = ctx.reader().getNumericDocValues(field); + assertTrue(id + "." + field, dv.advanceExact(doc)); + assertEquals(id + "." + field, expected, dv.longValue()); + return; + } + } + fail("id not found: " + id); + } + + /** Random set-only updates across many docs and fields, interleaved with reopens and merges. */ + public void testRandomizedSetOnlyUpdates() throws Exception { + try (Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, incrementalConfig())) { + int numDocs = atLeast(50); + Map expected = new HashMap<>(); + for (int i = 0; i < numDocs; i++) { + Document d = new Document(); + d.add(new StringField("id", "d" + i, StringField.Store.NO)); + d.add(new NumericDocValuesField("val", i)); + expected.put("d" + i, (long) i); + w.addDocument(d); + } + int updates = atLeast(200); + for (int i = 0; i < updates; i++) { + String id = "d" + random().nextInt(numDocs); + long v = random().nextLong(); + w.updateNumericDocValue(new Term("id", id), "val", v); + expected.put(id, v); + if (rarely()) { + w.commit(); + } + if (rarely()) { + w.forceMerge(1 + random().nextInt(3)); + } + } + try (DirectoryReader reader = DirectoryReader.open(w)) { + for (Map.Entry e : expected.entrySet()) { + assertNumeric(reader, e.getKey(), e.getValue()); + } + } + } + } + + /** + * A merge flattens the overlay back into a single dense column that still reads the latest + * values. + */ + public void testMergeFlattensOverlay() throws Exception { + try (Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, incrementalConfig())) { + for (int i = 0; i < 20; i++) { + Document d = new Document(); + d.add(new StringField("id", "d" + i, StringField.Store.NO)); + d.add(new NumericDocValuesField("val", i)); + w.addDocument(d); + } + for (int round = 0; round < 10; round++) { + for (int i = 0; i < 20; i++) { + w.updateNumericDocValue(new Term("id", "d" + i), "val", 1000L + round * 100 + i); + } + } + w.forceMerge(1); + try (DirectoryReader reader = DirectoryReader.open(w)) { + assertEquals(1, reader.leaves().size()); + for (int i = 0; i < 20; i++) { + assertNumeric(reader, "d" + i, 1000L + 9 * 100 + i); + } + } + } + } + + /** + * Removing a value (null update) falls back to the dense rewrite and is observed as "no value". + */ + public void testResetRemovesValue() throws Exception { + try (Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, incrementalConfig())) { + Document d = new Document(); + d.add(new StringField("id", "d0", StringField.Store.NO)); + d.add(new NumericDocValuesField("val", 5)); + w.addDocument(d); + w.updateNumericDocValue(new Term("id", "d0"), "val", 7); + w.updateDocValues(new Term("id", "d0"), new NumericDocValuesField("val", null)); + try (DirectoryReader reader = DirectoryReader.open(w)) { + LeafReaderContext ctx = reader.leaves().get(0); + NumericDocValues dv = ctx.reader().getNumericDocValues("val"); + assertFalse("value should have been removed", dv.advanceExact(0)); + } + } + } + + /** + * An index whose updates were written with the feature disabled keeps working when it is enabled. + */ + public void testContinuesIndexWrittenWithFeatureDisabled() throws Exception { + try (Directory dir = newDirectory()) { + try (IndexWriter w = + new IndexWriter( + dir, new IndexWriterConfig(new MockAnalyzer(random())).setMaxDocValuesOverlays(0))) { + for (int i = 0; i < 10; i++) { + Document d = new Document(); + d.add(new StringField("id", "d" + i, StringField.Store.NO)); + d.add(new NumericDocValuesField("val", i)); + w.addDocument(d); + } + // dense update generations, written by the classic path + for (int i = 0; i < 10; i++) { + w.updateNumericDocValue(new Term("id", "d" + i), "val", 100L + i); + } + w.commit(); + } + // reopen with the feature enabled and stack sparse deltas on top of the dense generations + try (IndexWriter w = new IndexWriter(dir, incrementalConfig())) { + for (int i = 0; i < 10; i++) { + w.updateNumericDocValue(new Term("id", "d" + i), "val", 200L + i); + } + try (DirectoryReader reader = DirectoryReader.open(w)) { + for (int i = 0; i < 10; i++) { + assertNumeric(reader, "d" + i, 200L + i); + } + } + } + } + } + + /** + * Repeatedly updating the whole corpus makes the delta generations cover the entire column; the + * writer then folds back to a single dense column (reclaiming the base) instead of overlaying an + * ever-denser delta. Checked here only for value correctness across the transition. + */ + public void testFoldsToDenseOnFullCorpusUpdates() throws Exception { + try (Directory dir = newDirectory(); + IndexWriter w = + new IndexWriter( + dir, + new IndexWriterConfig(new MockAnalyzer(random())).setMaxDocValuesOverlays(2))) { + int numDocs = 40; + for (int i = 0; i < numDocs; i++) { + Document d = new Document(); + d.add(new StringField("id", "d" + i, StringField.Store.NO)); + d.add(new NumericDocValuesField("val", i)); + w.addDocument(d); + } + long expected = 0; + for (int round = 0; round < 12; round++) { + expected = 1000L + round; + for (int i = 0; i < numDocs; i++) { + w.updateNumericDocValue(new Term("id", "d" + i), "val", expected + i); + } + w.commit(); + } + try (DirectoryReader reader = DirectoryReader.open(w)) { + for (int i = 0; i < numDocs; i++) { + assertNumeric(reader, "d" + i, expected + i); + } + } + } + } + + /** + * Several updatable fields on the same docs, updated independently. Each field's overlay is + * tracked separately. + */ + public void testMultipleUpdatableFields() throws Exception { + try (Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, incrementalConfig())) { + int numDocs = atLeast(40); + int numFields = 3; + long[][] expected = new long[numDocs][numFields]; + for (int i = 0; i < numDocs; i++) { + Document d = new Document(); + d.add(new StringField("id", "d" + i, StringField.Store.NO)); + for (int f = 0; f < numFields; f++) { + long v = i * 10L + f; + d.add(new NumericDocValuesField("f" + f, v)); + expected[i][f] = v; + } + w.addDocument(d); + } + int updates = atLeast(300); + for (int u = 0; u < updates; u++) { + int i = random().nextInt(numDocs); + int f = random().nextInt(numFields); + long v = random().nextLong(); + w.updateNumericDocValue(new Term("id", "d" + i), "f" + f, v); + expected[i][f] = v; + if (rarely()) { + w.commit(); + } + if (rarely()) { + w.forceMerge(1); + } + } + try (DirectoryReader reader = DirectoryReader.open(w)) { + for (int i = 0; i < numDocs; i++) { + for (int f = 0; f < numFields; f++) { + assertNumericField(reader, "d" + i, "f" + f, expected[i][f]); + } + } + } + } + } + + /** + * Updates interleaved with deletes: deleted docs drop out, surviving docs keep their latest + * value. + */ + public void testUpdatesInterleavedWithDeletes() throws Exception { + try (Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, incrementalConfig())) { + int numDocs = atLeast(60); + Map expected = new HashMap<>(); + for (int i = 0; i < numDocs; i++) { + Document d = new Document(); + d.add(new StringField("id", "d" + i, StringField.Store.NO)); + d.add(new NumericDocValuesField("val", i)); + expected.put("d" + i, (long) i); + w.addDocument(d); + } + int ops = atLeast(300); + for (int o = 0; o < ops; o++) { + String id = "d" + random().nextInt(numDocs); + if (expected.containsKey(id) && random().nextInt(6) == 0) { + w.deleteDocuments(new Term("id", id)); + expected.remove(id); + } else if (expected.containsKey(id)) { + long v = random().nextLong(); + w.updateNumericDocValue(new Term("id", id), "val", v); + expected.put(id, v); + } + if (rarely()) { + w.commit(); + } + } + try (DirectoryReader reader = DirectoryReader.open(w)) { + for (Map.Entry e : expected.entrySet()) { + assertNumeric(reader, e.getKey(), e.getValue()); + } + } + } + } + + /** + * Doc-values updates on an index that is sorted on a different field: overlay docs are in the + * sorted order. + */ + public void testSortedIndexWithUpdates() throws Exception { + try (Directory dir = newDirectory()) { + IndexWriterConfig conf = + incrementalConfig().setIndexSort(new Sort(new SortField("sortkey", SortField.Type.LONG))); + try (IndexWriter w = new IndexWriter(dir, conf)) { + int numDocs = atLeast(40); + Map expected = new HashMap<>(); + for (int i = 0; i < numDocs; i++) { + Document d = new Document(); + d.add(new StringField("id", "d" + i, StringField.Store.NO)); + d.add(new NumericDocValuesField("sortkey", random().nextInt(1000))); + d.add(new NumericDocValuesField("val", i)); + expected.put("d" + i, (long) i); + w.addDocument(d); + } + for (int u = 0; u < atLeast(200); u++) { + String id = "d" + random().nextInt(numDocs); + long v = random().nextLong(); + w.updateNumericDocValue(new Term("id", id), "val", v); + expected.put(id, v); + if (rarely()) { + w.commit(); + } + } + try (DirectoryReader reader = DirectoryReader.open(w)) { + for (Map.Entry e : expected.entrySet()) { + assertNumeric(reader, e.getKey(), e.getValue()); + } + } + } + } + } + + /** + * addIndexes(CodecReader...) flattens the source overlay into the destination, preserving the + * latest values. + */ + public void testAddIndexesFromUpdatedIndex() throws Exception { + try (Directory src = newDirectory(); + Directory dst = newDirectory()) { + Map expected = new HashMap<>(); + try (IndexWriter w = new IndexWriter(src, incrementalConfig())) { + int numDocs = atLeast(30); + for (int i = 0; i < numDocs; i++) { + Document d = new Document(); + d.add(new StringField("id", "d" + i, StringField.Store.NO)); + d.add(new NumericDocValuesField("val", i)); + expected.put("d" + i, (long) i); + w.addDocument(d); + } + for (int u = 0; u < atLeast(150); u++) { + String id = "d" + random().nextInt(numDocs); + long v = random().nextLong(); + w.updateNumericDocValue(new Term("id", id), "val", v); + expected.put(id, v); + } + w.commit(); + } + try (IndexWriter w = new IndexWriter(dst, incrementalConfig()); + DirectoryReader src2 = DirectoryReader.open(src)) { + CodecReader[] readers = new CodecReader[src2.leaves().size()]; + for (int i = 0; i < readers.length; i++) { + readers[i] = (CodecReader) src2.leaves().get(i).reader(); + } + w.addIndexes(readers); + try (DirectoryReader reader = DirectoryReader.open(w)) { + for (Map.Entry e : expected.entrySet()) { + assertNumeric(reader, e.getKey(), e.getValue()); + } + } + } + } + } + + public void testBinarySetOnlyUpdates() throws Exception { + try (Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, incrementalConfig())) { + int numDocs = atLeast(30); + Map expected = new HashMap<>(); + for (int i = 0; i < numDocs; i++) { + Document d = new Document(); + d.add(new StringField("id", "d" + i, StringField.Store.NO)); + d.add(new BinaryDocValuesField("val", new BytesRef("v" + i))); + expected.put("d" + i, "v" + i); + w.addDocument(d); + } + int updates = atLeast(120); + for (int i = 0; i < updates; i++) { + String id = "d" + random().nextInt(numDocs); + String v = "u" + random().nextInt(1_000_000); + w.updateBinaryDocValue(new Term("id", id), "val", new BytesRef(v)); + expected.put(id, v); + if (rarely()) { + w.forceMerge(1); + } + } + try (DirectoryReader reader = DirectoryReader.open(w)) { + for (Map.Entry e : expected.entrySet()) { + boolean found = false; + for (LeafReaderContext ctx : reader.leaves()) { + TermsEnum te = ctx.reader().terms("id").iterator(); + if (te.seekExact(new BytesRef(e.getKey()))) { + int doc = te.postings(null).nextDoc(); + BinaryDocValues dv = ctx.reader().getBinaryDocValues("val"); + assertTrue(e.getKey(), dv.advanceExact(doc)); + assertEquals(e.getKey(), new BytesRef(e.getValue()), dv.binaryValue()); + found = true; + break; + } + } + assertTrue("id not found: " + e.getKey(), found); + } + } + } + } + + /** + * Doc-values updates on a skip-indexed field are rejected up front by {@link IndexWriter}. That + * pre-existing restriction is what lets the overlay assume a field's skipper always has a single + * producer (skippers are never overlaid), so the incremental path needs no special handling for + * them. + */ + public void testCannotUpdateSkipIndexedField() throws Exception { + try (Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, incrementalConfig())) { + Document d = new Document(); + d.add(new StringField("id", "0", StringField.Store.NO)); + d.add(NumericDocValuesField.indexedField("val", 1L)); + w.addDocument(d); + w.commit(); + IllegalArgumentException e = + expectThrows( + IllegalArgumentException.class, + () -> w.updateNumericDocValue(new Term("id", "0"), "val", 2L)); + assertTrue(e.getMessage(), e.getMessage().contains("doc values skip index")); + } + } + + /** + * A reader that predates the feature rejects the index (before any codec) rather than misreading + * a delta as the whole column, and the overlay round-trips through the segments file. + */ + public void testOverlaySegmentRejectedByOlderReaders() throws Exception { + Directory dir = newDirectory(); + try (IndexWriter w = new IndexWriter(dir, incrementalConfig())) { + Document d = new Document(); + d.add(new StringField("id", "0", StringField.Store.NO)); + d.add(new NumericDocValuesField("val", 1L)); + w.addDocument(d); + w.commit(); + w.updateNumericDocValue(new Term("id", "0"), "val", 2L); // writes a sparse overlay generation + w.commit(); + } + // The commit records overlay generations, so its segments file is written at VERSION_10_6; a + // reader that only understands up to VERSION_86 rejects it with IndexFormatTooNewException. + assertOldReaderRejects(dir); + // And the current reader sees the overlay round-tripped through the segments file. + assertTrue( + SegmentInfos.readLatestCommit(dir).asList().stream() + .anyMatch(SegmentCommitInfo::hasDocValuesOverlays)); + dir.close(); + } + + /** + * After the deltas first fold back to a dense column, further updates must keep writing sparse + * deltas over that dense base rather than degrading to a full-column rewrite on every fold. + */ + public void testSparseFoldOverDenseBase() throws Exception { + int numDocs = 10; + try (Directory dir = newDirectory(); + IndexWriter w = + new IndexWriter( + dir, + new IndexWriterConfig(new MockAnalyzer(random())) + .setMaxDocValuesOverlays(1) + .setMergePolicy(NoMergePolicy.INSTANCE))) { + for (int i = 0; i < numDocs; i++) { + Document d = new Document(); + d.add(new StringField("id", "d" + i, StringField.Store.NO)); + d.add(new NumericDocValuesField("val", i)); + w.addDocument(d); + } + w.commit(); + // Update every doc twice: the second round's coverage reaches maxDoc and folds to a dense + // column, so the field's base becomes a dense generation rather than the core column. + for (int round = 1; round <= 2; round++) { + for (int i = 0; i < numDocs; i++) { + w.updateNumericDocValue(new Term("id", "d" + i), "val", 100L * round + i); + } + w.commit(); + } + // Now keep updating a single doc across two more folds over the dense base. + for (int round = 0; round < 2; round++) { + w.updateNumericDocValue(new Term("id", "d0"), "val", 999L + round); + w.commit(); + } + // The last fold over the dense base must stay sparse: an overlay whose base is a dense + // generation (!= -1) with a folded delta, not a full-column rewrite that clears the overlay. + SegmentInfos sis = SegmentInfos.readLatestCommit(dir); + assertEquals(1, sis.size()); + Map overlays = sis.info(0).getDocValuesOverlays(); + assertFalse("expected a sparse overlay over the dense base", overlays.isEmpty()); + long[] packed = overlays.values().iterator().next(); + assertTrue("base should be a dense generation, not the core column", packed[0] != -1); + assertTrue("expected at least one delta over the dense base", packed.length >= 2); + try (DirectoryReader reader = DirectoryReader.open(dir)) { + assertNumeric(reader, "d0", 1000L); + for (int i = 1; i < numDocs; i++) { + assertNumeric(reader, "d" + i, 200L + i); + } + } + TestUtil.checkIndex(dir); + } + } + + /** + * Soft deletes are numeric doc-values updates on the soft-deletes field, so they ride the overlay + * path too. Marking docs across many commits folds that field's overlay (and folds it to a dense + * column once coverage crosses the threshold); liveness and the surviving values must stay + * correct throughout. + */ + public void testSoftDeletesOverOverlay() throws Exception { + String softField = "__soft_deletes"; + int numDocs = 20; + try (Directory dir = newDirectory(); + IndexWriter w = + new IndexWriter( + dir, + new IndexWriterConfig(new MockAnalyzer(random())) + .setSoftDeletesField(softField) + .setMaxDocValuesOverlays(2) + .setMergePolicy(NoMergePolicy.INSTANCE))) { + for (int i = 0; i < numDocs; i++) { + Document d = new Document(); + d.add(new StringField("id", "d" + i, StringField.Store.NO)); + d.add(new NumericDocValuesField("val", i)); + w.addDocument(d); + } + w.commit(); + // Soft-delete every even doc, one commit each, so the soft-deletes field accrues deltas that + // fold and eventually fold to a dense column (coverage reaches half the segment). + for (int i = 0; i < numDocs; i += 2) { + w.updateDocValues(new Term("id", "d" + i), new NumericDocValuesField(softField, 1L)); + w.commit(); + } + try (DirectoryReader reader = DirectoryReader.open(w)) { + assertEquals(numDocs / 2, reader.numDocs()); + for (int i = 1; i < numDocs; i += 2) { + assertNumeric(reader, "d" + i, i); + } + } + TestUtil.checkIndex(dir); + } + } + + /** + * addIndexes(Directory...) copies segments as-is via copySegmentAsIs, so the copied segment must + * keep its doc-values overlay rather than flatten it the way the CodecReader path does. + */ + public void testAddIndexesDirectoryCarriesOverlay() throws Exception { + try (Directory src = newDirectory(); + Directory dst = newDirectory()) { + int numDocs = 10; + try (IndexWriter w = + new IndexWriter(src, incrementalConfig().setMergePolicy(NoMergePolicy.INSTANCE))) { + for (int i = 0; i < numDocs; i++) { + Document d = new Document(); + d.add(new StringField("id", "d" + i, StringField.Store.NO)); + d.add(new NumericDocValuesField("val", i)); + w.addDocument(d); + } + w.commit(); + for (int i = 0; i < numDocs; i++) { + w.updateNumericDocValue(new Term("id", "d" + i), "val", 100L + i); + } + w.commit(); + assertTrue("source segment should carry an overlay", hasOverlay(src)); + } + try (IndexWriter w = + new IndexWriter( + dst, + new IndexWriterConfig(new MockAnalyzer(random())) + .setMergePolicy(NoMergePolicy.INSTANCE))) { + w.addIndexes(src); + w.commit(); + assertTrue("copied segment should still carry the overlay", hasOverlay(dst)); + try (DirectoryReader reader = DirectoryReader.open(w)) { + for (int i = 0; i < numDocs; i++) { + assertNumeric(reader, "d" + i, 100L + i); + } + } + } + TestUtil.checkIndex(dst); + } + } + + private static boolean hasOverlay(Directory dir) throws IOException { + for (SegmentCommitInfo si : SegmentInfos.readLatestCommit(dir)) { + if (si.getDocValuesOverlays().isEmpty() == false) { + return true; + } + } + return false; + } + + private static void assertOldReaderRejects(Directory dir) throws IOException { + String segmentsFile = SegmentInfos.getLastCommitSegmentsFileName(dir); + try (ChecksumIndexInput in = dir.openChecksumInput(segmentsFile)) { + assertEquals(CodecUtil.CODEC_MAGIC, CodecUtil.readBEInt(in)); + expectThrows( + IndexFormatTooNewException.class, + () -> + CodecUtil.checkHeaderNoMagic( + in, "segments", SegmentInfos.VERSION_74, SegmentInfos.VERSION_86)); + } + } +} diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java index fff8623e3f70..a7c6dc4a0235 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java @@ -2010,6 +2010,13 @@ public void testBadDVUpdate() throws Exception { assertEquals( exc.getMessage(), "cannot update docvalues field involved in the index sort, field=foo, sort="); + exc = + expectThrows( + IllegalArgumentException.class, + () -> w.updateBinaryDocValue(new Term("id", "0"), "foo", newBytesRef("bar"))); + assertEquals( + exc.getMessage(), + "cannot update docvalues field involved in the index sort, field=foo, sort="); w.close(); dir.close(); } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java index d5f014b302d9..be2bbb6f9118 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java @@ -1733,7 +1733,11 @@ private void ensureConsistentFieldInfos(FieldInfos old, FieldInfos after) { public void testDeleteUnusedUpdatesFiles() throws Exception { Directory dir = newDirectory(); - IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random())); + // Asserts the dense-rewrite behavior of superseding a field's prior generation, so it disables + // the sparse incremental path, which intentionally retains prior overlays to layer them at read + // time. + IndexWriterConfig conf = + newIndexWriterConfig(new MockAnalyzer(random())).setMaxDocValuesOverlays(0); IndexWriter writer = new IndexWriter(dir, conf); Document doc = new Document(); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOverlayBinaryDocValues.java b/lucene/core/src/test/org/apache/lucene/index/TestOverlayBinaryDocValues.java new file mode 100644 index 000000000000..cbec7c504587 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/index/TestOverlayBinaryDocValues.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.util.BytesRef; + +public class TestOverlayBinaryDocValues extends LuceneTestCase { + + private static BinaryDocValues sparse(int[] docs, String[] values) { + return new BinaryDocValues() { + int i = -1; + int doc = -1; + + @Override + public BytesRef binaryValue() { + return new BytesRef(values[i]); + } + + @Override + public int docID() { + return doc; + } + + @Override + public int advance(int target) { + i++; + while (i < docs.length && docs[i] < target) { + i++; + } + doc = i < docs.length ? docs[i] : DocIdSetIterator.NO_MORE_DOCS; + return doc; + } + + @Override + public int nextDoc() { + return advance(doc + 1); + } + + @Override + public boolean advanceExact(int target) { + int j = Math.max(i, 0); + while (j < docs.length && docs[j] < target) { + j++; + } + i = j; + doc = target; + return i < docs.length && docs[i] == target; + } + + @Override + public long cost() { + return docs.length; + } + }; + } + + public void testAdvanceExactNewestWins() throws IOException { + BinaryDocValues base = sparse(new int[] {0, 1, 2, 3}, new String[] {"a0", "a1", "a2", "a3"}); + BinaryDocValues d1 = sparse(new int[] {1, 2}, new String[] {"b1", "b2"}); + BinaryDocValues d2 = sparse(new int[] {2}, new String[] {"c2"}); + String[] expected = {"a0", "b1", "c2", "a3"}; + + OverlayBinaryDocValues overlay = + new OverlayBinaryDocValues(new BinaryDocValues[] {d2, d1, base}); + for (int doc = 0; doc < 4; doc++) { + assertTrue("doc " + doc, overlay.advanceExact(doc)); + assertEquals("doc " + doc, new BytesRef(expected[doc]), overlay.binaryValue()); + } + } + + public void testNextDocUnionMerge() throws IOException { + BinaryDocValues base = sparse(new int[] {0, 4}, new String[] {"a0", "a4"}); + BinaryDocValues d1 = + sparse(new int[] {4, 6}, new String[] {"b4", "b6"}); // newest wins on doc 4 + OverlayBinaryDocValues overlay = new OverlayBinaryDocValues(new BinaryDocValues[] {d1, base}); + + List seen = new ArrayList<>(); + List vals = new ArrayList<>(); + for (int doc = overlay.nextDoc(); + doc != DocIdSetIterator.NO_MORE_DOCS; + doc = overlay.nextDoc()) { + seen.add(doc); + vals.add(overlay.binaryValue().utf8ToString()); + } + assertEquals(List.of(0, 4, 6), seen); + assertEquals(List.of("a0", "b4", "b6"), vals); + } +} diff --git a/lucene/core/src/test/org/apache/lucene/index/TestOverlayNumericDocValues.java b/lucene/core/src/test/org/apache/lucene/index/TestOverlayNumericDocValues.java new file mode 100644 index 000000000000..9947de969fa4 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/index/TestOverlayNumericDocValues.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.tests.util.LuceneTestCase; + +public class TestOverlayNumericDocValues extends LuceneTestCase { + + /** + * Sparse array-backed {@link NumericDocValues} over ascending docs; used in one access mode per + * instance. + */ + private static NumericDocValues sparse(int[] docs, long[] values) { + return new NumericDocValues() { + int i = -1; + int doc = -1; + + @Override + public long longValue() { + return values[i]; + } + + @Override + public int docID() { + return doc; + } + + @Override + public int advance(int target) { + i++; + while (i < docs.length && docs[i] < target) { + i++; + } + doc = i < docs.length ? docs[i] : DocIdSetIterator.NO_MORE_DOCS; + return doc; + } + + @Override + public int nextDoc() { + return advance(doc + 1); + } + + @Override + public boolean advanceExact(int target) { + int j = Math.max(i, 0); + while (j < docs.length && docs[j] < target) { + j++; + } + i = j; + doc = target; + return i < docs.length && docs[i] == target; + } + + @Override + public long cost() { + return docs.length; + } + }; + } + + /** base dense 0..9, an older delta on {2,5}, a newest delta on {5,7}: newest wins on doc 5. */ + public void testAdvanceExactNewestWins() throws IOException { + NumericDocValues base = + sparse( + new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + new long[] {100, 101, 102, 103, 104, 105, 106, 107, 108, 109}); + NumericDocValues d1 = sparse(new int[] {2, 5}, new long[] {201, 205}); + NumericDocValues d2 = sparse(new int[] {5, 7}, new long[] {305, 307}); + long[] expected = {100, 101, 201, 103, 104, 305, 106, 307, 108, 109}; + + OverlayNumericDocValues overlay = + new OverlayNumericDocValues(new NumericDocValues[] {d2, d1, base}); + for (int doc = 0; doc < 10; doc++) { + assertTrue("doc " + doc, overlay.advanceExact(doc)); + assertEquals("doc " + doc, expected[doc], overlay.longValue()); + } + } + + public void testNextDocUnionMerge() throws IOException { + NumericDocValues base = + sparse( + new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + new long[] {100, 101, 102, 103, 104, 105, 106, 107, 108, 109}); + NumericDocValues d1 = sparse(new int[] {2, 5}, new long[] {201, 205}); + NumericDocValues d2 = sparse(new int[] {5, 7}, new long[] {305, 307}); + long[] expected = {100, 101, 201, 103, 104, 305, 106, 307, 108, 109}; + + OverlayNumericDocValues overlay = + new OverlayNumericDocValues(new NumericDocValues[] {d2, d1, base}); + List seen = new ArrayList<>(); + for (int doc = overlay.nextDoc(); + doc != DocIdSetIterator.NO_MORE_DOCS; + doc = overlay.nextDoc()) { + seen.add(doc); + assertEquals("doc " + doc, expected[doc], overlay.longValue()); + } + assertEquals(List.of(0, 1, 2, 3, 4, 5, 6, 7, 8, 9), seen); + } + + /** Sparse base: docs not covered by any layer have no value; the union still merges correctly. */ + public void testSparseBaseNextDocAndAdvanceExact() throws IOException { + NumericDocValues base = sparse(new int[] {0, 4}, new long[] {100, 104}); + NumericDocValues d1 = sparse(new int[] {4, 6}, new long[] {204, 206}); // newest wins on doc 4 + OverlayNumericDocValues overlay = + new OverlayNumericDocValues(new NumericDocValues[] {d1, base}); + + List seen = new ArrayList<>(); + for (int doc = overlay.nextDoc(); + doc != DocIdSetIterator.NO_MORE_DOCS; + doc = overlay.nextDoc()) { + seen.add(doc); + } + assertEquals(List.of(0, 4, 6), seen); + + OverlayNumericDocValues ra = + new OverlayNumericDocValues( + new NumericDocValues[] { + sparse(new int[] {4, 6}, new long[] {204, 206}), + sparse(new int[] {0, 4}, new long[] {100, 104}) + }); + assertTrue(ra.advanceExact(0)); + assertEquals(100, ra.longValue()); + assertFalse("doc 1 has no value", ra.advanceExact(1)); + assertTrue(ra.advanceExact(4)); + assertEquals(204, ra.longValue()); // newest layer wins + assertFalse("doc 5 has no value", ra.advanceExact(5)); + assertTrue(ra.advanceExact(6)); + assertEquals(206, ra.longValue()); + } + + public void testAdvanceSkips() throws IOException { + NumericDocValues base = + sparse( + new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, + new long[] {100, 101, 102, 103, 104, 105, 106, 107, 108, 109}); + NumericDocValues d1 = sparse(new int[] {5}, new long[] {205}); + OverlayNumericDocValues overlay = + new OverlayNumericDocValues(new NumericDocValues[] {d1, base}); + assertEquals(5, overlay.advance(5)); + assertEquals(205, overlay.longValue()); + assertEquals(6, overlay.nextDoc()); + assertEquals(106, overlay.longValue()); + assertEquals(9, overlay.advance(9)); + assertEquals(109, overlay.longValue()); + assertEquals(DocIdSetIterator.NO_MORE_DOCS, overlay.nextDoc()); + } +} diff --git a/lucene/core/src/test/org/apache/lucene/index/TestReaderPool.java b/lucene/core/src/test/org/apache/lucene/index/TestReaderPool.java index 3c59c696763e..2635be05bb9e 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestReaderPool.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestReaderPool.java @@ -46,7 +46,15 @@ public void testDrop() throws IOException { ReaderPool pool = new ReaderPool( - directory, directory, segmentInfos, fieldNumbers, () -> 0l, null, null, null); + directory, + directory, + segmentInfos, + fieldNumbers, + () -> 0l, + null, + null, + null, + new IndexWriterConfig()); SegmentCommitInfo commitInfo = RandomPicks.randomFrom(random(), segmentInfos.asList()); ReadersAndUpdates readersAndUpdates = pool.get(commitInfo, true); assertSame(readersAndUpdates, pool.get(commitInfo, false)); @@ -67,7 +75,15 @@ public void testPoolReaders() throws IOException { ReaderPool pool = new ReaderPool( - directory, directory, segmentInfos, fieldNumbers, () -> 0l, null, null, null); + directory, + directory, + segmentInfos, + fieldNumbers, + () -> 0l, + null, + null, + null, + new IndexWriterConfig()); SegmentCommitInfo commitInfo = RandomPicks.randomFrom(random(), segmentInfos.asList()); assertFalse(pool.isReaderPoolingEnabled()); pool.release(pool.get(commitInfo, true), random().nextBoolean()); @@ -111,7 +127,8 @@ public void testUpdate() throws IOException { () -> 0l, new NullInfoStream(), null, - null); + null, + new IndexWriterConfig()); int id = random().nextInt(10); if (random().nextBoolean()) { pool.enableReaderPooling(); @@ -188,7 +205,8 @@ public void testDeletes() throws IOException { () -> 0l, new NullInfoStream(), null, - null); + null, + new IndexWriterConfig()); int id = random().nextInt(10); if (random().nextBoolean()) { pool.enableReaderPooling(); @@ -241,7 +259,8 @@ public void testPassReaderToMergePolicyConcurrently() throws Exception { () -> 0L, new NullInfoStream(), null, - null); + null, + new IndexWriterConfig()); if (random().nextBoolean()) { pool.enableReaderPooling(); } @@ -329,7 +348,8 @@ public void testGetReaderByRam() throws IOException { () -> 0l, new NullInfoStream(), null, - null); + null, + new IndexWriterConfig()); assertEquals(0, pool.getReadersByRam().size()); int ord = 0; diff --git a/lucene/luke/src/java/org/apache/lucene/luke/models/util/IndexUtils.java b/lucene/luke/src/java/org/apache/lucene/luke/models/util/IndexUtils.java index 1703209666a8..d9b5a26d390b 100644 --- a/lucene/luke/src/java/org/apache/lucene/luke/models/util/IndexUtils.java +++ b/lucene/luke/src/java/org/apache/lucene/luke/models/util/IndexUtils.java @@ -350,8 +350,10 @@ protected String doBody(String segmentFileName) throws IOException { format = "Lucene 7.4 or later"; } else if (actualVersion == SegmentInfos.VERSION_86) { format = "Lucene 8.6 or later"; - } else if (actualVersion > SegmentInfos.VERSION_86) { - format = "Lucene 8.6 or later (UNSUPPORTED)"; + } else if (actualVersion == SegmentInfos.VERSION_10_6) { + format = "Lucene 10.6 or later"; + } else if (actualVersion > SegmentInfos.VERSION_10_6) { + format = "Lucene 10.6 or later (UNSUPPORTED)"; } } else { format = "Lucene 6.x or prior (UNSUPPORTED)"; diff --git a/lucene/luke/src/test/org/apache/lucene/luke/models/overview/TestOverviewImpl.java b/lucene/luke/src/test/org/apache/lucene/luke/models/overview/TestOverviewImpl.java index bab9acef1973..f372782cb1ed 100644 --- a/lucene/luke/src/test/org/apache/lucene/luke/models/overview/TestOverviewImpl.java +++ b/lucene/luke/src/test/org/apache/lucene/luke/models/overview/TestOverviewImpl.java @@ -84,7 +84,7 @@ public void testGetIndexVersion() { @Test public void testGetIndexFormat() { OverviewImpl overview = new OverviewImpl(reader, indexDir.toString()); - assertEquals("Lucene 8.6 or later", overview.getIndexFormat().get()); + assertEquals("Lucene 10.6 or later", overview.getIndexFormat().get()); } @Test