From e7828521ccfbd2076100cfec6cd3397dac495038 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 9 Oct 2026 09:38:01 +0800 Subject: [PATCH 01/15] [Storage] Replace IDeviceID.isTableModel with context checks --- .../execution/fragment/FragmentInstanceContext.java | 4 +--- .../queryengine/execution/fragment/QueryContext.java | 11 ++++++++--- .../ReadChunkAlignedSeriesCompactionExecutor.java | 3 ++- .../recover/file/UnsealedTsFileRecoverPerformer.java | 7 +++++-- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index c76b15b0f0fc4..b7060be8cbc64 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@ -490,9 +490,7 @@ public List getPathModifications(TsFileResource tsFileResource, IDevic getQueryModificationLoader( tsFileResource, modification -> - deviceID.isTableModel() - ? modification.affects(deviceID) - : modification.affectsAll(deviceID), + isTableModel() ? modification.affects(deviceID) : modification.affectsAll(deviceID), mods -> getPathModifications(mods, deviceID))) { return modificationLoader.getPathModifications(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java index a9cea56e96e3a..1bd5f70081fd4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java @@ -145,7 +145,7 @@ public PatternTreeMap loadAllModificationsFromDisk( } protected boolean shouldSkipModification(ModEntry modification) { - if (tables != null && modification instanceof TableDeletionEntry) { + if (isTableModel() && modification instanceof TableDeletionEntry) { String tableName = ((TableDeletionEntry) modification).getTableName(); return !tables.contains(tableName); } @@ -170,7 +170,7 @@ public List getPathModifications( return Collections.emptyList(); } List modEntries = fileModEntries.getOverlapped(deviceID, measurement); - if (deviceID.isTableModel()) { + if (isTableModel()) { // the pattern tree has false-positive for table model deletion, so we do a further // filtering modEntries = @@ -200,7 +200,7 @@ public List getPathModifications( } List modEntries = fileModEntries.getOverlapped(deviceID, AlignedPath.VECTOR_PLACEHOLDER); - if (deviceID.isTableModel()) { + if (isTableModel()) { // the pattern tree has false-positive for table model deletion, so we do a further // filtering modEntries = @@ -274,6 +274,11 @@ public boolean isIgnoreAllNullRows() { return ignoreAllNullRows; } + // ignoreAllNullRows is true only for tree-model queries. + public boolean isTableModel() { + return !ignoreAllNullRows; + } + public void setIgnoreAllNullRows(boolean ignoreAllNullRows) { this.ignoreAllNullRows = ignoreAllNullRows; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java index 947a22ae60027..90181e111a2ca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.utils.MetadataUtils; +import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionLastTimeCheckFailedException; @@ -109,7 +110,7 @@ public ReadChunkAlignedSeriesCompactionExecutor( this.targetResource = targetResource; this.summary = summary; this.ignoreAllNullRows = ignoreAllNullRows; - if (device.isTableModel()) { + if (PathUtils.isTableModelDatabase(database)) { collectValueColumnSchemaListForTable(database); } else { collectValueColumnSchemaListForTree(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java index d861723419070..abce4d0ec0306 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.schema.table.TsFileTableSchemaUtil; +import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.exception.DataRegionException; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; @@ -84,6 +85,7 @@ public class UnsealedTsFileRecoverPerformer extends AbstractTsFileRecoverPerform private final WALRecoverListener recoverListener; private final String databaseName; private final String dataRegionId; + private final boolean isTableModel; public UnsealedTsFileRecoverPerformer( TsFileResource tsFileResource, @@ -92,6 +94,7 @@ public UnsealedTsFileRecoverPerformer( super(tsFileResource); this.databaseName = tsFileResource.getDatabaseName(); this.dataRegionId = tsFileResource.getDataRegionId(); + this.isTableModel = PathUtils.isTableModelDatabase(databaseName); this.sequence = sequence; this.callbackAfterUnsealedTsFileRecovered = callbackAfterUnsealedTsFileRecovered; this.walRedoer = new TsFilePlanRedoer(tsFileResource); @@ -183,7 +186,7 @@ public void redoLog(WALEntry walEntry) { // delete data already flushed in the MemTable to avoid duplicates for (IDeviceID device : tsFileResource.getDevices()) { // iterating the index, must present - if (device.isTableModel()) { + if (isTableModel) { memTable.delete( new TableDeletionEntry( new DeletionPredicate(device.getTableName(), new FullExactMatch(device)), @@ -200,7 +203,7 @@ public void redoLog(WALEntry walEntry) { } } for (IDeviceID deviceID : memTable.getMemTableMap().keySet()) { - if (deviceID.isTableModel()) { + if (isTableModel) { registerToTsFile(deviceID.getTableName()); } } From d05671245a632f4e1b4a1e19ca2d091cbe65c4d3 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 9 Oct 2026 11:39:23 +0800 Subject: [PATCH 02/15] [Storage] Pass table model explicitly in compaction paths --- .../impl/FastCompactionPerformer.java | 8 ++-- .../impl/ReadChunkCompactionPerformer.java | 2 +- .../impl/ReadPointCompactionPerformer.java | 2 +- .../execute/utils/CompactionPathUtils.java | 10 +++-- .../execute/utils/CompactionUtils.java | 15 ++++--- .../utils/MultiTsFileDeviceIterator.java | 45 ++++++++++++------- .../FastAlignedSeriesCompactionExecutor.java | 6 ++- ...astNonAlignedSeriesCompactionExecutor.java | 2 +- .../fast/SeriesCompactionExecutor.java | 5 ++- .../selector/impl/SettleSelectorImpl.java | 7 ++- .../iotdb/db/utils/ModificationUtils.java | 20 +++++---- .../compaction/utils/CompactionUtilsTest.java | 11 ++--- 12 files changed, 81 insertions(+), 52 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java index 9b57f56499a5d..4af319684885a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java @@ -183,7 +183,6 @@ public void perform() throws Exception { // actually exist but the judgment return device being existed. sortedSourceFiles.addAll(seqFiles); sortedSourceFiles.addAll(unseqFiles); - boolean isTreeModel = !isAligned || device.getTableName().startsWith("root."); long ttl = deviceIterator.getTTLForCurrentDevice(); sortedSourceFiles.removeIf(x -> x.definitelyNotContains(device)); // checked above @@ -193,7 +192,9 @@ public void perform() throws Exception { if (ttl != Long.MAX_VALUE) { ttlDeletion = CompactionUtils.convertTtlToDeletion( - device, deviceIterator.getTimeLowerBoundForCurrentDevice()); + device, + deviceIterator.getTimeLowerBoundForCurrentDevice(), + deviceIterator.isTableModel()); } compactionWriter.setTTLDeletion(ttlDeletion); @@ -205,7 +206,8 @@ public void perform() throws Exception { compactionWriter.startChunkGroup(device, isAligned); if (isAligned) { - compactAlignedSeries(device, deviceIterator, compactionWriter, isTreeModel); + compactAlignedSeries( + device, deviceIterator, compactionWriter, !deviceIterator.isTableModel()); } else { compactNonAlignedSeries(device, deviceIterator, compactionWriter); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java index da2e15db500d4..82bdbd3e6496d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java @@ -256,7 +256,7 @@ private void compactAlignedSeries( readerAndChunkMetadataList, writer, summary, - device.getTableName().startsWith("root.")); + !deviceIterator.isTableModel()); compactionExecutor.execute(); for (ChunkMetadata chunkMetadata : writer.getChunkMetadataListOfCurrentDeviceInMemory()) { if (chunkMetadata.getMeasurementUid().isEmpty()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java index 8ba6cebb82de0..517f88cda77fd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java @@ -223,7 +223,7 @@ private void compactAlignedSeries( .map(IMeasurementSchema::getMeasurementName) .collect(Collectors.toList()); - fragmentInstanceContext.setIgnoreAllNullRows(device.getTableName().startsWith("root.")); + fragmentInstanceContext.setIgnoreAllNullRows(!deviceIterator.isTableModel()); IDataBlockReader dataBlockReader = constructReader( device, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java index 898ea1729f09e..fcde517479a5d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java @@ -31,13 +31,15 @@ public class CompactionPathUtils { private CompactionPathUtils() {} - public static MeasurementPath getPath(final IDeviceID device, final String measurement) + public static MeasurementPath getPath( + final IDeviceID device, final String measurement, final boolean isTableModel) throws IllegalPathException { - return getPath(device).concatAsMeasurementPath(measurement); + return getPath(device, isTableModel).concatAsMeasurementPath(measurement); } - public static PartialPath getPath(final IDeviceID device) throws IllegalPathException { - if (device.isTableModel()) { + public static PartialPath getPath(final IDeviceID device, final boolean isTableModel) + throws IllegalPathException { + if (isTableModel) { final String[] nodes = new String[device.segmentNum()]; nodes[0] = device.getTableName(); for (int i = 0; i < device.segmentNum() - 1; i++) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java index d26ecdb2129be..55143751f1828 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java @@ -428,12 +428,13 @@ public static List getMatchedModifications( PatternTreeMap patternTreeMap, IDeviceID deviceID, String measurement, - ModEntry ttlDeletion) + ModEntry ttlDeletion, + boolean isTableModel) throws IllegalPathException { if ((patternTreeMap == null) || patternTreeMap.isEmpty()) { return ttlDeletion == null ? Collections.emptyList() : Collections.singletonList(ttlDeletion); } - PartialPath path = CompactionPathUtils.getPath(deviceID, measurement); + PartialPath path = CompactionPathUtils.getPath(deviceID, measurement, isTableModel); List modEntries = patternTreeMap.getOverlapped(path); if (ttlDeletion != null) { if (!(modEntries instanceof ArrayList)) { @@ -443,7 +444,7 @@ public static List getMatchedModifications( } modEntries.add(ttlDeletion); } - if (path.getIDeviceID().isTableModel()) { + if (isTableModel) { modEntries = modEntries.stream() .filter(e -> e.affects(path.getIDeviceID()) && e.affects(path.getMeasurement())) @@ -500,9 +501,9 @@ public static ArrayDeviceTimeIndex buildDeviceTimeIndex( return resource.buildDeviceTimeIndex(); } - public static ModEntry convertTtlToDeletion(IDeviceID deviceID, long timeLowerBound) - throws IllegalPathException { - if (!deviceID.isTableModel()) { + public static ModEntry convertTtlToDeletion( + IDeviceID deviceID, long timeLowerBound, boolean isTableModel) throws IllegalPathException { + if (!isTableModel) { return new TreeDeletionEntry( new MeasurementPath(deviceID, IoTDBConstant.ONE_LEVEL_PATH_WILDCARD), Long.MIN_VALUE, @@ -524,7 +525,7 @@ public static List getLatestMeasurementSchemasForTreeModel( PartialPath devicePath; PathPatternTree patternTree = new PathPatternTree(); try { - devicePath = CompactionPathUtils.getPath(deviceID); + devicePath = CompactionPathUtils.getPath(deviceID, false); for (String measurement : measurements) { patternTree.appendFullPath(devicePath, measurement); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java index b9c1224067b97..1b5e64f5717e7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.path.PatternTreeMap; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache; import org.apache.iotdb.db.schemaengine.lease.MetadataLeaseManager; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; @@ -82,10 +83,10 @@ public class MultiTsFileDeviceIterator implements AutoCloseable { private final Map> modificationCache = new HashMap<>(); private Pair currentDevice = null; - private boolean ignoreAllNullRows; private long ttlForCurrentDevice; private long timeLowerBoundForCurrentDevice; private final String databaseName; + private final boolean isTableModel; /** * Used for compaction with read chunk performer. @@ -94,6 +95,7 @@ public class MultiTsFileDeviceIterator implements AutoCloseable { */ public MultiTsFileDeviceIterator(List tsFileResources) throws IOException { this.databaseName = tsFileResources.get(0).getDatabaseName(); + this.isTableModel = PathUtils.isTableModelDatabase(databaseName); this.tsFileResourcesSortedByDesc = new ArrayList<>(tsFileResources); this.tsFileResourcesSortedByAsc = new ArrayList<>(tsFileResources); // sort the files from the oldest to the newest @@ -131,6 +133,7 @@ public MultiTsFileDeviceIterator( this.tsFileResourcesSortedByDesc = new ArrayList<>(seqResources); tsFileResourcesSortedByDesc.addAll(unseqResources); this.databaseName = tsFileResourcesSortedByDesc.get(0).getDatabaseName(); + this.isTableModel = PathUtils.isTableModelDatabase(databaseName); // sort the files from the newest to the oldest Collections.sort( this.tsFileResourcesSortedByDesc, TsFileResource::compareFileCreationOrderByDesc); @@ -156,6 +159,7 @@ public MultiTsFileDeviceIterator( this.tsFileResourcesSortedByDesc = new ArrayList<>(seqResources); tsFileResourcesSortedByDesc.addAll(unseqResources); this.databaseName = tsFileResourcesSortedByDesc.get(0).getDatabaseName(); + this.isTableModel = PathUtils.isTableModelDatabase(databaseName); // sort tsfiles from the newest to the oldest Collections.sort( this.tsFileResourcesSortedByDesc, TsFileResource::compareFileCreationOrderByDesc); @@ -236,7 +240,6 @@ public Pair nextDevice() throws IllegalPathException, IOExce IDeviceID deviceID = currentDevice.left; boolean isAligned = currentDevice.right; - ignoreAllNullRows = !isAligned || deviceID.getTableName().startsWith("root."); if (MetadataLeaseManager.getInstance().isFenced()) { // Metadata lease fenced: this DataNode may hold a stale TTL (it could have missed a // ConfigNode @@ -245,7 +248,7 @@ public Pair nextDevice() throws IllegalPathException, IOExce // deletes nothing by TTL while fenced, and real TTL deletion resumes once the lease recovers // and the cache resyncs. (Checked first so the table path also avoids the fenced cache.) ttlForCurrentDevice = Long.MAX_VALUE; - } else if (!ignoreAllNullRows) { + } else if (isTableModel) { ttlForCurrentDevice = DataNodeTTLCache.getInstance().getTTLForTable(databaseName, deviceID.getTableName()); } else { @@ -262,6 +265,10 @@ public String getDatabaseName() { return databaseName; } + public boolean isTableModel() { + return isTableModel; + } + public long getTTLForCurrentDevice() { return ttlForCurrentDevice; } @@ -271,9 +278,9 @@ public long getTimeLowerBoundForCurrentDevice() { } public Map getAllSchemasOfCurrentDevice() throws IOException { - return ignoreAllNullRows - ? getAllSchemasOfCurrentDeviceForTree() - : getAllSchemasOfCurrentDeviceForTable(); + return isTableModel + ? getAllSchemasOfCurrentDeviceForTable() + : getAllSchemasOfCurrentDeviceForTree(); } /** @@ -448,9 +455,9 @@ public Map getCompactionSeriesContextOfCurrentD */ public Map>>> getTimeseriesSchemaAndMetadataOffsetOfCurrentDevice() throws IOException { - return ignoreAllNullRows - ? getTimeseriesSchemaAndMetadataOffsetOfCurrentDeviceForTree() - : getTimeseriesSchemaAndMetadataOffsetOfCurrentDeviceForTable(); + return isTableModel + ? getTimeseriesSchemaAndMetadataOffsetOfCurrentDeviceForTable() + : getTimeseriesSchemaAndMetadataOffsetOfCurrentDeviceForTree(); } @SuppressWarnings({"checkstyle:AtclauseOrderCheck", "squid:S3824"}) @@ -613,7 +620,7 @@ public Map getCompactionSeriesContextOfCurrentD TsFileSequenceReader reader = readerMap.get(tsFileResource); List alignedChunkMetadataList = reader.getAlignedChunkMetadataByMetadataIndexNode( - currentDevice.left, firstMeasurementNodeOfCurrentDevice, ignoreAllNullRows); + currentDevice.left, firstMeasurementNodeOfCurrentDevice, !isTableModel); applyModificationForAlignedChunkMetadataList(tsFileResource, alignedChunkMetadataList); readerAndChunkMetadataList.add(new Pair<>(reader, alignedChunkMetadataList)); } @@ -638,7 +645,9 @@ private void applyModificationForAlignedChunkMetadataList( ModEntry ttlDeletion = null; Optional startTime = tsFileResource.getStartTime(device); if (startTime.isPresent() && startTime.get() < timeLowerBoundForCurrentDevice) { - ttlDeletion = CompactionUtils.convertTtlToDeletion(device, timeLowerBoundForCurrentDevice); + ttlDeletion = + CompactionUtils.convertTtlToDeletion( + device, timeLowerBoundForCurrentDevice, isTableModel); } PatternTreeMap modifications = @@ -652,7 +661,7 @@ private void applyModificationForAlignedChunkMetadataList( // match time column modifications List modificationForTimeColumn = CompactionUtils.getMatchedModifications( - modifications, device, AlignedPath.VECTOR_PLACEHOLDER, ttlDeletion); + modifications, device, AlignedPath.VECTOR_PLACEHOLDER, ttlDeletion, isTableModel); // match value column modifications List> modificationForValueColumns = new ArrayList<>(); @@ -663,7 +672,7 @@ private void applyModificationForAlignedChunkMetadataList( } List modificationList = CompactionUtils.getMatchedModifications( - modifications, device, valueChunkMetadata.getMeasurementUid(), null); + modifications, device, valueChunkMetadata.getMeasurementUid(), null, isTableModel); modificationForValueColumns.add( modificationList.isEmpty() ? Collections.emptyList() : modificationList); } @@ -672,7 +681,7 @@ private void applyModificationForAlignedChunkMetadataList( alignedChunkMetadataList, modificationForTimeColumn, modificationForValueColumns, - ignoreAllNullRows); + !isTableModel); } public Map getReaderMap() { @@ -884,7 +893,11 @@ public String nextSeries() throws IllegalPathException { // collect the modifications for current series List modificationForCurrentSeries = CompactionUtils.getMatchedModifications( - modificationsInThisResource, device, currentCompactingSeries, ttlDeletion); + modificationsInThisResource, + device, + currentCompactingSeries, + ttlDeletion, + isTableModel); // if there are modifications of current series, apply them to the chunk metadata if (!modificationForCurrentSeries.isEmpty()) { @@ -902,7 +915,7 @@ public String nextSeries() throws IllegalPathException { // skip data of deleted table private boolean isCurrentDeviceDataInDeprecatedTable(TsFileResource resource) { - if (ignoreAllNullRows) { + if (!isTableModel) { return false; } String tableName = currentDevice.getLeft().getTableName(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java index cf9ed65b8194f..718373058f63c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java @@ -247,7 +247,8 @@ protected List getAlignedChunkMetadataList(TsFileR // get time modifications of this file List timeModifications = - getModificationsFromCache(resource, deviceId, AlignedPath.VECTOR_PLACEHOLDER); + getModificationsFromCache( + resource, deviceId, AlignedPath.VECTOR_PLACEHOLDER, !ignoreAllNullRows); // get value modifications of this file List> valueModifications = new ArrayList<>(); alignedChunkMetadataList @@ -260,7 +261,8 @@ protected List getAlignedChunkMetadataList(TsFileR valueModifications.add(null); } else { valueModifications.add( - getModificationsFromCache(resource, deviceId, x.getMeasurementUid())); + getModificationsFromCache( + resource, deviceId, x.getMeasurementUid(), !ignoreAllNullRows)); } } catch (IllegalPathException e) { throw new RuntimeException(e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java index 49cbcb14582f3..b187a568b8416 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java @@ -160,7 +160,7 @@ void deserializeFileIntoChunkMetadataQueue(List fileElements) ModificationUtils.modifyChunkMetaData( iChunkMetadataList, getModificationsFromCache( - resource, deviceId, iChunkMetadataList.get(0).getMeasurementUid())); + resource, deviceId, iChunkMetadataList.get(0).getMeasurementUid(), false)); if (iChunkMetadataList.isEmpty()) { // all chunks has been deleted in this file, just remove it removeFile(fileElement); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java index b3073bd3d258d..1452b27f239c3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java @@ -474,7 +474,7 @@ protected void removeFile(FileElement fileElement) throws IllegalPathException, * modification from ttl cache. */ protected List getModificationsFromCache( - TsFileResource tsFileResource, IDeviceID deviceId, String measurement) + TsFileResource tsFileResource, IDeviceID deviceId, String measurement, boolean isTableModel) throws IllegalPathException { PatternTreeMap allModifications = modificationCacheMap.get(tsFileResource.getTsFile().getName()); @@ -482,7 +482,8 @@ protected List getModificationsFromCache( allModifications, deviceId, measurement, - compactionWriter.getTTLLowerBoundForCurrentDevice()); + compactionWriter.getTTLLowerBoundForCurrentDevice(), + isTableModel); } @SuppressWarnings("squid:S3776") diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SettleSelectorImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SettleSelectorImpl.java index 61f8229cd3ca3..774a1d3cfe9bd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SettleSelectorImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SettleSelectorImpl.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.path.AlignedPath; import org.apache.iotdb.commons.path.PatternTreeMap; import org.apache.iotdb.commons.utils.CommonDateTimeUtils; +import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.StorageEngineMessages; @@ -67,6 +68,7 @@ public class SettleSelectorImpl implements ISettleSelector { private final boolean heavySelect; private final String storageGroupName; private final String dataRegionId; + private final boolean isTableModel; private final long timePartition; private final TsFileManager tsFileManager; private boolean isSeq; @@ -83,6 +85,7 @@ public SettleSelectorImpl( this.heavySelect = heavySelect; this.storageGroupName = storageGroupName; this.dataRegionId = dataRegionId; + this.isTableModel = PathUtils.isTableModelDatabase(storageGroupName); this.timePartition = timePartition; this.tsFileManager = tsFileManager; this.context = context; @@ -261,7 +264,7 @@ private FileDirtyInfo selectFileBaseOnDirtyData(TsFileResource resource) long ttl; String tableName = device.getTableName(); boolean hasSetTTL; - if (tableName.startsWith("root.")) { + if (!isTableModel) { ttl = DataNodeTTLCache.getInstance().getTTLForTree(device); hasSetTTL = ttl != Long.MAX_VALUE; } else { @@ -327,7 +330,7 @@ private boolean isDeviceDeletedByMods( throws IllegalPathException { return ModificationUtils.isAllDeletedByMods( CompactionUtils.getMatchedModifications( - modifications, device, AlignedPath.VECTOR_PLACEHOLDER, null), + modifications, device, AlignedPath.VECTOR_PLACEHOLDER, null, isTableModel), startTime, endTime); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java index 0ea1abee866e6..e6dd6170ae6e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java @@ -280,16 +280,17 @@ public static boolean isAllDeletedByMods( final PatternTreeMap modifications, final IDeviceID device, final long startTime, - final long endTime) + final long endTime, + final boolean isTableModel) throws IllegalPathException { final List mods = modifications.getOverlapped( - CompactionPathUtils.getPath(device, AlignedPath.VECTOR_PLACEHOLDER)); + CompactionPathUtils.getPath(device, AlignedPath.VECTOR_PLACEHOLDER, isTableModel)); return mods.stream() .anyMatch( modification -> modification.getTimeRange().contains(startTime, endTime) - && (!device.isTableModel() || modification.affects(device))); + && (!isTableModel || modification.affects(device))); } public static boolean isAllDeletedByMods( @@ -310,15 +311,16 @@ public static boolean isTimeSeriesDeletedByMods( final IDeviceID device, final String measurement, final long startTime, - final long endTime) + final long endTime, + final boolean isTableModel) throws IllegalPathException { final List mods = - modifications.getOverlapped(CompactionPathUtils.getPath(device, measurement)); + modifications.getOverlapped(CompactionPathUtils.getPath(device, measurement, isTableModel)); return mods.stream() .anyMatch( modification -> modification.getTimeRange().contains(startTime, endTime) - && (!device.isTableModel() + && (!isTableModel || modification.affects(device) && modification.affects(measurement))); } @@ -452,7 +454,8 @@ public static List sortAndMerge(List modifications) { public static boolean isDeviceDeletedByMods( final PatternTreeMap currentModifications, final ITimeIndex currentTimeIndex, - final IDeviceID device) + final IDeviceID device, + final boolean isTableModel) throws IllegalPathException { return isAllDeletedByMods( currentModifications, @@ -462,6 +465,7 @@ public static boolean isDeviceDeletedByMods( : currentTimeIndex.getStartTime(device).orElse(Long.MIN_VALUE), Objects.isNull(currentTimeIndex) ? Long.MAX_VALUE - : currentTimeIndex.getEndTime(device).orElse(Long.MAX_VALUE)); + : currentTimeIndex.getEndTime(device).orElse(Long.MAX_VALUE), + isTableModel); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionUtilsTest.java index 2a0890e627760..f7c0ab4105384 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionUtilsTest.java @@ -59,10 +59,11 @@ public void testCompactionPathUtilsForTableModelDevice() throws Exception { IDeviceID deviceID = new StringArrayDeviceID(new String[] {"db.table1", null, "tag1"}); Assert.assertArrayEquals( - new String[] {"db.table1", null, "tag1"}, CompactionPathUtils.getPath(deviceID).getNodes()); + new String[] {"db.table1", null, "tag1"}, + CompactionPathUtils.getPath(deviceID, true).getNodes()); Assert.assertArrayEquals( new String[] {"db.table1", null, "tag1", "s1"}, - CompactionPathUtils.getPath(deviceID, "s1").getNodes()); + CompactionPathUtils.getPath(deviceID, "s1", true).getNodes()); } @Test @@ -71,9 +72,9 @@ public void testCompactionPathUtilsForTwoLevelTreeDevice() throws Exception { IDeviceID deviceID = deletionPath.getIDeviceID(); Assert.assertArrayEquals( - new String[] {"root", "repro"}, CompactionPathUtils.getPath(deviceID).getNodes()); + new String[] {"root", "repro"}, CompactionPathUtils.getPath(deviceID, false).getNodes()); Assert.assertArrayEquals( - deletionPath.getNodes(), CompactionPathUtils.getPath(deviceID, "s1").getNodes()); + deletionPath.getNodes(), CompactionPathUtils.getPath(deviceID, "s1", false).getNodes()); TreeDeletionEntry deletion = new TreeDeletionEntry(deletionPath, Long.MIN_VALUE, Long.MAX_VALUE); @@ -82,7 +83,7 @@ public void testCompactionPathUtilsForTwoLevelTreeDevice() throws Exception { modifications.append(deletion.keyOfPatternTree(), deletion); List matchedModifications = - CompactionUtils.getMatchedModifications(modifications, deviceID, "s1", null); + CompactionUtils.getMatchedModifications(modifications, deviceID, "s1", null, false); Assert.assertEquals(1, matchedModifications.size()); Assert.assertEquals( deletionPath, ((TreeDeletionEntry) matchedModifications.get(0)).getPathPattern()); From 0e2d974e90529ec207212a2b90656d4121b8d31d Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 9 Oct 2026 11:49:58 +0800 Subject: [PATCH 03/15] [Storage] Pass table model explicitly to remaining callers --- .../TsFileInsertionEventQueryParser.java | 3 ++- ...sertionEventQueryParserTabletIterator.java | 2 +- .../scan/TsFileInsertionEventScanParser.java | 10 +++++--- ...sertionEventTableParserTabletIterator.java | 7 +++--- .../tsfile/parser/util/ModsOperationUtil.java | 24 ++++++++++++------- .../cache/partition/PartitionCache.java | 13 +++++----- .../load/LoadTsFileTableSchemaCache.java | 2 +- .../load/LoadTsFileTreeSchemaCache.java | 6 +++-- .../utils/TreeDiskUsageStatisticUtil.java | 2 +- 9 files changed, 42 insertions(+), 27 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java index c26910b426f18..e48fb71de7bfa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java @@ -320,7 +320,8 @@ public TsFileInsertionEventQueryParser( measurement, meta.getStatistics().getStartTime(), meta.getStatistics().getEndTime(), - currentModifications); + currentModifications, + false); } catch (IOException e) { LOGGER.warn( DataNodePipeMessages.FAILED_TO_READ_METADATA_FOR_DEVICEID_MEASUREMENT, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java index 281247e1529de..ee27eda4a5354 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java @@ -115,7 +115,7 @@ public class TsFileInsertionEventQueryParserTabletIterator implements Iterator modifications) { + PatternTreeMap modifications, + boolean isTableModel) { if (modifications == null) { return false; } - final List mods = getOverlappedMods(deviceID, measurementID, modifications); + final List mods = + getOverlappedMods(deviceID, measurementID, modifications, isTableModel); if (mods == null || mods.isEmpty()) { return false; } // Different logic for tree model and table model - if (deviceID.isTableModel()) { + if (isTableModel) { // For table model: check if any modification affects the device and covers the time range return mods.stream() .anyMatch( @@ -119,17 +122,20 @@ public static boolean isAllDeletedByMods( * @param deviceID device ID * @param measurements measurement list * @param modifications modification records + * @param isTableModel whether the device belongs to table model * @return mapping from measurement ID to mods list and index */ public static List initializeMeasurementMods( IDeviceID deviceID, List measurements, - PatternTreeMap modifications) { + PatternTreeMap modifications, + boolean isTableModel) { List modsInfos = new ArrayList<>(measurements.size()); for (final String measurement : measurements) { - final List mods = getOverlappedMods(deviceID, measurement, modifications); + final List mods = + getOverlappedMods(deviceID, measurement, modifications, isTableModel); if (mods == null || mods.isEmpty()) { // No mods, use empty list and index 0 modsInfos.add(new ModsInfo(Collections.emptyList(), 0)); @@ -139,7 +145,7 @@ public static List initializeMeasurementMods( // Sort by time range for efficient lookup // Different filtering logic for tree model and table model final List filteredMods; - if (deviceID.isTableModel()) { + if (isTableModel) { // For table model: filter modifications that affect the device filteredMods = mods.stream() @@ -161,9 +167,11 @@ public static List initializeMeasurementMods( private static List getOverlappedMods( final IDeviceID deviceID, final String measurement, - final PatternTreeMap modifications) { + final PatternTreeMap modifications, + final boolean isTableModel) { try { - return modifications.getOverlapped(CompactionPathUtils.getPath(deviceID, measurement)); + return modifications.getOverlapped( + CompactionPathUtils.getPath(deviceID, measurement, isTableModel)); } catch (final IllegalPathException e) { throw new PipeException(e.getMessage(), e); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java index 0b8dfbed002f0..03f2d809c0edf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java @@ -180,7 +180,7 @@ public void put(final IDeviceID device, final String databaseName) { map.computeIfAbsent(databaseName, k -> new ArrayList<>()).add(device); } }; - getDatabaseCacheResult(result, deviceIDs, tryToFetch, isAutoCreate, userName); + getDatabaseCacheResult(result, deviceIDs, tryToFetch, isAutoCreate, userName, false); return result.getMap(); } @@ -205,7 +205,7 @@ public void put(final IDeviceID device, final String databaseName) { map.put(device, databaseName); } }; - getDatabaseCacheResult(result, deviceIDs, tryToFetch, isAutoCreate, userName); + getDatabaseCacheResult(result, deviceIDs, tryToFetch, isAutoCreate, userName, false); return result.getMap(); } @@ -502,7 +502,8 @@ private void getDatabaseCacheResult( final List deviceIDs, final boolean tryToFetch, final boolean isAutoCreate, - final String userName) { + final String userName, + final boolean isTableModel) { if (!isAutoCreate) { // TODO: avoid IDeviceID contains "*" // miss when deviceId contains * @@ -530,13 +531,11 @@ private void getDatabaseCacheResult( } else { // check if it is to auto create the system or audit database for (IDeviceID deviceID : deviceIDs) { - if (!deviceID.isTableModel() - && deviceID.startWith("root." + SystemConstant.SYSTEM_PREFIX_KEY)) { + if (!isTableModel && deviceID.startWith("root." + SystemConstant.SYSTEM_PREFIX_KEY)) { createDatabaseAndUpdateCache(result, Collections.singletonList(deviceID), userName); break; } - if (!deviceID.isTableModel() - && deviceID.startWith("root." + SystemConstant.AUDIT_PREFIX_KEY)) { + if (!isTableModel && deviceID.startWith("root." + SystemConstant.AUDIT_PREFIX_KEY)) { createDatabaseAndUpdateCache(result, Collections.singletonList(deviceID), userName); break; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java index 322a86acfd3b3..63a8adcaea955 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java @@ -171,7 +171,7 @@ public void autoCreateAndVerify(final IDeviceID device) throws LoadAnalyzeExcept public boolean isDeviceDeletedByMods(final IDeviceID device) { try { return ModificationUtils.isDeviceDeletedByMods( - currentModifications, currentTimeIndex, device); + currentModifications, currentTimeIndex, device, true); } catch (final IllegalPathException e) { LOGGER.warn( DataNodeQueryMessages diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java index 87375b5dfd427..b2ffa01590bda 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java @@ -196,7 +196,8 @@ public void setCurrentTimeIndex(final ITimeIndex timeIndex) { } public boolean isDeviceDeletedByMods(IDeviceID device) throws IllegalPathException { - return ModificationUtils.isDeviceDeletedByMods(currentModifications, currentTimeIndex, device); + return ModificationUtils.isDeviceDeletedByMods( + currentModifications, currentTimeIndex, device, false); } public boolean isTimeSeriesDeletedByMods(IDeviceID device, TimeseriesMetadata timeseriesMetadata) @@ -206,7 +207,8 @@ public boolean isTimeSeriesDeletedByMods(IDeviceID device, TimeseriesMetadata ti device, timeseriesMetadata.getMeasurementId(), timeseriesMetadata.getStatistics().getStartTime(), - timeseriesMetadata.getStatistics().getEndTime()); + timeseriesMetadata.getStatistics().getEndTime(), + false); } public void addAlreadySetDatabase(PartialPath database) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TreeDiskUsageStatisticUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TreeDiskUsageStatisticUtil.java index ec4d8f03d8683..5317f8628d83f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TreeDiskUsageStatisticUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TreeDiskUsageStatisticUtil.java @@ -165,6 +165,6 @@ private long calculatePathPatternSize( } private boolean matchPathPattern(IDeviceID deviceID) throws IllegalPathException { - return pathPattern.matchFullPath(CompactionPathUtils.getPath(deviceID)); + return pathPattern.matchFullPath(CompactionPathUtils.getPath(deviceID, false)); } } From e1a540722a65ba7ee973a5f40f17c8f12fc3ba1d Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 9 Oct 2026 18:24:50 +0800 Subject: [PATCH 04/15] [Storage] Pass table model explicitly in memtable, flush and WAL replay --- .../iotdb/db/i18n/ImportWALMessages.java | 6 +- .../iotdb/db/i18n/ImportWALMessages.java | 6 +- .../PipeTableModelTsFileBuilderV2.java | 3 +- .../builder/PipeTreeModelTsFileBuilderV2.java | 3 +- .../utils/ResourceByPathUtils.java | 5 +- .../dataregion/flush/MemTableFlushTask.java | 22 +++++-- .../dataregion/memtable/AbstractMemTable.java | 54 +++++++++------- .../memtable/AbstractWritableMemChunk.java | 9 ++- .../memtable/AlignedWritableMemChunk.java | 52 ++++++++-------- .../AlignedWritableMemChunkGroup.java | 37 ++++++----- .../dataregion/memtable/IMemTable.java | 8 ++- .../memtable/IWritableMemChunk.java | 14 +++-- .../memtable/IWritableMemChunkGroup.java | 12 ++-- .../dataregion/memtable/TsFileProcessor.java | 12 +++- .../dataregion/memtable/WritableMemChunk.java | 27 +++++--- .../memtable/WritableMemChunkGroup.java | 26 ++++---- .../wal/recover/file/TsFilePlanRedoer.java | 9 ++- .../file/UnsealedTsFileRecoverPerformer.java | 13 ++-- .../org/apache/iotdb/db/tools/ImportWAL.java | 61 ++++++++++++++----- .../FragmentInstanceExecutionTest.java | 10 +-- ...BitmapMemoryAccountingPerformanceTest.java | 6 +- .../memtable/MemChunkDeserializeTest.java | 12 ++-- .../memtable/MemTableFlushTaskTest.java | 28 ++++----- .../memtable/PrimitiveMemTableTest.java | 27 +++++--- .../WritableMemChunkRegionScanTest.java | 8 ++- .../apache/iotdb/db/tools/ImportWALTest.java | 54 ++++++++++++++-- 26 files changed, 338 insertions(+), 186 deletions(-) diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/ImportWALMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/ImportWALMessages.java index d28ca7dbcfacf..fefbfffe8693c 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/ImportWALMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/ImportWALMessages.java @@ -136,14 +136,16 @@ public final class ImportWALMessages { public static final String MESSAGE_SKIPPED_ARG_CORRUPTED_WAL_FILES_SOURCE_FILES_RETAINED_A889CCE2 = "Skipped %d corrupted WAL files; source files retained."; - public static final String MESSAGE_TARGET_DATABASE_FOR_TABLE_MODEL_WAL_ENTRIES_IF_OMITTED_INFER_FROM_THE_WAL_PARENT_DIRECTORY_AND_ASK_FOR_CONFIRMATION_4B1E409D = - "Target database for table-model WAL entries. If omitted, infer from the WAL parent directory and ask for confirmation."; + public static final String MESSAGE_TARGET_DATABASE_FOR_WAL_REPLAY_IF_OMITTED_INFER_FROM_THE_WAL_PARENT_DIRECTORY_IF_INFERENCE_FAILS_DB_DATABASE_IS_REQUIRED_6EEBC019 = + "Target database for WAL replay. If omitted, infer from the WAL parent directory; if inference fails, -db/--database is required."; public static final String MESSAGE_INFERRED_TABLE_DATABASE_ARG_FROM_WAL_DIRECTORY_ARG_REPLAY_INTO_THIS_DATABASE_Y_YES_A_ACCEPT_ALL_INFERRED_DATABASES_N_QUIT_5B59D833 = "Inferred table database %s from WAL directory %s. Replay into this database? [y] yes, [a] accept all inferred databases, [N] quit: "; public static final String EXCEPTION_DATABASE_CONFIRMATION_REQUIRED_FOR_WAL_DIRECTORY_ARG_INFERRED_DATABASE_ARG_SPECIFY_DB_DATABASE_OR_SKIP_DB_CONFIRMATION_WHEN_INTERACTIVE_INPUT_IS_UNAVAILABLE_14DF6D36 = "Database confirmation required for WAL directory %s (inferred database: %s). Specify -db/--database or --skip_db_confirmation when interactive input is unavailable."; public static final String EXCEPTION_REPLAY_INTO_INFERRED_DATABASE_ARG_WAS_NOT_CONFIRMED_SPECIFY_DB_DATABASE_TO_SELECT_THE_TARGET_EXPLICITLY_86F81190 = "Replay into inferred database %s was not confirmed. Specify -db/--database to select the target explicitly."; + public static final String EXCEPTION_CANNOT_DETERMINE_THE_TARGET_DATABASE_OF_WAL_DIRECTORIES_ARG_SPECIFY_DB_DATABASE_WHICH_APPLIES_TO_ALL_IMPORTED_DIRECTORIES_55B174E4 = + "Cannot determine the target database of WAL directories %s. Specify -db/--database, which applies to all imported directories."; public static final String MESSAGE_ACCEPT_ALL_INFERRED_DATABASE_NAMES_WITHOUT_CONFIRMATION_DB_DATABASE_STILL_TAKES_PRECEDENCE_FA49A73C = "Accept all inferred database names without confirmation; -db/--database still takes precedence."; diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/ImportWALMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/ImportWALMessages.java index 5f9b6e18b0299..fc65b1bec5bc5 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/ImportWALMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/ImportWALMessages.java @@ -135,14 +135,16 @@ public final class ImportWALMessages { public static final String MESSAGE_SKIPPED_ARG_CORRUPTED_WAL_FILES_SOURCE_FILES_RETAINED_A889CCE2 = "已跳过 %d 个损坏的 WAL 文件；源文件已保留。"; - public static final String MESSAGE_TARGET_DATABASE_FOR_TABLE_MODEL_WAL_ENTRIES_IF_OMITTED_INFER_FROM_THE_WAL_PARENT_DIRECTORY_AND_ASK_FOR_CONFIRMATION_4B1E409D = - "表模型 WAL 条目的目标数据库。省略时尝试从 WAL 文件的父目录名推断，并请求确认。"; + public static final String MESSAGE_TARGET_DATABASE_FOR_WAL_REPLAY_IF_OMITTED_INFER_FROM_THE_WAL_PARENT_DIRECTORY_IF_INFERENCE_FAILS_DB_DATABASE_IS_REQUIRED_6EEBC019 = + "WAL 重放的目标数据库。省略时尝试从 WAL 文件的父目录名推断；无法推断时必须指定 -db/--database。"; public static final String MESSAGE_INFERRED_TABLE_DATABASE_ARG_FROM_WAL_DIRECTORY_ARG_REPLAY_INTO_THIS_DATABASE_Y_YES_A_ACCEPT_ALL_INFERRED_DATABASES_N_QUIT_5B59D833 = "推断目标表模型数据库为 %s，来源 WAL 目录为 %s。是否向此数据库重放？[y] 同意，[a] 全部同意推断的数据库，[N] 退出："; public static final String EXCEPTION_DATABASE_CONFIRMATION_REQUIRED_FOR_WAL_DIRECTORY_ARG_INFERRED_DATABASE_ARG_SPECIFY_DB_DATABASE_OR_SKIP_DB_CONFIRMATION_WHEN_INTERACTIVE_INPUT_IS_UNAVAILABLE_14DF6D36 = "需要确认 WAL 目录 %s 的目标数据库（推断结果：%s）。无交互终端时请指定 -db/--database 或 --skip_db_confirmation。"; public static final String EXCEPTION_REPLAY_INTO_INFERRED_DATABASE_ARG_WAS_NOT_CONFIRMED_SPECIFY_DB_DATABASE_TO_SELECT_THE_TARGET_EXPLICITLY_86F81190 = "未确认向推断出的数据库 %s 重放。请使用 -db/--database 显式选择目标。"; + public static final String EXCEPTION_CANNOT_DETERMINE_THE_TARGET_DATABASE_OF_WAL_DIRECTORIES_ARG_SPECIFY_DB_DATABASE_WHICH_APPLIES_TO_ALL_IMPORTED_DIRECTORIES_55B174E4 = + "无法推断 WAL 目录 %s 的目标数据库，请使用 -db/--database 显式指定（-db 会作用于本次导入的所有目录）。"; public static final String MESSAGE_ACCEPT_ALL_INFERRED_DATABASE_NAMES_WITHOUT_CONFIRMATION_DB_DATABASE_STILL_TAKES_PRECEDENCE_FA49A73C = "自动接受所有推断出的数据库名，不再询问确认；-db/--database 仍优先。"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTableModelTsFileBuilderV2.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTableModelTsFileBuilderV2.java index ca4efdadb9802..dc1a58ff0121e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTableModelTsFileBuilderV2.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTableModelTsFileBuilderV2.java @@ -329,7 +329,8 @@ private void writeTabletsIntoOneFile( } } - final MemTableFlushTask memTableFlushTask = new MemTableFlushTask(memTable, writer, null, null); + final MemTableFlushTask memTableFlushTask = + new MemTableFlushTask(memTable, writer, null, null, true); memTableFlushTask.syncFlushMemTable(); writer.endFile(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTreeModelTsFileBuilderV2.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTreeModelTsFileBuilderV2.java index 9d9d8b0fb0a83..21dd3bb9cb052 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTreeModelTsFileBuilderV2.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTreeModelTsFileBuilderV2.java @@ -257,7 +257,8 @@ private void writeTabletsIntoOneFile( } } - final MemTableFlushTask memTableFlushTask = new MemTableFlushTask(memTable, writer, null, null); + final MemTableFlushTask memTableFlushTask = + new MemTableFlushTask(memTable, writer, null, null, false); memTableFlushTask.syncFlushMemTable(); writer.endFile(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java index f95fcfda473b7..f72711387249b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java @@ -816,7 +816,10 @@ public ReadOnlyMemChunk getReadOnlyMemChunkFromMemTable( return null; } IWritableMemChunk memChunk = - memTableMap.get(deviceID).getMemChunkMap().get(fullPath.getMeasurement()); + memTableMap + .get(deviceID) + .getMemChunkMap(context.isIgnoreAllNullRows()) + .get(fullPath.getMeasurement()); // check If data type matches if (memChunk.getSchema().getType() != fullPath.getMeasurementSchema().getType() && !fullPath diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java index 68727f03aa73d..4a2dfd88cd604 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java @@ -85,6 +85,12 @@ public class MemTableFlushTask { private String storageGroup; private String dataRegionId; + /** + * Whether rows whose values are all null should be ignored when flushing. All-null rows are only + * meaningful for table model databases, so it is the negation of {@code isTableModel}. + */ + private final boolean ignoreAllNullRows; + private IMemTable memTable; private volatile long memSerializeTime = 0L; @@ -97,16 +103,21 @@ public class MemTableFlushTask { * @param memTable the memTable to flush * @param writer the writer where memTable will be flushed to (current tsfile writer or vm writer) * @param storageGroup current database + * @param dataRegionId current data region id + * @param isTableModel whether the memTable belongs to a table model database, it cannot be + * inferred when the caller does not know the database, e.g. the pipe tsfile builders */ public MemTableFlushTask( IMemTable memTable, RestorableTsFileIOWriter writer, String storageGroup, - String dataRegionId) { + String dataRegionId, + boolean isTableModel) { this.memTable = memTable; this.writer = writer; this.storageGroup = storageGroup; this.dataRegionId = dataRegionId; + this.ignoreAllNullRows = !isTableModel; this.encodingTaskFuture = SUB_TASK_POOL_MANAGER.submit(encodingTask); this.ioTaskFuture = SUB_TASK_POOL_MANAGER.submit(ioTask); @@ -160,9 +171,10 @@ public void syncFlushMemTable() throws ExecutionException, InterruptedException // sort the IDeviceID in lexicographical order Collections.sort(deviceIDList); for (IDeviceID deviceID : deviceIDList) { - final Map value = memTableMap.get(deviceID).getMemChunkMap(); + final Map value = + memTableMap.get(deviceID).getMemChunkMap(ignoreAllNullRows); // skip the empty device/chunk group - if (memTableMap.get(deviceID).isEmpty() || value.isEmpty()) { + if (memTableMap.get(deviceID).isEmpty(ignoreAllNullRows) || value.isEmpty()) { continue; } encodingTaskQueue.put(new StartFlushGroupIOTask(deviceID)); @@ -171,7 +183,7 @@ public void syncFlushMemTable() throws ExecutionException, InterruptedException for (String seriesId : seriesInOrder) { long startTime = System.currentTimeMillis(); IWritableMemChunk series = value.get(seriesId); - if (series.count() == 0) { + if (series.count(ignoreAllNullRows) == 0) { continue; } /* @@ -278,7 +290,7 @@ public void run() { if (writableMemChunk instanceof AlignedWritableMemChunk && times == null) { times = new long[MAX_NUMBER_OF_POINTS_IN_PAGE]; } - writableMemChunk.encode(ioTaskQueue, encodeInfo, times); + writableMemChunk.encode(ioTaskQueue, encodeInfo, times, ignoreAllNullRows); long subTaskTime = System.currentTimeMillis() - starTime; WRITING_METRICS.recordFlushSubTaskCost(WritingMetrics.ENCODING_TASK, subTaskTime); memSerializeTime += subTaskTime; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java index 35a3696630e50..2bdc6c2903507 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java @@ -187,7 +187,7 @@ private IWritableMemChunkGroup createAlignedMemChunkGroupIfNotExistAndGet( deviceId, k -> { seriesNumber += filteredSchemaList.size(); - return new AlignedWritableMemChunkGroup(filteredSchemaList, k.isTableModel()); + return new AlignedWritableMemChunkGroup(filteredSchemaList); }); for (IMeasurementSchema schema : schemaList) { if (schema != null && !memChunkGroup.contains(schema.getMeasurementName())) { @@ -562,10 +562,10 @@ public double getNullValueRatio() { } @Override - public long size() { + public long size(boolean ignoreAllNullRows) { long sum = 0; for (IWritableMemChunkGroup writableMemChunkGroup : memTableMap.values()) { - sum += writableMemChunkGroup.count(); + sum += writableMemChunkGroup.count(ignoreAllNullRows); } return sum; } @@ -617,6 +617,7 @@ public void queryForSeriesRegionScan( Filter globalTimeFilter) { IDeviceID deviceID = fullPath.getDeviceId(); + // Rows whose values are all null can never satisfy this read path, so they are always ignored. if (fullPath instanceof NonAlignedFullPath) { String measurementId = ((NonAlignedFullPath) fullPath).getMeasurement(); @@ -637,6 +638,7 @@ public void queryForSeriesRegionScan( chunkMetaDataMap, memChunkHandleMap, deletionList, + true, globalTimeFilter); } else { // check If MemTable Contains this path @@ -660,6 +662,7 @@ public void queryForSeriesRegionScan( chunkMetaDataMap, memChunkHandleMap, deletionList, + true, globalTimeFilter); } } @@ -682,6 +685,7 @@ public void queryForDeviceRegionScan( } IWritableMemChunkGroup writableMemChunkGroup = memTableMap.get(deviceID); + // Rows whose values are all null can never satisfy this read path, so they are always ignored. if (isAligned) { getMemAlignedChunkHandleFromMemTable( deviceID, @@ -690,6 +694,7 @@ public void queryForDeviceRegionScan( memChunkHandleMap, ttlLowerBound, modsToMemTabled, + true, globalTimeFilter); } else { getMemChunkHandleFromMemTable( @@ -699,6 +704,7 @@ public void queryForDeviceRegionScan( memChunkHandleMap, ttlLowerBound, modsToMemTabled, + true, globalTimeFilter); } } @@ -709,10 +715,12 @@ private void getMemChunkHandleFromMemTable( Map> chunkMetadataMap, Map> memChunkHandleMap, List deletionList, + boolean ignoreAllNullRows, Filter globalTimeFilter) { WritableMemChunk memChunk = - (WritableMemChunk) memTableMap.get(deviceID).getMemChunkMap().get(measurementId); + (WritableMemChunk) + memTableMap.get(deviceID).getMemChunkMap(ignoreAllNullRows).get(measurementId); if (memChunk == null) { return; @@ -740,6 +748,7 @@ private void getMemAlignedChunkHandleFromMemTable( Map> chunkMetadataList, Map> memChunkHandleMap, List> deletionList, + boolean ignoreAllNullRows, Filter globalTimeFilter) { AlignedWritableMemChunk alignedMemChunk = @@ -758,7 +767,8 @@ private void getMemAlignedChunkHandleFromMemTable( List bitMaps = new ArrayList<>(); long[] timestamps = - alignedMemChunk.getAnySatisfiedTimestamp(deletionList, bitMaps, true, globalTimeFilter); + alignedMemChunk.getAnySatisfiedTimestamp( + deletionList, bitMaps, ignoreAllNullRows, globalTimeFilter); if (timestamps.length == 0) { return; } @@ -780,6 +790,7 @@ private void getMemAlignedChunkHandleFromMemTable( Map> memChunkHandleMap, long ttlLowerBound, List> modsToMemTabled, + boolean ignoreAllNullRows, Filter globalTimeFilter) { AlignedWritableMemChunk memChunk = writableMemChunkGroup.getAlignedMemChunk(); @@ -796,7 +807,8 @@ private void getMemAlignedChunkHandleFromMemTable( List bitMaps = new ArrayList<>(); long[] timestamps = - memChunk.getAnySatisfiedTimestamp(deletionList, bitMaps, true, globalTimeFilter); + memChunk.getAnySatisfiedTimestamp( + deletionList, bitMaps, ignoreAllNullRows, globalTimeFilter); if (timestamps.length == 0) { return; } @@ -817,10 +829,11 @@ private void getMemChunkHandleFromMemTable( Map> memChunkHandleMap, long ttlLowerBound, List> modsToMemTabled, + boolean ignoreAllNullRows, Filter globalTimeFilter) { for (Entry entry : - writableMemChunkGroup.getMemChunkMap().entrySet()) { + writableMemChunkGroup.getMemChunkMap(ignoreAllNullRows).entrySet()) { String measurementId = entry.getKey(); WritableMemChunk writableMemChunk = (WritableMemChunk) entry.getValue(); @@ -921,7 +934,7 @@ private IChunkMetadata buildFakeChunkMetaDataForFakeMemoryChunk( } @Override - public long delete(ModEntry modEntry) { + public long delete(ModEntry modEntry, boolean ignoreAllNullRows) { List> targetDeviceList = new ArrayList<>(); for (Entry entry : memTableMap.entrySet()) { if (modEntry.affects(entry.getKey())) { @@ -932,11 +945,11 @@ public long delete(ModEntry modEntry) { long pointDeleted = 0; for (Pair pair : targetDeviceList) { if (modEntry.affectsAll(pair.left)) { - pointDeleted += pair.right.deleteTime(modEntry); + pointDeleted += pair.right.deleteTime(modEntry, ignoreAllNullRows); } else { - pointDeleted += pair.right.delete(modEntry); + pointDeleted += pair.right.delete(modEntry, ignoreAllNullRows); } - if (pair.right.isEmpty()) { + if (pair.right.isEmpty(ignoreAllNullRows)) { memTableMap.remove(pair.left).release(); } } @@ -1087,15 +1100,13 @@ protected void deserialize(DataInputStream stream, boolean multiTvListInMemChunk IWritableMemChunkGroup memChunkGroup; if (multiTvListInMemChunk) { if (isAligned) { - memChunkGroup = AlignedWritableMemChunkGroup.deserialize(stream, deviceID.isTableModel()); + memChunkGroup = AlignedWritableMemChunkGroup.deserialize(stream); } else { memChunkGroup = WritableMemChunkGroup.deserialize(stream); } } else { if (isAligned) { - memChunkGroup = - AlignedWritableMemChunkGroup.deserializeSingleTVListMemChunks( - stream, deviceID.isTableModel()); + memChunkGroup = AlignedWritableMemChunkGroup.deserializeSingleTVListMemChunks(stream); } else { memChunkGroup = WritableMemChunkGroup.deserializeSingleTVListMemChunks(stream); } @@ -1130,15 +1141,13 @@ public void deserializeFromOldMemTableSnapshot( IWritableMemChunkGroup memChunkGroup; if (multiTvListInMemChunk) { if (isAligned) { - memChunkGroup = AlignedWritableMemChunkGroup.deserialize(stream, deviceID.isTableModel()); + memChunkGroup = AlignedWritableMemChunkGroup.deserialize(stream); } else { memChunkGroup = WritableMemChunkGroup.deserialize(stream); } } else { if (isAligned) { - memChunkGroup = - AlignedWritableMemChunkGroup.deserializeSingleTVListMemChunks( - stream, deviceID.isTableModel()); + memChunkGroup = AlignedWritableMemChunkGroup.deserializeSingleTVListMemChunks(stream); } else { memChunkGroup = WritableMemChunkGroup.deserializeSingleTVListMemChunks(stream); } @@ -1148,11 +1157,12 @@ public void deserializeFromOldMemTableSnapshot( } @Override - public Map getMaxTime() { + public Map getMaxTime(boolean ignoreAllNullRows) { Map latestTimeForEachDevice = new HashMap<>(); for (Entry entry : memTableMap.entrySet()) { - if (entry.getValue().count() > 0 && !entry.getValue().isEmpty()) { - latestTimeForEachDevice.put(entry.getKey(), entry.getValue().getMaxTime()); + if (entry.getValue().count(ignoreAllNullRows) > 0 + && !entry.getValue().isEmpty(ignoreAllNullRows)) { + latestTimeForEachDevice.put(entry.getKey(), entry.getValue().getMaxTime(ignoreAllNullRows)); } } return latestTimeForEachDevice; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java index 4f2bb0abd1163..327944c297551 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java @@ -222,7 +222,7 @@ public abstract void writeAlignedTablet( TSStatus[] results); @Override - public abstract long count(); + public abstract long count(boolean ignoreAllNullRows); @Override public abstract long rowCount(); @@ -284,13 +284,16 @@ public void releaseTemporaryTvListForFlush() { @Override public abstract void encode( - BlockingQueue ioTaskQueue, BatchEncodeInfo encodeInfo, long[] times); + BlockingQueue ioTaskQueue, + BatchEncodeInfo encodeInfo, + long[] times, + boolean ignoreAllNullRows); @Override public abstract void release(); @Override - public abstract boolean isEmpty(); + public abstract boolean isEmpty(boolean ignoreAllNullRows); @Override public abstract List getSortedList(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java index ebdc024aac84b..54b481de66af6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java @@ -75,7 +75,6 @@ public class AlignedWritableMemChunk extends AbstractWritableMemChunk { private AlignedTVList list; private List sortedList; private long sortedRowCount = 0; - private final boolean ignoreAllNullRows; private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); private final int TVLIST_SORT_THRESHOLD = CONFIG.getTvListSortThreshold(); @@ -84,14 +83,12 @@ public class AlignedWritableMemChunk extends AbstractWritableMemChunk { private EncryptParameter encryptParameter; - public AlignedWritableMemChunk(List schemaList, boolean isTableModel) { - this(schemaList, isTableModel, EncryptUtils.getEncryptParameter()); + public AlignedWritableMemChunk(List schemaList) { + this(schemaList, EncryptUtils.getEncryptParameter()); } public AlignedWritableMemChunk( - List schemaList, - boolean isTableModel, - EncryptParameter encryptParameter) { + List schemaList, EncryptParameter encryptParameter) { this.measurementIndexMap = new LinkedHashMap<>(); this.dataTypes = new ArrayList<>(); this.schemaList = schemaList; @@ -101,19 +98,16 @@ public AlignedWritableMemChunk( } this.list = AlignedTVList.newAlignedList(dataTypes); this.sortedList = new ArrayList<>(); - this.ignoreAllNullRows = !isTableModel; this.encryptParameter = encryptParameter; } - private AlignedWritableMemChunk( - List schemaList, AlignedTVList list, boolean isTableModel) { - this(schemaList, list, isTableModel, EncryptUtils.getEncryptParameter()); + private AlignedWritableMemChunk(List schemaList, AlignedTVList list) { + this(schemaList, list, EncryptUtils.getEncryptParameter()); } private AlignedWritableMemChunk( List schemaList, AlignedTVList list, - boolean isTableModel, EncryptParameter encryptParameter) { this.measurementIndexMap = new LinkedHashMap<>(); this.schemaList = schemaList; @@ -123,7 +117,6 @@ private AlignedWritableMemChunk( this.list = list; this.dataTypes = list.getTsDataTypes(); this.sortedList = new ArrayList<>(); - this.ignoreAllNullRows = !isTableModel; this.encryptParameter = encryptParameter; } @@ -465,7 +458,7 @@ public void setWorkingTVList(TVList list) { } @Override - public long count() { + public long count(boolean ignoreAllNullRows) { if (!ignoreAllNullRows && measurementIndexMap.isEmpty()) { return rowCount(); } @@ -487,8 +480,8 @@ public IMeasurementSchema getSchema() { } @Override - public long getMaxTime() { - if (isEmpty()) { + public long getMaxTime(boolean ignoreAllNullRows) { + if (isEmpty(ignoreAllNullRows)) { return Long.MIN_VALUE; } long maxTime = list.getMaxTime(); @@ -582,7 +575,8 @@ private List getDataTypes(List schemaList) { public void encodeWorkingAlignedTVList( BlockingQueue ioTaskQueue, long maxNumberOfPointsInChunk, - int maxNumberOfPointsInPage) { + int maxNumberOfPointsInPage, + boolean ignoreAllNullRows) { List activeSchemaList = getActiveSchemaList(); if (activeSchemaList.isEmpty() && ignoreAllNullRows) { return; @@ -1065,7 +1059,11 @@ private void handleEncodingWithDeletedMeasurements( } @Override - public void encode(BlockingQueue ioTaskQueue, BatchEncodeInfo encodeInfo, long[] times) { + public void encode( + BlockingQueue ioTaskQueue, + BatchEncodeInfo encodeInfo, + long[] times, + boolean ignoreAllNullRows) { encodeInfo.maxNumberOfPointsInChunk = Math.min( encodeInfo.maxNumberOfPointsInChunk, @@ -1073,7 +1071,10 @@ public void encode(BlockingQueue ioTaskQueue, BatchEncodeInfo encodeInfo if (TVLIST_SORT_THRESHOLD == 0) { encodeWorkingAlignedTVList( - ioTaskQueue, encodeInfo.maxNumberOfPointsInChunk, encodeInfo.maxNumberOfPointsInPage); + ioTaskQueue, + encodeInfo.maxNumberOfPointsInChunk, + encodeInfo.maxNumberOfPointsInPage, + ignoreAllNullRows); return; } @@ -1151,15 +1152,15 @@ public long getFirstPoint() { } @Override - public long getLastPoint() { + public long getLastPoint(boolean ignoreAllNullRows) { if (rowCount() == 0) { return Long.MIN_VALUE; } - return getMaxTime(); + return getMaxTime(ignoreAllNullRows); } @Override - public boolean isEmpty() { + public boolean isEmpty(boolean ignoreAllNullRows) { if (rowCount() == 0) { return true; } @@ -1215,8 +1216,7 @@ public void serializeToWAL(IWALByteBufferView buffer) { list.serializeToWAL(buffer); } - public static AlignedWritableMemChunk deserialize(DataInputStream stream, boolean isTableModel) - throws IOException { + public static AlignedWritableMemChunk deserialize(DataInputStream stream) throws IOException { int schemaListSize = stream.readInt(); List schemaList = new ArrayList<>(schemaListSize); for (int i = 0; i < schemaListSize; i++) { @@ -1230,13 +1230,13 @@ public static AlignedWritableMemChunk deserialize(DataInputStream stream, boolea sortedList.add(tvList); } AlignedTVList list = AlignedTVList.deserialize(stream); - AlignedWritableMemChunk chunk = new AlignedWritableMemChunk(schemaList, list, isTableModel); + AlignedWritableMemChunk chunk = new AlignedWritableMemChunk(schemaList, list); chunk.sortedList = sortedList; return chunk; } public static AlignedWritableMemChunk deserializeSingleTVListMemChunks( - DataInputStream stream, boolean isTableModel) throws IOException { + DataInputStream stream) throws IOException { int schemaListSize = stream.readInt(); List schemaList = new ArrayList<>(schemaListSize); for (int i = 0; i < schemaListSize; i++) { @@ -1245,7 +1245,7 @@ public static AlignedWritableMemChunk deserializeSingleTVListMemChunks( } AlignedTVList list = AlignedTVList.deserialize(stream); - return new AlignedWritableMemChunk(schemaList, list, isTableModel); + return new AlignedWritableMemChunk(schemaList, list); } public List getSchemaList() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java index a27973acdb1dd..e1235ca935b0d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java @@ -46,13 +46,13 @@ public class AlignedWritableMemChunkGroup implements IWritableMemChunkGroup { private EncryptParameter encryptParameter; - public AlignedWritableMemChunkGroup(List schemaList, boolean isTableModel) { - memChunk = new AlignedWritableMemChunk(schemaList, isTableModel); + public AlignedWritableMemChunkGroup(List schemaList) { + memChunk = new AlignedWritableMemChunk(schemaList); } @TestOnly public AlignedWritableMemChunkGroup( - AlignedWritableMemChunk memChunk, List schemaList, boolean isTableModel) { + AlignedWritableMemChunk memChunk, List schemaList) { this.memChunk = memChunk; } @@ -78,8 +78,8 @@ public void release() { } @Override - public long count() { - return memChunk.count(); + public long count(boolean ignoreAllNullRows) { + return memChunk.count(ignoreAllNullRows); } /** @@ -101,20 +101,20 @@ public void writeRow(long insertTime, Object[] objectValue, List getMemChunkMap() { - if (memChunk.count() == 0) { + public Map getMemChunkMap(boolean ignoreAllNullRows) { + if (memChunk.count(ignoreAllNullRows) == 0) { return Collections.emptyMap(); } return Collections.singletonMap("", memChunk); } @Override - public boolean isEmpty() { - return memChunk.isEmpty() || memChunk.isAllDeleted(); + public boolean isEmpty(boolean ignoreAllNullRows) { + return memChunk.isEmpty(ignoreAllNullRows) || memChunk.isAllDeleted(); } @Override - public long delete(ModEntry modEntry) { + public long delete(ModEntry modEntry, boolean ignoreAllNullRows) { int deletedPointsNumber = 0; Set measurements = memChunk.getAllMeasurements(); List columnsToBeRemoved = new ArrayList<>(); @@ -138,7 +138,7 @@ public long delete(ModEntry modEntry) { return deletedPointsNumber; } - public long deleteTime(ModEntry modEntry) { + public long deleteTime(ModEntry modEntry, boolean ignoreAllNullRows) { return memChunk.deleteTime(modEntry.getStartTime(), modEntry.getEndTime()); } @@ -148,8 +148,8 @@ public IWritableMemChunk getWritableMemChunk(String measurement) { } @Override - public long getMaxTime() { - return memChunk.getMaxTime(); + public long getMaxTime(boolean ignoreAllNullRows) { + return memChunk.getMaxTime(ignoreAllNullRows); } public AlignedWritableMemChunk getAlignedMemChunk() { @@ -172,10 +172,10 @@ public void setEncryptParameter(EncryptParameter encryptParameter) { memChunk.setEncryptParameter(encryptParameter); } - protected static AlignedWritableMemChunkGroup deserialize( - DataInputStream stream, boolean isTableModel) throws IOException { + protected static AlignedWritableMemChunkGroup deserialize(DataInputStream stream) + throws IOException { AlignedWritableMemChunkGroup memChunkGroup = new AlignedWritableMemChunkGroup(); - memChunkGroup.memChunk = AlignedWritableMemChunk.deserialize(stream, isTableModel); + memChunkGroup.memChunk = AlignedWritableMemChunk.deserialize(stream); return memChunkGroup; } @@ -185,10 +185,9 @@ public void checkDataType(InsertNode node) throws DataTypeInconsistentException } protected static AlignedWritableMemChunkGroup deserializeSingleTVListMemChunks( - DataInputStream stream, boolean isTableModel) throws IOException { + DataInputStream stream) throws IOException { AlignedWritableMemChunkGroup memChunkGroup = new AlignedWritableMemChunkGroup(); - memChunkGroup.memChunk = - AlignedWritableMemChunk.deserializeSingleTVListMemChunks(stream, isTableModel); + memChunkGroup.memChunk = AlignedWritableMemChunk.deserializeSingleTVListMemChunks(stream); return memChunkGroup; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java index afa24083ff6d3..48d43644c8dc3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java @@ -68,9 +68,11 @@ void writeAlignedRow( Object[] objectValue); /** + * @param ignoreAllNullRows whether rows whose values are all null should be ignored, it is + * determined by the data model of the database the memtable belongs to * @return the number of points */ - long size(); + long size(boolean ignoreAllNullRows); /** * @return memory usage @@ -170,7 +172,7 @@ void queryForDeviceRegionScan( * * @return */ - long delete(ModEntry modEntry); + long delete(ModEntry modEntry, boolean ignoreAllNullRows); /** * Make a copy of this MemTable. @@ -214,7 +216,7 @@ void queryForDeviceRegionScan( void setFlushStatus(FlushStatus flushStatus); - Map getMaxTime(); + Map getMaxTime(boolean ignoreAllNullRows); String getDatabase(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java index e57f30680f9aa..71f205ef601b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java @@ -84,7 +84,7 @@ void writeAlignedTablet( int end, TSStatus[] results); - long count(); + long count(boolean ignoreAllNullRows); long rowCount(); @@ -100,7 +100,7 @@ void writeAlignedTablet( void releaseTemporaryTvListForFlush(); - default long getMaxTime() { + default long getMaxTime(boolean ignoreAllNullRows) { return Long.MAX_VALUE; } @@ -115,15 +115,19 @@ default long getMinTime() { IChunkWriter createIChunkWriter(); - void encode(BlockingQueue ioTaskQueue, BatchEncodeInfo encodeInfo, long[] times); + void encode( + BlockingQueue ioTaskQueue, + BatchEncodeInfo encodeInfo, + long[] times, + boolean ignoreAllNullRows); void release(); long getFirstPoint(); - long getLastPoint(); + long getLastPoint(boolean ignoreAllNullRows); - boolean isEmpty(); + boolean isEmpty(boolean ignoreAllNullRows); List getSortedList(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java index 46a21f05840f6..0cafd43632941 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java @@ -47,21 +47,21 @@ void writeTablet( void release(); - long count(); + long count(boolean ignoreAllNullRows); boolean contains(String measurement); - Map getMemChunkMap(); + Map getMemChunkMap(boolean ignoreAllNullRows); - boolean isEmpty(); + boolean isEmpty(boolean ignoreAllNullRows); - long delete(ModEntry modEntry); + long delete(ModEntry modEntry, boolean ignoreAllNullRows); - long deleteTime(ModEntry modEntry); + long deleteTime(ModEntry modEntry, boolean ignoreAllNullRows); IWritableMemChunk getWritableMemChunk(String measurement); - long getMaxTime(); + long getMaxTime(boolean ignoreAllNullRows); void setEncryptParameter(EncryptParameter encryptParameter); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 78c48a2702ba7..7a89a07a9af52 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -1584,6 +1584,11 @@ private void rollbackMemoryInfoIfNeeded(final long[] memIncrements) { } } + /** Whether the data region of this processor belongs to a table model database. */ + private boolean isTableModel() { + return dataRegionInfo.getDataRegion().isTableModel(); + } + /** * Delete data which belongs to the timeseries `deviceId.measurementId` and the timestamp of which * <= 'timestamp' in the deletion.
@@ -1599,7 +1604,7 @@ public boolean deleteDataInMemory(ModEntry deletion) { try { boolean deleted = false; if (workMemTable != null) { - long pointDeleted = workMemTable.delete(deletion); + long pointDeleted = workMemTable.delete(deletion, !isTableModel()); logger.info( StorageEngineMessages .STORAGE_LOG_DELETION_DELETION_WITH_IN_WORKMEMTABLE_POINTS_DELETED_00EA995A, @@ -1767,7 +1772,7 @@ public void asyncFlush() { * flushManager again. */ private Future addAMemtableIntoFlushingList(IMemTable tobeFlushed) throws IOException { - final Map lastTimeForEachDevice = tobeFlushed.getMaxTime(); + final Map lastTimeForEachDevice = tobeFlushed.getMaxTime(!isTableModel()); // If some devices have been removed in MemTable, the number of device in MemTable and // tsFileResource will not be the same. And the endTime of these devices in resource will be @@ -1911,7 +1916,8 @@ public void flushOneMemTable() { memTableToFlush, writer, dataRegionName, - dataRegionInfo.getDataRegion().getDataRegionIdString()); + dataRegionInfo.getDataRegion().getDataRegionIdString(), + isTableModel()); flushTask.syncFlushMemTable(); memTableFlushPointCount = memTableToFlush.getTotalPointsNum(); } catch (Throwable e) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java index 7e54296c6a8bf..929108e924690 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java @@ -222,7 +222,12 @@ public void setWorkingTVList(TVList list) { } @Override - public long count() { + public long count(boolean ignoreAllNullRows) { + // a non-aligned chunk never stores all-null rows, so ignoreAllNullRows makes no difference + return pointCount(); + } + + private long pointCount() { long count = list.count(); for (TVList tvList : sortedList) { count += tvList.count(); @@ -241,7 +246,7 @@ public IMeasurementSchema getSchema() { } @Override - public long getMaxTime() { + public long getMaxTime(boolean ignoreAllNullRows) { long maxTime = list.getMaxTime(); for (TVList tvList : sortedList) { maxTime = Math.max(maxTime, tvList.getMaxTime()); @@ -260,23 +265,23 @@ public long getMinTime() { @Override public long getFirstPoint() { - if (count() == 0) { + if (pointCount() == 0) { return Long.MAX_VALUE; } return getMinTime(); } @Override - public long getLastPoint() { - if (count() == 0) { + public long getLastPoint(boolean ignoreAllNullRows) { + if (pointCount() == 0) { return Long.MIN_VALUE; } - return getMaxTime(); + return getMaxTime(ignoreAllNullRows); } @Override - public boolean isEmpty() { - return count() == 0; + public boolean isEmpty(boolean ignoreAllNullRows) { + return pointCount() == 0; } @Override @@ -389,7 +394,11 @@ public void encodeWorkingTVList( } @Override - public void encode(BlockingQueue ioTaskQueue, BatchEncodeInfo encodeInfo, long[] times) { + public void encode( + BlockingQueue ioTaskQueue, + BatchEncodeInfo encodeInfo, + long[] times, + boolean ignoreAllNullRows) { if (TVLIST_SORT_THRESHOLD == 0) { encodeWorkingTVList( ioTaskQueue, encodeInfo.maxNumberOfPointsInChunk, encodeInfo.targetChunkSize); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java index 6f5c63824c491..9f27648b0aef8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java @@ -93,10 +93,10 @@ public void release() { } @Override - public long count() { + public long count(boolean ignoreAllNullRows) { long count = 0; for (IWritableMemChunk memChunk : memChunkMap.values()) { - count += memChunk.count(); + count += memChunk.count(ignoreAllNullRows); } return count; } @@ -118,17 +118,19 @@ public void writeRow(long insertTime, Object[] objectValue, List getMemChunkMap() { + public Map getMemChunkMap(boolean ignoreAllNullRows) { + // a group of non-aligned chunks never stores all-null rows, so ignoreAllNullRows makes no + // difference return memChunkMap; } @Override - public boolean isEmpty() { - return memChunkMap.isEmpty() || count() == 0; + public boolean isEmpty(boolean ignoreAllNullRows) { + return memChunkMap.isEmpty() || count(ignoreAllNullRows) == 0; } @Override - public long delete(ModEntry modEntry) { + public long delete(ModEntry modEntry, boolean ignoreAllNullRows) { Iterator> iter = memChunkMap.entrySet().iterator(); long deletedPointsNumber = 0; while (iter.hasNext()) { @@ -140,11 +142,11 @@ public long delete(ModEntry modEntry) { IWritableMemChunk chunk = entry.getValue(); if (modEntry.getStartTime() == Long.MIN_VALUE && modEntry.getEndTime() == Long.MAX_VALUE) { iter.remove(); - deletedPointsNumber += chunk.count(); + deletedPointsNumber += chunk.count(ignoreAllNullRows); chunk.release(); } else { deletedPointsNumber += chunk.delete(modEntry.getStartTime(), modEntry.getEndTime()); - if (chunk.count() == 0) { + if (chunk.count(ignoreAllNullRows) == 0) { iter.remove(); } } @@ -153,8 +155,8 @@ public long delete(ModEntry modEntry) { } @Override - public long deleteTime(ModEntry modEntry) { - return delete(modEntry); + public long deleteTime(ModEntry modEntry, boolean ignoreAllNullRows) { + return delete(modEntry, ignoreAllNullRows); } @Override @@ -166,10 +168,10 @@ public IWritableMemChunk getWritableMemChunk(String measurement) { } @Override - public long getMaxTime() { + public long getMaxTime(boolean ignoreAllNullRows) { long maxTime = Long.MIN_VALUE; for (IWritableMemChunk memChunk : memChunkMap.values()) { - maxTime = Math.max(maxTime, memChunk.getMaxTime()); + maxTime = Math.max(maxTime, memChunk.getMaxTime(ignoreAllNullRows)); } return maxTime; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java index 221d39cdfccb3..8e191d475ba5e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; @@ -55,11 +56,15 @@ */ public class TsFilePlanRedoer { private final TsFileResource tsFileResource; + // whether rows whose values are all null should be ignored, which depends on the data model of + // the database this tsfile belongs to + private final boolean ignoreAllNullRows; // store data when redoing logs private IMemTable recoveryMemTable; public TsFilePlanRedoer(TsFileResource tsFileResource) { this.tsFileResource = tsFileResource; + this.ignoreAllNullRows = !PathUtils.isTableModelDatabase(tsFileResource.getDatabaseName()); this.recoveryMemTable = new PrimitiveMemTable(tsFileResource.getDatabaseName(), tsFileResource.getDataRegionId()); WritingMetrics.getInstance().recordActiveMemTableCount(tsFileResource.getDataRegionId(), 1); @@ -73,7 +78,7 @@ void redoDelete(DeleteDataNode deleteDataNode) throws IOException { TreeDeletionEntry deletionEntry = new TreeDeletionEntry( path, deleteDataNode.getDeleteStartTime(), deleteDataNode.getDeleteEndTime()); - recoveryMemTable.delete(deletionEntry); + recoveryMemTable.delete(deletionEntry, ignoreAllNullRows); deletionEntries.add(deletionEntry); } tsFileResource.getModFileForWrite().write(deletionEntries); @@ -81,7 +86,7 @@ void redoDelete(DeleteDataNode deleteDataNode) throws IOException { void redoDelete(RelationalDeleteDataNode node) throws IOException { for (TableDeletionEntry modEntry : node.getModEntries()) { - recoveryMemTable.delete(modEntry); + recoveryMemTable.delete(modEntry, ignoreAllNullRows); } tsFileResource.getModFileForWrite().write(node.getModEntries()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java index abce4d0ec0306..d8369e2c406dd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java @@ -192,13 +192,15 @@ public void redoLog(WALEntry walEntry) { new DeletionPredicate(device.getTableName(), new FullExactMatch(device)), new TimeRange( tsFileResource.getStartTime(device).get(), - tsFileResource.getEndTime(device).get()))); + tsFileResource.getEndTime(device).get())), + !isTableModel); } else { memTable.delete( new TreeDeletionEntry( new MeasurementPath(device, "*"), tsFileResource.getStartTime(device).get(), - tsFileResource.getEndTime(device).get())); + tsFileResource.getEndTime(device).get()), + !isTableModel); } } } @@ -269,10 +271,10 @@ public void endRecovery() throws WALRecoverException { for (Map.Entry deviceEntry : memTableMap.entrySet()) { IDeviceID deviceId = deviceEntry.getKey(); for (Map.Entry measurementEntry : - deviceEntry.getValue().getMemChunkMap().entrySet()) { + deviceEntry.getValue().getMemChunkMap(!isTableModel).entrySet()) { IWritableMemChunk memChunk = measurementEntry.getValue(); tsFileResource.updateStartTime(deviceId, memChunk.getFirstPoint()); - tsFileResource.updateEndTime(deviceId, memChunk.getLastPoint()); + tsFileResource.updateEndTime(deviceId, memChunk.getLastPoint(!isTableModel)); } } // flush memTable @@ -283,7 +285,8 @@ public void endRecovery() throws WALRecoverException { recoveryMemTable, writer, databaseName + FILE_NAME_SEPARATOR + dataRegionId, - dataRegionId); + dataRegionId, + isTableModel); tableFlushTask.syncFlushMemTable(); tsFileResource.updatePlanIndexes(recoveryMemTable.getMinPlanIndex()); tsFileResource.updatePlanIndexes(recoveryMemTable.getMaxPlanIndex()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java index 265c027e8ac4d..005b78c5a0cb9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java @@ -163,6 +163,7 @@ static int run(final String[] args, final PrintStream out, final PrintStream err .MESSAGE_INFERRED_TABLE_DATABASE_ARG_FROM_WAL_DIRECTORY_ARG_REPLAY_INTO_THIS_DATABASE_Y_YES_A_ACCEPT_ALL_INFERRED_DATABASES_N_QUIT_5B59D833, inferredDatabase, directory)); + requireResolvedDatabases(directoryDatabases); final ReplayStatistics statistics = replayWALDirectories( walFiles, @@ -255,7 +256,7 @@ static Options createOptions() { .hasArg() .desc( ImportWALMessages - .MESSAGE_TARGET_DATABASE_FOR_TABLE_MODEL_WAL_ENTRIES_IF_OMITTED_INFER_FROM_THE_WAL_PARENT_DIRECTORY_AND_ASK_FOR_CONFIRMATION_4B1E409D) + .MESSAGE_TARGET_DATABASE_FOR_WAL_REPLAY_IF_OMITTED_INFER_FROM_THE_WAL_PARENT_DIRECTORY_IF_INFERENCE_FAILS_DB_DATABASE_IS_REQUIRED_6EEBC019) .build()); options.addOption( Option.builder() @@ -444,8 +445,9 @@ private static WALReplayWorker createWALReplayWorker( final WALReplayer.ReplayDecisionController replayDecisionController) throws IOException { final Session treeSession = createSession(host, port, username, password, null); + final boolean isTableModel = database != null && PathUtils.isTableModelDatabase(database); final Session tableSession = - database == null ? null : createSession(host, port, username, password, database); + isTableModel ? createSession(host, port, username, password, database) : null; try { treeSession.open(false); if (tableSession != null) { @@ -576,6 +578,26 @@ static Map resolveDirectoryDatabases( return databases; } + /** + * The data model of every directory must be known before any replay starts, otherwise a WAL + * snapshot could only be replayed with a guessed model. + */ + static void requireResolvedDatabases(final Map directoryDatabases) { + final List unresolvedDirectories = new ArrayList<>(); + for (final Map.Entry entry : directoryDatabases.entrySet()) { + if (entry.getValue() == null) { + unresolvedDirectories.add(entry.getKey().toString()); + } + } + if (!unresolvedDirectories.isEmpty()) { + throw new IllegalArgumentException( + String.format( + ImportWALMessages + .EXCEPTION_CANNOT_DETERMINE_THE_TARGET_DATABASE_OF_WAL_DIRECTORIES_ARG_SPECIFY_DB_DATABASE_WHICH_APPLIES_TO_ALL_IMPORTED_DIRECTORIES_55B174E4, + String.join(", ", unresolvedDirectories))); + } + } + private static final Comparator WAL_FILE_COMPARATOR = Comparator.comparing((Path path) -> Objects.toString(path.getParent(), "")) .thenComparingLong(ImportWAL::getWALVersion) @@ -936,6 +958,7 @@ static class WALReplayer implements WALReplayWorker { private final Session treeSession; private final Session tableSession; + private final boolean isTableModel; private final ConsensusLogToTabletConverter converter; private final ReplayDecisionPrompt replayDecisionPrompt; private final Map tableSchemas = new HashMap<>(); @@ -949,6 +972,15 @@ static class WALReplayer implements WALReplayWorker { new ReplayDecisionController(System.console())); } + /** + * @param treeSession the session which replays tree model entries, it carries no database + * @param tableSession the session which replays table model entries, null when no table + * database is declared + * @param tableDatabaseName the target database resolved before replay; null means no table + * database is declared, so the entries are replayed with the tree model only + * @param replayDecisionPrompt the prompt which decides how to handle entries that cannot be + * replayed automatically + */ WALReplayer( final Session treeSession, final Session tableSession, @@ -956,10 +988,12 @@ static class WALReplayer implements WALReplayWorker { final ReplayDecisionPrompt replayDecisionPrompt) { this.treeSession = treeSession; this.tableSession = tableSession; + this.isTableModel = + tableDatabaseName != null && PathUtils.isTableModelDatabase(tableDatabaseName); this.replayDecisionPrompt = replayDecisionPrompt; converter = new ConsensusLogToTabletConverter( - null, null, ColumnFilterMatcher.matchAll(), tableDatabaseName); + null, null, ColumnFilterMatcher.matchAll(), isTableModel ? tableDatabaseName : null); } @Override @@ -972,9 +1006,8 @@ public ReplayResult replay(final WALEntry entry) throws IoTDBConnectionException, StatementExecutionException { if (entry.getType() == WALEntryType.MEMORY_TABLE_SNAPSHOT || entry.getType() == WALEntryType.OLD_MEMORY_TABLE_SNAPSHOT) { - return replayMemTableSnapshot((IMemTable) entry.getValue()) - ? ReplayResult.REPLAYED - : ReplayResult.IGNORED; + final IMemTable memTable = (IMemTable) entry.getValue(); + return replayMemTableSnapshot(memTable) ? ReplayResult.REPLAYED : ReplayResult.IGNORED; } if (entry.getValue() instanceof InsertNode insertNode) { replayInsert(insertNode); @@ -1318,8 +1351,8 @@ private boolean replayMemTableSnapshot(final IMemTable memTable) memTable.getMemTableMap().entrySet()) { final IDeviceID deviceId = deviceEntry.getKey(); final IWritableMemChunkGroup group = deviceEntry.getValue(); - for (IWritableMemChunk chunk : group.getMemChunkMap().values()) { - if (chunk == null || chunk.isEmpty()) { + for (IWritableMemChunk chunk : group.getMemChunkMap(!isTableModel).values()) { + if (chunk == null || chunk.isEmpty(!isTableModel)) { continue; } if (chunk instanceof AlignedWritableMemChunk) { @@ -1336,8 +1369,7 @@ private boolean replayNonAlignedMemChunk( final IDeviceID deviceId, final IWritableMemChunk chunk) throws IoTDBConnectionException, StatementExecutionException { final List schemas = Collections.singletonList(chunk.getSchema()); - final boolean tableModel = deviceId.isTableModel(); - requireTableSessionIfNeeded(tableModel); + requireTableSessionIfNeeded(isTableModel); final TableTabletSchema tabletSchema = createTableTabletSchema(deviceId, schemas); final List lists = new ArrayList<>(); lists.addAll(chunk.getSortedList()); @@ -1354,7 +1386,7 @@ private boolean replayNonAlignedMemChunk( final int end = Math.min(start + SNAPSHOT_TABLET_ROW_LIMIT, list.rowCount()); final Tablet tablet = buildNonAlignedTablet(deviceId, tabletSchema, schemas, list, start, end); - sendTablet(tablet, tableModel, false); + sendTablet(tablet, isTableModel, false); replayed = true; } } @@ -1364,8 +1396,7 @@ private boolean replayNonAlignedMemChunk( private boolean replayAlignedMemChunk( final IDeviceID deviceId, final AlignedWritableMemChunk chunk) throws IoTDBConnectionException, StatementExecutionException { - final boolean tableModel = deviceId.isTableModel(); - requireTableSessionIfNeeded(tableModel); + requireTableSessionIfNeeded(isTableModel); final List schemas = chunk.getSchemaList(); final TableTabletSchema tabletSchema = createTableTabletSchema(deviceId, schemas); final List lists = new ArrayList<>(); @@ -1384,7 +1415,7 @@ private boolean replayAlignedMemChunk( final int end = Math.min(start + SNAPSHOT_TABLET_ROW_LIMIT, replayableRows.size()); final Tablet tablet = buildAlignedTablet(deviceId, tabletSchema, schemas, list, replayableRows, start, end); - sendTablet(tablet, tableModel, true); + sendTablet(tablet, isTableModel, true); replayed = true; } } @@ -1412,7 +1443,7 @@ private void requireTableSessionIfNeeded(final boolean tableModel) private TableTabletSchema createTableTabletSchema( final IDeviceID deviceId, final List fieldSchemas) throws IoTDBConnectionException, StatementExecutionException { - if (!deviceId.isTableModel()) { + if (!isTableModel) { return new TableTabletSchema(fieldSchemas, null, 0); } final List tagSchemas = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java index 79e66f92d9e66..0caf1692f635b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java @@ -209,7 +209,7 @@ public void testTVListOwnerTransferTimeOnlyAlignedReleasesAllValueColumns() new MeasurementSchema("s0", TSDataType.INT64, TSEncoding.PLAIN), new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN), new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.PLAIN))); - AlignedWritableMemChunk memChunk = new AlignedWritableMemChunk(schemas, true); + AlignedWritableMemChunk memChunk = new AlignedWritableMemChunk(schemas); for (int i = 0; i < 100; i++) { memChunk.putAlignedRow(i, new Object[] {(long) i, (long) i * 2, (long) i * 3}); } @@ -265,8 +265,8 @@ public void testTVListCloneForQuery() throws InterruptedException { IMemTable memTable = createMemTable(deviceId, measurementId); assertEquals(1, memTable.getMemTableMap().size()); IWritableMemChunkGroup memChunkGroup = memTable.getMemTableMap().values().iterator().next(); - assertEquals(1, memChunkGroup.getMemChunkMap().size()); - IWritableMemChunk memChunk = memChunkGroup.getMemChunkMap().values().iterator().next(); + assertEquals(1, memChunkGroup.getMemChunkMap(true).size()); + IWritableMemChunk memChunk = memChunkGroup.getMemChunkMap(true).values().iterator().next(); TVList tvList = memChunk.getWorkingTVList(); assertFalse(tvList.isSorted()); @@ -330,8 +330,8 @@ public void testAlignedTVListPartialColumnCloneEndToEnd() throws InterruptedExce assertEquals(1, memTable.getMemTableMap().size()); IWritableMemChunkGroup memChunkGroup = memTable.getMemTableMap().values().iterator().next(); - assertEquals(1, memChunkGroup.getMemChunkMap().size()); - IWritableMemChunk memChunk = memChunkGroup.getMemChunkMap().values().iterator().next(); + assertEquals(1, memChunkGroup.getMemChunkMap(true).size()); + IWritableMemChunk memChunk = memChunkGroup.getMemChunkMap(true).values().iterator().next(); AlignedTVList workingTvList = (AlignedTVList) memChunk.getWorkingTVList(); assertFalse(workingTvList.isSorted()); assertEquals(100, workingTvList.rowCount()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedBitmapMemoryAccountingPerformanceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedBitmapMemoryAccountingPerformanceTest.java index b18b543a2d53b..b9b9fd5368554 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedBitmapMemoryAccountingPerformanceTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedBitmapMemoryAccountingPerformanceTest.java @@ -266,14 +266,14 @@ private static void runWrite(Scenario scenario, AlignedWritableMemChunk[] memChu private static AlignedWritableMemChunk[] createMemChunks(Scenario scenario, int count) { AlignedWritableMemChunk[] memChunks = new AlignedWritableMemChunk[count]; for (int i = 0; i < count; i++) { - memChunks[i] = new AlignedWritableMemChunk(new ArrayList<>(scenario.schemas), false); + memChunks[i] = new AlignedWritableMemChunk(new ArrayList<>(scenario.schemas)); } return memChunks; } private static AccountingTarget createAccountingTarget(Scenario scenario) { AlignedWritableMemChunk memChunk = - new AlignedWritableMemChunk(new ArrayList<>(scenario.schemas), false); + new AlignedWritableMemChunk(new ArrayList<>(scenario.schemas)); memChunk.writeAlignedTablet( scenario.times, scenario.columns, @@ -290,7 +290,7 @@ private static AccountingTarget createAccountingTarget(Scenario scenario) { Collections.singletonMap( deviceId, new AlignedWritableMemChunkGroup( - memChunk, new ArrayList<>(scenario.schemas), false))); + memChunk, new ArrayList<>(scenario.schemas)))); return new AccountingTarget(memTable, deviceId); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkDeserializeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkDeserializeTest.java index 5595390752a39..682b4b287d289 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkDeserializeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkDeserializeTest.java @@ -239,7 +239,7 @@ public void testAlignedSeries() throws IOException, QueryProcessException, Metad new MeasurementSchema("s4", TSDataType.FLOAT), new MeasurementSchema("s5", TSDataType.DOUBLE), new MeasurementSchema("s6", TSDataType.TEXT)); - AlignedWritableMemChunk series = new AlignedWritableMemChunk(schemaList, false); + AlignedWritableMemChunk series = new AlignedWritableMemChunk(schemaList); int count = 1000; for (int i = 0; i < count; i++) { @@ -261,7 +261,7 @@ public void testAlignedSeries() throws IOException, QueryProcessException, Metad series.serializeToWAL(walBuffer); DataInputStream inputStream = new DataInputStream(new ByteArrayInputStream(walBuffer.getBuffer().array())); - AlignedWritableMemChunk memChunk = AlignedWritableMemChunk.deserialize(inputStream, false); + AlignedWritableMemChunk memChunk = AlignedWritableMemChunk.deserialize(inputStream); AlignedReadOnlyMemChunk readableChunk = (AlignedReadOnlyMemChunk) getAlignedReadOnlyChunk(memChunk, schemaList, measurementList); @@ -293,7 +293,7 @@ public void testNonAlignedMemChunkGroupSerializedSizeWithNonAsciiMeasurement() series.writeNonAlignedPoint(1, 1); WritableMemChunkGroup group = new WritableMemChunkGroup(); - group.getMemChunkMap().put(measurement, series); + group.getMemChunkMap(true).put(measurement, series); WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.allocate(group.serializedSize())); @@ -303,7 +303,7 @@ public void testNonAlignedMemChunkGroupSerializedSizeWithNonAsciiMeasurement() DataInputStream inputStream = new DataInputStream(new ByteArrayInputStream(walBuffer.getBuffer().array())); WritableMemChunkGroup deserialized = WritableMemChunkGroup.deserialize(inputStream); - Assert.assertTrue(deserialized.getMemChunkMap().containsKey(measurement)); + Assert.assertTrue(deserialized.getMemChunkMap(true).containsKey(measurement)); } private WritableMemChunk createWritableMemChunkFromBytes(WritableMemChunk series) @@ -319,7 +319,7 @@ private WritableMemChunk createWritableMemChunkFromBytes(WritableMemChunk series private ReadOnlyMemChunk getReadOnlyChunk(WritableMemChunk memChunk, TSDataType dataType) throws QueryProcessException, IOException, MetadataException { WritableMemChunkGroup memChunkGroup = new WritableMemChunkGroup(); - memChunkGroup.getMemChunkMap().put("s1", memChunk); + memChunkGroup.getMemChunkMap(true).put("s1", memChunk); Map memTableMap = new HashMap<>(); memTableMap.put(deviceID, memChunkGroup); IMemTable memTable = new PrimitiveMemTable(storageGroup, dataRegionId, memTableMap); @@ -343,7 +343,7 @@ private ReadOnlyMemChunk getAlignedReadOnlyChunk( List measurementList) throws QueryProcessException, IOException, MetadataException { AlignedWritableMemChunkGroup memChunkGroup = - new AlignedWritableMemChunkGroup(memChunk, schemaList, false); + new AlignedWritableMemChunkGroup(memChunk, schemaList); Map memTableMap = new HashMap<>(); memTableMap.put(deviceID, memChunkGroup); IMemTable memTable = new PrimitiveMemTable(storageGroup, dataRegionId, memTableMap); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemTableFlushTaskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemTableFlushTaskTest.java index afe0384839bcb..ef4c520b13554 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemTableFlushTaskTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemTableFlushTaskTest.java @@ -97,7 +97,7 @@ public void testFlushMemTable() MemTableTestUtils.measurementId0, MemTableTestUtils.dataType0); MemTableFlushTask memTableFlushTask = - new MemTableFlushTask(memTable, writer, storageGroup, dataRegionId); + new MemTableFlushTask(memTable, writer, storageGroup, dataRegionId, true); assertTrue( writer .getVisibleMetadataList( @@ -134,7 +134,7 @@ public void testFlushVectorMemTable() throws ExecutionException, InterruptedException, IllegalPathException, WriteProcessException { MemTableTestUtils.produceVectorData(memTable); MemTableFlushTask memTableFlushTask = - new MemTableFlushTask(memTable, writer, storageGroup, dataRegionId); + new MemTableFlushTask(memTable, writer, storageGroup, dataRegionId, true); assertTrue( writer .getVisibleMetadataList(MemTableTestUtils.deviceId0, "sensor0", TSDataType.BOOLEAN) @@ -162,7 +162,7 @@ public void testFlushNullableVectorMemTable() throws ExecutionException, InterruptedException, IllegalPathException, WriteProcessException { MemTableTestUtils.produceNullableVectorData(memTable); MemTableFlushTask memTableFlushTask = - new MemTableFlushTask(memTable, writer, storageGroup, dataRegionId); + new MemTableFlushTask(memTable, writer, storageGroup, dataRegionId, true); assertTrue( writer .getVisibleMetadataList(MemTableTestUtils.deviceId0, "sensor0", TSDataType.BOOLEAN) @@ -192,7 +192,7 @@ public void testAlignedFlushWithoutDeletedMeasurementsSkipsColumnMapping() { memChunk.sortTvListForFlush(); BlockingQueue ioTaskQueue = new LinkedBlockingQueue<>(); - memChunk.encodeWorkingAlignedTVList(ioTaskQueue, 100, 100); + memChunk.encodeWorkingAlignedTVList(ioTaskQueue, 100, 100, true); assertFalse(memChunk.isColumnMappingBuilt()); assertFalse(ioTaskQueue.isEmpty()); @@ -206,7 +206,7 @@ public void testAlignedFlushWithDeletedMeasurementsKeepsColumnMapping() { memChunk.sortTvListForFlush(); BlockingQueue ioTaskQueue = new LinkedBlockingQueue<>(); - memChunk.encodeWorkingAlignedTVList(ioTaskQueue, 100, 100); + memChunk.encodeWorkingAlignedTVList(ioTaskQueue, 100, 100, true); assertTrue(memChunk.isColumnMappingBuilt()); assertFalse(ioTaskQueue.isEmpty()); @@ -228,7 +228,7 @@ private void checkAlignedFlushMergesDuplicateTimestamps(boolean removeSecondMeas Arrays.asList( new MeasurementSchema("s0", TSDataType.INT64, TSEncoding.PLAIN), new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN)); - AlignedWritableMemChunk memChunk = new AlignedWritableMemChunk(schemas, true); + AlignedWritableMemChunk memChunk = new AlignedWritableMemChunk(schemas); String alignedFilePath = TestConstant.OUTPUT_DATA_DIR.concat( "duplicateAlignedRows-" + removeSecondMeasurement + ".tsfile"); @@ -245,7 +245,7 @@ private void checkAlignedFlushMergesDuplicateTimestamps(boolean removeSecondMeas memChunk.sortTvListForFlush(); BlockingQueue ioTaskQueue = new LinkedBlockingQueue<>(); - memChunk.encodeWorkingAlignedTVList(ioTaskQueue, 100, 100); + memChunk.encodeWorkingAlignedTVList(ioTaskQueue, 100, 100, false); try (TsFileIOWriter alignedWriter = new TsFileIOWriter(new File(alignedFilePath))) { alignedWriter.startChunkGroup(IDeviceID.Factory.DEFAULT_FACTORY.create("root.d")); Object task; @@ -304,7 +304,7 @@ public void testAlignedFastPathKeepsPagesAndValuesAlignedAfterPartialSegmentSort Arrays.asList( new MeasurementSchema("s0", TSDataType.INT64, TSEncoding.PLAIN), new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN)); - AlignedWritableMemChunk memChunk = new AlignedWritableMemChunk(schemas, false); + AlignedWritableMemChunk memChunk = new AlignedWritableMemChunk(schemas); String alignedFilePath = TestConstant.OUTPUT_DATA_DIR.concat("testAlignedFastPath.tsfile"); try { @@ -320,7 +320,7 @@ public void testAlignedFastPathKeepsPagesAndValuesAlignedAfterPartialSegmentSort memChunk.sortTvListForFlush(); BlockingQueue ioTaskQueue = new LinkedBlockingQueue<>(); - memChunk.encodeWorkingAlignedTVList(ioTaskQueue, rowCount, 1024); + memChunk.encodeWorkingAlignedTVList(ioTaskQueue, rowCount, 1024, true); try (TsFileIOWriter alignedWriter = new TsFileIOWriter(new File(alignedFilePath))) { alignedWriter.startChunkGroup(IDeviceID.Factory.DEFAULT_FACTORY.create("root.d")); Object task; @@ -373,7 +373,7 @@ private void checkAlignedFlushAfterTimeDeletion(boolean removeColumn) throws IOE Arrays.asList( new MeasurementSchema("s0", TSDataType.INT32, TSEncoding.PLAIN), new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN)); - AlignedWritableMemChunk memChunk = new AlignedWritableMemChunk(schemas, false); + AlignedWritableMemChunk memChunk = new AlignedWritableMemChunk(schemas); String alignedFilePath = TestConstant.OUTPUT_DATA_DIR.concat("testAlignedTimeDeletion" + pageSize + ".tsfile"); try { @@ -389,7 +389,7 @@ private void checkAlignedFlushAfterTimeDeletion(boolean removeColumn) throws IOE BlockingQueue ioTaskQueue = new LinkedBlockingQueue<>(); // Cover a single page and boundaries between pages and chunks. - memChunk.encodeWorkingAlignedTVList(ioTaskQueue, pageSize + 1, pageSize); + memChunk.encodeWorkingAlignedTVList(ioTaskQueue, pageSize + 1, pageSize, true); try (TsFileIOWriter alignedWriter = new TsFileIOWriter(new File(alignedFilePath))) { alignedWriter.startChunkGroup(IDeviceID.Factory.DEFAULT_FACTORY.create("root.d")); Object task; @@ -464,7 +464,7 @@ private void checkUnmaterializedSegments(boolean moved) throws Exception { schemas.add(new MeasurementSchema("s" + column, types.get(column), TSEncoding.PLAIN)); } schemas.add(new MeasurementSchema("empty", TSDataType.INT64, TSEncoding.PLAIN)); - AlignedWritableMemChunk chunk = new AlignedWritableMemChunk(schemas, true); + AlignedWritableMemChunk chunk = new AlignedWritableMemChunk(schemas); Object[][] expected = new Object[rowCount][types.size()]; String path = TestConstant.OUTPUT_DATA_DIR.concat("unmaterialized-" + moved + ".tsfile"); try { @@ -502,7 +502,7 @@ private void checkUnmaterializedSegments(boolean moved) throws Exception { } chunk.sortTvListForFlush(); BlockingQueue queue = new LinkedBlockingQueue<>(); - chunk.encodeWorkingAlignedTVList(queue, ARRAY_SIZE * 5 + 3, ARRAY_SIZE + 3); + chunk.encodeWorkingAlignedTVList(queue, ARRAY_SIZE * 5 + 3, ARRAY_SIZE + 3, false); try (TsFileIOWriter fileWriter = new TsFileIOWriter(new File(path))) { fileWriter.startChunkGroup(IDeviceID.Factory.DEFAULT_FACTORY.create("root.d")); Object task; @@ -559,7 +559,7 @@ private static class TrackingAlignedWritableMemChunk extends AlignedWritableMemC private boolean columnMappingBuilt; private TrackingAlignedWritableMemChunk(List schemaList) { - super(schemaList, false); + super(schemaList); } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java index dffd4d941d057..5a0f8a1e800f3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java @@ -183,11 +183,14 @@ public void testWriteDuringPrepareTVListAndActualQueryExecution() new StringArrayDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i}); } memTable.delete( - new TreeDeletionEntry(new MeasurementPath("root.test.d1.s1", TSDataType.INT32), 150, 160)); + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s1", TSDataType.INT32), 150, 160), + true); memTable.delete( - new TreeDeletionEntry(new MeasurementPath("root.test.d1.s2", TSDataType.INT32), 150, 160)); + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s2", TSDataType.INT32), 150, 160), + true); memTable.delete( - new TreeDeletionEntry(new MeasurementPath("root.test.d1.s3", TSDataType.INT32), 150, 160)); + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s3", TSDataType.INT32), 150, 160), + true); ResourceByPathUtils resourcesByPathUtils = ResourceByPathUtils.getResourceInstance( new AlignedFullPath( @@ -230,11 +233,14 @@ public void testWriteAndFlushSortDuringQuerySortTVListAndActualQueryExecution() new StringArrayDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i}); } memTable.delete( - new TreeDeletionEntry(new MeasurementPath("root.test.d1.s1", TSDataType.INT32), 150, 160)); + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s1", TSDataType.INT32), 150, 160), + true); memTable.delete( - new TreeDeletionEntry(new MeasurementPath("root.test.d1.s2", TSDataType.INT32), 150, 160)); + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s2", TSDataType.INT32), 150, 160), + true); memTable.delete( - new TreeDeletionEntry(new MeasurementPath("root.test.d1.s3", TSDataType.INT32), 150, 160)); + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s3", TSDataType.INT32), 150, 160), + true); ResourceByPathUtils resourcesByPathUtils = ResourceByPathUtils.getResourceInstance( new AlignedFullPath( @@ -290,11 +296,14 @@ public void testFlushingQueryDoesNotSortWorkingTVListUsedByPreviousQuery() memTable.writeAlignedRow(deviceID, measurementSchemas, i, new Object[] {i, i, i}); } memTable.delete( - new TreeDeletionEntry(new MeasurementPath("root.test.d1.s1", TSDataType.INT32), 1, 10)); + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s1", TSDataType.INT32), 1, 10), + true); memTable.delete( - new TreeDeletionEntry(new MeasurementPath("root.test.d1.s2", TSDataType.INT32), 1, 10)); + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s2", TSDataType.INT32), 1, 10), + true); memTable.delete( - new TreeDeletionEntry(new MeasurementPath("root.test.d1.s3", TSDataType.INT32), 1, 10)); + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s3", TSDataType.INT32), 1, 10), + true); Assert.assertFalse(originalWorkingList.isSorted()); AlignedReadOnlyMemChunk flushingQueryMemChunk = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java index a1ba4cee07463..d22e41c35f53a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java @@ -266,7 +266,8 @@ public void testAlignedWritableMemChunkRegionScan2() throws IllegalPathException memTable.delete( new TreeDeletionEntry( new MeasurementPath(new StringArrayDeviceID("root.test.d1"), "s1"), - new TimeRange(1, 1500))); + new TimeRange(1, 1500)), + true); writableMemChunk = (AlignedWritableMemChunk) memTable.getWritableMemChunk(new StringArrayDeviceID("root.test.d1"), ""); @@ -296,7 +297,7 @@ public void testTableWritableMemChunkRegionScan() { new MeasurementSchema("s2", TSDataType.INT32), new MeasurementSchema("s3", TSDataType.INT32)); AlignedWritableMemChunk writableMemChunk = - new AlignedWritableMemChunk(measurementSchemas, true); + new AlignedWritableMemChunk(measurementSchemas); int size = 100000; for (int i = 0; i < size; i++) { if (i <= 10000) { @@ -429,7 +430,8 @@ public void testNonAlignedWritableMemChunkRegionScan2() throws IllegalPathExcept memTable.delete( new TreeDeletionEntry( new MeasurementPath(new StringArrayDeviceID("root.test.d1"), "s1"), - new TimeRange(1, 1500))); + new TimeRange(1, 1500)), + true); WritableMemChunk writableMemChunk = (WritableMemChunk) memTable.getWritableMemChunk(new StringArrayDeviceID("root.test.d1"), "s1"); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/ImportWALTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/ImportWALTest.java index aa71595d4439b..bcc5ee2df456d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/ImportWALTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/ImportWALTest.java @@ -304,6 +304,49 @@ public void testUnrecognizedDirectoryDoesNotInferFromAncestor() { assertTrue(databases.values().stream().allMatch(database -> database == null)); } + /** + * Every directory must declare a database before any replay starts, otherwise the data model of a + * WAL snapshot could only be guessed. + */ + @Test + public void testUnresolvedDirectoryRequiresExplicitDatabase() { + final List files = + Arrays.asList( + Paths.get("factory-3/one.wal"), + Paths.get("0/two.wal"), + Paths.get("root.sg-4/three.wal")); + ImportWAL.requireResolvedDatabases( + ImportWAL.resolveDirectoryDatabases(files, "target", true, null)); + + final Map unresolved = + ImportWAL.resolveDirectoryDatabases(files, null, true, null); + final IllegalArgumentException failure = + assertThrows( + IllegalArgumentException.class, + () -> ImportWAL.requireResolvedDatabases(unresolved)); + // The inferable directory is not reported, the others are listed in directory order. + assertTrue(failure.getMessage().contains("0, root.sg-4")); + assertTrue(failure.getMessage().contains("-db/--database")); + } + + /** A directory whose database cannot be inferred must fail before any Session is opened. */ + @Test + public void testUnresolvedDatabaseFailsBeforeReplay() throws Exception { + final Path directory = temporaryFolder.newFolder("0").toPath(); + createWALFile(directory, 0); + final ByteArrayOutputStream error = new ByteArrayOutputStream(); + + assertEquals( + 1, + ImportWAL.run( + new String[] {"-f", directory.toString(), "-pw", "password"}, + new PrintStream(error), + new PrintStream(error))); + + assertTrue(error.toString().contains(directory.toString())); + assertTrue(error.toString().contains("-db/--database")); + } + /** * One thread may visit several databases; each directory needs its own Session and schema cache. */ @@ -1547,7 +1590,8 @@ public void testReplayNonAlignedMemTableSnapshotAsTablets() throws Exception { new Object[] {10, null, DateUtils.parseDateExpressionToInt(firstDate)}); final Session treeSession = mock(Session.class); - new ImportWAL.WALReplayer(treeSession, null, null).replay(new WALInfoEntry(1, memTable)); + new ImportWAL.WALReplayer(treeSession, null, "root.sg") + .replay(new WALInfoEntry(1, memTable)); final ArgumentCaptor tabletCaptor = ArgumentCaptor.forClass(Tablet.class); verify(treeSession, times(3)).insertTablet(tabletCaptor.capture()); @@ -1593,7 +1637,8 @@ public void testReplayAlignedMemTableSnapshotPreservesNulls() throws Exception { new Object[] {10, 100L, DateUtils.parseDateExpressionToInt(firstDate)}); final Session treeSession = mock(Session.class); - new ImportWAL.WALReplayer(treeSession, null, null).replay(new WALInfoEntry(1, memTable)); + new ImportWAL.WALReplayer(treeSession, null, "root.sg") + .replay(new WALInfoEntry(1, memTable)); final ArgumentCaptor tabletCaptor = ArgumentCaptor.forClass(Tablet.class); verify(treeSession).insertAlignedTablet(tabletCaptor.capture()); @@ -1623,7 +1668,7 @@ public void testReplaySerializedMemTableSnapshot() throws Exception { final ImportWAL.ReplayStatistics statistics = ImportWAL.replayWALFiles( Collections.singletonList(walFile.toPath()), - new ImportWAL.WALReplayer(treeSession, null, null)); + new ImportWAL.WALReplayer(treeSession, null, "root.sg")); assertEquals(1, statistics.getReplayedOperationCount()); final ArgumentCaptor tabletCaptor = ArgumentCaptor.forClass(Tablet.class); @@ -1644,7 +1689,8 @@ public void testReplayMemTableSnapshotSplitsLargeChunk() throws Exception { } final Session treeSession = mock(Session.class); - new ImportWAL.WALReplayer(treeSession, null, null).replay(new WALInfoEntry(1, memTable)); + new ImportWAL.WALReplayer(treeSession, null, "root.sg") + .replay(new WALInfoEntry(1, memTable)); final ArgumentCaptor tabletCaptor = ArgumentCaptor.forClass(Tablet.class); verify(treeSession, times(2)).insertTablet(tabletCaptor.capture()); From 8814a3c9dca41e4b2e65bdc70f6439fdca16e3a2 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 9 Oct 2026 18:28:40 +0800 Subject: [PATCH 05/15] [Storage] Fix table TTL lookup in query time bound - take the data model from the data region instead of the "root." device prefix - look up the table TTL with the database name of the data region, because the table cache is keyed by database and table - spotless formatting for the files touched by the previous commit --- .../memtable/AlignedWritableMemChunk.java | 8 +++----- .../dataregion/memtable/TsFileProcessor.java | 13 ++++++++----- ...lignedBitmapMemoryAccountingPerformanceTest.java | 3 +-- .../memtable/WritableMemChunkRegionScanTest.java | 3 +-- .../org/apache/iotdb/db/tools/ImportWALTest.java | 12 ++++-------- 5 files changed, 17 insertions(+), 22 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java index 54b481de66af6..df493067eb09b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java @@ -106,9 +106,7 @@ private AlignedWritableMemChunk(List schemaList, AlignedTVLi } private AlignedWritableMemChunk( - List schemaList, - AlignedTVList list, - EncryptParameter encryptParameter) { + List schemaList, AlignedTVList list, EncryptParameter encryptParameter) { this.measurementIndexMap = new LinkedHashMap<>(); this.schemaList = schemaList; for (int i = 0; i < schemaList.size(); i++) { @@ -1235,8 +1233,8 @@ public static AlignedWritableMemChunk deserialize(DataInputStream stream) throws return chunk; } - public static AlignedWritableMemChunk deserializeSingleTVListMemChunks( - DataInputStream stream) throws IOException { + public static AlignedWritableMemChunk deserializeSingleTVListMemChunks(DataInputStream stream) + throws IOException { int schemaListSize = stream.readInt(); List schemaList = new ArrayList<>(schemaListSize); for (int i = 0; i < schemaListSize; i++) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 7a89a07a9af52..c22dfcf58d1f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -2644,13 +2644,16 @@ private void query( } private long getQueryTimeLowerBound(IDeviceID deviceID) { - long ttl; - if (deviceID.getTableName().startsWith("root.")) { - ttl = DataNodeTTLCache.getInstance().getTTLForTree(deviceID); - } else { + final long ttl; + if (isTableModel()) { + // The table TTL is cached by the database and the table, both known from the data region of + // this processor. ttl = DataNodeTTLCache.getInstance() - .getTTLForTable(this.dataRegionName, deviceID.getTableName()); + .getTTLForTable( + dataRegionInfo.getDataRegion().getDatabaseName(), deviceID.getTableName()); + } else { + ttl = DataNodeTTLCache.getInstance().getTTLForTree(deviceID); } return ttl != Long.MAX_VALUE ? CommonUtils.getTTLLowerBound(ttl) : Long.MIN_VALUE; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedBitmapMemoryAccountingPerformanceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedBitmapMemoryAccountingPerformanceTest.java index b9b9fd5368554..8da57414e4a74 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedBitmapMemoryAccountingPerformanceTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedBitmapMemoryAccountingPerformanceTest.java @@ -289,8 +289,7 @@ private static AccountingTarget createAccountingTarget(Scenario scenario) { "0", Collections.singletonMap( deviceId, - new AlignedWritableMemChunkGroup( - memChunk, new ArrayList<>(scenario.schemas)))); + new AlignedWritableMemChunkGroup(memChunk, new ArrayList<>(scenario.schemas)))); return new AccountingTarget(memTable, deviceId); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java index d22e41c35f53a..7a4eedccf74c2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java @@ -296,8 +296,7 @@ public void testTableWritableMemChunkRegionScan() { new MeasurementSchema("s1", TSDataType.INT32), new MeasurementSchema("s2", TSDataType.INT32), new MeasurementSchema("s3", TSDataType.INT32)); - AlignedWritableMemChunk writableMemChunk = - new AlignedWritableMemChunk(measurementSchemas); + AlignedWritableMemChunk writableMemChunk = new AlignedWritableMemChunk(measurementSchemas); int size = 100000; for (int i = 0; i < size; i++) { if (i <= 10000) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/ImportWALTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/ImportWALTest.java index bcc5ee2df456d..00fb827e2e1bd 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/ImportWALTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/ImportWALTest.java @@ -322,8 +322,7 @@ public void testUnresolvedDirectoryRequiresExplicitDatabase() { ImportWAL.resolveDirectoryDatabases(files, null, true, null); final IllegalArgumentException failure = assertThrows( - IllegalArgumentException.class, - () -> ImportWAL.requireResolvedDatabases(unresolved)); + IllegalArgumentException.class, () -> ImportWAL.requireResolvedDatabases(unresolved)); // The inferable directory is not reported, the others are listed in directory order. assertTrue(failure.getMessage().contains("0, root.sg-4")); assertTrue(failure.getMessage().contains("-db/--database")); @@ -1590,8 +1589,7 @@ public void testReplayNonAlignedMemTableSnapshotAsTablets() throws Exception { new Object[] {10, null, DateUtils.parseDateExpressionToInt(firstDate)}); final Session treeSession = mock(Session.class); - new ImportWAL.WALReplayer(treeSession, null, "root.sg") - .replay(new WALInfoEntry(1, memTable)); + new ImportWAL.WALReplayer(treeSession, null, "root.sg").replay(new WALInfoEntry(1, memTable)); final ArgumentCaptor tabletCaptor = ArgumentCaptor.forClass(Tablet.class); verify(treeSession, times(3)).insertTablet(tabletCaptor.capture()); @@ -1637,8 +1635,7 @@ public void testReplayAlignedMemTableSnapshotPreservesNulls() throws Exception { new Object[] {10, 100L, DateUtils.parseDateExpressionToInt(firstDate)}); final Session treeSession = mock(Session.class); - new ImportWAL.WALReplayer(treeSession, null, "root.sg") - .replay(new WALInfoEntry(1, memTable)); + new ImportWAL.WALReplayer(treeSession, null, "root.sg").replay(new WALInfoEntry(1, memTable)); final ArgumentCaptor tabletCaptor = ArgumentCaptor.forClass(Tablet.class); verify(treeSession).insertAlignedTablet(tabletCaptor.capture()); @@ -1689,8 +1686,7 @@ public void testReplayMemTableSnapshotSplitsLargeChunk() throws Exception { } final Session treeSession = mock(Session.class); - new ImportWAL.WALReplayer(treeSession, null, "root.sg") - .replay(new WALInfoEntry(1, memTable)); + new ImportWAL.WALReplayer(treeSession, null, "root.sg").replay(new WALInfoEntry(1, memTable)); final ArgumentCaptor tabletCaptor = ArgumentCaptor.forClass(Tablet.class); verify(treeSession, times(2)).insertTablet(tabletCaptor.capture()); From 0bfbf03472a0f051d94ae68e425795ae555c5e79 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 9 Oct 2026 18:51:05 +0800 Subject: [PATCH 06/15] [Pipe] Take the data model from the context instead of the device name - CachedSchemaPatternMatcher matches the sources by the model declared in the event - PipeHistoricalDataRegionTsFileAndDeletionSource decides the model by the tsfile database - PipeRawTabletInsertionEvent picks the event parser by isTableModelEvent() - TreeViewTabletProjector drops the device name prefix check --- .../tablet/PipeRawTabletInsertionEvent.java | 39 +++++++++++++++---- ...icalDataRegionTsFileAndDeletionSource.java | 16 ++------ .../matcher/CachedSchemaPatternMatcher.java | 25 +++++------- .../columnfilter/TreeViewTabletProjector.java | 1 - .../event/PipeTabletInsertionEventTest.java | 36 ++++++++--------- .../CachedSchemaPatternMatcherTest.java | 1 + 6 files changed, 64 insertions(+), 54 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java index dc2ab1d381fd8..6ebff4375600d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java @@ -200,9 +200,10 @@ public PipeRawTabletInsertionEvent( } @TestOnly - public PipeRawTabletInsertionEvent(final Tablet tablet, final boolean isAligned) { + public PipeRawTabletInsertionEvent( + final boolean isTableModel, final Tablet tablet, final boolean isAligned) { this( - null, + isTableModel, null, null, null, @@ -225,9 +226,12 @@ public PipeRawTabletInsertionEvent(final Tablet tablet, final boolean isAligned) @TestOnly public PipeRawTabletInsertionEvent( - final Tablet tablet, final boolean isAligned, final TreePattern treePattern) { + final boolean isTableModel, + final Tablet tablet, + final boolean isAligned, + final TreePattern treePattern) { this( - null, + isTableModel, null, null, null, @@ -250,10 +254,27 @@ public PipeRawTabletInsertionEvent( @TestOnly public PipeRawTabletInsertionEvent( - final Tablet tablet, final long startTime, final long endTime) { + final boolean isTableModel, final Tablet tablet, final long startTime, final long endTime) { this( - null, null, null, null, tablet, false, null, false, null, 0, null, null, null, null, null, - null, true, startTime, endTime); + isTableModel, + null, + null, + null, + tablet, + false, + null, + false, + null, + 0, + null, + null, + null, + null, + null, + null, + true, + startTime, + endTime); } @Override @@ -479,8 +500,10 @@ public Tablet convertToTablet() { private TabletInsertionEventParser initEventParser() { if (eventParser == null) { + // The data model of the payload is the one declared by the event, it must never be derived + // from the device name of the tablet. eventParser = - tablet.getDeviceId().startsWith("root.") + !isTableModelEvent() ? new TabletInsertionEventTreePatternParser( pipeTaskMeta, this, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java index e542fc5171915..a28598ddec158 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java @@ -71,7 +71,6 @@ import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.file.metadata.PlainDeviceID; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -124,8 +123,6 @@ import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_START_TIME_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_TSFILE_PARSER_KEY; import static org.apache.iotdb.commons.pipe.source.IoTDBSource.getSkipIfNoPrivileges; -import static org.apache.tsfile.common.constant.TsFileConstant.PATH_ROOT; -import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; public class PipeHistoricalDataRegionTsFileAndDeletionSource implements PipeHistoricalDataRegionSource { @@ -133,8 +130,6 @@ public class PipeHistoricalDataRegionTsFileAndDeletionSource private static final Logger LOGGER = LoggerFactory.getLogger(PipeHistoricalDataRegionTsFileAndDeletionSource.class); - private static final String TREE_MODEL_EVENT_TABLE_NAME_PREFIX = PATH_ROOT + PATH_SEPARATOR; - private String pipeName; private long creationTime; private String pipeNameWithCreationTime; @@ -1026,7 +1021,7 @@ private boolean mayTsFileResourceOverlappedWithPattern(final TsFileResource reso .anyMatch( deviceID -> { if (!isModelDetected) { - detectModel(resource, deviceID); + detectModel(resource); isModelDetected = true; } @@ -1039,13 +1034,10 @@ private boolean mayTsFileResourceOverlappedWithPattern(final TsFileResource reso }); } - private void detectModel(final TsFileResource resource, final IDeviceID deviceID) { - this.isTableModel = - !(deviceID instanceof PlainDeviceID - || deviceID.getTableName().startsWith(TREE_MODEL_EVENT_TABLE_NAME_PREFIX) - || deviceID.getTableName().equals(PATH_ROOT)); - + private void detectModel(final TsFileResource resource) { + // One source serves one data region, so the data model is decided by its database. final String databaseName = resource.getDatabaseName(); + this.isTableModel = PathUtils.isTableModelDatabase(databaseName); isDbNameCoveredByPattern = isTableModel ? tablePattern.isTableModelDataAllowedToBeCaptured() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/matcher/CachedSchemaPatternMatcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/matcher/CachedSchemaPatternMatcher.java index d8f427868f04e..6d7071fe93b1d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/matcher/CachedSchemaPatternMatcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/matcher/CachedSchemaPatternMatcher.java @@ -33,7 +33,6 @@ import org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegionSource; import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.file.metadata.PlainDeviceID; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,15 +46,10 @@ import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.locks.ReentrantReadWriteLock; -import static org.apache.tsfile.common.constant.TsFileConstant.PATH_ROOT; -import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; - public class CachedSchemaPatternMatcher implements PipeDataRegionMatcher { protected static final Logger LOGGER = LoggerFactory.getLogger(CachedSchemaPatternMatcher.class); - protected static final String TREE_MODEL_EVENT_TABLE_NAME_PREFIX = PATH_ROOT + PATH_SEPARATOR; - protected final ReentrantReadWriteLock lock; protected final Set sources; @@ -149,23 +143,24 @@ public Pair, Set final boolean isTableModelTsFileEvent = event.getEvent() instanceof PipeTsFileInsertionEvent && ((PipeTsFileInsertionEvent) event.getEvent()).isTableModelEvent(); + // The whole event comes from one data region, so all of its devices share the data model + // declared by the event. It must never be derived from the device name. + final boolean isTableModelEvent = + event.getEvent() instanceof PipeInsertionEvent + && ((PipeInsertionEvent) event.getEvent()).isTableModelEvent(); final Set tableNames = new HashSet<>(); for (final Map.Entry entry : event.getSchemaInfo().entrySet()) { final IDeviceID deviceID = entry.getKey(); - // TODO: Check the role to determine whether to match with tree model or table model - if (deviceID instanceof PlainDeviceID - || deviceID.getTableName().startsWith(TREE_MODEL_EVENT_TABLE_NAME_PREFIX) - || deviceID.getTableName().equals(PATH_ROOT)) { + if (!isTableModelEvent) { matchTreeModelEvent(deviceID, entry.getValue(), matchedSources); } else { final String tableName = deviceID.getTableName(); if (tableNames.add(tableName) && matchedSources.size() < sources.size()) { - final String tableModelDatabaseName = - event.getEvent() instanceof PipeInsertionEvent - ? ((PipeInsertionEvent) event.getEvent()).getTableModelDatabaseName() - : null; - matchTableModelEvent(tableModelDatabaseName, tableName, matchedSources); + matchTableModelEvent( + ((PipeInsertionEvent) event.getEvent()).getTableModelDatabaseName(), + tableName, + matchedSources); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/TreeViewTabletProjector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/TreeViewTabletProjector.java index d95fb42b11652..3a09af7e48d62 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/TreeViewTabletProjector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/TreeViewTabletProjector.java @@ -66,7 +66,6 @@ public String getDatabaseName() { public Tablet project(final Tablet treeTablet) { if (Objects.isNull(treeTablet) || Objects.isNull(treeTablet.getDeviceId()) - || !treeTablet.getDeviceId().startsWith("root.") || Objects.isNull(treeTablet.getValues())) { return null; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java index f5c3f1fa05c8c..847f48e939327 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java @@ -338,14 +338,14 @@ public void convertToTabletForTest() throws Exception { Assert.assertFalse(isAligned2); PipeRawTabletInsertionEvent event3 = - new PipeRawTabletInsertionEvent(tablet1, false, new PrefixTreePattern(pattern)); + new PipeRawTabletInsertionEvent(false, tablet1, false, new PrefixTreePattern(pattern)); Tablet tablet3 = event3.convertToTablet(); boolean isAligned3 = event3.isAligned(); Assert.assertEquals(tablet1, tablet3); Assert.assertFalse(isAligned3); PipeRawTabletInsertionEvent event4 = - new PipeRawTabletInsertionEvent(tablet2, false, new PrefixTreePattern(pattern)); + new PipeRawTabletInsertionEvent(false, tablet2, false, new PrefixTreePattern(pattern)); Tablet tablet4 = event4.convertToTablet(); boolean isAligned4 = event4.isAligned(); Assert.assertEquals(tablet2, tablet4); @@ -371,14 +371,14 @@ public void convertToAlignedTabletForTest() throws Exception { Assert.assertTrue(isAligned2); PipeRawTabletInsertionEvent event3 = - new PipeRawTabletInsertionEvent(tablet1, true, new PrefixTreePattern(pattern)); + new PipeRawTabletInsertionEvent(false, tablet1, true, new PrefixTreePattern(pattern)); Tablet tablet3 = event3.convertToTablet(); boolean isAligned3 = event3.isAligned(); Assert.assertEquals(tablet1, tablet3); Assert.assertTrue(isAligned3); PipeRawTabletInsertionEvent event4 = - new PipeRawTabletInsertionEvent(tablet2, true, new PrefixTreePattern(pattern)); + new PipeRawTabletInsertionEvent(false, tablet2, true, new PrefixTreePattern(pattern)); Tablet tablet4 = event4.convertToTablet(); boolean isAligned4 = event4.isAligned(); Assert.assertEquals(tablet2, tablet4); @@ -389,7 +389,7 @@ public void convertToAlignedTabletForTest() throws Exception { public void processAlignedTabletWithCollectPreservesAlignmentForTest() { final PipeRawTabletInsertionEvent event = new PipeRawTabletInsertionEvent( - tabletForInsertTabletNode, true, new PrefixTreePattern(pattern)); + false, tabletForInsertTabletNode, true, new PrefixTreePattern(pattern)); final List events = new ArrayList<>(); event @@ -543,7 +543,7 @@ public void convertToTabletWithFilteredRowsForTest() throws Exception { TabletInsertionEventTreePatternParser container1 = new TabletInsertionEventTreePatternParser( null, - new PipeRawTabletInsertionEvent(tabletForInsertRowNode, 111L, 113L), + new PipeRawTabletInsertionEvent(false, tabletForInsertRowNode, 111L, 113L), insertRowNode, new PrefixTreePattern(pattern), null); @@ -555,7 +555,7 @@ public void convertToTabletWithFilteredRowsForTest() throws Exception { TabletInsertionEventTreePatternParser container2 = new TabletInsertionEventTreePatternParser( null, - new PipeRawTabletInsertionEvent(tabletForInsertRowNode, 110L, 110L), + new PipeRawTabletInsertionEvent(false, tabletForInsertRowNode, 110L, 110L), insertRowNode, new PrefixTreePattern(pattern), null); @@ -567,7 +567,7 @@ public void convertToTabletWithFilteredRowsForTest() throws Exception { TabletInsertionEventTreePatternParser container3 = new TabletInsertionEventTreePatternParser( null, - new PipeRawTabletInsertionEvent(tabletForInsertTabletNode, 111L, 113L), + new PipeRawTabletInsertionEvent(false, tabletForInsertTabletNode, 111L, 113L), insertTabletNode, new PrefixTreePattern(pattern), null); @@ -579,7 +579,7 @@ public void convertToTabletWithFilteredRowsForTest() throws Exception { TabletInsertionEventTreePatternParser container4 = new TabletInsertionEventTreePatternParser( null, - new PipeRawTabletInsertionEvent(tabletForInsertTabletNode, Long.MIN_VALUE, 109L), + new PipeRawTabletInsertionEvent(false, tabletForInsertTabletNode, Long.MIN_VALUE, 109L), insertTabletNode, new PrefixTreePattern(pattern), null); @@ -593,20 +593,20 @@ public void convertToTabletWithFilteredRowsForTest() throws Exception { public void isEventTimeOverlappedWithTimeRangeTest() { PipeRawTabletInsertionEvent event; - event = new PipeRawTabletInsertionEvent(tabletForInsertRowNode, 111L, 113L); + event = new PipeRawTabletInsertionEvent(false, tabletForInsertRowNode, 111L, 113L); Assert.assertFalse(event.mayEventTimeOverlappedWithTimeRange()); - event = new PipeRawTabletInsertionEvent(tabletForInsertRowNode, 110L, 110L); + event = new PipeRawTabletInsertionEvent(false, tabletForInsertRowNode, 110L, 110L); Assert.assertTrue(event.mayEventTimeOverlappedWithTimeRange()); - event = new PipeRawTabletInsertionEvent(tabletForInsertTabletNode, 111L, 113L); + event = new PipeRawTabletInsertionEvent(false, tabletForInsertTabletNode, 111L, 113L); Assert.assertTrue(event.mayEventTimeOverlappedWithTimeRange()); - event = new PipeRawTabletInsertionEvent(tabletForInsertTabletNode, Long.MIN_VALUE, 110L); + event = new PipeRawTabletInsertionEvent(false, tabletForInsertTabletNode, Long.MIN_VALUE, 110L); Assert.assertTrue(event.mayEventTimeOverlappedWithTimeRange()); - event = new PipeRawTabletInsertionEvent(tabletForInsertTabletNode, 114L, Long.MAX_VALUE); + event = new PipeRawTabletInsertionEvent(false, tabletForInsertTabletNode, 114L, Long.MAX_VALUE); Assert.assertTrue(event.mayEventTimeOverlappedWithTimeRange()); - event = new PipeRawTabletInsertionEvent(tabletForInsertTabletNode, Long.MIN_VALUE, 109L); + event = new PipeRawTabletInsertionEvent(false, tabletForInsertTabletNode, Long.MIN_VALUE, 109L); Assert.assertFalse(event.mayEventTimeOverlappedWithTimeRange()); - event = new PipeRawTabletInsertionEvent(tabletForInsertTabletNode, 115L, Long.MAX_VALUE); + event = new PipeRawTabletInsertionEvent(false, tabletForInsertTabletNode, 115L, Long.MAX_VALUE); Assert.assertFalse(event.mayEventTimeOverlappedWithTimeRange()); } @@ -619,9 +619,9 @@ public void isEventTimeOverlappedWithTimeRangeUsesActualRowSizeForTest() throws partialTablet.setRowSize(3); PipeRawTabletInsertionEvent rawEvent = - new PipeRawTabletInsertionEvent(partialTablet, 111L, 112L); + new PipeRawTabletInsertionEvent(false, partialTablet, 111L, 112L); Assert.assertTrue(rawEvent.mayEventTimeOverlappedWithTimeRange()); - rawEvent = new PipeRawTabletInsertionEvent(partialTablet, 113L, Long.MAX_VALUE); + rawEvent = new PipeRawTabletInsertionEvent(false, partialTablet, 113L, Long.MAX_VALUE); Assert.assertFalse(rawEvent.mayEventTimeOverlappedWithTimeRange()); final InsertTabletNode partialInsertTabletNode = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/CachedSchemaPatternMatcherTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/CachedSchemaPatternMatcherTest.java index 832bae3935256..3e99576bee146 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/CachedSchemaPatternMatcherTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/CachedSchemaPatternMatcherTest.java @@ -223,6 +223,7 @@ public void testTableModelMatchesEachTableOncePerEvent() throws Exception { countingMatcher.register(new PipeRealtimeDataRegionFakeSource()); final PipeInsertionEvent insertionEvent = Mockito.mock(PipeInsertionEvent.class); + Mockito.when(insertionEvent.isTableModelEvent()).thenReturn(true); Mockito.when(insertionEvent.getTableModelDatabaseName()).thenReturn("db"); final Map schemaInfo = new LinkedHashMap<>(); schemaInfo.put(new StringArrayDeviceID("table1", "tag1"), new String[0]); From df3eb05cee0ed12997ba881e4ce70781f62b9227 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Fri, 9 Oct 2026 18:56:51 +0800 Subject: [PATCH 07/15] [Query] Mark the tree/table model only APIs in PartitionCache - annotate the tree model only device -> database lookups with @TreeModel - annotate the tree model only database creation with @TreeModel and the table model one with @TableModel - document why the isTableModel flag of the device -> database lookup is false for both callers --- .../cache/partition/PartitionCache.java | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java index 03f2d809c0edf..32ee6032f4d3d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java @@ -162,11 +162,15 @@ protected void failIfMetadataLeaseFenced() { /** * get database to device map * + *

Tree model only: a tree model device path contains its database, while a table model device + * (table + tags) can only be resolved with a known database. + * * @param deviceIDs the devices that need to hit * @param tryToFetch whether try to get all database from config node * @param isAutoCreate whether auto create database when cache miss * @param userName the userName */ + @TreeModel public Map> getDatabaseToDevice( final List deviceIDs, final boolean tryToFetch, @@ -180,6 +184,7 @@ public void put(final IDeviceID device, final String databaseName) { map.computeIfAbsent(databaseName, k -> new ArrayList<>()).add(device); } }; + // Tree model only, see the javadoc above. getDatabaseCacheResult(result, deviceIDs, tryToFetch, isAutoCreate, userName, false); return result.getMap(); } @@ -187,11 +192,15 @@ public void put(final IDeviceID device, final String databaseName) { /** * get device to database map * + *

Tree model only: a tree model device path contains its database, while a table model device + * (table + tags) can only be resolved with a known database. + * * @param deviceIDs the devices that need to hit * @param tryToFetch whether try to get all database from config node * @param isAutoCreate whether auto create database when cache miss * @param userName the userName */ + @TreeModel public Map getDeviceToDatabase( final List deviceIDs, final boolean tryToFetch, @@ -205,6 +214,7 @@ public void put(final IDeviceID device, final String databaseName) { map.put(device, databaseName); } }; + // Tree model only, see the javadoc above. getDatabaseCacheResult(result, deviceIDs, tryToFetch, isAutoCreate, userName, false); return result.getMap(); } @@ -242,9 +252,12 @@ private boolean containsDatabase(final String database) { /** * get all database from configNode and update database cache * + *

Tree model only: the databases requested from the config node are the tree model ones. + * * @param result the result of get database cache * @param deviceIDs the devices that need to hit */ + @TreeModel private void fetchDatabaseAndUpdateCache( final DatabaseCacheResult result, final List deviceIDs) throws ClientManagerException, TException { @@ -290,7 +303,12 @@ public boolean isNeedLastCache(final String database) { return Objects.isNull(needLastCache) || needLastCache; } - /** get all database from configNode and update database cache. */ + /** + * get all database from configNode and update database cache. + * + * @param isTableModel the data model of the requested databases, it must be the model of the + * caller, i.e. {@code false} for a tree model caller and {@code true} for a table model one + */ private void fetchDatabaseAndUpdateCache(final boolean isTableModel) throws ClientManagerException, TException { databaseCacheLock.writeLock().lock(); @@ -313,11 +331,15 @@ private void fetchDatabaseAndUpdateCache(final boolean isTableModel) /** * create not existed database and update database cache * + *

Tree model only: only tree model databases, i.e. {@code root.__system} and {@code + * root.__audit}, are created here. + * * @param result the result of get database cache * @param deviceIDs the devices that need to hit * @param userName the username * @throws RuntimeException if failed to create database */ + @TreeModel private void createDatabaseAndUpdateCache( final DatabaseCacheResult result, final List deviceIDs, @@ -399,10 +421,13 @@ private void createDatabaseAndUpdateCache( /** * create not existed database and update database cache * + *

Table model only: the database created here is a table model database. + * * @param database the database * @param userName the username * @throws RuntimeException if failed to create database */ + @TableModel private void createDatabaseAndUpdateCache(final String database, final String userName) throws ClientManagerException, TException { databaseCacheLock.writeLock().lock(); @@ -496,6 +521,8 @@ private void getDatabaseMap( * @param tryToFetch whether try to get all database from confignode * @param isAutoCreate whether auto create database when device miss * @param userName + * @param isTableModel the data model of the queried devices; the current callers resolve tree + * model devices, because only a tree model device path contains its database */ private void getDatabaseCacheResult( final DatabaseCacheResult result, From 36683baf5968a50f00321c3f146cc8c53ca8e8d1 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Sat, 10 Oct 2026 09:56:29 +0800 Subject: [PATCH 08/15] [Query] Keep the table filter guard when loading modifications A compaction context is a table model context but it never collects a table filter, so reading a table deletion from the mods file threw a NullPointerException. Skip a modification only when a table filter exists, i.e. restore the null guard. --- .../db/queryengine/execution/fragment/QueryContext.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java index 1bd5f70081fd4..b25dc7a083ee2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java @@ -145,7 +145,10 @@ public PatternTreeMap loadAllModificationsFromDisk( } protected boolean shouldSkipModification(ModEntry modification) { - if (isTableModel() && modification instanceof TableDeletionEntry) { + // tables is only collected by the table model query planning. Compaction and other contexts + // leave it null, which means that there is no table filter at all, so no modification can be + // skipped. + if (tables != null && modification instanceof TableDeletionEntry) { String tableName = ((TableDeletionEntry) modification).getTableName(); return !tables.contains(tableName); } From 4ab1874c18925185f6527d8b881841e886e23ba8 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Sat, 10 Oct 2026 10:02:09 +0800 Subject: [PATCH 09/15] [Pipe] Use a real database name in the realtime extract test PipeRealtimeExtractTest passed the data region id as the database name of the inserted events. The data model of such an event is derived from that string, so "1" resolved to the table model and the tree model sources never matched the tree devices, which made the extract listeners time out. Pass the database the test actually writes to. --- .../db/pipe/source/PipeRealtimeExtractTest.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java index fc4a15a885b5e..ea3642777dd29 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java @@ -86,6 +86,7 @@ public class PipeRealtimeExtractTest { private static final String TEST_PIPE_NAME = "test_degraded_status_pipe"; private static final long TEST_PIPE_CREATION_TIME = 1L; private static final String TEST_REFERENCE_HOLDER = PipeRealtimeExtractTest.class.getName(); + private static final String DATABASE = "root.sg"; private final int dataRegion1 = 1; private final int dataRegion2 = 2; @@ -114,7 +115,7 @@ public void setUp() throws Exception { + File.separator + IoTDBConstant.SEQUENCE_FOLDER_NAME + File.separator - + "root.sg"); + + DATABASE); } @After @@ -314,7 +315,7 @@ public void testListenToTsFileSkipsAssignerWithoutTsFileSource() throws Exceptio resource.close(); PipeInsertionDataNodeListener.getInstance() - .listenToTsFile(dataRegion1, Integer.toString(dataRegion1), resource, false); + .listenToTsFile(dataRegion1, DATABASE, resource, false); final long deadline = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(1); while (System.currentTimeMillis() < deadline @@ -353,7 +354,7 @@ public void testHybridSourceReportsTsFileEpochDegradedStatus() throws Exception final TsFileResource resource = createTsFileResource(dataRegion1, "100-100-0-0.tsfile"); final PipeRealtimeEvent tabletEvent = PipeRealtimeEventFactory.createRealtimeEvent( - false, "root.sg", createInsertRowNode("degraded-tablet", "a"), resource); + false, DATABASE, createInsertRowNode("degraded-tablet", "a"), resource); Assert.assertTrue(tabletEvent.increaseReferenceCount(TEST_REFERENCE_HOLDER)); extractor.extract(tabletEvent); @@ -364,7 +365,7 @@ public void testHybridSourceReportsTsFileEpochDegradedStatus() throws Exception Assert.assertEquals(Boolean.TRUE, getGlobalTsFileEpochDegraded()); final PipeRealtimeEvent tsFileEvent = - PipeRealtimeEventFactory.createRealtimeEvent(false, "root.sg", resource, false); + PipeRealtimeEventFactory.createRealtimeEvent(false, DATABASE, resource, false); Assert.assertTrue(tsFileEvent.increaseReferenceCount(TEST_REFERENCE_HOLDER)); extractor.extract(tsFileEvent); @@ -411,7 +412,7 @@ private Future write2DataRegion( PipeInsertionDataNodeListener.getInstance() .listenToInsertNode( dataRegionId, - Integer.toString(dataRegionId), + DATABASE, new InsertRowNode( new PlanNodeId(String.valueOf(i)), new PartialPath(device), @@ -425,7 +426,7 @@ private Future write2DataRegion( PipeInsertionDataNodeListener.getInstance() .listenToInsertNode( dataRegionId, - Integer.toString(dataRegionId), + DATABASE, new InsertRowNode( new PlanNodeId(String.valueOf(i)), new PartialPath(device), @@ -437,7 +438,7 @@ private Future write2DataRegion( false), resource); PipeInsertionDataNodeListener.getInstance() - .listenToTsFile(dataRegionId, Integer.toString(dataRegionId), resource, false); + .listenToTsFile(dataRegionId, DATABASE, resource, false); } }); } From 7c5009481099ad2fd76f8437269ae0dbc8fca11f Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Sat, 10 Oct 2026 10:38:13 +0800 Subject: [PATCH 10/15] [Storage] Give the table model compaction tests a table model database A compaction input takes its data model from the database directory of the source files, so the table model fixtures, which write table model devices, must not place their files in the tree model database root.testsg. Point them at the table model database testsg, and drop the cases that mix tree and table files in one task, which cannot occur because a database holds a single data model. --- .../compaction/AbstractCompactionTest.java | 121 +++++++------- .../CompactionDataTypeAlterTableTest.java | 37 ++++- .../AbstractTableModelCompactionTest.java | 84 ++++++++++ .../CompactionWithAllNullRowsTest.java | 3 +- .../TableModelCompactionWithTTLTest.java | 11 +- ...TableModelFastCompactionPerformerTest.java | 61 +------ ...ModelReadChunkCompactionPerformerTest.java | 155 +----------------- ...ModelReadPointCompactionPerformerTest.java | 61 +------ 8 files changed, 187 insertions(+), 346 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/AbstractTableModelCompactionTest.java diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java index 951585605df82..b51910f84efdc 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java @@ -163,30 +163,16 @@ public class AbstractCompactionTest { + "sequence" + File.separator + COMPACTION_TEST_SG); - protected static File SEQ_DIRS = - new File( - TestConstant.BASE_OUTPUT_PATH - + "data" - + File.separator - + "sequence" - + File.separator - + COMPACTION_TEST_SG - + File.separator - + "0" - + File.separator - + "0"); - protected static File UNSEQ_DIRS = + protected static File UNSEQ_STORAGE_GROUP_DIR = new File( TestConstant.BASE_OUTPUT_PATH + "data" + File.separator + "unsequence" + File.separator - + COMPACTION_TEST_SG - + File.separator - + "0" - + File.separator - + "0"); + + COMPACTION_TEST_SG); + protected static File SEQ_DIRS = new File(STORAGE_GROUP_DIR, "0" + File.separator + "0"); + protected static File UNSEQ_DIRS = new File(UNSEQ_STORAGE_GROUP_DIR, "0" + File.separator + "0"); protected Map> registeredTimePartitionDirs = new HashMap<>(); @@ -194,21 +180,55 @@ public class AbstractCompactionTest { private int fileCount = 0; - protected TsFileManager tsFileManager = - new TsFileManager(COMPACTION_TEST_SG, "0", STORAGE_GROUP_DIR.getPath()); + protected TsFileManager tsFileManager; protected CompactionFakeSchemaFetcherImpl schemaFetcher; + /** + * The database of the files created by this fixture and the directories that hold them. The + * defaults are the tree model database used by most of the compaction tests. A fixture that + * writes files of another data model overrides them, because the data model of the compaction + * input is resolved from the database directory of its source files: a table model database name + * must not start with {@code root.} + */ + protected String getTestStorageGroup() { + return COMPACTION_TEST_SG; + } + + /** The directory of the sequence files of the database of this fixture. */ + protected File getSeqStorageGroupDir() { + return STORAGE_GROUP_DIR; + } + + /** The directory of the unsequence files of the database of this fixture. */ + protected File getUnseqStorageGroupDir() { + return UNSEQ_STORAGE_GROUP_DIR; + } + + /** The directory of the data region 0, time partition 0 of the sequence files. */ + protected File getSeqDirs() { + return SEQ_DIRS; + } + + /** The directory of the data region 0, time partition 0 of the unsequence files. */ + protected File getUnseqDirs() { + return UNSEQ_DIRS; + } + + protected TsFileManager createTsFileManager() { + return new TsFileManager(getTestStorageGroup(), "0", getSeqStorageGroupDir().getPath()); + } + public void setUp() throws IOException, WriteProcessException, MetadataException, InterruptedException { MetadataLeaseManager.getInstance().updateFenceThresholdMs(Long.MAX_VALUE); MetadataLeaseManager.getInstance().recoveryLeaseForTest(true); fileCount = 0; - if (!SEQ_DIRS.exists()) { - Assert.assertTrue(SEQ_DIRS.mkdirs()); + if (!getSeqDirs().exists()) { + Assert.assertTrue(getSeqDirs().mkdirs()); } - if (!UNSEQ_DIRS.exists()) { - Assert.assertTrue(UNSEQ_DIRS.mkdirs()); + if (!getUnseqDirs().exists()) { + Assert.assertTrue(getUnseqDirs().mkdirs()); } dataType = TSDataType.INT64; CompactionTaskManager.getInstance().restart(); @@ -217,11 +237,12 @@ public void setUp() ChunkCache.getInstance().clear(); TimeSeriesMetadataCache.getInstance().clear(); BloomFilterCache.getInstance().clear(); + tsFileManager = createTsFileManager(); tsFileManager.getOrCreateSequenceListByTimePartition(0); tsFileManager.getOrCreateUnsequenceListByTimePartition(0); - registeredTimePartitionDirs.put(0L, new Pair<>(SEQ_DIRS, UNSEQ_DIRS)); + registeredTimePartitionDirs.put(0L, new Pair<>(getSeqDirs(), getUnseqDirs())); schemaFetcher = new CompactionFakeSchemaFetcherImpl(); - schemaFetcher.getSchemaTree().setDatabases(Collections.singleton(COMPACTION_TEST_SG)); + schemaFetcher.getSchemaTree().setDatabases(Collections.singleton(getTestStorageGroup())); CompactionUtils.setSchemaFetcher(schemaFetcher); } @@ -230,30 +251,10 @@ protected void createTimePartitionDirIfNotExist(long timePartition) { return; } File seqTimePartitionDir = - new File( - TestConstant.BASE_OUTPUT_PATH - + "data" - + File.separator - + "sequence" - + File.separator - + COMPACTION_TEST_SG - + File.separator - + "0" - + File.separator - + timePartition); + new File(getSeqStorageGroupDir(), "0" + File.separator + timePartition); seqTimePartitionDir.mkdirs(); File unseqTimePartitionDir = - new File( - TestConstant.BASE_OUTPUT_PATH - + "data" - + File.separator - + "unsequence" - + File.separator - + COMPACTION_TEST_SG - + File.separator - + "0" - + File.separator - + timePartition); + new File(getUnseqStorageGroupDir(), "0" + File.separator + timePartition); unseqTimePartitionDir.mkdirs(); registeredTimePartitionDirs.put( timePartition, new Pair(seqTimePartitionDir, unseqTimePartitionDir)); @@ -293,9 +294,9 @@ protected void createFiles( timestamp[fileCount++] + FilePathUtils.FILE_NAME_SEPARATOR + fileVersion + "-0-0.tsfile"; String filePath; if (isSeq) { - filePath = SEQ_DIRS.getPath() + File.separator + fileName; + filePath = getSeqDirs().getPath() + File.separator + fileName; } else { - filePath = UNSEQ_DIRS.getPath() + File.separator + fileName; + filePath = getUnseqDirs().getPath() + File.separator + fileName; } File file; if (isAlign) { @@ -365,9 +366,9 @@ protected void createFilesWithTextValue( timestamp[fileCount++] + FilePathUtils.FILE_NAME_SEPARATOR + fileVersion + "-0-0.tsfile"; String filePath; if (isSeq) { - filePath = SEQ_DIRS.getPath() + File.separator + fileName; + filePath = getSeqDirs().getPath() + File.separator + fileName; } else { - filePath = UNSEQ_DIRS.getPath() + File.separator + fileName; + filePath = getUnseqDirs().getPath() + File.separator + fileName; } File file; if (isAlign) { @@ -511,11 +512,11 @@ public void tearDown() throws IOException, StorageEngineException { EnvironmentUtils.cleanAllDir(); - if (SEQ_DIRS.exists()) { - FileUtils.deleteDirectory(SEQ_DIRS); + if (getSeqDirs().exists()) { + FileUtils.deleteDirectory(getSeqDirs()); } - if (UNSEQ_DIRS.exists()) { - FileUtils.deleteDirectory(UNSEQ_DIRS); + if (getUnseqDirs().exists()) { + FileUtils.deleteDirectory(getUnseqDirs()); } for (Map.Entry> entry : registeredTimePartitionDirs.entrySet()) { File seqDir = entry.getValue().left; @@ -738,9 +739,9 @@ protected TsFileResource createEmptyFileAndResource(boolean isSeq, int innerComp + String.format("-%d-0.tsfile", innerCompactionCnt); String filePath; if (isSeq) { - filePath = SEQ_DIRS.getPath() + File.separator + fileName; + filePath = getSeqDirs().getPath() + File.separator + fileName; } else { - filePath = UNSEQ_DIRS.getPath() + File.separator + fileName; + filePath = getUnseqDirs().getPath() + File.separator + fileName; } TsFileResource resource = new TsFileResource(new File(filePath)); resource.updatePlanIndexes(fileVersion); @@ -752,9 +753,9 @@ protected TsFileResource createEmptyFileAndResourceWithName( String fileName, boolean isSeq, int innerCompactionCnt) { String filePath; if (isSeq) { - filePath = SEQ_DIRS.getPath() + File.separator + fileName; + filePath = getSeqDirs().getPath() + File.separator + fileName; } else { - filePath = UNSEQ_DIRS.getPath() + File.separator + fileName; + filePath = getUnseqDirs().getPath() + File.separator + fileName; } TsFileResource resource = new TsFileResource(new File(filePath)); resource.setStatusForTest(TsFileResourceStatus.NORMAL); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java index 24551efa8d2cf..e9ea0025250ed 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask; +import org.apache.iotdb.db.storageengine.dataregion.compaction.tablemodel.AbstractTableModelCompactionTest; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; @@ -50,10 +51,13 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import static org.apache.iotdb.db.storageengine.dataregion.compaction.tablemodel.AbstractTableModelCompactionTest.TABLE_MODEL_TEST_SG; + @RunWith(Parameterized.class) public class CompactionDataTypeAlterTableTest extends AbstractCompactionAlterDataTypeTest { @@ -80,19 +84,44 @@ public static Collection data() { }); } + @Override + protected String getTestStorageGroup() { + return TABLE_MODEL_TEST_SG; + } + + @Override + protected File getSeqStorageGroupDir() { + return AbstractTableModelCompactionTest.TABLE_MODEL_SEQ_STORAGE_GROUP_DIR; + } + + @Override + protected File getUnseqStorageGroupDir() { + return AbstractTableModelCompactionTest.TABLE_MODEL_UNSEQ_STORAGE_GROUP_DIR; + } + + @Override + protected File getSeqDirs() { + return AbstractTableModelCompactionTest.TABLE_MODEL_SEQ_DIRS; + } + + @Override + protected File getUnseqDirs() { + return AbstractTableModelCompactionTest.TABLE_MODEL_UNSEQ_DIRS; + } + @Before @Override public void setUp() throws IOException, WriteProcessException, MetadataException, InterruptedException { super.setUp(); - DataNodeTableCache.getInstance().invalid(COMPACTION_TEST_SG); + DataNodeTableCache.getInstance().invalid(TABLE_MODEL_TEST_SG); } @After @Override public void tearDown() throws IOException, StorageEngineException { super.tearDown(); - DataNodeTableCache.getInstance().invalid(COMPACTION_TEST_SG); + DataNodeTableCache.getInstance().invalid(TABLE_MODEL_TEST_SG); } @Test @@ -159,8 +188,8 @@ private void createTable(String tableName, TSDataType dataType) { tsTable.addColumnSchema(new TagColumnSchema("id_column", TSDataType.STRING)); tsTable.addColumnSchema( new FieldColumnSchema("s1", dataType, TSEncoding.PLAIN, CompressionType.LZ4)); - DataNodeTableCache.getInstance().preUpdateTable(this.COMPACTION_TEST_SG, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(this.COMPACTION_TEST_SG, tableName, null); + DataNodeTableCache.getInstance().preUpdateTable(TABLE_MODEL_TEST_SG, tsTable, null); + DataNodeTableCache.getInstance().commitUpdateTable(TABLE_MODEL_TEST_SG, tableName, null); } private TsFileResource generateInt32TableFile(TimeRange timeRange, boolean seq) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/AbstractTableModelCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/AbstractTableModelCompactionTest.java new file mode 100644 index 0000000000000..159c4d229a0d4 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/AbstractTableModelCompactionTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.compaction.tablemodel; + +import org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest; +import org.apache.iotdb.db.utils.constant.TestConstant; + +import java.io.File; + +/** + * The base class of the compaction tests whose source files contain table model data. + * + *

The data model of the compaction input is resolved from the database directory of the source + * files, and a table model database name must not start with {@code root.}, therefore these tests + * use a table model database instead of the tree model database used by the default fixture. + */ +public abstract class AbstractTableModelCompactionTest extends AbstractCompactionTest { + + /** A table model database name, which must not start with "root.". */ + public static final String TABLE_MODEL_TEST_SG = "testsg"; + + public static final File TABLE_MODEL_SEQ_STORAGE_GROUP_DIR = + new File( + TestConstant.BASE_OUTPUT_PATH + + "data" + + File.separator + + "sequence" + + File.separator + + TABLE_MODEL_TEST_SG); + public static final File TABLE_MODEL_UNSEQ_STORAGE_GROUP_DIR = + new File( + TestConstant.BASE_OUTPUT_PATH + + "data" + + File.separator + + "unsequence" + + File.separator + + TABLE_MODEL_TEST_SG); + public static final File TABLE_MODEL_SEQ_DIRS = + new File(TABLE_MODEL_SEQ_STORAGE_GROUP_DIR, "0" + File.separator + "0"); + public static final File TABLE_MODEL_UNSEQ_DIRS = + new File(TABLE_MODEL_UNSEQ_STORAGE_GROUP_DIR, "0" + File.separator + "0"); + + @Override + protected String getTestStorageGroup() { + return TABLE_MODEL_TEST_SG; + } + + @Override + protected File getSeqStorageGroupDir() { + return TABLE_MODEL_SEQ_STORAGE_GROUP_DIR; + } + + @Override + protected File getUnseqStorageGroupDir() { + return TABLE_MODEL_UNSEQ_STORAGE_GROUP_DIR; + } + + @Override + protected File getSeqDirs() { + return TABLE_MODEL_SEQ_DIRS; + } + + @Override + protected File getUnseqDirs() { + return TABLE_MODEL_UNSEQ_DIRS; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java index 69e83e769a3ce..2de0d95bbc98c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java @@ -26,7 +26,6 @@ import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache; import org.apache.iotdb.db.storageengine.buffer.ChunkCache; import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; -import org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.constant.InnerSeqCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.constant.InnerUnseqCompactionPerformer; @@ -58,7 +57,7 @@ import java.util.Collections; @RunWith(Parameterized.class) -public class CompactionWithAllNullRowsTest extends AbstractCompactionTest { +public class CompactionWithAllNullRowsTest extends AbstractTableModelCompactionTest { private final String performerType; private String threadName; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java index 6746f0a274697..877756b73d5b7 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java @@ -25,7 +25,6 @@ import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; -import org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -47,7 +46,7 @@ import java.util.concurrent.TimeUnit; @RunWith(Parameterized.class) -public class TableModelCompactionWithTTLTest extends AbstractCompactionTest { +public class TableModelCompactionWithTTLTest extends AbstractTableModelCompactionTest { private final String performerType; private String threadName; @@ -57,7 +56,7 @@ public void setUp() throws IOException, WriteProcessException, MetadataException, InterruptedException { this.threadName = Thread.currentThread().getName(); Thread.currentThread().setName("pool-1-IoTDB-Compaction-Worker-1"); - DataNodeTableCache.getInstance().invalid(this.COMPACTION_TEST_SG); + DataNodeTableCache.getInstance().invalid(TABLE_MODEL_TEST_SG); super.setUp(); } @@ -65,7 +64,7 @@ public void setUp() public void tearDown() throws IOException, StorageEngineException { super.tearDown(); Thread.currentThread().setName(threadName); - DataNodeTableCache.getInstance().invalid(this.COMPACTION_TEST_SG); + DataNodeTableCache.getInstance().invalid(TABLE_MODEL_TEST_SG); } public TableModelCompactionWithTTLTest(String performerType) { @@ -170,7 +169,7 @@ public void createTable(String tableName, long ttl) { tsTable.addColumnSchema( new FieldColumnSchema("s1", TSDataType.STRING, TSEncoding.PLAIN, CompressionType.LZ4)); tsTable.addProp(TsTable.TTL_PROPERTY, ttl + ""); - DataNodeTableCache.getInstance().preUpdateTable(this.COMPACTION_TEST_SG, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(this.COMPACTION_TEST_SG, tableName, null); + DataNodeTableCache.getInstance().preUpdateTable(TABLE_MODEL_TEST_SG, tsTable, null); + DataNodeTableCache.getInstance().commitUpdateTable(TABLE_MODEL_TEST_SG, tableName, null); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelFastCompactionPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelFastCompactionPerformerTest.java index 778220d826389..8096003b9c916 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelFastCompactionPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelFastCompactionPerformerTest.java @@ -21,7 +21,6 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.db.exception.StorageEngineException; -import org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CrossSpaceCompactionTask; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask; @@ -38,14 +37,12 @@ import org.junit.Before; import org.junit.Test; -import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.util.Arrays; import java.util.Collections; import java.util.List; -public class TableModelFastCompactionPerformerTest extends AbstractCompactionTest { +public class TableModelFastCompactionPerformerTest extends AbstractTableModelCompactionTest { @Before public void setUp() @@ -143,62 +140,6 @@ public void testCrossSpaceCompactionOfTwoTableModelWithFastCompactionPerformer() } } - @Test - public void testCompactionWithV3Tsfile() throws IOException { - String pathStr = - this.getClass().getClassLoader().getResource("v3tsfile/compaction-test-tsfile").getFile(); - File v3TsFile = new File(pathStr); - File v3TsFileResource = new File(pathStr + "-resource"); - TsFileResource resource1 = createEmptyFileAndResource(true); - Files.copy(v3TsFile.toPath(), resource1.getTsFile().toPath()); - Files.copy( - v3TsFileResource.toPath(), new File(resource1.getTsFilePath() + ".resource").toPath()); - resource1.deserialize(); - - TsFileResource resource2 = createEmptyFileAndResource(true); - try (CompactionTableModelTestFileWriter writer = - new CompactionTableModelTestFileWriter(resource2)) { - writer.registerTableSchema("db1.t1", Arrays.asList("id1", "id2")); - writer.startChunkGroup("db1.t1", Arrays.asList("id_field1", "id_field2")); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Collections.singletonList("s1"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - - writer.startChunkGroup("d1"); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Collections.singletonList("s1"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - - writer.startChunkGroup("node1.node2.device"); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Collections.singletonList("s1"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - - writer.endFile(); - } - - seqResources.add(resource1); - seqResources.add(resource2); - InnerSpaceCompactionTask task = - new InnerSpaceCompactionTask( - 0, tsFileManager, seqResources, true, new FastCompactionPerformer(false), 0); - Assert.assertTrue(task.start()); - try (TsFileSequenceReader reader = - new TsFileSequenceReader( - tsFileManager.getTsFileList(true).get(0).getTsFile().getAbsolutePath())) { - Assert.assertEquals(1, reader.getTableSchemaMap().size()); - } - } - @Test public void testCompactionWithMultiTableSchema() throws IOException { TsFileResource resource1 = createEmptyFileAndResource(true); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadChunkCompactionPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadChunkCompactionPerformerTest.java index f2721652f99c5..06fff63881799 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadChunkCompactionPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadChunkCompactionPerformerTest.java @@ -21,10 +21,8 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.db.exception.StorageEngineException; -import org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadChunkCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask; -import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionTestFileWriter; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -42,15 +40,13 @@ import org.junit.Before; import org.junit.Test; -import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; -public class TableModelReadChunkCompactionPerformerTest extends AbstractCompactionTest { +public class TableModelReadChunkCompactionPerformerTest extends AbstractTableModelCompactionTest { private final String oldThreadName = Thread.currentThread().getName(); @@ -120,92 +116,6 @@ public void testSequenceInnerSpaceCompactionOfTwoTableModel() throws IOException } } - @Test - public void testSequenceInnerSpaceCompactionOfTwoV4TreeModel() throws IOException { - TsFileResource resource1 = createEmptyFileAndResource(true); - try (CompactionTestFileWriter writer = new CompactionTestFileWriter(resource1)) { - writer.startChunkGroup("d1"); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Collections.singletonList("s1"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - writer.endFile(); - } - TsFileResource resource2 = createEmptyFileAndResource(true); - try (CompactionTestFileWriter writer = new CompactionTestFileWriter(resource2)) { - writer.startChunkGroup("d1"); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Arrays.asList("s1", "s2"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(20, 22)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - writer.endFile(); - } - seqResources.add(resource1); - seqResources.add(resource2); - tsFileManager.addAll(seqResources, true); - - InnerSpaceCompactionTask task = - new InnerSpaceCompactionTask( - 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); - Assert.assertTrue(task.start()); - TsFileResource targetResource = tsFileManager.getTsFileList(true).get(0); - try (TsFileSequenceReader reader = - new TsFileSequenceReader(targetResource.getTsFile().getAbsolutePath())) { - Assert.assertTrue(reader.getTableSchemaMap().isEmpty()); - Map> allTimeseriesMetadata = - reader.getAllTimeseriesMetadata(true); - for (Map.Entry> entry : - allTimeseriesMetadata.entrySet()) { - Assert.assertEquals(3, entry.getValue().size()); - } - } - } - - @Test - public void testSequenceInnerSpaceCompactionOfTableModelAndTreeModel() throws IOException { - TsFileResource resource1 = createEmptyFileAndResource(true); - try (CompactionTestFileWriter writer = new CompactionTestFileWriter(resource1)) { - writer.startChunkGroup("d1"); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Collections.singletonList("s1"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - writer.endFile(); - } - TsFileResource resource2 = createEmptyFileAndResource(true); - try (CompactionTableModelTestFileWriter writer = - new CompactionTableModelTestFileWriter(resource2)) { - writer.registerTableSchema("t1", Arrays.asList("id1", "id2")); - writer.startChunkGroup("t1", Arrays.asList("id_field1", "id_field2")); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Collections.singletonList("s1"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - writer.endFile(); - } - seqResources.add(resource1); - seqResources.add(resource2); - tsFileManager.addAll(seqResources, true); - - InnerSpaceCompactionTask task = - new InnerSpaceCompactionTask( - 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); - Assert.assertTrue(task.start()); - TsFileResource targetResource = tsFileManager.getTsFileList(true).get(0); - try (TsFileSequenceReader reader = - new TsFileSequenceReader(targetResource.getTsFile().getAbsolutePath())) { - Assert.assertEquals(1, reader.getTableSchemaMap().size()); - } - } - @Test public void testSequenceInnerSpaceCompactionOfTableModelCanNotMatchTableSchema() throws IOException { @@ -271,67 +181,4 @@ public void testSequenceInnerSpaceCompactionOfTableModelCanNotMatchTableSchema() Assert.assertEquals("s2", tableSchema.get(3).getMeasurementName()); } } - - @Test - public void testCompactionWithV3Tsfile() throws IOException { - String pathStr = - this.getClass().getClassLoader().getResource("v3tsfile/compaction-test-tsfile").getFile(); - File v3TsFile = new File(pathStr); - File v3TsFileResource = new File(pathStr + "-resource"); - TsFileResource resource1 = createEmptyFileAndResource(true); - Files.copy(v3TsFile.toPath(), resource1.getTsFile().toPath()); - Files.copy( - v3TsFileResource.toPath(), new File(resource1.getTsFilePath() + ".resource").toPath()); - resource1.deserialize(); - - TsFileResource resource2 = createEmptyFileAndResource(true); - try (CompactionTableModelTestFileWriter writer = - new CompactionTableModelTestFileWriter(resource2)) { - writer.registerTableSchema("db1.t1", Arrays.asList("id1", "id2")); - writer.startChunkGroup("db1.t1", Arrays.asList("id_field1", "id_field2")); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Collections.singletonList("s1"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - - writer.startChunkGroup("d3"); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Collections.singletonList("s1"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - - writer.startChunkGroup("node1.node2.device"); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Collections.singletonList("s1"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - - writer.startChunkGroup("node1.node2.node3.device"); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Collections.singletonList("s1"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - writer.endFile(); - } - - seqResources.add(resource1); - seqResources.add(resource2); - InnerSpaceCompactionTask task = - new InnerSpaceCompactionTask( - 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); - Assert.assertTrue(task.start()); - try (TsFileSequenceReader reader = - new TsFileSequenceReader( - tsFileManager.getTsFileList(true).get(0).getTsFile().getAbsolutePath())) { - Assert.assertEquals(1, reader.getTableSchemaMap().size()); - } - } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadPointCompactionPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadPointCompactionPerformerTest.java index 740c2434fdb31..54882c30285b5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadPointCompactionPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadPointCompactionPerformerTest.java @@ -21,7 +21,6 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.db.exception.StorageEngineException; -import org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CrossSpaceCompactionTask; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask; @@ -38,12 +37,10 @@ import org.junit.Before; import org.junit.Test; -import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.util.Arrays; -public class TableModelReadPointCompactionPerformerTest extends AbstractCompactionTest { +public class TableModelReadPointCompactionPerformerTest extends AbstractTableModelCompactionTest { private final String oldThreadName = Thread.currentThread().getName(); @@ -167,62 +164,6 @@ public void testCrossSpaceCompactionOfTwoTableModelWithReadPointCompactionPerfor } } - @Test - public void testCompactionWithV3Tsfile() throws IOException { - String pathStr = - this.getClass().getClassLoader().getResource("v3tsfile/compaction-test-tsfile").getFile(); - File v3TsFile = new File(pathStr); - File v3TsFileResource = new File(pathStr + "-resource"); - TsFileResource resource1 = createEmptyFileAndResource(true); - Files.copy(v3TsFile.toPath(), resource1.getTsFile().toPath()); - Files.copy( - v3TsFileResource.toPath(), new File(resource1.getTsFilePath() + ".resource").toPath()); - resource1.deserialize(); - - TsFileResource resource2 = createEmptyFileAndResource(true); - try (CompactionTableModelTestFileWriter writer = - new CompactionTableModelTestFileWriter(resource2)) { - writer.registerTableSchema("db1.db1.t1", Arrays.asList("id1", "id2")); - writer.startChunkGroup("db1.db1.t1", Arrays.asList("id_field1", "id_field2")); - writer.generateSimpleNonAlignedSeriesToCurrentDevice( - "s1", - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - - writer.startChunkGroup("d1"); - writer.generateSimpleNonAlignedSeriesToCurrentDevice( - "s1", - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - - writer.startChunkGroup("node1.node2.device"); - writer.generateSimpleNonAlignedSeriesToCurrentDevice( - "s1", - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - - writer.endFile(); - } - - seqResources.add(resource1); - seqResources.add(resource2); - InnerSpaceCompactionTask task = - new InnerSpaceCompactionTask( - 0, tsFileManager, seqResources, true, new ReadPointCompactionPerformer(), 0); - Assert.assertTrue(task.start()); - try (TsFileSequenceReader reader = - new TsFileSequenceReader( - tsFileManager.getTsFileList(true).get(0).getTsFile().getAbsolutePath())) { - Assert.assertEquals(1, reader.getTableSchemaMap().size()); - } - } - @Test public void testCompactionWithMultiTableSchema() throws IOException { TsFileResource resource1 = createEmptyFileAndResource(true); From 765929a472f74356ac5af78c72e7f19e243ca27a Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Sat, 10 Oct 2026 10:38:13 +0800 Subject: [PATCH 11/15] [Tools] Reject WAL snapshot replay without a declared target database ImportWAL resolves the data model of a WAL directory from its target database. Without one the replayer fell back to the tree model, so a table model snapshot was replayed into the tree session instead of failing. Reject a snapshot whose data model is unknown until -db/--database is declared. --- .../iotdb/db/i18n/ImportWALMessages.java | 3 ++ .../iotdb/db/i18n/ImportWALMessages.java | 3 ++ .../org/apache/iotdb/db/tools/ImportWAL.java | 28 +++++++++++++------ 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/ImportWALMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/ImportWALMessages.java index fefbfffe8693c..d10d1028538b4 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/ImportWALMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/ImportWALMessages.java @@ -80,6 +80,9 @@ public final class ImportWALMessages { "Unsupported on_success value: %s. Expected none or delete."; public static final String EXCEPTION_TABLE_MODEL_WAL_ENTRIES_REQUIRE_DB_DATABASE_F7597726 = "Table-model WAL entries require -db/--database."; + public static final String + EXCEPTION_A_WAL_SNAPSHOT_REQUIRES_A_DECLARED_TARGET_DATABASE_TO_DETERMINE_ITS_DATA_MODEL_SPECIFY_DB_DATABASE_382FC74C = + "A WAL snapshot requires a declared target database to determine its data model. Specify -db/--database."; public static final String EXCEPTION_UNSUPPORTED_WAL_OPERATION_ARG_ABD227A0 = "Unsupported WAL operation: %s"; public static final String diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/ImportWALMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/ImportWALMessages.java index fc65b1bec5bc5..9b9f843987739 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/ImportWALMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/ImportWALMessages.java @@ -79,6 +79,9 @@ public final class ImportWALMessages { "不支持的 on_success 值：%s。应为 none 或 delete。"; public static final String EXCEPTION_TABLE_MODEL_WAL_ENTRIES_REQUIRE_DB_DATABASE_F7597726 = "表模型 WAL 条目要求指定 -db/--database。"; + public static final String + EXCEPTION_A_WAL_SNAPSHOT_REQUIRES_A_DECLARED_TARGET_DATABASE_TO_DETERMINE_ITS_DATA_MODEL_SPECIFY_DB_DATABASE_382FC74C = + "WAL 快照需要先声明目标数据库才能确定其数据模型。请指定 -db/--database。"; public static final String EXCEPTION_UNSUPPORTED_WAL_OPERATION_ARG_ABD227A0 = "不支持的 WAL 操作：%s"; public static final String diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java index 005b78c5a0cb9..ddcf55d5562e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java @@ -940,9 +940,9 @@ private static class SessionWALReplayer extends WALReplayer { private SessionWALReplayer( final Session treeSession, final Session tableSession, - final String tableDatabaseName, + final String targetDatabaseName, final WALReplayer.ReplayDecisionController replayDecisionController) { - super(treeSession, tableSession, tableDatabaseName, replayDecisionController); + super(treeSession, tableSession, targetDatabaseName, replayDecisionController); this.treeSession = treeSession; this.tableSession = tableSession; } @@ -958,17 +958,18 @@ static class WALReplayer implements WALReplayWorker { private final Session treeSession; private final Session tableSession; + private final String targetDatabaseName; private final boolean isTableModel; private final ConsensusLogToTabletConverter converter; private final ReplayDecisionPrompt replayDecisionPrompt; private final Map tableSchemas = new HashMap<>(); WALReplayer( - final Session treeSession, final Session tableSession, final String tableDatabaseName) { + final Session treeSession, final Session tableSession, final String targetDatabaseName) { this( treeSession, tableSession, - tableDatabaseName, + targetDatabaseName, new ReplayDecisionController(System.console())); } @@ -976,24 +977,26 @@ static class WALReplayer implements WALReplayWorker { * @param treeSession the session which replays tree model entries, it carries no database * @param tableSession the session which replays table model entries, null when no table * database is declared - * @param tableDatabaseName the target database resolved before replay; null means no table - * database is declared, so the entries are replayed with the tree model only + * @param targetDatabaseName the target database resolved before replay; null means no target + * database is resolved, so the data model is unknown and snapshot entries are rejected + * until -db/--database is declared * @param replayDecisionPrompt the prompt which decides how to handle entries that cannot be * replayed automatically */ WALReplayer( final Session treeSession, final Session tableSession, - final String tableDatabaseName, + final String targetDatabaseName, final ReplayDecisionPrompt replayDecisionPrompt) { this.treeSession = treeSession; this.tableSession = tableSession; + this.targetDatabaseName = targetDatabaseName; this.isTableModel = - tableDatabaseName != null && PathUtils.isTableModelDatabase(tableDatabaseName); + targetDatabaseName != null && PathUtils.isTableModelDatabase(targetDatabaseName); this.replayDecisionPrompt = replayDecisionPrompt; converter = new ConsensusLogToTabletConverter( - null, null, ColumnFilterMatcher.matchAll(), isTableModel ? tableDatabaseName : null); + null, null, ColumnFilterMatcher.matchAll(), isTableModel ? targetDatabaseName : null); } @Override @@ -1346,6 +1349,13 @@ private boolean replayMemTableSnapshot(final IMemTable memTable) if (memTable == null || memTable.isSignalMemTable()) { return false; } + if (targetDatabaseName == null) { + // Without a resolved target database the data model of the snapshot is unknown, and + // guessing it would replay table model data into the tree session. + throw new StatementExecutionException( + ImportWALMessages + .EXCEPTION_A_WAL_SNAPSHOT_REQUIRES_A_DECLARED_TARGET_DATABASE_TO_DETERMINE_ITS_DATA_MODEL_SPECIFY_DB_DATABASE_382FC74C); + } boolean replayed = false; for (Map.Entry deviceEntry : memTable.getMemTableMap().entrySet()) { From e9dc55e6f7d1e3ca9d27dd3b3f64d8e33c0c388f Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Sat, 10 Oct 2026 10:57:16 +0800 Subject: [PATCH 12/15] [Build] Compile against the tsfile version without IDeviceID.isTableModel This change no longer calls IDeviceID.isTableModel, so it needs the tsfile snapshot that removes it: bump tsfile.version to 2.4.1-261009-SNAPSHOT. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fd936192e878a..e088f9397f99d 100644 --- a/pom.xml +++ b/pom.xml @@ -145,7 +145,7 @@ 0.24.0 1.9 1.5.6-3 - 2.4.1-260915-SNAPSHOT + 2.4.1-261009-SNAPSHOT en From 2df256119d0cacb5cf7441d0a0d7534d0fd7f727 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Sat, 10 Oct 2026 11:02:28 +0800 Subject: [PATCH 13/15] [Tools] Only require -db for WAL snapshots that carry data An empty memtable snapshot has nothing to replay, so it must stay ignorable without a declared target database; otherwise a WAL that only holds signals, separators and an empty snapshot stops being deletable after replay. --- .../src/main/java/org/apache/iotdb/db/tools/ImportWAL.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java index ddcf55d5562e2..483cef7f172b7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java @@ -1349,9 +1349,10 @@ private boolean replayMemTableSnapshot(final IMemTable memTable) if (memTable == null || memTable.isSignalMemTable()) { return false; } - if (targetDatabaseName == null) { + if (targetDatabaseName == null && memTable.size(false) > 0) { // Without a resolved target database the data model of the snapshot is unknown, and - // guessing it would replay table model data into the tree session. + // guessing it would replay table model data into the tree session. An empty snapshot + // carries no data, so it stays ignorable. throw new StatementExecutionException( ImportWALMessages .EXCEPTION_A_WAL_SNAPSHOT_REQUIRES_A_DECLARED_TARGET_DATABASE_TO_DETERMINE_ITS_DATA_MODEL_SPECIFY_DB_DATABASE_382FC74C); From 648c29196091022689c84b226f675da8ac100dfa Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Sat, 10 Oct 2026 12:03:56 +0800 Subject: [PATCH 14/15] [Tools] Declare the WAL node allocation strategies that cannot be imported The ElasticStrategy and RoundRobinStrategy share WAL nodes between regions, so a WAL directory has neither a single database nor a single data model. State that such WALs cannot be imported when the help is printed, and repeat the declaration whenever the target database of a WAL directory cannot be determined. --- .../org/apache/iotdb/db/i18n/ImportWALMessages.java | 3 +++ .../org/apache/iotdb/db/i18n/ImportWALMessages.java | 3 +++ .../java/org/apache/iotdb/db/tools/ImportWAL.java | 13 +++++++++++-- .../org/apache/iotdb/db/tools/ImportWALTest.java | 3 +++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/ImportWALMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/ImportWALMessages.java index d10d1028538b4..d8fbc121146b7 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/ImportWALMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/ImportWALMessages.java @@ -23,6 +23,9 @@ public final class ImportWALMessages { public static final String MESSAGE_IMPORT_WAL_5E42804E = "import-wal"; + public static final String + MESSAGE_UNSUPPORTED_WALS_1_WALS_WRITTEN_WITH_THE_ELASTICSTRATEGY_2_WALS_WRITTEN_WITH_THE_ROUNDROBINSTRATEGY_REPLAYING_THEM_MAY_FAIL_C542F812 = + "Unsupported WALs: (1) WALs written with the ElasticStrategy; (2) WALs written with the RoundRobinStrategy. Replaying them may fail."; public static final String MESSAGE_PATH_OF_A_WAL_FILE_OR_A_DIRECTORY_CONTAINING_WAL_FILES_473D0554 = "Path of a WAL file or a directory containing WAL files."; diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/ImportWALMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/ImportWALMessages.java index 9b9f843987739..c0c3877e3cf51 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/ImportWALMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/ImportWALMessages.java @@ -23,6 +23,9 @@ public final class ImportWALMessages { public static final String MESSAGE_IMPORT_WAL_5E42804E = "import-wal"; + public static final String + MESSAGE_UNSUPPORTED_WALS_1_WALS_WRITTEN_WITH_THE_ELASTICSTRATEGY_2_WALS_WRITTEN_WITH_THE_ROUNDROBINSTRATEGY_REPLAYING_THEM_MAY_FAIL_C542F812 = + "不支持导入以下两类 WAL：(1) 使用 ElasticStrategy 写入的 WAL；(2) 使用 RoundRobinStrategy 写入的 WAL。强行导入可能会出错。"; public static final String MESSAGE_PATH_OF_A_WAL_FILE_OR_A_DIRECTORY_CONTAINING_WAL_FILES_473D0554 = "WAL 文件或包含 WAL 文件的目录路径。"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java index 483cef7f172b7..50dffbf96c86b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java @@ -163,7 +163,15 @@ static int run(final String[] args, final PrintStream out, final PrintStream err .MESSAGE_INFERRED_TABLE_DATABASE_ARG_FROM_WAL_DIRECTORY_ARG_REPLAY_INTO_THIS_DATABASE_Y_YES_A_ACCEPT_ALL_INFERRED_DATABASES_N_QUIT_5B59D833, inferredDatabase, directory)); - requireResolvedDatabases(directoryDatabases); + try { + requireResolvedDatabases(directoryDatabases); + } catch (final IllegalArgumentException e) { + // The ElasticStrategy and RoundRobinStrategy WAL node allocation strategies share WAL nodes + // between regions, so such a WAL has no per-directory database or data model; repeat the + // declaration where the operator has to decide. + err.println(ImportWALMessages.MESSAGE_UNSUPPORTED_WALS_1_WALS_WRITTEN_WITH_THE_ELASTICSTRATEGY_2_WALS_WRITTEN_WITH_THE_ROUNDROBINSTRATEGY_REPLAYING_THEM_MAY_FAIL_C542F812); + throw e; + } final ReplayStatistics statistics = replayWALDirectories( walFiles, @@ -384,7 +392,8 @@ private static void printHelp(final Options options, final PrintStream stream) { new PrintWriter(stream, true), 120, ImportWALMessages.MESSAGE_IMPORT_WAL_5E42804E, - null, + ImportWALMessages + .MESSAGE_UNSUPPORTED_WALS_1_WALS_WRITTEN_WITH_THE_ELASTICSTRATEGY_2_WALS_WRITTEN_WITH_THE_ROUNDROBINSTRATEGY_REPLAYING_THEM_MAY_FAIL_C542F812, options, 2, 2, diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/ImportWALTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/ImportWALTest.java index 00fb827e2e1bd..7e8c40acf3ac7 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/ImportWALTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/ImportWALTest.java @@ -344,6 +344,7 @@ public void testUnresolvedDatabaseFailsBeforeReplay() throws Exception { assertTrue(error.toString().contains(directory.toString())); assertTrue(error.toString().contains("-db/--database")); + assertTrue(error.toString().contains("ElasticStrategy")); } /** @@ -445,6 +446,8 @@ public void testHelpDescribesDeleteSourceOption() { assertTrue(output.toString().contains("--on_unsupported")); assertTrue(output.toString().contains("--on_corrupted")); assertTrue(output.toString().contains("--skip_db_confirmation")); + assertTrue(output.toString().contains("ElasticStrategy")); + assertTrue(output.toString().contains("RoundRobinStrategy")); } @Test From b565399a554bcce5729930d227534396decaf320 Mon Sep 17 00:00:00 2001 From: shuwenwei Date: Sat, 10 Oct 2026 12:12:33 +0800 Subject: [PATCH 15/15] spotless --- .../src/main/java/org/apache/iotdb/db/tools/ImportWAL.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java index 50dffbf96c86b..27078dad188de 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java @@ -169,7 +169,9 @@ static int run(final String[] args, final PrintStream out, final PrintStream err // The ElasticStrategy and RoundRobinStrategy WAL node allocation strategies share WAL nodes // between regions, so such a WAL has no per-directory database or data model; repeat the // declaration where the operator has to decide. - err.println(ImportWALMessages.MESSAGE_UNSUPPORTED_WALS_1_WALS_WRITTEN_WITH_THE_ELASTICSTRATEGY_2_WALS_WRITTEN_WITH_THE_ROUNDROBINSTRATEGY_REPLAYING_THEM_MAY_FAIL_C542F812); + err.println( + ImportWALMessages + .MESSAGE_UNSUPPORTED_WALS_1_WALS_WRITTEN_WITH_THE_ELASTICSTRATEGY_2_WALS_WRITTEN_WITH_THE_ROUNDROBINSTRATEGY_REPLAYING_THEM_MAY_FAIL_C542F812); throw e; } final ReplayStatistics statistics =