diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/ImportWALMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/ImportWALMessages.java index d28ca7dbcfacf..d8fbc121146b7 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/ImportWALMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/ImportWALMessages.java @@ -23,6 +23,9 @@ public final class ImportWALMessages { public static final String MESSAGE_IMPORT_WAL_5E42804E = "import-wal"; + public static final String + MESSAGE_UNSUPPORTED_WALS_1_WALS_WRITTEN_WITH_THE_ELASTICSTRATEGY_2_WALS_WRITTEN_WITH_THE_ROUNDROBINSTRATEGY_REPLAYING_THEM_MAY_FAIL_C542F812 = + "Unsupported WALs: (1) WALs written with the ElasticStrategy; (2) WALs written with the RoundRobinStrategy. Replaying them may fail."; public static final String MESSAGE_PATH_OF_A_WAL_FILE_OR_A_DIRECTORY_CONTAINING_WAL_FILES_473D0554 = "Path of a WAL file or a directory containing WAL files."; @@ -80,6 +83,9 @@ public final class ImportWALMessages { "Unsupported on_success value: %s. Expected none or delete."; public static final String EXCEPTION_TABLE_MODEL_WAL_ENTRIES_REQUIRE_DB_DATABASE_F7597726 = "Table-model WAL entries require -db/--database."; + public static final String + EXCEPTION_A_WAL_SNAPSHOT_REQUIRES_A_DECLARED_TARGET_DATABASE_TO_DETERMINE_ITS_DATA_MODEL_SPECIFY_DB_DATABASE_382FC74C = + "A WAL snapshot requires a declared target database to determine its data model. Specify -db/--database."; public static final String EXCEPTION_UNSUPPORTED_WAL_OPERATION_ARG_ABD227A0 = "Unsupported WAL operation: %s"; public static final String @@ -136,14 +142,16 @@ public final class ImportWALMessages { public static final String MESSAGE_SKIPPED_ARG_CORRUPTED_WAL_FILES_SOURCE_FILES_RETAINED_A889CCE2 = "Skipped %d corrupted WAL files; source files retained."; - public static final String MESSAGE_TARGET_DATABASE_FOR_TABLE_MODEL_WAL_ENTRIES_IF_OMITTED_INFER_FROM_THE_WAL_PARENT_DIRECTORY_AND_ASK_FOR_CONFIRMATION_4B1E409D = - "Target database for table-model WAL entries. If omitted, infer from the WAL parent directory and ask for confirmation."; + public static final String MESSAGE_TARGET_DATABASE_FOR_WAL_REPLAY_IF_OMITTED_INFER_FROM_THE_WAL_PARENT_DIRECTORY_IF_INFERENCE_FAILS_DB_DATABASE_IS_REQUIRED_6EEBC019 = + "Target database for WAL replay. If omitted, infer from the WAL parent directory; if inference fails, -db/--database is required."; public static final String MESSAGE_INFERRED_TABLE_DATABASE_ARG_FROM_WAL_DIRECTORY_ARG_REPLAY_INTO_THIS_DATABASE_Y_YES_A_ACCEPT_ALL_INFERRED_DATABASES_N_QUIT_5B59D833 = "Inferred table database %s from WAL directory %s. Replay into this database? [y] yes, [a] accept all inferred databases, [N] quit: "; public static final String EXCEPTION_DATABASE_CONFIRMATION_REQUIRED_FOR_WAL_DIRECTORY_ARG_INFERRED_DATABASE_ARG_SPECIFY_DB_DATABASE_OR_SKIP_DB_CONFIRMATION_WHEN_INTERACTIVE_INPUT_IS_UNAVAILABLE_14DF6D36 = "Database confirmation required for WAL directory %s (inferred database: %s). Specify -db/--database or --skip_db_confirmation when interactive input is unavailable."; public static final String EXCEPTION_REPLAY_INTO_INFERRED_DATABASE_ARG_WAS_NOT_CONFIRMED_SPECIFY_DB_DATABASE_TO_SELECT_THE_TARGET_EXPLICITLY_86F81190 = "Replay into inferred database %s was not confirmed. Specify -db/--database to select the target explicitly."; + public static final String EXCEPTION_CANNOT_DETERMINE_THE_TARGET_DATABASE_OF_WAL_DIRECTORIES_ARG_SPECIFY_DB_DATABASE_WHICH_APPLIES_TO_ALL_IMPORTED_DIRECTORIES_55B174E4 = + "Cannot determine the target database of WAL directories %s. Specify -db/--database, which applies to all imported directories."; public static final String MESSAGE_ACCEPT_ALL_INFERRED_DATABASE_NAMES_WITHOUT_CONFIRMATION_DB_DATABASE_STILL_TAKES_PRECEDENCE_FA49A73C = "Accept all inferred database names without confirmation; -db/--database still takes precedence."; diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/ImportWALMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/ImportWALMessages.java index 5f9b6e18b0299..c0c3877e3cf51 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/ImportWALMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/ImportWALMessages.java @@ -23,6 +23,9 @@ public final class ImportWALMessages { public static final String MESSAGE_IMPORT_WAL_5E42804E = "import-wal"; + public static final String + MESSAGE_UNSUPPORTED_WALS_1_WALS_WRITTEN_WITH_THE_ELASTICSTRATEGY_2_WALS_WRITTEN_WITH_THE_ROUNDROBINSTRATEGY_REPLAYING_THEM_MAY_FAIL_C542F812 = + "不支持导入以下两类 WAL：(1) 使用 ElasticStrategy 写入的 WAL；(2) 使用 RoundRobinStrategy 写入的 WAL。强行导入可能会出错。"; public static final String MESSAGE_PATH_OF_A_WAL_FILE_OR_A_DIRECTORY_CONTAINING_WAL_FILES_473D0554 = "WAL 文件或包含 WAL 文件的目录路径。"; @@ -79,6 +82,9 @@ public final class ImportWALMessages { "不支持的 on_success 值：%s。应为 none 或 delete。"; public static final String EXCEPTION_TABLE_MODEL_WAL_ENTRIES_REQUIRE_DB_DATABASE_F7597726 = "表模型 WAL 条目要求指定 -db/--database。"; + public static final String + EXCEPTION_A_WAL_SNAPSHOT_REQUIRES_A_DECLARED_TARGET_DATABASE_TO_DETERMINE_ITS_DATA_MODEL_SPECIFY_DB_DATABASE_382FC74C = + "WAL 快照需要先声明目标数据库才能确定其数据模型。请指定 -db/--database。"; public static final String EXCEPTION_UNSUPPORTED_WAL_OPERATION_ARG_ABD227A0 = "不支持的 WAL 操作：%s"; public static final String @@ -135,14 +141,16 @@ public final class ImportWALMessages { public static final String MESSAGE_SKIPPED_ARG_CORRUPTED_WAL_FILES_SOURCE_FILES_RETAINED_A889CCE2 = "已跳过 %d 个损坏的 WAL 文件；源文件已保留。"; - public static final String MESSAGE_TARGET_DATABASE_FOR_TABLE_MODEL_WAL_ENTRIES_IF_OMITTED_INFER_FROM_THE_WAL_PARENT_DIRECTORY_AND_ASK_FOR_CONFIRMATION_4B1E409D = - "表模型 WAL 条目的目标数据库。省略时尝试从 WAL 文件的父目录名推断，并请求确认。"; + public static final String MESSAGE_TARGET_DATABASE_FOR_WAL_REPLAY_IF_OMITTED_INFER_FROM_THE_WAL_PARENT_DIRECTORY_IF_INFERENCE_FAILS_DB_DATABASE_IS_REQUIRED_6EEBC019 = + "WAL 重放的目标数据库。省略时尝试从 WAL 文件的父目录名推断；无法推断时必须指定 -db/--database。"; public static final String MESSAGE_INFERRED_TABLE_DATABASE_ARG_FROM_WAL_DIRECTORY_ARG_REPLAY_INTO_THIS_DATABASE_Y_YES_A_ACCEPT_ALL_INFERRED_DATABASES_N_QUIT_5B59D833 = "推断目标表模型数据库为 %s，来源 WAL 目录为 %s。是否向此数据库重放？[y] 同意，[a] 全部同意推断的数据库，[N] 退出："; public static final String EXCEPTION_DATABASE_CONFIRMATION_REQUIRED_FOR_WAL_DIRECTORY_ARG_INFERRED_DATABASE_ARG_SPECIFY_DB_DATABASE_OR_SKIP_DB_CONFIRMATION_WHEN_INTERACTIVE_INPUT_IS_UNAVAILABLE_14DF6D36 = "需要确认 WAL 目录 %s 的目标数据库（推断结果：%s）。无交互终端时请指定 -db/--database 或 --skip_db_confirmation。"; public static final String EXCEPTION_REPLAY_INTO_INFERRED_DATABASE_ARG_WAS_NOT_CONFIRMED_SPECIFY_DB_DATABASE_TO_SELECT_THE_TARGET_EXPLICITLY_86F81190 = "未确认向推断出的数据库 %s 重放。请使用 -db/--database 显式选择目标。"; + public static final String EXCEPTION_CANNOT_DETERMINE_THE_TARGET_DATABASE_OF_WAL_DIRECTORIES_ARG_SPECIFY_DB_DATABASE_WHICH_APPLIES_TO_ALL_IMPORTED_DIRECTORIES_55B174E4 = + "无法推断 WAL 目录 %s 的目标数据库，请使用 -db/--database 显式指定（-db 会作用于本次导入的所有目录）。"; public static final String MESSAGE_ACCEPT_ALL_INFERRED_DATABASE_NAMES_WITHOUT_CONFIRMATION_DB_DATABASE_STILL_TAKES_PRECEDENCE_FA49A73C = "自动接受所有推断出的数据库名，不再询问确认；-db/--database 仍优先。"; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java index dc2ab1d381fd8..6ebff4375600d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java @@ -200,9 +200,10 @@ public PipeRawTabletInsertionEvent( } @TestOnly - public PipeRawTabletInsertionEvent(final Tablet tablet, final boolean isAligned) { + public PipeRawTabletInsertionEvent( + final boolean isTableModel, final Tablet tablet, final boolean isAligned) { this( - null, + isTableModel, null, null, null, @@ -225,9 +226,12 @@ public PipeRawTabletInsertionEvent(final Tablet tablet, final boolean isAligned) @TestOnly public PipeRawTabletInsertionEvent( - final Tablet tablet, final boolean isAligned, final TreePattern treePattern) { + final boolean isTableModel, + final Tablet tablet, + final boolean isAligned, + final TreePattern treePattern) { this( - null, + isTableModel, null, null, null, @@ -250,10 +254,27 @@ public PipeRawTabletInsertionEvent( @TestOnly public PipeRawTabletInsertionEvent( - final Tablet tablet, final long startTime, final long endTime) { + final boolean isTableModel, final Tablet tablet, final long startTime, final long endTime) { this( - null, null, null, null, tablet, false, null, false, null, 0, null, null, null, null, null, - null, true, startTime, endTime); + isTableModel, + null, + null, + null, + tablet, + false, + null, + false, + null, + 0, + null, + null, + null, + null, + null, + null, + true, + startTime, + endTime); } @Override @@ -479,8 +500,10 @@ public Tablet convertToTablet() { private TabletInsertionEventParser initEventParser() { if (eventParser == null) { + // The data model of the payload is the one declared by the event, it must never be derived + // from the device name of the tablet. eventParser = - tablet.getDeviceId().startsWith("root.") + !isTableModelEvent() ? new TabletInsertionEventTreePatternParser( pipeTaskMeta, this, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java index c26910b426f18..e48fb71de7bfa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParser.java @@ -320,7 +320,8 @@ public TsFileInsertionEventQueryParser( measurement, meta.getStatistics().getStartTime(), meta.getStatistics().getEndTime(), - currentModifications); + currentModifications, + false); } catch (IOException e) { LOGGER.warn( DataNodePipeMessages.FAILED_TO_READ_METADATA_FOR_DEVICEID_MEASUREMENT, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java index 281247e1529de..ee27eda4a5354 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/query/TsFileInsertionEventQueryParserTabletIterator.java @@ -115,7 +115,7 @@ public class TsFileInsertionEventQueryParserTabletIterator implements Iterator modifications) { + PatternTreeMap modifications, + boolean isTableModel) { if (modifications == null) { return false; } - final List mods = getOverlappedMods(deviceID, measurementID, modifications); + final List mods = + getOverlappedMods(deviceID, measurementID, modifications, isTableModel); if (mods == null || mods.isEmpty()) { return false; } // Different logic for tree model and table model - if (deviceID.isTableModel()) { + if (isTableModel) { // For table model: check if any modification affects the device and covers the time range return mods.stream() .anyMatch( @@ -119,17 +122,20 @@ public static boolean isAllDeletedByMods( * @param deviceID device ID * @param measurements measurement list * @param modifications modification records + * @param isTableModel whether the device belongs to table model * @return mapping from measurement ID to mods list and index */ public static List initializeMeasurementMods( IDeviceID deviceID, List measurements, - PatternTreeMap modifications) { + PatternTreeMap modifications, + boolean isTableModel) { List modsInfos = new ArrayList<>(measurements.size()); for (final String measurement : measurements) { - final List mods = getOverlappedMods(deviceID, measurement, modifications); + final List mods = + getOverlappedMods(deviceID, measurement, modifications, isTableModel); if (mods == null || mods.isEmpty()) { // No mods, use empty list and index 0 modsInfos.add(new ModsInfo(Collections.emptyList(), 0)); @@ -139,7 +145,7 @@ public static List initializeMeasurementMods( // Sort by time range for efficient lookup // Different filtering logic for tree model and table model final List filteredMods; - if (deviceID.isTableModel()) { + if (isTableModel) { // For table model: filter modifications that affect the device filteredMods = mods.stream() @@ -161,9 +167,11 @@ public static List initializeMeasurementMods( private static List getOverlappedMods( final IDeviceID deviceID, final String measurement, - final PatternTreeMap modifications) { + final PatternTreeMap modifications, + final boolean isTableModel) { try { - return modifications.getOverlapped(CompactionPathUtils.getPath(deviceID, measurement)); + return modifications.getOverlapped( + CompactionPathUtils.getPath(deviceID, measurement, isTableModel)); } catch (final IllegalPathException e) { throw new PipeException(e.getMessage(), e); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTableModelTsFileBuilderV2.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTableModelTsFileBuilderV2.java index ca4efdadb9802..dc1a58ff0121e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTableModelTsFileBuilderV2.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTableModelTsFileBuilderV2.java @@ -329,7 +329,8 @@ private void writeTabletsIntoOneFile( } } - final MemTableFlushTask memTableFlushTask = new MemTableFlushTask(memTable, writer, null, null); + final MemTableFlushTask memTableFlushTask = + new MemTableFlushTask(memTable, writer, null, null, true); memTableFlushTask.syncFlushMemTable(); writer.endFile(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTreeModelTsFileBuilderV2.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTreeModelTsFileBuilderV2.java index 9d9d8b0fb0a83..21dd3bb9cb052 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTreeModelTsFileBuilderV2.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/builder/PipeTreeModelTsFileBuilderV2.java @@ -257,7 +257,8 @@ private void writeTabletsIntoOneFile( } } - final MemTableFlushTask memTableFlushTask = new MemTableFlushTask(memTable, writer, null, null); + final MemTableFlushTask memTableFlushTask = + new MemTableFlushTask(memTable, writer, null, null, false); memTableFlushTask.syncFlushMemTable(); writer.endFile(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java index e542fc5171915..a28598ddec158 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java @@ -71,7 +71,6 @@ import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.file.metadata.PlainDeviceID; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -124,8 +123,6 @@ import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_START_TIME_KEY; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.SOURCE_TSFILE_PARSER_KEY; import static org.apache.iotdb.commons.pipe.source.IoTDBSource.getSkipIfNoPrivileges; -import static org.apache.tsfile.common.constant.TsFileConstant.PATH_ROOT; -import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; public class PipeHistoricalDataRegionTsFileAndDeletionSource implements PipeHistoricalDataRegionSource { @@ -133,8 +130,6 @@ public class PipeHistoricalDataRegionTsFileAndDeletionSource private static final Logger LOGGER = LoggerFactory.getLogger(PipeHistoricalDataRegionTsFileAndDeletionSource.class); - private static final String TREE_MODEL_EVENT_TABLE_NAME_PREFIX = PATH_ROOT + PATH_SEPARATOR; - private String pipeName; private long creationTime; private String pipeNameWithCreationTime; @@ -1026,7 +1021,7 @@ private boolean mayTsFileResourceOverlappedWithPattern(final TsFileResource reso .anyMatch( deviceID -> { if (!isModelDetected) { - detectModel(resource, deviceID); + detectModel(resource); isModelDetected = true; } @@ -1039,13 +1034,10 @@ private boolean mayTsFileResourceOverlappedWithPattern(final TsFileResource reso }); } - private void detectModel(final TsFileResource resource, final IDeviceID deviceID) { - this.isTableModel = - !(deviceID instanceof PlainDeviceID - || deviceID.getTableName().startsWith(TREE_MODEL_EVENT_TABLE_NAME_PREFIX) - || deviceID.getTableName().equals(PATH_ROOT)); - + private void detectModel(final TsFileResource resource) { + // One source serves one data region, so the data model is decided by its database. final String databaseName = resource.getDatabaseName(); + this.isTableModel = PathUtils.isTableModelDatabase(databaseName); isDbNameCoveredByPattern = isTableModel ? tablePattern.isTableModelDataAllowedToBeCaptured() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/matcher/CachedSchemaPatternMatcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/matcher/CachedSchemaPatternMatcher.java index d8f427868f04e..6d7071fe93b1d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/matcher/CachedSchemaPatternMatcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/matcher/CachedSchemaPatternMatcher.java @@ -33,7 +33,6 @@ import org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegionSource; import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.file.metadata.PlainDeviceID; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,15 +46,10 @@ import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.locks.ReentrantReadWriteLock; -import static org.apache.tsfile.common.constant.TsFileConstant.PATH_ROOT; -import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; - public class CachedSchemaPatternMatcher implements PipeDataRegionMatcher { protected static final Logger LOGGER = LoggerFactory.getLogger(CachedSchemaPatternMatcher.class); - protected static final String TREE_MODEL_EVENT_TABLE_NAME_PREFIX = PATH_ROOT + PATH_SEPARATOR; - protected final ReentrantReadWriteLock lock; protected final Set sources; @@ -149,23 +143,24 @@ public Pair, Set final boolean isTableModelTsFileEvent = event.getEvent() instanceof PipeTsFileInsertionEvent && ((PipeTsFileInsertionEvent) event.getEvent()).isTableModelEvent(); + // The whole event comes from one data region, so all of its devices share the data model + // declared by the event. It must never be derived from the device name. + final boolean isTableModelEvent = + event.getEvent() instanceof PipeInsertionEvent + && ((PipeInsertionEvent) event.getEvent()).isTableModelEvent(); final Set tableNames = new HashSet<>(); for (final Map.Entry entry : event.getSchemaInfo().entrySet()) { final IDeviceID deviceID = entry.getKey(); - // TODO: Check the role to determine whether to match with tree model or table model - if (deviceID instanceof PlainDeviceID - || deviceID.getTableName().startsWith(TREE_MODEL_EVENT_TABLE_NAME_PREFIX) - || deviceID.getTableName().equals(PATH_ROOT)) { + if (!isTableModelEvent) { matchTreeModelEvent(deviceID, entry.getValue(), matchedSources); } else { final String tableName = deviceID.getTableName(); if (tableNames.add(tableName) && matchedSources.size() < sources.size()) { - final String tableModelDatabaseName = - event.getEvent() instanceof PipeInsertionEvent - ? ((PipeInsertionEvent) event.getEvent()).getTableModelDatabaseName() - : null; - matchTableModelEvent(tableModelDatabaseName, tableName, matchedSources); + matchTableModelEvent( + ((PipeInsertionEvent) event.getEvent()).getTableModelDatabaseName(), + tableName, + matchedSources); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index c76b15b0f0fc4..b7060be8cbc64 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@ -490,9 +490,7 @@ public List getPathModifications(TsFileResource tsFileResource, IDevic getQueryModificationLoader( tsFileResource, modification -> - deviceID.isTableModel() - ? modification.affects(deviceID) - : modification.affectsAll(deviceID), + isTableModel() ? modification.affects(deviceID) : modification.affectsAll(deviceID), mods -> getPathModifications(mods, deviceID))) { return modificationLoader.getPathModifications(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java index a9cea56e96e3a..b25dc7a083ee2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java @@ -145,6 +145,9 @@ public PatternTreeMap loadAllModificationsFromDisk( } protected boolean shouldSkipModification(ModEntry modification) { + // tables is only collected by the table model query planning. Compaction and other contexts + // leave it null, which means that there is no table filter at all, so no modification can be + // skipped. if (tables != null && modification instanceof TableDeletionEntry) { String tableName = ((TableDeletionEntry) modification).getTableName(); return !tables.contains(tableName); @@ -170,7 +173,7 @@ public List getPathModifications( return Collections.emptyList(); } List modEntries = fileModEntries.getOverlapped(deviceID, measurement); - if (deviceID.isTableModel()) { + if (isTableModel()) { // the pattern tree has false-positive for table model deletion, so we do a further // filtering modEntries = @@ -200,7 +203,7 @@ public List getPathModifications( } List modEntries = fileModEntries.getOverlapped(deviceID, AlignedPath.VECTOR_PLACEHOLDER); - if (deviceID.isTableModel()) { + if (isTableModel()) { // the pattern tree has false-positive for table model deletion, so we do a further // filtering modEntries = @@ -274,6 +277,11 @@ public boolean isIgnoreAllNullRows() { return ignoreAllNullRows; } + // ignoreAllNullRows is true only for tree-model queries. + public boolean isTableModel() { + return !ignoreAllNullRows; + } + public void setIgnoreAllNullRows(boolean ignoreAllNullRows) { this.ignoreAllNullRows = ignoreAllNullRows; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java index 0b8dfbed002f0..32ee6032f4d3d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java @@ -162,11 +162,15 @@ protected void failIfMetadataLeaseFenced() { /** * get database to device map * + *

Tree model only: a tree model device path contains its database, while a table model device + * (table + tags) can only be resolved with a known database. + * * @param deviceIDs the devices that need to hit * @param tryToFetch whether try to get all database from config node * @param isAutoCreate whether auto create database when cache miss * @param userName the userName */ + @TreeModel public Map> getDatabaseToDevice( final List deviceIDs, final boolean tryToFetch, @@ -180,18 +184,23 @@ public void put(final IDeviceID device, final String databaseName) { map.computeIfAbsent(databaseName, k -> new ArrayList<>()).add(device); } }; - getDatabaseCacheResult(result, deviceIDs, tryToFetch, isAutoCreate, userName); + // Tree model only, see the javadoc above. + getDatabaseCacheResult(result, deviceIDs, tryToFetch, isAutoCreate, userName, false); return result.getMap(); } /** * get device to database map * + *

Tree model only: a tree model device path contains its database, while a table model device + * (table + tags) can only be resolved with a known database. + * * @param deviceIDs the devices that need to hit * @param tryToFetch whether try to get all database from config node * @param isAutoCreate whether auto create database when cache miss * @param userName the userName */ + @TreeModel public Map getDeviceToDatabase( final List deviceIDs, final boolean tryToFetch, @@ -205,7 +214,8 @@ public void put(final IDeviceID device, final String databaseName) { map.put(device, databaseName); } }; - getDatabaseCacheResult(result, deviceIDs, tryToFetch, isAutoCreate, userName); + // Tree model only, see the javadoc above. + getDatabaseCacheResult(result, deviceIDs, tryToFetch, isAutoCreate, userName, false); return result.getMap(); } @@ -242,9 +252,12 @@ private boolean containsDatabase(final String database) { /** * get all database from configNode and update database cache * + *

Tree model only: the databases requested from the config node are the tree model ones. + * * @param result the result of get database cache * @param deviceIDs the devices that need to hit */ + @TreeModel private void fetchDatabaseAndUpdateCache( final DatabaseCacheResult result, final List deviceIDs) throws ClientManagerException, TException { @@ -290,7 +303,12 @@ public boolean isNeedLastCache(final String database) { return Objects.isNull(needLastCache) || needLastCache; } - /** get all database from configNode and update database cache. */ + /** + * get all database from configNode and update database cache. + * + * @param isTableModel the data model of the requested databases, it must be the model of the + * caller, i.e. {@code false} for a tree model caller and {@code true} for a table model one + */ private void fetchDatabaseAndUpdateCache(final boolean isTableModel) throws ClientManagerException, TException { databaseCacheLock.writeLock().lock(); @@ -313,11 +331,15 @@ private void fetchDatabaseAndUpdateCache(final boolean isTableModel) /** * create not existed database and update database cache * + *

Tree model only: only tree model databases, i.e. {@code root.__system} and {@code + * root.__audit}, are created here. + * * @param result the result of get database cache * @param deviceIDs the devices that need to hit * @param userName the username * @throws RuntimeException if failed to create database */ + @TreeModel private void createDatabaseAndUpdateCache( final DatabaseCacheResult result, final List deviceIDs, @@ -399,10 +421,13 @@ private void createDatabaseAndUpdateCache( /** * create not existed database and update database cache * + *

Table model only: the database created here is a table model database. + * * @param database the database * @param userName the username * @throws RuntimeException if failed to create database */ + @TableModel private void createDatabaseAndUpdateCache(final String database, final String userName) throws ClientManagerException, TException { databaseCacheLock.writeLock().lock(); @@ -496,13 +521,16 @@ private void getDatabaseMap( * @param tryToFetch whether try to get all database from confignode * @param isAutoCreate whether auto create database when device miss * @param userName + * @param isTableModel the data model of the queried devices; the current callers resolve tree + * model devices, because only a tree model device path contains its database */ private void getDatabaseCacheResult( final DatabaseCacheResult result, final List deviceIDs, final boolean tryToFetch, final boolean isAutoCreate, - final String userName) { + final String userName, + final boolean isTableModel) { if (!isAutoCreate) { // TODO: avoid IDeviceID contains "*" // miss when deviceId contains * @@ -530,13 +558,11 @@ private void getDatabaseCacheResult( } else { // check if it is to auto create the system or audit database for (IDeviceID deviceID : deviceIDs) { - if (!deviceID.isTableModel() - && deviceID.startWith("root." + SystemConstant.SYSTEM_PREFIX_KEY)) { + if (!isTableModel && deviceID.startWith("root." + SystemConstant.SYSTEM_PREFIX_KEY)) { createDatabaseAndUpdateCache(result, Collections.singletonList(deviceID), userName); break; } - if (!deviceID.isTableModel() - && deviceID.startWith("root." + SystemConstant.AUDIT_PREFIX_KEY)) { + if (!isTableModel && deviceID.startWith("root." + SystemConstant.AUDIT_PREFIX_KEY)) { createDatabaseAndUpdateCache(result, Collections.singletonList(deviceID), userName); break; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java index 322a86acfd3b3..63a8adcaea955 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java @@ -171,7 +171,7 @@ public void autoCreateAndVerify(final IDeviceID device) throws LoadAnalyzeExcept public boolean isDeviceDeletedByMods(final IDeviceID device) { try { return ModificationUtils.isDeviceDeletedByMods( - currentModifications, currentTimeIndex, device); + currentModifications, currentTimeIndex, device, true); } catch (final IllegalPathException e) { LOGGER.warn( DataNodeQueryMessages diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java index 87375b5dfd427..b2ffa01590bda 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java @@ -196,7 +196,8 @@ public void setCurrentTimeIndex(final ITimeIndex timeIndex) { } public boolean isDeviceDeletedByMods(IDeviceID device) throws IllegalPathException { - return ModificationUtils.isDeviceDeletedByMods(currentModifications, currentTimeIndex, device); + return ModificationUtils.isDeviceDeletedByMods( + currentModifications, currentTimeIndex, device, false); } public boolean isTimeSeriesDeletedByMods(IDeviceID device, TimeseriesMetadata timeseriesMetadata) @@ -206,7 +207,8 @@ public boolean isTimeSeriesDeletedByMods(IDeviceID device, TimeseriesMetadata ti device, timeseriesMetadata.getMeasurementId(), timeseriesMetadata.getStatistics().getStartTime(), - timeseriesMetadata.getStatistics().getEndTime()); + timeseriesMetadata.getStatistics().getEndTime(), + false); } public void addAlreadySetDatabase(PartialPath database) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java index f95fcfda473b7..f72711387249b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java @@ -816,7 +816,10 @@ public ReadOnlyMemChunk getReadOnlyMemChunkFromMemTable( return null; } IWritableMemChunk memChunk = - memTableMap.get(deviceID).getMemChunkMap().get(fullPath.getMeasurement()); + memTableMap + .get(deviceID) + .getMemChunkMap(context.isIgnoreAllNullRows()) + .get(fullPath.getMeasurement()); // check If data type matches if (memChunk.getSchema().getType() != fullPath.getMeasurementSchema().getType() && !fullPath diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java index 9b57f56499a5d..4af319684885a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java @@ -183,7 +183,6 @@ public void perform() throws Exception { // actually exist but the judgment return device being existed. sortedSourceFiles.addAll(seqFiles); sortedSourceFiles.addAll(unseqFiles); - boolean isTreeModel = !isAligned || device.getTableName().startsWith("root."); long ttl = deviceIterator.getTTLForCurrentDevice(); sortedSourceFiles.removeIf(x -> x.definitelyNotContains(device)); // checked above @@ -193,7 +192,9 @@ public void perform() throws Exception { if (ttl != Long.MAX_VALUE) { ttlDeletion = CompactionUtils.convertTtlToDeletion( - device, deviceIterator.getTimeLowerBoundForCurrentDevice()); + device, + deviceIterator.getTimeLowerBoundForCurrentDevice(), + deviceIterator.isTableModel()); } compactionWriter.setTTLDeletion(ttlDeletion); @@ -205,7 +206,8 @@ public void perform() throws Exception { compactionWriter.startChunkGroup(device, isAligned); if (isAligned) { - compactAlignedSeries(device, deviceIterator, compactionWriter, isTreeModel); + compactAlignedSeries( + device, deviceIterator, compactionWriter, !deviceIterator.isTableModel()); } else { compactNonAlignedSeries(device, deviceIterator, compactionWriter); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java index da2e15db500d4..82bdbd3e6496d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java @@ -256,7 +256,7 @@ private void compactAlignedSeries( readerAndChunkMetadataList, writer, summary, - device.getTableName().startsWith("root.")); + !deviceIterator.isTableModel()); compactionExecutor.execute(); for (ChunkMetadata chunkMetadata : writer.getChunkMetadataListOfCurrentDeviceInMemory()) { if (chunkMetadata.getMeasurementUid().isEmpty()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java index 8ba6cebb82de0..517f88cda77fd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadPointCompactionPerformer.java @@ -223,7 +223,7 @@ private void compactAlignedSeries( .map(IMeasurementSchema::getMeasurementName) .collect(Collectors.toList()); - fragmentInstanceContext.setIgnoreAllNullRows(device.getTableName().startsWith("root.")); + fragmentInstanceContext.setIgnoreAllNullRows(!deviceIterator.isTableModel()); IDataBlockReader dataBlockReader = constructReader( device, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java index 898ea1729f09e..fcde517479a5d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionPathUtils.java @@ -31,13 +31,15 @@ public class CompactionPathUtils { private CompactionPathUtils() {} - public static MeasurementPath getPath(final IDeviceID device, final String measurement) + public static MeasurementPath getPath( + final IDeviceID device, final String measurement, final boolean isTableModel) throws IllegalPathException { - return getPath(device).concatAsMeasurementPath(measurement); + return getPath(device, isTableModel).concatAsMeasurementPath(measurement); } - public static PartialPath getPath(final IDeviceID device) throws IllegalPathException { - if (device.isTableModel()) { + public static PartialPath getPath(final IDeviceID device, final boolean isTableModel) + throws IllegalPathException { + if (isTableModel) { final String[] nodes = new String[device.segmentNum()]; nodes[0] = device.getTableName(); for (int i = 0; i < device.segmentNum() - 1; i++) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java index d26ecdb2129be..55143751f1828 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java @@ -428,12 +428,13 @@ public static List getMatchedModifications( PatternTreeMap patternTreeMap, IDeviceID deviceID, String measurement, - ModEntry ttlDeletion) + ModEntry ttlDeletion, + boolean isTableModel) throws IllegalPathException { if ((patternTreeMap == null) || patternTreeMap.isEmpty()) { return ttlDeletion == null ? Collections.emptyList() : Collections.singletonList(ttlDeletion); } - PartialPath path = CompactionPathUtils.getPath(deviceID, measurement); + PartialPath path = CompactionPathUtils.getPath(deviceID, measurement, isTableModel); List modEntries = patternTreeMap.getOverlapped(path); if (ttlDeletion != null) { if (!(modEntries instanceof ArrayList)) { @@ -443,7 +444,7 @@ public static List getMatchedModifications( } modEntries.add(ttlDeletion); } - if (path.getIDeviceID().isTableModel()) { + if (isTableModel) { modEntries = modEntries.stream() .filter(e -> e.affects(path.getIDeviceID()) && e.affects(path.getMeasurement())) @@ -500,9 +501,9 @@ public static ArrayDeviceTimeIndex buildDeviceTimeIndex( return resource.buildDeviceTimeIndex(); } - public static ModEntry convertTtlToDeletion(IDeviceID deviceID, long timeLowerBound) - throws IllegalPathException { - if (!deviceID.isTableModel()) { + public static ModEntry convertTtlToDeletion( + IDeviceID deviceID, long timeLowerBound, boolean isTableModel) throws IllegalPathException { + if (!isTableModel) { return new TreeDeletionEntry( new MeasurementPath(deviceID, IoTDBConstant.ONE_LEVEL_PATH_WILDCARD), Long.MIN_VALUE, @@ -524,7 +525,7 @@ public static List getLatestMeasurementSchemasForTreeModel( PartialPath devicePath; PathPatternTree patternTree = new PathPatternTree(); try { - devicePath = CompactionPathUtils.getPath(deviceID); + devicePath = CompactionPathUtils.getPath(deviceID, false); for (String measurement : measurements) { patternTree.appendFullPath(devicePath, measurement); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java index b9c1224067b97..1b5e64f5717e7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.path.PatternTreeMap; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; +import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache; import org.apache.iotdb.db.schemaengine.lease.MetadataLeaseManager; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; @@ -82,10 +83,10 @@ public class MultiTsFileDeviceIterator implements AutoCloseable { private final Map> modificationCache = new HashMap<>(); private Pair currentDevice = null; - private boolean ignoreAllNullRows; private long ttlForCurrentDevice; private long timeLowerBoundForCurrentDevice; private final String databaseName; + private final boolean isTableModel; /** * Used for compaction with read chunk performer. @@ -94,6 +95,7 @@ public class MultiTsFileDeviceIterator implements AutoCloseable { */ public MultiTsFileDeviceIterator(List tsFileResources) throws IOException { this.databaseName = tsFileResources.get(0).getDatabaseName(); + this.isTableModel = PathUtils.isTableModelDatabase(databaseName); this.tsFileResourcesSortedByDesc = new ArrayList<>(tsFileResources); this.tsFileResourcesSortedByAsc = new ArrayList<>(tsFileResources); // sort the files from the oldest to the newest @@ -131,6 +133,7 @@ public MultiTsFileDeviceIterator( this.tsFileResourcesSortedByDesc = new ArrayList<>(seqResources); tsFileResourcesSortedByDesc.addAll(unseqResources); this.databaseName = tsFileResourcesSortedByDesc.get(0).getDatabaseName(); + this.isTableModel = PathUtils.isTableModelDatabase(databaseName); // sort the files from the newest to the oldest Collections.sort( this.tsFileResourcesSortedByDesc, TsFileResource::compareFileCreationOrderByDesc); @@ -156,6 +159,7 @@ public MultiTsFileDeviceIterator( this.tsFileResourcesSortedByDesc = new ArrayList<>(seqResources); tsFileResourcesSortedByDesc.addAll(unseqResources); this.databaseName = tsFileResourcesSortedByDesc.get(0).getDatabaseName(); + this.isTableModel = PathUtils.isTableModelDatabase(databaseName); // sort tsfiles from the newest to the oldest Collections.sort( this.tsFileResourcesSortedByDesc, TsFileResource::compareFileCreationOrderByDesc); @@ -236,7 +240,6 @@ public Pair nextDevice() throws IllegalPathException, IOExce IDeviceID deviceID = currentDevice.left; boolean isAligned = currentDevice.right; - ignoreAllNullRows = !isAligned || deviceID.getTableName().startsWith("root."); if (MetadataLeaseManager.getInstance().isFenced()) { // Metadata lease fenced: this DataNode may hold a stale TTL (it could have missed a // ConfigNode @@ -245,7 +248,7 @@ public Pair nextDevice() throws IllegalPathException, IOExce // deletes nothing by TTL while fenced, and real TTL deletion resumes once the lease recovers // and the cache resyncs. (Checked first so the table path also avoids the fenced cache.) ttlForCurrentDevice = Long.MAX_VALUE; - } else if (!ignoreAllNullRows) { + } else if (isTableModel) { ttlForCurrentDevice = DataNodeTTLCache.getInstance().getTTLForTable(databaseName, deviceID.getTableName()); } else { @@ -262,6 +265,10 @@ public String getDatabaseName() { return databaseName; } + public boolean isTableModel() { + return isTableModel; + } + public long getTTLForCurrentDevice() { return ttlForCurrentDevice; } @@ -271,9 +278,9 @@ public long getTimeLowerBoundForCurrentDevice() { } public Map getAllSchemasOfCurrentDevice() throws IOException { - return ignoreAllNullRows - ? getAllSchemasOfCurrentDeviceForTree() - : getAllSchemasOfCurrentDeviceForTable(); + return isTableModel + ? getAllSchemasOfCurrentDeviceForTable() + : getAllSchemasOfCurrentDeviceForTree(); } /** @@ -448,9 +455,9 @@ public Map getCompactionSeriesContextOfCurrentD */ public Map>>> getTimeseriesSchemaAndMetadataOffsetOfCurrentDevice() throws IOException { - return ignoreAllNullRows - ? getTimeseriesSchemaAndMetadataOffsetOfCurrentDeviceForTree() - : getTimeseriesSchemaAndMetadataOffsetOfCurrentDeviceForTable(); + return isTableModel + ? getTimeseriesSchemaAndMetadataOffsetOfCurrentDeviceForTable() + : getTimeseriesSchemaAndMetadataOffsetOfCurrentDeviceForTree(); } @SuppressWarnings({"checkstyle:AtclauseOrderCheck", "squid:S3824"}) @@ -613,7 +620,7 @@ public Map getCompactionSeriesContextOfCurrentD TsFileSequenceReader reader = readerMap.get(tsFileResource); List alignedChunkMetadataList = reader.getAlignedChunkMetadataByMetadataIndexNode( - currentDevice.left, firstMeasurementNodeOfCurrentDevice, ignoreAllNullRows); + currentDevice.left, firstMeasurementNodeOfCurrentDevice, !isTableModel); applyModificationForAlignedChunkMetadataList(tsFileResource, alignedChunkMetadataList); readerAndChunkMetadataList.add(new Pair<>(reader, alignedChunkMetadataList)); } @@ -638,7 +645,9 @@ private void applyModificationForAlignedChunkMetadataList( ModEntry ttlDeletion = null; Optional startTime = tsFileResource.getStartTime(device); if (startTime.isPresent() && startTime.get() < timeLowerBoundForCurrentDevice) { - ttlDeletion = CompactionUtils.convertTtlToDeletion(device, timeLowerBoundForCurrentDevice); + ttlDeletion = + CompactionUtils.convertTtlToDeletion( + device, timeLowerBoundForCurrentDevice, isTableModel); } PatternTreeMap modifications = @@ -652,7 +661,7 @@ private void applyModificationForAlignedChunkMetadataList( // match time column modifications List modificationForTimeColumn = CompactionUtils.getMatchedModifications( - modifications, device, AlignedPath.VECTOR_PLACEHOLDER, ttlDeletion); + modifications, device, AlignedPath.VECTOR_PLACEHOLDER, ttlDeletion, isTableModel); // match value column modifications List> modificationForValueColumns = new ArrayList<>(); @@ -663,7 +672,7 @@ private void applyModificationForAlignedChunkMetadataList( } List modificationList = CompactionUtils.getMatchedModifications( - modifications, device, valueChunkMetadata.getMeasurementUid(), null); + modifications, device, valueChunkMetadata.getMeasurementUid(), null, isTableModel); modificationForValueColumns.add( modificationList.isEmpty() ? Collections.emptyList() : modificationList); } @@ -672,7 +681,7 @@ private void applyModificationForAlignedChunkMetadataList( alignedChunkMetadataList, modificationForTimeColumn, modificationForValueColumns, - ignoreAllNullRows); + !isTableModel); } public Map getReaderMap() { @@ -884,7 +893,11 @@ public String nextSeries() throws IllegalPathException { // collect the modifications for current series List modificationForCurrentSeries = CompactionUtils.getMatchedModifications( - modificationsInThisResource, device, currentCompactingSeries, ttlDeletion); + modificationsInThisResource, + device, + currentCompactingSeries, + ttlDeletion, + isTableModel); // if there are modifications of current series, apply them to the chunk metadata if (!modificationForCurrentSeries.isEmpty()) { @@ -902,7 +915,7 @@ public String nextSeries() throws IllegalPathException { // skip data of deleted table private boolean isCurrentDeviceDataInDeprecatedTable(TsFileResource resource) { - if (ignoreAllNullRows) { + if (!isTableModel) { return false; } String tableName = currentDevice.getLeft().getTableName(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java index cf9ed65b8194f..718373058f63c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java @@ -247,7 +247,8 @@ protected List getAlignedChunkMetadataList(TsFileR // get time modifications of this file List timeModifications = - getModificationsFromCache(resource, deviceId, AlignedPath.VECTOR_PLACEHOLDER); + getModificationsFromCache( + resource, deviceId, AlignedPath.VECTOR_PLACEHOLDER, !ignoreAllNullRows); // get value modifications of this file List> valueModifications = new ArrayList<>(); alignedChunkMetadataList @@ -260,7 +261,8 @@ protected List getAlignedChunkMetadataList(TsFileR valueModifications.add(null); } else { valueModifications.add( - getModificationsFromCache(resource, deviceId, x.getMeasurementUid())); + getModificationsFromCache( + resource, deviceId, x.getMeasurementUid(), !ignoreAllNullRows)); } } catch (IllegalPathException e) { throw new RuntimeException(e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java index 49cbcb14582f3..b187a568b8416 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java @@ -160,7 +160,7 @@ void deserializeFileIntoChunkMetadataQueue(List fileElements) ModificationUtils.modifyChunkMetaData( iChunkMetadataList, getModificationsFromCache( - resource, deviceId, iChunkMetadataList.get(0).getMeasurementUid())); + resource, deviceId, iChunkMetadataList.get(0).getMeasurementUid(), false)); if (iChunkMetadataList.isEmpty()) { // all chunks has been deleted in this file, just remove it removeFile(fileElement); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java index b3073bd3d258d..1452b27f239c3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java @@ -474,7 +474,7 @@ protected void removeFile(FileElement fileElement) throws IllegalPathException, * modification from ttl cache. */ protected List getModificationsFromCache( - TsFileResource tsFileResource, IDeviceID deviceId, String measurement) + TsFileResource tsFileResource, IDeviceID deviceId, String measurement, boolean isTableModel) throws IllegalPathException { PatternTreeMap allModifications = modificationCacheMap.get(tsFileResource.getTsFile().getName()); @@ -482,7 +482,8 @@ protected List getModificationsFromCache( allModifications, deviceId, measurement, - compactionWriter.getTTLLowerBoundForCurrentDevice()); + compactionWriter.getTTLLowerBoundForCurrentDevice(), + isTableModel); } @SuppressWarnings("squid:S3776") diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java index 947a22ae60027..90181e111a2ca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.utils.MetadataUtils; +import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionLastTimeCheckFailedException; @@ -109,7 +110,7 @@ public ReadChunkAlignedSeriesCompactionExecutor( this.targetResource = targetResource; this.summary = summary; this.ignoreAllNullRows = ignoreAllNullRows; - if (device.isTableModel()) { + if (PathUtils.isTableModelDatabase(database)) { collectValueColumnSchemaListForTable(database); } else { collectValueColumnSchemaListForTree(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SettleSelectorImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SettleSelectorImpl.java index 61f8229cd3ca3..774a1d3cfe9bd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SettleSelectorImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SettleSelectorImpl.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.path.AlignedPath; import org.apache.iotdb.commons.path.PatternTreeMap; import org.apache.iotdb.commons.utils.CommonDateTimeUtils; +import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.StorageEngineMessages; @@ -67,6 +68,7 @@ public class SettleSelectorImpl implements ISettleSelector { private final boolean heavySelect; private final String storageGroupName; private final String dataRegionId; + private final boolean isTableModel; private final long timePartition; private final TsFileManager tsFileManager; private boolean isSeq; @@ -83,6 +85,7 @@ public SettleSelectorImpl( this.heavySelect = heavySelect; this.storageGroupName = storageGroupName; this.dataRegionId = dataRegionId; + this.isTableModel = PathUtils.isTableModelDatabase(storageGroupName); this.timePartition = timePartition; this.tsFileManager = tsFileManager; this.context = context; @@ -261,7 +264,7 @@ private FileDirtyInfo selectFileBaseOnDirtyData(TsFileResource resource) long ttl; String tableName = device.getTableName(); boolean hasSetTTL; - if (tableName.startsWith("root.")) { + if (!isTableModel) { ttl = DataNodeTTLCache.getInstance().getTTLForTree(device); hasSetTTL = ttl != Long.MAX_VALUE; } else { @@ -327,7 +330,7 @@ private boolean isDeviceDeletedByMods( throws IllegalPathException { return ModificationUtils.isAllDeletedByMods( CompactionUtils.getMatchedModifications( - modifications, device, AlignedPath.VECTOR_PLACEHOLDER, null), + modifications, device, AlignedPath.VECTOR_PLACEHOLDER, null, isTableModel), startTime, endTime); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java index 68727f03aa73d..4a2dfd88cd604 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java @@ -85,6 +85,12 @@ public class MemTableFlushTask { private String storageGroup; private String dataRegionId; + /** + * Whether rows whose values are all null should be ignored when flushing. All-null rows are only + * meaningful for table model databases, so it is the negation of {@code isTableModel}. + */ + private final boolean ignoreAllNullRows; + private IMemTable memTable; private volatile long memSerializeTime = 0L; @@ -97,16 +103,21 @@ public class MemTableFlushTask { * @param memTable the memTable to flush * @param writer the writer where memTable will be flushed to (current tsfile writer or vm writer) * @param storageGroup current database + * @param dataRegionId current data region id + * @param isTableModel whether the memTable belongs to a table model database, it cannot be + * inferred when the caller does not know the database, e.g. the pipe tsfile builders */ public MemTableFlushTask( IMemTable memTable, RestorableTsFileIOWriter writer, String storageGroup, - String dataRegionId) { + String dataRegionId, + boolean isTableModel) { this.memTable = memTable; this.writer = writer; this.storageGroup = storageGroup; this.dataRegionId = dataRegionId; + this.ignoreAllNullRows = !isTableModel; this.encodingTaskFuture = SUB_TASK_POOL_MANAGER.submit(encodingTask); this.ioTaskFuture = SUB_TASK_POOL_MANAGER.submit(ioTask); @@ -160,9 +171,10 @@ public void syncFlushMemTable() throws ExecutionException, InterruptedException // sort the IDeviceID in lexicographical order Collections.sort(deviceIDList); for (IDeviceID deviceID : deviceIDList) { - final Map value = memTableMap.get(deviceID).getMemChunkMap(); + final Map value = + memTableMap.get(deviceID).getMemChunkMap(ignoreAllNullRows); // skip the empty device/chunk group - if (memTableMap.get(deviceID).isEmpty() || value.isEmpty()) { + if (memTableMap.get(deviceID).isEmpty(ignoreAllNullRows) || value.isEmpty()) { continue; } encodingTaskQueue.put(new StartFlushGroupIOTask(deviceID)); @@ -171,7 +183,7 @@ public void syncFlushMemTable() throws ExecutionException, InterruptedException for (String seriesId : seriesInOrder) { long startTime = System.currentTimeMillis(); IWritableMemChunk series = value.get(seriesId); - if (series.count() == 0) { + if (series.count(ignoreAllNullRows) == 0) { continue; } /* @@ -278,7 +290,7 @@ public void run() { if (writableMemChunk instanceof AlignedWritableMemChunk && times == null) { times = new long[MAX_NUMBER_OF_POINTS_IN_PAGE]; } - writableMemChunk.encode(ioTaskQueue, encodeInfo, times); + writableMemChunk.encode(ioTaskQueue, encodeInfo, times, ignoreAllNullRows); long subTaskTime = System.currentTimeMillis() - starTime; WRITING_METRICS.recordFlushSubTaskCost(WritingMetrics.ENCODING_TASK, subTaskTime); memSerializeTime += subTaskTime; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java index 35a3696630e50..2bdc6c2903507 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractMemTable.java @@ -187,7 +187,7 @@ private IWritableMemChunkGroup createAlignedMemChunkGroupIfNotExistAndGet( deviceId, k -> { seriesNumber += filteredSchemaList.size(); - return new AlignedWritableMemChunkGroup(filteredSchemaList, k.isTableModel()); + return new AlignedWritableMemChunkGroup(filteredSchemaList); }); for (IMeasurementSchema schema : schemaList) { if (schema != null && !memChunkGroup.contains(schema.getMeasurementName())) { @@ -562,10 +562,10 @@ public double getNullValueRatio() { } @Override - public long size() { + public long size(boolean ignoreAllNullRows) { long sum = 0; for (IWritableMemChunkGroup writableMemChunkGroup : memTableMap.values()) { - sum += writableMemChunkGroup.count(); + sum += writableMemChunkGroup.count(ignoreAllNullRows); } return sum; } @@ -617,6 +617,7 @@ public void queryForSeriesRegionScan( Filter globalTimeFilter) { IDeviceID deviceID = fullPath.getDeviceId(); + // Rows whose values are all null can never satisfy this read path, so they are always ignored. if (fullPath instanceof NonAlignedFullPath) { String measurementId = ((NonAlignedFullPath) fullPath).getMeasurement(); @@ -637,6 +638,7 @@ public void queryForSeriesRegionScan( chunkMetaDataMap, memChunkHandleMap, deletionList, + true, globalTimeFilter); } else { // check If MemTable Contains this path @@ -660,6 +662,7 @@ public void queryForSeriesRegionScan( chunkMetaDataMap, memChunkHandleMap, deletionList, + true, globalTimeFilter); } } @@ -682,6 +685,7 @@ public void queryForDeviceRegionScan( } IWritableMemChunkGroup writableMemChunkGroup = memTableMap.get(deviceID); + // Rows whose values are all null can never satisfy this read path, so they are always ignored. if (isAligned) { getMemAlignedChunkHandleFromMemTable( deviceID, @@ -690,6 +694,7 @@ public void queryForDeviceRegionScan( memChunkHandleMap, ttlLowerBound, modsToMemTabled, + true, globalTimeFilter); } else { getMemChunkHandleFromMemTable( @@ -699,6 +704,7 @@ public void queryForDeviceRegionScan( memChunkHandleMap, ttlLowerBound, modsToMemTabled, + true, globalTimeFilter); } } @@ -709,10 +715,12 @@ private void getMemChunkHandleFromMemTable( Map> chunkMetadataMap, Map> memChunkHandleMap, List deletionList, + boolean ignoreAllNullRows, Filter globalTimeFilter) { WritableMemChunk memChunk = - (WritableMemChunk) memTableMap.get(deviceID).getMemChunkMap().get(measurementId); + (WritableMemChunk) + memTableMap.get(deviceID).getMemChunkMap(ignoreAllNullRows).get(measurementId); if (memChunk == null) { return; @@ -740,6 +748,7 @@ private void getMemAlignedChunkHandleFromMemTable( Map> chunkMetadataList, Map> memChunkHandleMap, List> deletionList, + boolean ignoreAllNullRows, Filter globalTimeFilter) { AlignedWritableMemChunk alignedMemChunk = @@ -758,7 +767,8 @@ private void getMemAlignedChunkHandleFromMemTable( List bitMaps = new ArrayList<>(); long[] timestamps = - alignedMemChunk.getAnySatisfiedTimestamp(deletionList, bitMaps, true, globalTimeFilter); + alignedMemChunk.getAnySatisfiedTimestamp( + deletionList, bitMaps, ignoreAllNullRows, globalTimeFilter); if (timestamps.length == 0) { return; } @@ -780,6 +790,7 @@ private void getMemAlignedChunkHandleFromMemTable( Map> memChunkHandleMap, long ttlLowerBound, List> modsToMemTabled, + boolean ignoreAllNullRows, Filter globalTimeFilter) { AlignedWritableMemChunk memChunk = writableMemChunkGroup.getAlignedMemChunk(); @@ -796,7 +807,8 @@ private void getMemAlignedChunkHandleFromMemTable( List bitMaps = new ArrayList<>(); long[] timestamps = - memChunk.getAnySatisfiedTimestamp(deletionList, bitMaps, true, globalTimeFilter); + memChunk.getAnySatisfiedTimestamp( + deletionList, bitMaps, ignoreAllNullRows, globalTimeFilter); if (timestamps.length == 0) { return; } @@ -817,10 +829,11 @@ private void getMemChunkHandleFromMemTable( Map> memChunkHandleMap, long ttlLowerBound, List> modsToMemTabled, + boolean ignoreAllNullRows, Filter globalTimeFilter) { for (Entry entry : - writableMemChunkGroup.getMemChunkMap().entrySet()) { + writableMemChunkGroup.getMemChunkMap(ignoreAllNullRows).entrySet()) { String measurementId = entry.getKey(); WritableMemChunk writableMemChunk = (WritableMemChunk) entry.getValue(); @@ -921,7 +934,7 @@ private IChunkMetadata buildFakeChunkMetaDataForFakeMemoryChunk( } @Override - public long delete(ModEntry modEntry) { + public long delete(ModEntry modEntry, boolean ignoreAllNullRows) { List> targetDeviceList = new ArrayList<>(); for (Entry entry : memTableMap.entrySet()) { if (modEntry.affects(entry.getKey())) { @@ -932,11 +945,11 @@ public long delete(ModEntry modEntry) { long pointDeleted = 0; for (Pair pair : targetDeviceList) { if (modEntry.affectsAll(pair.left)) { - pointDeleted += pair.right.deleteTime(modEntry); + pointDeleted += pair.right.deleteTime(modEntry, ignoreAllNullRows); } else { - pointDeleted += pair.right.delete(modEntry); + pointDeleted += pair.right.delete(modEntry, ignoreAllNullRows); } - if (pair.right.isEmpty()) { + if (pair.right.isEmpty(ignoreAllNullRows)) { memTableMap.remove(pair.left).release(); } } @@ -1087,15 +1100,13 @@ protected void deserialize(DataInputStream stream, boolean multiTvListInMemChunk IWritableMemChunkGroup memChunkGroup; if (multiTvListInMemChunk) { if (isAligned) { - memChunkGroup = AlignedWritableMemChunkGroup.deserialize(stream, deviceID.isTableModel()); + memChunkGroup = AlignedWritableMemChunkGroup.deserialize(stream); } else { memChunkGroup = WritableMemChunkGroup.deserialize(stream); } } else { if (isAligned) { - memChunkGroup = - AlignedWritableMemChunkGroup.deserializeSingleTVListMemChunks( - stream, deviceID.isTableModel()); + memChunkGroup = AlignedWritableMemChunkGroup.deserializeSingleTVListMemChunks(stream); } else { memChunkGroup = WritableMemChunkGroup.deserializeSingleTVListMemChunks(stream); } @@ -1130,15 +1141,13 @@ public void deserializeFromOldMemTableSnapshot( IWritableMemChunkGroup memChunkGroup; if (multiTvListInMemChunk) { if (isAligned) { - memChunkGroup = AlignedWritableMemChunkGroup.deserialize(stream, deviceID.isTableModel()); + memChunkGroup = AlignedWritableMemChunkGroup.deserialize(stream); } else { memChunkGroup = WritableMemChunkGroup.deserialize(stream); } } else { if (isAligned) { - memChunkGroup = - AlignedWritableMemChunkGroup.deserializeSingleTVListMemChunks( - stream, deviceID.isTableModel()); + memChunkGroup = AlignedWritableMemChunkGroup.deserializeSingleTVListMemChunks(stream); } else { memChunkGroup = WritableMemChunkGroup.deserializeSingleTVListMemChunks(stream); } @@ -1148,11 +1157,12 @@ public void deserializeFromOldMemTableSnapshot( } @Override - public Map getMaxTime() { + public Map getMaxTime(boolean ignoreAllNullRows) { Map latestTimeForEachDevice = new HashMap<>(); for (Entry entry : memTableMap.entrySet()) { - if (entry.getValue().count() > 0 && !entry.getValue().isEmpty()) { - latestTimeForEachDevice.put(entry.getKey(), entry.getValue().getMaxTime()); + if (entry.getValue().count(ignoreAllNullRows) > 0 + && !entry.getValue().isEmpty(ignoreAllNullRows)) { + latestTimeForEachDevice.put(entry.getKey(), entry.getValue().getMaxTime(ignoreAllNullRows)); } } return latestTimeForEachDevice; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java index 4f2bb0abd1163..327944c297551 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AbstractWritableMemChunk.java @@ -222,7 +222,7 @@ public abstract void writeAlignedTablet( TSStatus[] results); @Override - public abstract long count(); + public abstract long count(boolean ignoreAllNullRows); @Override public abstract long rowCount(); @@ -284,13 +284,16 @@ public void releaseTemporaryTvListForFlush() { @Override public abstract void encode( - BlockingQueue ioTaskQueue, BatchEncodeInfo encodeInfo, long[] times); + BlockingQueue ioTaskQueue, + BatchEncodeInfo encodeInfo, + long[] times, + boolean ignoreAllNullRows); @Override public abstract void release(); @Override - public abstract boolean isEmpty(); + public abstract boolean isEmpty(boolean ignoreAllNullRows); @Override public abstract List getSortedList(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java index ebdc024aac84b..df493067eb09b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java @@ -75,7 +75,6 @@ public class AlignedWritableMemChunk extends AbstractWritableMemChunk { private AlignedTVList list; private List sortedList; private long sortedRowCount = 0; - private final boolean ignoreAllNullRows; private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); private final int TVLIST_SORT_THRESHOLD = CONFIG.getTvListSortThreshold(); @@ -84,14 +83,12 @@ public class AlignedWritableMemChunk extends AbstractWritableMemChunk { private EncryptParameter encryptParameter; - public AlignedWritableMemChunk(List schemaList, boolean isTableModel) { - this(schemaList, isTableModel, EncryptUtils.getEncryptParameter()); + public AlignedWritableMemChunk(List schemaList) { + this(schemaList, EncryptUtils.getEncryptParameter()); } public AlignedWritableMemChunk( - List schemaList, - boolean isTableModel, - EncryptParameter encryptParameter) { + List schemaList, EncryptParameter encryptParameter) { this.measurementIndexMap = new LinkedHashMap<>(); this.dataTypes = new ArrayList<>(); this.schemaList = schemaList; @@ -101,20 +98,15 @@ public AlignedWritableMemChunk( } this.list = AlignedTVList.newAlignedList(dataTypes); this.sortedList = new ArrayList<>(); - this.ignoreAllNullRows = !isTableModel; this.encryptParameter = encryptParameter; } - private AlignedWritableMemChunk( - List schemaList, AlignedTVList list, boolean isTableModel) { - this(schemaList, list, isTableModel, EncryptUtils.getEncryptParameter()); + private AlignedWritableMemChunk(List schemaList, AlignedTVList list) { + this(schemaList, list, EncryptUtils.getEncryptParameter()); } private AlignedWritableMemChunk( - List schemaList, - AlignedTVList list, - boolean isTableModel, - EncryptParameter encryptParameter) { + List schemaList, AlignedTVList list, EncryptParameter encryptParameter) { this.measurementIndexMap = new LinkedHashMap<>(); this.schemaList = schemaList; for (int i = 0; i < schemaList.size(); i++) { @@ -123,7 +115,6 @@ private AlignedWritableMemChunk( this.list = list; this.dataTypes = list.getTsDataTypes(); this.sortedList = new ArrayList<>(); - this.ignoreAllNullRows = !isTableModel; this.encryptParameter = encryptParameter; } @@ -465,7 +456,7 @@ public void setWorkingTVList(TVList list) { } @Override - public long count() { + public long count(boolean ignoreAllNullRows) { if (!ignoreAllNullRows && measurementIndexMap.isEmpty()) { return rowCount(); } @@ -487,8 +478,8 @@ public IMeasurementSchema getSchema() { } @Override - public long getMaxTime() { - if (isEmpty()) { + public long getMaxTime(boolean ignoreAllNullRows) { + if (isEmpty(ignoreAllNullRows)) { return Long.MIN_VALUE; } long maxTime = list.getMaxTime(); @@ -582,7 +573,8 @@ private List getDataTypes(List schemaList) { public void encodeWorkingAlignedTVList( BlockingQueue ioTaskQueue, long maxNumberOfPointsInChunk, - int maxNumberOfPointsInPage) { + int maxNumberOfPointsInPage, + boolean ignoreAllNullRows) { List activeSchemaList = getActiveSchemaList(); if (activeSchemaList.isEmpty() && ignoreAllNullRows) { return; @@ -1065,7 +1057,11 @@ private void handleEncodingWithDeletedMeasurements( } @Override - public void encode(BlockingQueue ioTaskQueue, BatchEncodeInfo encodeInfo, long[] times) { + public void encode( + BlockingQueue ioTaskQueue, + BatchEncodeInfo encodeInfo, + long[] times, + boolean ignoreAllNullRows) { encodeInfo.maxNumberOfPointsInChunk = Math.min( encodeInfo.maxNumberOfPointsInChunk, @@ -1073,7 +1069,10 @@ public void encode(BlockingQueue ioTaskQueue, BatchEncodeInfo encodeInfo if (TVLIST_SORT_THRESHOLD == 0) { encodeWorkingAlignedTVList( - ioTaskQueue, encodeInfo.maxNumberOfPointsInChunk, encodeInfo.maxNumberOfPointsInPage); + ioTaskQueue, + encodeInfo.maxNumberOfPointsInChunk, + encodeInfo.maxNumberOfPointsInPage, + ignoreAllNullRows); return; } @@ -1151,15 +1150,15 @@ public long getFirstPoint() { } @Override - public long getLastPoint() { + public long getLastPoint(boolean ignoreAllNullRows) { if (rowCount() == 0) { return Long.MIN_VALUE; } - return getMaxTime(); + return getMaxTime(ignoreAllNullRows); } @Override - public boolean isEmpty() { + public boolean isEmpty(boolean ignoreAllNullRows) { if (rowCount() == 0) { return true; } @@ -1215,8 +1214,7 @@ public void serializeToWAL(IWALByteBufferView buffer) { list.serializeToWAL(buffer); } - public static AlignedWritableMemChunk deserialize(DataInputStream stream, boolean isTableModel) - throws IOException { + public static AlignedWritableMemChunk deserialize(DataInputStream stream) throws IOException { int schemaListSize = stream.readInt(); List schemaList = new ArrayList<>(schemaListSize); for (int i = 0; i < schemaListSize; i++) { @@ -1230,13 +1228,13 @@ public static AlignedWritableMemChunk deserialize(DataInputStream stream, boolea sortedList.add(tvList); } AlignedTVList list = AlignedTVList.deserialize(stream); - AlignedWritableMemChunk chunk = new AlignedWritableMemChunk(schemaList, list, isTableModel); + AlignedWritableMemChunk chunk = new AlignedWritableMemChunk(schemaList, list); chunk.sortedList = sortedList; return chunk; } - public static AlignedWritableMemChunk deserializeSingleTVListMemChunks( - DataInputStream stream, boolean isTableModel) throws IOException { + public static AlignedWritableMemChunk deserializeSingleTVListMemChunks(DataInputStream stream) + throws IOException { int schemaListSize = stream.readInt(); List schemaList = new ArrayList<>(schemaListSize); for (int i = 0; i < schemaListSize; i++) { @@ -1245,7 +1243,7 @@ public static AlignedWritableMemChunk deserializeSingleTVListMemChunks( } AlignedTVList list = AlignedTVList.deserialize(stream); - return new AlignedWritableMemChunk(schemaList, list, isTableModel); + return new AlignedWritableMemChunk(schemaList, list); } public List getSchemaList() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java index a27973acdb1dd..e1235ca935b0d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunkGroup.java @@ -46,13 +46,13 @@ public class AlignedWritableMemChunkGroup implements IWritableMemChunkGroup { private EncryptParameter encryptParameter; - public AlignedWritableMemChunkGroup(List schemaList, boolean isTableModel) { - memChunk = new AlignedWritableMemChunk(schemaList, isTableModel); + public AlignedWritableMemChunkGroup(List schemaList) { + memChunk = new AlignedWritableMemChunk(schemaList); } @TestOnly public AlignedWritableMemChunkGroup( - AlignedWritableMemChunk memChunk, List schemaList, boolean isTableModel) { + AlignedWritableMemChunk memChunk, List schemaList) { this.memChunk = memChunk; } @@ -78,8 +78,8 @@ public void release() { } @Override - public long count() { - return memChunk.count(); + public long count(boolean ignoreAllNullRows) { + return memChunk.count(ignoreAllNullRows); } /** @@ -101,20 +101,20 @@ public void writeRow(long insertTime, Object[] objectValue, List getMemChunkMap() { - if (memChunk.count() == 0) { + public Map getMemChunkMap(boolean ignoreAllNullRows) { + if (memChunk.count(ignoreAllNullRows) == 0) { return Collections.emptyMap(); } return Collections.singletonMap("", memChunk); } @Override - public boolean isEmpty() { - return memChunk.isEmpty() || memChunk.isAllDeleted(); + public boolean isEmpty(boolean ignoreAllNullRows) { + return memChunk.isEmpty(ignoreAllNullRows) || memChunk.isAllDeleted(); } @Override - public long delete(ModEntry modEntry) { + public long delete(ModEntry modEntry, boolean ignoreAllNullRows) { int deletedPointsNumber = 0; Set measurements = memChunk.getAllMeasurements(); List columnsToBeRemoved = new ArrayList<>(); @@ -138,7 +138,7 @@ public long delete(ModEntry modEntry) { return deletedPointsNumber; } - public long deleteTime(ModEntry modEntry) { + public long deleteTime(ModEntry modEntry, boolean ignoreAllNullRows) { return memChunk.deleteTime(modEntry.getStartTime(), modEntry.getEndTime()); } @@ -148,8 +148,8 @@ public IWritableMemChunk getWritableMemChunk(String measurement) { } @Override - public long getMaxTime() { - return memChunk.getMaxTime(); + public long getMaxTime(boolean ignoreAllNullRows) { + return memChunk.getMaxTime(ignoreAllNullRows); } public AlignedWritableMemChunk getAlignedMemChunk() { @@ -172,10 +172,10 @@ public void setEncryptParameter(EncryptParameter encryptParameter) { memChunk.setEncryptParameter(encryptParameter); } - protected static AlignedWritableMemChunkGroup deserialize( - DataInputStream stream, boolean isTableModel) throws IOException { + protected static AlignedWritableMemChunkGroup deserialize(DataInputStream stream) + throws IOException { AlignedWritableMemChunkGroup memChunkGroup = new AlignedWritableMemChunkGroup(); - memChunkGroup.memChunk = AlignedWritableMemChunk.deserialize(stream, isTableModel); + memChunkGroup.memChunk = AlignedWritableMemChunk.deserialize(stream); return memChunkGroup; } @@ -185,10 +185,9 @@ public void checkDataType(InsertNode node) throws DataTypeInconsistentException } protected static AlignedWritableMemChunkGroup deserializeSingleTVListMemChunks( - DataInputStream stream, boolean isTableModel) throws IOException { + DataInputStream stream) throws IOException { AlignedWritableMemChunkGroup memChunkGroup = new AlignedWritableMemChunkGroup(); - memChunkGroup.memChunk = - AlignedWritableMemChunk.deserializeSingleTVListMemChunks(stream, isTableModel); + memChunkGroup.memChunk = AlignedWritableMemChunk.deserializeSingleTVListMemChunks(stream); return memChunkGroup; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java index afa24083ff6d3..48d43644c8dc3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IMemTable.java @@ -68,9 +68,11 @@ void writeAlignedRow( Object[] objectValue); /** + * @param ignoreAllNullRows whether rows whose values are all null should be ignored, it is + * determined by the data model of the database the memtable belongs to * @return the number of points */ - long size(); + long size(boolean ignoreAllNullRows); /** * @return memory usage @@ -170,7 +172,7 @@ void queryForDeviceRegionScan( * * @return */ - long delete(ModEntry modEntry); + long delete(ModEntry modEntry, boolean ignoreAllNullRows); /** * Make a copy of this MemTable. @@ -214,7 +216,7 @@ void queryForDeviceRegionScan( void setFlushStatus(FlushStatus flushStatus); - Map getMaxTime(); + Map getMaxTime(boolean ignoreAllNullRows); String getDatabase(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java index e57f30680f9aa..71f205ef601b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java @@ -84,7 +84,7 @@ void writeAlignedTablet( int end, TSStatus[] results); - long count(); + long count(boolean ignoreAllNullRows); long rowCount(); @@ -100,7 +100,7 @@ void writeAlignedTablet( void releaseTemporaryTvListForFlush(); - default long getMaxTime() { + default long getMaxTime(boolean ignoreAllNullRows) { return Long.MAX_VALUE; } @@ -115,15 +115,19 @@ default long getMinTime() { IChunkWriter createIChunkWriter(); - void encode(BlockingQueue ioTaskQueue, BatchEncodeInfo encodeInfo, long[] times); + void encode( + BlockingQueue ioTaskQueue, + BatchEncodeInfo encodeInfo, + long[] times, + boolean ignoreAllNullRows); void release(); long getFirstPoint(); - long getLastPoint(); + long getLastPoint(boolean ignoreAllNullRows); - boolean isEmpty(); + boolean isEmpty(boolean ignoreAllNullRows); List getSortedList(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java index 46a21f05840f6..0cafd43632941 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunkGroup.java @@ -47,21 +47,21 @@ void writeTablet( void release(); - long count(); + long count(boolean ignoreAllNullRows); boolean contains(String measurement); - Map getMemChunkMap(); + Map getMemChunkMap(boolean ignoreAllNullRows); - boolean isEmpty(); + boolean isEmpty(boolean ignoreAllNullRows); - long delete(ModEntry modEntry); + long delete(ModEntry modEntry, boolean ignoreAllNullRows); - long deleteTime(ModEntry modEntry); + long deleteTime(ModEntry modEntry, boolean ignoreAllNullRows); IWritableMemChunk getWritableMemChunk(String measurement); - long getMaxTime(); + long getMaxTime(boolean ignoreAllNullRows); void setEncryptParameter(EncryptParameter encryptParameter); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 78c48a2702ba7..c22dfcf58d1f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -1584,6 +1584,11 @@ private void rollbackMemoryInfoIfNeeded(final long[] memIncrements) { } } + /** Whether the data region of this processor belongs to a table model database. */ + private boolean isTableModel() { + return dataRegionInfo.getDataRegion().isTableModel(); + } + /** * Delete data which belongs to the timeseries `deviceId.measurementId` and the timestamp of which * <= 'timestamp' in the deletion.
@@ -1599,7 +1604,7 @@ public boolean deleteDataInMemory(ModEntry deletion) { try { boolean deleted = false; if (workMemTable != null) { - long pointDeleted = workMemTable.delete(deletion); + long pointDeleted = workMemTable.delete(deletion, !isTableModel()); logger.info( StorageEngineMessages .STORAGE_LOG_DELETION_DELETION_WITH_IN_WORKMEMTABLE_POINTS_DELETED_00EA995A, @@ -1767,7 +1772,7 @@ public void asyncFlush() { * flushManager again. */ private Future addAMemtableIntoFlushingList(IMemTable tobeFlushed) throws IOException { - final Map lastTimeForEachDevice = tobeFlushed.getMaxTime(); + final Map lastTimeForEachDevice = tobeFlushed.getMaxTime(!isTableModel()); // If some devices have been removed in MemTable, the number of device in MemTable and // tsFileResource will not be the same. And the endTime of these devices in resource will be @@ -1911,7 +1916,8 @@ public void flushOneMemTable() { memTableToFlush, writer, dataRegionName, - dataRegionInfo.getDataRegion().getDataRegionIdString()); + dataRegionInfo.getDataRegion().getDataRegionIdString(), + isTableModel()); flushTask.syncFlushMemTable(); memTableFlushPointCount = memTableToFlush.getTotalPointsNum(); } catch (Throwable e) { @@ -2638,13 +2644,16 @@ private void query( } private long getQueryTimeLowerBound(IDeviceID deviceID) { - long ttl; - if (deviceID.getTableName().startsWith("root.")) { - ttl = DataNodeTTLCache.getInstance().getTTLForTree(deviceID); - } else { + final long ttl; + if (isTableModel()) { + // The table TTL is cached by the database and the table, both known from the data region of + // this processor. ttl = DataNodeTTLCache.getInstance() - .getTTLForTable(this.dataRegionName, deviceID.getTableName()); + .getTTLForTable( + dataRegionInfo.getDataRegion().getDatabaseName(), deviceID.getTableName()); + } else { + ttl = DataNodeTTLCache.getInstance().getTTLForTree(deviceID); } return ttl != Long.MAX_VALUE ? CommonUtils.getTTLLowerBound(ttl) : Long.MIN_VALUE; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java index 7e54296c6a8bf..929108e924690 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java @@ -222,7 +222,12 @@ public void setWorkingTVList(TVList list) { } @Override - public long count() { + public long count(boolean ignoreAllNullRows) { + // a non-aligned chunk never stores all-null rows, so ignoreAllNullRows makes no difference + return pointCount(); + } + + private long pointCount() { long count = list.count(); for (TVList tvList : sortedList) { count += tvList.count(); @@ -241,7 +246,7 @@ public IMeasurementSchema getSchema() { } @Override - public long getMaxTime() { + public long getMaxTime(boolean ignoreAllNullRows) { long maxTime = list.getMaxTime(); for (TVList tvList : sortedList) { maxTime = Math.max(maxTime, tvList.getMaxTime()); @@ -260,23 +265,23 @@ public long getMinTime() { @Override public long getFirstPoint() { - if (count() == 0) { + if (pointCount() == 0) { return Long.MAX_VALUE; } return getMinTime(); } @Override - public long getLastPoint() { - if (count() == 0) { + public long getLastPoint(boolean ignoreAllNullRows) { + if (pointCount() == 0) { return Long.MIN_VALUE; } - return getMaxTime(); + return getMaxTime(ignoreAllNullRows); } @Override - public boolean isEmpty() { - return count() == 0; + public boolean isEmpty(boolean ignoreAllNullRows) { + return pointCount() == 0; } @Override @@ -389,7 +394,11 @@ public void encodeWorkingTVList( } @Override - public void encode(BlockingQueue ioTaskQueue, BatchEncodeInfo encodeInfo, long[] times) { + public void encode( + BlockingQueue ioTaskQueue, + BatchEncodeInfo encodeInfo, + long[] times, + boolean ignoreAllNullRows) { if (TVLIST_SORT_THRESHOLD == 0) { encodeWorkingTVList( ioTaskQueue, encodeInfo.maxNumberOfPointsInChunk, encodeInfo.targetChunkSize); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java index 6f5c63824c491..9f27648b0aef8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkGroup.java @@ -93,10 +93,10 @@ public void release() { } @Override - public long count() { + public long count(boolean ignoreAllNullRows) { long count = 0; for (IWritableMemChunk memChunk : memChunkMap.values()) { - count += memChunk.count(); + count += memChunk.count(ignoreAllNullRows); } return count; } @@ -118,17 +118,19 @@ public void writeRow(long insertTime, Object[] objectValue, List getMemChunkMap() { + public Map getMemChunkMap(boolean ignoreAllNullRows) { + // a group of non-aligned chunks never stores all-null rows, so ignoreAllNullRows makes no + // difference return memChunkMap; } @Override - public boolean isEmpty() { - return memChunkMap.isEmpty() || count() == 0; + public boolean isEmpty(boolean ignoreAllNullRows) { + return memChunkMap.isEmpty() || count(ignoreAllNullRows) == 0; } @Override - public long delete(ModEntry modEntry) { + public long delete(ModEntry modEntry, boolean ignoreAllNullRows) { Iterator> iter = memChunkMap.entrySet().iterator(); long deletedPointsNumber = 0; while (iter.hasNext()) { @@ -140,11 +142,11 @@ public long delete(ModEntry modEntry) { IWritableMemChunk chunk = entry.getValue(); if (modEntry.getStartTime() == Long.MIN_VALUE && modEntry.getEndTime() == Long.MAX_VALUE) { iter.remove(); - deletedPointsNumber += chunk.count(); + deletedPointsNumber += chunk.count(ignoreAllNullRows); chunk.release(); } else { deletedPointsNumber += chunk.delete(modEntry.getStartTime(), modEntry.getEndTime()); - if (chunk.count() == 0) { + if (chunk.count(ignoreAllNullRows) == 0) { iter.remove(); } } @@ -153,8 +155,8 @@ public long delete(ModEntry modEntry) { } @Override - public long deleteTime(ModEntry modEntry) { - return delete(modEntry); + public long deleteTime(ModEntry modEntry, boolean ignoreAllNullRows) { + return delete(modEntry, ignoreAllNullRows); } @Override @@ -166,10 +168,10 @@ public IWritableMemChunk getWritableMemChunk(String measurement) { } @Override - public long getMaxTime() { + public long getMaxTime(boolean ignoreAllNullRows) { long maxTime = Long.MIN_VALUE; for (IWritableMemChunk memChunk : memChunkMap.values()) { - maxTime = Math.max(maxTime, memChunk.getMaxTime()); + maxTime = Math.max(maxTime, memChunk.getMaxTime(ignoreAllNullRows)); } return maxTime; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TreeDiskUsageStatisticUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TreeDiskUsageStatisticUtil.java index ec4d8f03d8683..5317f8628d83f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TreeDiskUsageStatisticUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TreeDiskUsageStatisticUtil.java @@ -165,6 +165,6 @@ private long calculatePathPatternSize( } private boolean matchPathPattern(IDeviceID deviceID) throws IllegalPathException { - return pathPattern.matchFullPath(CompactionPathUtils.getPath(deviceID)); + return pathPattern.matchFullPath(CompactionPathUtils.getPath(deviceID, false)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java index 221d39cdfccb3..8e191d475ba5e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/TsFilePlanRedoer.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; @@ -55,11 +56,15 @@ */ public class TsFilePlanRedoer { private final TsFileResource tsFileResource; + // whether rows whose values are all null should be ignored, which depends on the data model of + // the database this tsfile belongs to + private final boolean ignoreAllNullRows; // store data when redoing logs private IMemTable recoveryMemTable; public TsFilePlanRedoer(TsFileResource tsFileResource) { this.tsFileResource = tsFileResource; + this.ignoreAllNullRows = !PathUtils.isTableModelDatabase(tsFileResource.getDatabaseName()); this.recoveryMemTable = new PrimitiveMemTable(tsFileResource.getDatabaseName(), tsFileResource.getDataRegionId()); WritingMetrics.getInstance().recordActiveMemTableCount(tsFileResource.getDataRegionId(), 1); @@ -73,7 +78,7 @@ void redoDelete(DeleteDataNode deleteDataNode) throws IOException { TreeDeletionEntry deletionEntry = new TreeDeletionEntry( path, deleteDataNode.getDeleteStartTime(), deleteDataNode.getDeleteEndTime()); - recoveryMemTable.delete(deletionEntry); + recoveryMemTable.delete(deletionEntry, ignoreAllNullRows); deletionEntries.add(deletionEntry); } tsFileResource.getModFileForWrite().write(deletionEntries); @@ -81,7 +86,7 @@ void redoDelete(DeleteDataNode deleteDataNode) throws IOException { void redoDelete(RelationalDeleteDataNode node) throws IOException { for (TableDeletionEntry modEntry : node.getModEntries()) { - recoveryMemTable.delete(modEntry); + recoveryMemTable.delete(modEntry, ignoreAllNullRows); } tsFileResource.getModFileForWrite().write(node.getModEntries()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java index d861723419070..d8369e2c406dd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.schema.table.TsFileTableSchemaUtil; +import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.exception.DataRegionException; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; @@ -84,6 +85,7 @@ public class UnsealedTsFileRecoverPerformer extends AbstractTsFileRecoverPerform private final WALRecoverListener recoverListener; private final String databaseName; private final String dataRegionId; + private final boolean isTableModel; public UnsealedTsFileRecoverPerformer( TsFileResource tsFileResource, @@ -92,6 +94,7 @@ public UnsealedTsFileRecoverPerformer( super(tsFileResource); this.databaseName = tsFileResource.getDatabaseName(); this.dataRegionId = tsFileResource.getDataRegionId(); + this.isTableModel = PathUtils.isTableModelDatabase(databaseName); this.sequence = sequence; this.callbackAfterUnsealedTsFileRecovered = callbackAfterUnsealedTsFileRecovered; this.walRedoer = new TsFilePlanRedoer(tsFileResource); @@ -183,24 +186,26 @@ public void redoLog(WALEntry walEntry) { // delete data already flushed in the MemTable to avoid duplicates for (IDeviceID device : tsFileResource.getDevices()) { // iterating the index, must present - if (device.isTableModel()) { + if (isTableModel) { memTable.delete( new TableDeletionEntry( new DeletionPredicate(device.getTableName(), new FullExactMatch(device)), new TimeRange( tsFileResource.getStartTime(device).get(), - tsFileResource.getEndTime(device).get()))); + tsFileResource.getEndTime(device).get())), + !isTableModel); } else { memTable.delete( new TreeDeletionEntry( new MeasurementPath(device, "*"), tsFileResource.getStartTime(device).get(), - tsFileResource.getEndTime(device).get())); + tsFileResource.getEndTime(device).get()), + !isTableModel); } } } for (IDeviceID deviceID : memTable.getMemTableMap().keySet()) { - if (deviceID.isTableModel()) { + if (isTableModel) { registerToTsFile(deviceID.getTableName()); } } @@ -266,10 +271,10 @@ public void endRecovery() throws WALRecoverException { for (Map.Entry deviceEntry : memTableMap.entrySet()) { IDeviceID deviceId = deviceEntry.getKey(); for (Map.Entry measurementEntry : - deviceEntry.getValue().getMemChunkMap().entrySet()) { + deviceEntry.getValue().getMemChunkMap(!isTableModel).entrySet()) { IWritableMemChunk memChunk = measurementEntry.getValue(); tsFileResource.updateStartTime(deviceId, memChunk.getFirstPoint()); - tsFileResource.updateEndTime(deviceId, memChunk.getLastPoint()); + tsFileResource.updateEndTime(deviceId, memChunk.getLastPoint(!isTableModel)); } } // flush memTable @@ -280,7 +285,8 @@ public void endRecovery() throws WALRecoverException { recoveryMemTable, writer, databaseName + FILE_NAME_SEPARATOR + dataRegionId, - dataRegionId); + dataRegionId, + isTableModel); tableFlushTask.syncFlushMemTable(); tsFileResource.updatePlanIndexes(recoveryMemTable.getMinPlanIndex()); tsFileResource.updatePlanIndexes(recoveryMemTable.getMaxPlanIndex()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/TreeViewTabletProjector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/TreeViewTabletProjector.java index d95fb42b11652..3a09af7e48d62 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/TreeViewTabletProjector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/columnfilter/TreeViewTabletProjector.java @@ -66,7 +66,6 @@ public String getDatabaseName() { public Tablet project(final Tablet treeTablet) { if (Objects.isNull(treeTablet) || Objects.isNull(treeTablet.getDeviceId()) - || !treeTablet.getDeviceId().startsWith("root.") || Objects.isNull(treeTablet.getValues())) { return null; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java index 265c027e8ac4d..27078dad188de 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/ImportWAL.java @@ -163,6 +163,17 @@ static int run(final String[] args, final PrintStream out, final PrintStream err .MESSAGE_INFERRED_TABLE_DATABASE_ARG_FROM_WAL_DIRECTORY_ARG_REPLAY_INTO_THIS_DATABASE_Y_YES_A_ACCEPT_ALL_INFERRED_DATABASES_N_QUIT_5B59D833, inferredDatabase, directory)); + try { + requireResolvedDatabases(directoryDatabases); + } catch (final IllegalArgumentException e) { + // The ElasticStrategy and RoundRobinStrategy WAL node allocation strategies share WAL nodes + // between regions, so such a WAL has no per-directory database or data model; repeat the + // declaration where the operator has to decide. + err.println( + ImportWALMessages + .MESSAGE_UNSUPPORTED_WALS_1_WALS_WRITTEN_WITH_THE_ELASTICSTRATEGY_2_WALS_WRITTEN_WITH_THE_ROUNDROBINSTRATEGY_REPLAYING_THEM_MAY_FAIL_C542F812); + throw e; + } final ReplayStatistics statistics = replayWALDirectories( walFiles, @@ -255,7 +266,7 @@ static Options createOptions() { .hasArg() .desc( ImportWALMessages - .MESSAGE_TARGET_DATABASE_FOR_TABLE_MODEL_WAL_ENTRIES_IF_OMITTED_INFER_FROM_THE_WAL_PARENT_DIRECTORY_AND_ASK_FOR_CONFIRMATION_4B1E409D) + .MESSAGE_TARGET_DATABASE_FOR_WAL_REPLAY_IF_OMITTED_INFER_FROM_THE_WAL_PARENT_DIRECTORY_IF_INFERENCE_FAILS_DB_DATABASE_IS_REQUIRED_6EEBC019) .build()); options.addOption( Option.builder() @@ -383,7 +394,8 @@ private static void printHelp(final Options options, final PrintStream stream) { new PrintWriter(stream, true), 120, ImportWALMessages.MESSAGE_IMPORT_WAL_5E42804E, - null, + ImportWALMessages + .MESSAGE_UNSUPPORTED_WALS_1_WALS_WRITTEN_WITH_THE_ELASTICSTRATEGY_2_WALS_WRITTEN_WITH_THE_ROUNDROBINSTRATEGY_REPLAYING_THEM_MAY_FAIL_C542F812, options, 2, 2, @@ -444,8 +456,9 @@ private static WALReplayWorker createWALReplayWorker( final WALReplayer.ReplayDecisionController replayDecisionController) throws IOException { final Session treeSession = createSession(host, port, username, password, null); + final boolean isTableModel = database != null && PathUtils.isTableModelDatabase(database); final Session tableSession = - database == null ? null : createSession(host, port, username, password, database); + isTableModel ? createSession(host, port, username, password, database) : null; try { treeSession.open(false); if (tableSession != null) { @@ -576,6 +589,26 @@ static Map resolveDirectoryDatabases( return databases; } + /** + * The data model of every directory must be known before any replay starts, otherwise a WAL + * snapshot could only be replayed with a guessed model. + */ + static void requireResolvedDatabases(final Map directoryDatabases) { + final List unresolvedDirectories = new ArrayList<>(); + for (final Map.Entry entry : directoryDatabases.entrySet()) { + if (entry.getValue() == null) { + unresolvedDirectories.add(entry.getKey().toString()); + } + } + if (!unresolvedDirectories.isEmpty()) { + throw new IllegalArgumentException( + String.format( + ImportWALMessages + .EXCEPTION_CANNOT_DETERMINE_THE_TARGET_DATABASE_OF_WAL_DIRECTORIES_ARG_SPECIFY_DB_DATABASE_WHICH_APPLIES_TO_ALL_IMPORTED_DIRECTORIES_55B174E4, + String.join(", ", unresolvedDirectories))); + } + } + private static final Comparator WAL_FILE_COMPARATOR = Comparator.comparing((Path path) -> Objects.toString(path.getParent(), "")) .thenComparingLong(ImportWAL::getWALVersion) @@ -918,9 +951,9 @@ private static class SessionWALReplayer extends WALReplayer { private SessionWALReplayer( final Session treeSession, final Session tableSession, - final String tableDatabaseName, + final String targetDatabaseName, final WALReplayer.ReplayDecisionController replayDecisionController) { - super(treeSession, tableSession, tableDatabaseName, replayDecisionController); + super(treeSession, tableSession, targetDatabaseName, replayDecisionController); this.treeSession = treeSession; this.tableSession = tableSession; } @@ -936,30 +969,45 @@ static class WALReplayer implements WALReplayWorker { private final Session treeSession; private final Session tableSession; + private final String targetDatabaseName; + private final boolean isTableModel; private final ConsensusLogToTabletConverter converter; private final ReplayDecisionPrompt replayDecisionPrompt; private final Map tableSchemas = new HashMap<>(); WALReplayer( - final Session treeSession, final Session tableSession, final String tableDatabaseName) { + final Session treeSession, final Session tableSession, final String targetDatabaseName) { this( treeSession, tableSession, - tableDatabaseName, + targetDatabaseName, new ReplayDecisionController(System.console())); } + /** + * @param treeSession the session which replays tree model entries, it carries no database + * @param tableSession the session which replays table model entries, null when no table + * database is declared + * @param targetDatabaseName the target database resolved before replay; null means no target + * database is resolved, so the data model is unknown and snapshot entries are rejected + * until -db/--database is declared + * @param replayDecisionPrompt the prompt which decides how to handle entries that cannot be + * replayed automatically + */ WALReplayer( final Session treeSession, final Session tableSession, - final String tableDatabaseName, + final String targetDatabaseName, final ReplayDecisionPrompt replayDecisionPrompt) { this.treeSession = treeSession; this.tableSession = tableSession; + this.targetDatabaseName = targetDatabaseName; + this.isTableModel = + targetDatabaseName != null && PathUtils.isTableModelDatabase(targetDatabaseName); this.replayDecisionPrompt = replayDecisionPrompt; converter = new ConsensusLogToTabletConverter( - null, null, ColumnFilterMatcher.matchAll(), tableDatabaseName); + null, null, ColumnFilterMatcher.matchAll(), isTableModel ? targetDatabaseName : null); } @Override @@ -972,9 +1020,8 @@ public ReplayResult replay(final WALEntry entry) throws IoTDBConnectionException, StatementExecutionException { if (entry.getType() == WALEntryType.MEMORY_TABLE_SNAPSHOT || entry.getType() == WALEntryType.OLD_MEMORY_TABLE_SNAPSHOT) { - return replayMemTableSnapshot((IMemTable) entry.getValue()) - ? ReplayResult.REPLAYED - : ReplayResult.IGNORED; + final IMemTable memTable = (IMemTable) entry.getValue(); + return replayMemTableSnapshot(memTable) ? ReplayResult.REPLAYED : ReplayResult.IGNORED; } if (entry.getValue() instanceof InsertNode insertNode) { replayInsert(insertNode); @@ -1313,13 +1360,21 @@ private boolean replayMemTableSnapshot(final IMemTable memTable) if (memTable == null || memTable.isSignalMemTable()) { return false; } + if (targetDatabaseName == null && memTable.size(false) > 0) { + // Without a resolved target database the data model of the snapshot is unknown, and + // guessing it would replay table model data into the tree session. An empty snapshot + // carries no data, so it stays ignorable. + throw new StatementExecutionException( + ImportWALMessages + .EXCEPTION_A_WAL_SNAPSHOT_REQUIRES_A_DECLARED_TARGET_DATABASE_TO_DETERMINE_ITS_DATA_MODEL_SPECIFY_DB_DATABASE_382FC74C); + } boolean replayed = false; for (Map.Entry deviceEntry : memTable.getMemTableMap().entrySet()) { final IDeviceID deviceId = deviceEntry.getKey(); final IWritableMemChunkGroup group = deviceEntry.getValue(); - for (IWritableMemChunk chunk : group.getMemChunkMap().values()) { - if (chunk == null || chunk.isEmpty()) { + for (IWritableMemChunk chunk : group.getMemChunkMap(!isTableModel).values()) { + if (chunk == null || chunk.isEmpty(!isTableModel)) { continue; } if (chunk instanceof AlignedWritableMemChunk) { @@ -1336,8 +1391,7 @@ private boolean replayNonAlignedMemChunk( final IDeviceID deviceId, final IWritableMemChunk chunk) throws IoTDBConnectionException, StatementExecutionException { final List schemas = Collections.singletonList(chunk.getSchema()); - final boolean tableModel = deviceId.isTableModel(); - requireTableSessionIfNeeded(tableModel); + requireTableSessionIfNeeded(isTableModel); final TableTabletSchema tabletSchema = createTableTabletSchema(deviceId, schemas); final List lists = new ArrayList<>(); lists.addAll(chunk.getSortedList()); @@ -1354,7 +1408,7 @@ private boolean replayNonAlignedMemChunk( final int end = Math.min(start + SNAPSHOT_TABLET_ROW_LIMIT, list.rowCount()); final Tablet tablet = buildNonAlignedTablet(deviceId, tabletSchema, schemas, list, start, end); - sendTablet(tablet, tableModel, false); + sendTablet(tablet, isTableModel, false); replayed = true; } } @@ -1364,8 +1418,7 @@ private boolean replayNonAlignedMemChunk( private boolean replayAlignedMemChunk( final IDeviceID deviceId, final AlignedWritableMemChunk chunk) throws IoTDBConnectionException, StatementExecutionException { - final boolean tableModel = deviceId.isTableModel(); - requireTableSessionIfNeeded(tableModel); + requireTableSessionIfNeeded(isTableModel); final List schemas = chunk.getSchemaList(); final TableTabletSchema tabletSchema = createTableTabletSchema(deviceId, schemas); final List lists = new ArrayList<>(); @@ -1384,7 +1437,7 @@ private boolean replayAlignedMemChunk( final int end = Math.min(start + SNAPSHOT_TABLET_ROW_LIMIT, replayableRows.size()); final Tablet tablet = buildAlignedTablet(deviceId, tabletSchema, schemas, list, replayableRows, start, end); - sendTablet(tablet, tableModel, true); + sendTablet(tablet, isTableModel, true); replayed = true; } } @@ -1412,7 +1465,7 @@ private void requireTableSessionIfNeeded(final boolean tableModel) private TableTabletSchema createTableTabletSchema( final IDeviceID deviceId, final List fieldSchemas) throws IoTDBConnectionException, StatementExecutionException { - if (!deviceId.isTableModel()) { + if (!isTableModel) { return new TableTabletSchema(fieldSchemas, null, 0); } final List tagSchemas = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java index 0ea1abee866e6..e6dd6170ae6e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java @@ -280,16 +280,17 @@ public static boolean isAllDeletedByMods( final PatternTreeMap modifications, final IDeviceID device, final long startTime, - final long endTime) + final long endTime, + final boolean isTableModel) throws IllegalPathException { final List mods = modifications.getOverlapped( - CompactionPathUtils.getPath(device, AlignedPath.VECTOR_PLACEHOLDER)); + CompactionPathUtils.getPath(device, AlignedPath.VECTOR_PLACEHOLDER, isTableModel)); return mods.stream() .anyMatch( modification -> modification.getTimeRange().contains(startTime, endTime) - && (!device.isTableModel() || modification.affects(device))); + && (!isTableModel || modification.affects(device))); } public static boolean isAllDeletedByMods( @@ -310,15 +311,16 @@ public static boolean isTimeSeriesDeletedByMods( final IDeviceID device, final String measurement, final long startTime, - final long endTime) + final long endTime, + final boolean isTableModel) throws IllegalPathException { final List mods = - modifications.getOverlapped(CompactionPathUtils.getPath(device, measurement)); + modifications.getOverlapped(CompactionPathUtils.getPath(device, measurement, isTableModel)); return mods.stream() .anyMatch( modification -> modification.getTimeRange().contains(startTime, endTime) - && (!device.isTableModel() + && (!isTableModel || modification.affects(device) && modification.affects(measurement))); } @@ -452,7 +454,8 @@ public static List sortAndMerge(List modifications) { public static boolean isDeviceDeletedByMods( final PatternTreeMap currentModifications, final ITimeIndex currentTimeIndex, - final IDeviceID device) + final IDeviceID device, + final boolean isTableModel) throws IllegalPathException { return isAllDeletedByMods( currentModifications, @@ -462,6 +465,7 @@ public static boolean isDeviceDeletedByMods( : currentTimeIndex.getStartTime(device).orElse(Long.MIN_VALUE), Objects.isNull(currentTimeIndex) ? Long.MAX_VALUE - : currentTimeIndex.getEndTime(device).orElse(Long.MAX_VALUE)); + : currentTimeIndex.getEndTime(device).orElse(Long.MAX_VALUE), + isTableModel); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java index f5c3f1fa05c8c..847f48e939327 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java @@ -338,14 +338,14 @@ public void convertToTabletForTest() throws Exception { Assert.assertFalse(isAligned2); PipeRawTabletInsertionEvent event3 = - new PipeRawTabletInsertionEvent(tablet1, false, new PrefixTreePattern(pattern)); + new PipeRawTabletInsertionEvent(false, tablet1, false, new PrefixTreePattern(pattern)); Tablet tablet3 = event3.convertToTablet(); boolean isAligned3 = event3.isAligned(); Assert.assertEquals(tablet1, tablet3); Assert.assertFalse(isAligned3); PipeRawTabletInsertionEvent event4 = - new PipeRawTabletInsertionEvent(tablet2, false, new PrefixTreePattern(pattern)); + new PipeRawTabletInsertionEvent(false, tablet2, false, new PrefixTreePattern(pattern)); Tablet tablet4 = event4.convertToTablet(); boolean isAligned4 = event4.isAligned(); Assert.assertEquals(tablet2, tablet4); @@ -371,14 +371,14 @@ public void convertToAlignedTabletForTest() throws Exception { Assert.assertTrue(isAligned2); PipeRawTabletInsertionEvent event3 = - new PipeRawTabletInsertionEvent(tablet1, true, new PrefixTreePattern(pattern)); + new PipeRawTabletInsertionEvent(false, tablet1, true, new PrefixTreePattern(pattern)); Tablet tablet3 = event3.convertToTablet(); boolean isAligned3 = event3.isAligned(); Assert.assertEquals(tablet1, tablet3); Assert.assertTrue(isAligned3); PipeRawTabletInsertionEvent event4 = - new PipeRawTabletInsertionEvent(tablet2, true, new PrefixTreePattern(pattern)); + new PipeRawTabletInsertionEvent(false, tablet2, true, new PrefixTreePattern(pattern)); Tablet tablet4 = event4.convertToTablet(); boolean isAligned4 = event4.isAligned(); Assert.assertEquals(tablet2, tablet4); @@ -389,7 +389,7 @@ public void convertToAlignedTabletForTest() throws Exception { public void processAlignedTabletWithCollectPreservesAlignmentForTest() { final PipeRawTabletInsertionEvent event = new PipeRawTabletInsertionEvent( - tabletForInsertTabletNode, true, new PrefixTreePattern(pattern)); + false, tabletForInsertTabletNode, true, new PrefixTreePattern(pattern)); final List events = new ArrayList<>(); event @@ -543,7 +543,7 @@ public void convertToTabletWithFilteredRowsForTest() throws Exception { TabletInsertionEventTreePatternParser container1 = new TabletInsertionEventTreePatternParser( null, - new PipeRawTabletInsertionEvent(tabletForInsertRowNode, 111L, 113L), + new PipeRawTabletInsertionEvent(false, tabletForInsertRowNode, 111L, 113L), insertRowNode, new PrefixTreePattern(pattern), null); @@ -555,7 +555,7 @@ public void convertToTabletWithFilteredRowsForTest() throws Exception { TabletInsertionEventTreePatternParser container2 = new TabletInsertionEventTreePatternParser( null, - new PipeRawTabletInsertionEvent(tabletForInsertRowNode, 110L, 110L), + new PipeRawTabletInsertionEvent(false, tabletForInsertRowNode, 110L, 110L), insertRowNode, new PrefixTreePattern(pattern), null); @@ -567,7 +567,7 @@ public void convertToTabletWithFilteredRowsForTest() throws Exception { TabletInsertionEventTreePatternParser container3 = new TabletInsertionEventTreePatternParser( null, - new PipeRawTabletInsertionEvent(tabletForInsertTabletNode, 111L, 113L), + new PipeRawTabletInsertionEvent(false, tabletForInsertTabletNode, 111L, 113L), insertTabletNode, new PrefixTreePattern(pattern), null); @@ -579,7 +579,7 @@ public void convertToTabletWithFilteredRowsForTest() throws Exception { TabletInsertionEventTreePatternParser container4 = new TabletInsertionEventTreePatternParser( null, - new PipeRawTabletInsertionEvent(tabletForInsertTabletNode, Long.MIN_VALUE, 109L), + new PipeRawTabletInsertionEvent(false, tabletForInsertTabletNode, Long.MIN_VALUE, 109L), insertTabletNode, new PrefixTreePattern(pattern), null); @@ -593,20 +593,20 @@ public void convertToTabletWithFilteredRowsForTest() throws Exception { public void isEventTimeOverlappedWithTimeRangeTest() { PipeRawTabletInsertionEvent event; - event = new PipeRawTabletInsertionEvent(tabletForInsertRowNode, 111L, 113L); + event = new PipeRawTabletInsertionEvent(false, tabletForInsertRowNode, 111L, 113L); Assert.assertFalse(event.mayEventTimeOverlappedWithTimeRange()); - event = new PipeRawTabletInsertionEvent(tabletForInsertRowNode, 110L, 110L); + event = new PipeRawTabletInsertionEvent(false, tabletForInsertRowNode, 110L, 110L); Assert.assertTrue(event.mayEventTimeOverlappedWithTimeRange()); - event = new PipeRawTabletInsertionEvent(tabletForInsertTabletNode, 111L, 113L); + event = new PipeRawTabletInsertionEvent(false, tabletForInsertTabletNode, 111L, 113L); Assert.assertTrue(event.mayEventTimeOverlappedWithTimeRange()); - event = new PipeRawTabletInsertionEvent(tabletForInsertTabletNode, Long.MIN_VALUE, 110L); + event = new PipeRawTabletInsertionEvent(false, tabletForInsertTabletNode, Long.MIN_VALUE, 110L); Assert.assertTrue(event.mayEventTimeOverlappedWithTimeRange()); - event = new PipeRawTabletInsertionEvent(tabletForInsertTabletNode, 114L, Long.MAX_VALUE); + event = new PipeRawTabletInsertionEvent(false, tabletForInsertTabletNode, 114L, Long.MAX_VALUE); Assert.assertTrue(event.mayEventTimeOverlappedWithTimeRange()); - event = new PipeRawTabletInsertionEvent(tabletForInsertTabletNode, Long.MIN_VALUE, 109L); + event = new PipeRawTabletInsertionEvent(false, tabletForInsertTabletNode, Long.MIN_VALUE, 109L); Assert.assertFalse(event.mayEventTimeOverlappedWithTimeRange()); - event = new PipeRawTabletInsertionEvent(tabletForInsertTabletNode, 115L, Long.MAX_VALUE); + event = new PipeRawTabletInsertionEvent(false, tabletForInsertTabletNode, 115L, Long.MAX_VALUE); Assert.assertFalse(event.mayEventTimeOverlappedWithTimeRange()); } @@ -619,9 +619,9 @@ public void isEventTimeOverlappedWithTimeRangeUsesActualRowSizeForTest() throws partialTablet.setRowSize(3); PipeRawTabletInsertionEvent rawEvent = - new PipeRawTabletInsertionEvent(partialTablet, 111L, 112L); + new PipeRawTabletInsertionEvent(false, partialTablet, 111L, 112L); Assert.assertTrue(rawEvent.mayEventTimeOverlappedWithTimeRange()); - rawEvent = new PipeRawTabletInsertionEvent(partialTablet, 113L, Long.MAX_VALUE); + rawEvent = new PipeRawTabletInsertionEvent(false, partialTablet, 113L, Long.MAX_VALUE); Assert.assertFalse(rawEvent.mayEventTimeOverlappedWithTimeRange()); final InsertTabletNode partialInsertTabletNode = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/CachedSchemaPatternMatcherTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/CachedSchemaPatternMatcherTest.java index 832bae3935256..3e99576bee146 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/CachedSchemaPatternMatcherTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/CachedSchemaPatternMatcherTest.java @@ -223,6 +223,7 @@ public void testTableModelMatchesEachTableOncePerEvent() throws Exception { countingMatcher.register(new PipeRealtimeDataRegionFakeSource()); final PipeInsertionEvent insertionEvent = Mockito.mock(PipeInsertionEvent.class); + Mockito.when(insertionEvent.isTableModelEvent()).thenReturn(true); Mockito.when(insertionEvent.getTableModelDatabaseName()).thenReturn("db"); final Map schemaInfo = new LinkedHashMap<>(); schemaInfo.put(new StringArrayDeviceID("table1", "tag1"), new String[0]); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java index fc4a15a885b5e..ea3642777dd29 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java @@ -86,6 +86,7 @@ public class PipeRealtimeExtractTest { private static final String TEST_PIPE_NAME = "test_degraded_status_pipe"; private static final long TEST_PIPE_CREATION_TIME = 1L; private static final String TEST_REFERENCE_HOLDER = PipeRealtimeExtractTest.class.getName(); + private static final String DATABASE = "root.sg"; private final int dataRegion1 = 1; private final int dataRegion2 = 2; @@ -114,7 +115,7 @@ public void setUp() throws Exception { + File.separator + IoTDBConstant.SEQUENCE_FOLDER_NAME + File.separator - + "root.sg"); + + DATABASE); } @After @@ -314,7 +315,7 @@ public void testListenToTsFileSkipsAssignerWithoutTsFileSource() throws Exceptio resource.close(); PipeInsertionDataNodeListener.getInstance() - .listenToTsFile(dataRegion1, Integer.toString(dataRegion1), resource, false); + .listenToTsFile(dataRegion1, DATABASE, resource, false); final long deadline = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(1); while (System.currentTimeMillis() < deadline @@ -353,7 +354,7 @@ public void testHybridSourceReportsTsFileEpochDegradedStatus() throws Exception final TsFileResource resource = createTsFileResource(dataRegion1, "100-100-0-0.tsfile"); final PipeRealtimeEvent tabletEvent = PipeRealtimeEventFactory.createRealtimeEvent( - false, "root.sg", createInsertRowNode("degraded-tablet", "a"), resource); + false, DATABASE, createInsertRowNode("degraded-tablet", "a"), resource); Assert.assertTrue(tabletEvent.increaseReferenceCount(TEST_REFERENCE_HOLDER)); extractor.extract(tabletEvent); @@ -364,7 +365,7 @@ public void testHybridSourceReportsTsFileEpochDegradedStatus() throws Exception Assert.assertEquals(Boolean.TRUE, getGlobalTsFileEpochDegraded()); final PipeRealtimeEvent tsFileEvent = - PipeRealtimeEventFactory.createRealtimeEvent(false, "root.sg", resource, false); + PipeRealtimeEventFactory.createRealtimeEvent(false, DATABASE, resource, false); Assert.assertTrue(tsFileEvent.increaseReferenceCount(TEST_REFERENCE_HOLDER)); extractor.extract(tsFileEvent); @@ -411,7 +412,7 @@ private Future write2DataRegion( PipeInsertionDataNodeListener.getInstance() .listenToInsertNode( dataRegionId, - Integer.toString(dataRegionId), + DATABASE, new InsertRowNode( new PlanNodeId(String.valueOf(i)), new PartialPath(device), @@ -425,7 +426,7 @@ private Future write2DataRegion( PipeInsertionDataNodeListener.getInstance() .listenToInsertNode( dataRegionId, - Integer.toString(dataRegionId), + DATABASE, new InsertRowNode( new PlanNodeId(String.valueOf(i)), new PartialPath(device), @@ -437,7 +438,7 @@ private Future write2DataRegion( false), resource); PipeInsertionDataNodeListener.getInstance() - .listenToTsFile(dataRegionId, Integer.toString(dataRegionId), resource, false); + .listenToTsFile(dataRegionId, DATABASE, resource, false); } }); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java index 79e66f92d9e66..0caf1692f635b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceExecutionTest.java @@ -209,7 +209,7 @@ public void testTVListOwnerTransferTimeOnlyAlignedReleasesAllValueColumns() new MeasurementSchema("s0", TSDataType.INT64, TSEncoding.PLAIN), new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN), new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.PLAIN))); - AlignedWritableMemChunk memChunk = new AlignedWritableMemChunk(schemas, true); + AlignedWritableMemChunk memChunk = new AlignedWritableMemChunk(schemas); for (int i = 0; i < 100; i++) { memChunk.putAlignedRow(i, new Object[] {(long) i, (long) i * 2, (long) i * 3}); } @@ -265,8 +265,8 @@ public void testTVListCloneForQuery() throws InterruptedException { IMemTable memTable = createMemTable(deviceId, measurementId); assertEquals(1, memTable.getMemTableMap().size()); IWritableMemChunkGroup memChunkGroup = memTable.getMemTableMap().values().iterator().next(); - assertEquals(1, memChunkGroup.getMemChunkMap().size()); - IWritableMemChunk memChunk = memChunkGroup.getMemChunkMap().values().iterator().next(); + assertEquals(1, memChunkGroup.getMemChunkMap(true).size()); + IWritableMemChunk memChunk = memChunkGroup.getMemChunkMap(true).values().iterator().next(); TVList tvList = memChunk.getWorkingTVList(); assertFalse(tvList.isSorted()); @@ -330,8 +330,8 @@ public void testAlignedTVListPartialColumnCloneEndToEnd() throws InterruptedExce assertEquals(1, memTable.getMemTableMap().size()); IWritableMemChunkGroup memChunkGroup = memTable.getMemTableMap().values().iterator().next(); - assertEquals(1, memChunkGroup.getMemChunkMap().size()); - IWritableMemChunk memChunk = memChunkGroup.getMemChunkMap().values().iterator().next(); + assertEquals(1, memChunkGroup.getMemChunkMap(true).size()); + IWritableMemChunk memChunk = memChunkGroup.getMemChunkMap(true).values().iterator().next(); AlignedTVList workingTvList = (AlignedTVList) memChunk.getWorkingTVList(); assertFalse(workingTvList.isSorted()); assertEquals(100, workingTvList.rowCount()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java index 951585605df82..b51910f84efdc 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/AbstractCompactionTest.java @@ -163,30 +163,16 @@ public class AbstractCompactionTest { + "sequence" + File.separator + COMPACTION_TEST_SG); - protected static File SEQ_DIRS = - new File( - TestConstant.BASE_OUTPUT_PATH - + "data" - + File.separator - + "sequence" - + File.separator - + COMPACTION_TEST_SG - + File.separator - + "0" - + File.separator - + "0"); - protected static File UNSEQ_DIRS = + protected static File UNSEQ_STORAGE_GROUP_DIR = new File( TestConstant.BASE_OUTPUT_PATH + "data" + File.separator + "unsequence" + File.separator - + COMPACTION_TEST_SG - + File.separator - + "0" - + File.separator - + "0"); + + COMPACTION_TEST_SG); + protected static File SEQ_DIRS = new File(STORAGE_GROUP_DIR, "0" + File.separator + "0"); + protected static File UNSEQ_DIRS = new File(UNSEQ_STORAGE_GROUP_DIR, "0" + File.separator + "0"); protected Map> registeredTimePartitionDirs = new HashMap<>(); @@ -194,21 +180,55 @@ public class AbstractCompactionTest { private int fileCount = 0; - protected TsFileManager tsFileManager = - new TsFileManager(COMPACTION_TEST_SG, "0", STORAGE_GROUP_DIR.getPath()); + protected TsFileManager tsFileManager; protected CompactionFakeSchemaFetcherImpl schemaFetcher; + /** + * The database of the files created by this fixture and the directories that hold them. The + * defaults are the tree model database used by most of the compaction tests. A fixture that + * writes files of another data model overrides them, because the data model of the compaction + * input is resolved from the database directory of its source files: a table model database name + * must not start with {@code root.} + */ + protected String getTestStorageGroup() { + return COMPACTION_TEST_SG; + } + + /** The directory of the sequence files of the database of this fixture. */ + protected File getSeqStorageGroupDir() { + return STORAGE_GROUP_DIR; + } + + /** The directory of the unsequence files of the database of this fixture. */ + protected File getUnseqStorageGroupDir() { + return UNSEQ_STORAGE_GROUP_DIR; + } + + /** The directory of the data region 0, time partition 0 of the sequence files. */ + protected File getSeqDirs() { + return SEQ_DIRS; + } + + /** The directory of the data region 0, time partition 0 of the unsequence files. */ + protected File getUnseqDirs() { + return UNSEQ_DIRS; + } + + protected TsFileManager createTsFileManager() { + return new TsFileManager(getTestStorageGroup(), "0", getSeqStorageGroupDir().getPath()); + } + public void setUp() throws IOException, WriteProcessException, MetadataException, InterruptedException { MetadataLeaseManager.getInstance().updateFenceThresholdMs(Long.MAX_VALUE); MetadataLeaseManager.getInstance().recoveryLeaseForTest(true); fileCount = 0; - if (!SEQ_DIRS.exists()) { - Assert.assertTrue(SEQ_DIRS.mkdirs()); + if (!getSeqDirs().exists()) { + Assert.assertTrue(getSeqDirs().mkdirs()); } - if (!UNSEQ_DIRS.exists()) { - Assert.assertTrue(UNSEQ_DIRS.mkdirs()); + if (!getUnseqDirs().exists()) { + Assert.assertTrue(getUnseqDirs().mkdirs()); } dataType = TSDataType.INT64; CompactionTaskManager.getInstance().restart(); @@ -217,11 +237,12 @@ public void setUp() ChunkCache.getInstance().clear(); TimeSeriesMetadataCache.getInstance().clear(); BloomFilterCache.getInstance().clear(); + tsFileManager = createTsFileManager(); tsFileManager.getOrCreateSequenceListByTimePartition(0); tsFileManager.getOrCreateUnsequenceListByTimePartition(0); - registeredTimePartitionDirs.put(0L, new Pair<>(SEQ_DIRS, UNSEQ_DIRS)); + registeredTimePartitionDirs.put(0L, new Pair<>(getSeqDirs(), getUnseqDirs())); schemaFetcher = new CompactionFakeSchemaFetcherImpl(); - schemaFetcher.getSchemaTree().setDatabases(Collections.singleton(COMPACTION_TEST_SG)); + schemaFetcher.getSchemaTree().setDatabases(Collections.singleton(getTestStorageGroup())); CompactionUtils.setSchemaFetcher(schemaFetcher); } @@ -230,30 +251,10 @@ protected void createTimePartitionDirIfNotExist(long timePartition) { return; } File seqTimePartitionDir = - new File( - TestConstant.BASE_OUTPUT_PATH - + "data" - + File.separator - + "sequence" - + File.separator - + COMPACTION_TEST_SG - + File.separator - + "0" - + File.separator - + timePartition); + new File(getSeqStorageGroupDir(), "0" + File.separator + timePartition); seqTimePartitionDir.mkdirs(); File unseqTimePartitionDir = - new File( - TestConstant.BASE_OUTPUT_PATH - + "data" - + File.separator - + "unsequence" - + File.separator - + COMPACTION_TEST_SG - + File.separator - + "0" - + File.separator - + timePartition); + new File(getUnseqStorageGroupDir(), "0" + File.separator + timePartition); unseqTimePartitionDir.mkdirs(); registeredTimePartitionDirs.put( timePartition, new Pair(seqTimePartitionDir, unseqTimePartitionDir)); @@ -293,9 +294,9 @@ protected void createFiles( timestamp[fileCount++] + FilePathUtils.FILE_NAME_SEPARATOR + fileVersion + "-0-0.tsfile"; String filePath; if (isSeq) { - filePath = SEQ_DIRS.getPath() + File.separator + fileName; + filePath = getSeqDirs().getPath() + File.separator + fileName; } else { - filePath = UNSEQ_DIRS.getPath() + File.separator + fileName; + filePath = getUnseqDirs().getPath() + File.separator + fileName; } File file; if (isAlign) { @@ -365,9 +366,9 @@ protected void createFilesWithTextValue( timestamp[fileCount++] + FilePathUtils.FILE_NAME_SEPARATOR + fileVersion + "-0-0.tsfile"; String filePath; if (isSeq) { - filePath = SEQ_DIRS.getPath() + File.separator + fileName; + filePath = getSeqDirs().getPath() + File.separator + fileName; } else { - filePath = UNSEQ_DIRS.getPath() + File.separator + fileName; + filePath = getUnseqDirs().getPath() + File.separator + fileName; } File file; if (isAlign) { @@ -511,11 +512,11 @@ public void tearDown() throws IOException, StorageEngineException { EnvironmentUtils.cleanAllDir(); - if (SEQ_DIRS.exists()) { - FileUtils.deleteDirectory(SEQ_DIRS); + if (getSeqDirs().exists()) { + FileUtils.deleteDirectory(getSeqDirs()); } - if (UNSEQ_DIRS.exists()) { - FileUtils.deleteDirectory(UNSEQ_DIRS); + if (getUnseqDirs().exists()) { + FileUtils.deleteDirectory(getUnseqDirs()); } for (Map.Entry> entry : registeredTimePartitionDirs.entrySet()) { File seqDir = entry.getValue().left; @@ -738,9 +739,9 @@ protected TsFileResource createEmptyFileAndResource(boolean isSeq, int innerComp + String.format("-%d-0.tsfile", innerCompactionCnt); String filePath; if (isSeq) { - filePath = SEQ_DIRS.getPath() + File.separator + fileName; + filePath = getSeqDirs().getPath() + File.separator + fileName; } else { - filePath = UNSEQ_DIRS.getPath() + File.separator + fileName; + filePath = getUnseqDirs().getPath() + File.separator + fileName; } TsFileResource resource = new TsFileResource(new File(filePath)); resource.updatePlanIndexes(fileVersion); @@ -752,9 +753,9 @@ protected TsFileResource createEmptyFileAndResourceWithName( String fileName, boolean isSeq, int innerCompactionCnt) { String filePath; if (isSeq) { - filePath = SEQ_DIRS.getPath() + File.separator + fileName; + filePath = getSeqDirs().getPath() + File.separator + fileName; } else { - filePath = UNSEQ_DIRS.getPath() + File.separator + fileName; + filePath = getUnseqDirs().getPath() + File.separator + fileName; } TsFileResource resource = new TsFileResource(new File(filePath)); resource.setStatusForTest(TsFileResourceStatus.NORMAL); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java index 24551efa8d2cf..e9ea0025250ed 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/alterDataType/CompactionDataTypeAlterTableTest.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask; +import org.apache.iotdb.db.storageengine.dataregion.compaction.tablemodel.AbstractTableModelCompactionTest; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; @@ -50,10 +51,13 @@ import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.Collection; +import static org.apache.iotdb.db.storageengine.dataregion.compaction.tablemodel.AbstractTableModelCompactionTest.TABLE_MODEL_TEST_SG; + @RunWith(Parameterized.class) public class CompactionDataTypeAlterTableTest extends AbstractCompactionAlterDataTypeTest { @@ -80,19 +84,44 @@ public static Collection data() { }); } + @Override + protected String getTestStorageGroup() { + return TABLE_MODEL_TEST_SG; + } + + @Override + protected File getSeqStorageGroupDir() { + return AbstractTableModelCompactionTest.TABLE_MODEL_SEQ_STORAGE_GROUP_DIR; + } + + @Override + protected File getUnseqStorageGroupDir() { + return AbstractTableModelCompactionTest.TABLE_MODEL_UNSEQ_STORAGE_GROUP_DIR; + } + + @Override + protected File getSeqDirs() { + return AbstractTableModelCompactionTest.TABLE_MODEL_SEQ_DIRS; + } + + @Override + protected File getUnseqDirs() { + return AbstractTableModelCompactionTest.TABLE_MODEL_UNSEQ_DIRS; + } + @Before @Override public void setUp() throws IOException, WriteProcessException, MetadataException, InterruptedException { super.setUp(); - DataNodeTableCache.getInstance().invalid(COMPACTION_TEST_SG); + DataNodeTableCache.getInstance().invalid(TABLE_MODEL_TEST_SG); } @After @Override public void tearDown() throws IOException, StorageEngineException { super.tearDown(); - DataNodeTableCache.getInstance().invalid(COMPACTION_TEST_SG); + DataNodeTableCache.getInstance().invalid(TABLE_MODEL_TEST_SG); } @Test @@ -159,8 +188,8 @@ private void createTable(String tableName, TSDataType dataType) { tsTable.addColumnSchema(new TagColumnSchema("id_column", TSDataType.STRING)); tsTable.addColumnSchema( new FieldColumnSchema("s1", dataType, TSEncoding.PLAIN, CompressionType.LZ4)); - DataNodeTableCache.getInstance().preUpdateTable(this.COMPACTION_TEST_SG, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(this.COMPACTION_TEST_SG, tableName, null); + DataNodeTableCache.getInstance().preUpdateTable(TABLE_MODEL_TEST_SG, tsTable, null); + DataNodeTableCache.getInstance().commitUpdateTable(TABLE_MODEL_TEST_SG, tableName, null); } private TsFileResource generateInt32TableFile(TimeRange timeRange, boolean seq) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/AbstractTableModelCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/AbstractTableModelCompactionTest.java new file mode 100644 index 0000000000000..159c4d229a0d4 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/AbstractTableModelCompactionTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.compaction.tablemodel; + +import org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest; +import org.apache.iotdb.db.utils.constant.TestConstant; + +import java.io.File; + +/** + * The base class of the compaction tests whose source files contain table model data. + * + *

The data model of the compaction input is resolved from the database directory of the source + * files, and a table model database name must not start with {@code root.}, therefore these tests + * use a table model database instead of the tree model database used by the default fixture. + */ +public abstract class AbstractTableModelCompactionTest extends AbstractCompactionTest { + + /** A table model database name, which must not start with "root.". */ + public static final String TABLE_MODEL_TEST_SG = "testsg"; + + public static final File TABLE_MODEL_SEQ_STORAGE_GROUP_DIR = + new File( + TestConstant.BASE_OUTPUT_PATH + + "data" + + File.separator + + "sequence" + + File.separator + + TABLE_MODEL_TEST_SG); + public static final File TABLE_MODEL_UNSEQ_STORAGE_GROUP_DIR = + new File( + TestConstant.BASE_OUTPUT_PATH + + "data" + + File.separator + + "unsequence" + + File.separator + + TABLE_MODEL_TEST_SG); + public static final File TABLE_MODEL_SEQ_DIRS = + new File(TABLE_MODEL_SEQ_STORAGE_GROUP_DIR, "0" + File.separator + "0"); + public static final File TABLE_MODEL_UNSEQ_DIRS = + new File(TABLE_MODEL_UNSEQ_STORAGE_GROUP_DIR, "0" + File.separator + "0"); + + @Override + protected String getTestStorageGroup() { + return TABLE_MODEL_TEST_SG; + } + + @Override + protected File getSeqStorageGroupDir() { + return TABLE_MODEL_SEQ_STORAGE_GROUP_DIR; + } + + @Override + protected File getUnseqStorageGroupDir() { + return TABLE_MODEL_UNSEQ_STORAGE_GROUP_DIR; + } + + @Override + protected File getSeqDirs() { + return TABLE_MODEL_SEQ_DIRS; + } + + @Override + protected File getUnseqDirs() { + return TABLE_MODEL_UNSEQ_DIRS; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java index 69e83e769a3ce..2de0d95bbc98c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java @@ -26,7 +26,6 @@ import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache; import org.apache.iotdb.db.storageengine.buffer.ChunkCache; import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; -import org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.constant.InnerSeqCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.constant.InnerUnseqCompactionPerformer; @@ -58,7 +57,7 @@ import java.util.Collections; @RunWith(Parameterized.class) -public class CompactionWithAllNullRowsTest extends AbstractCompactionTest { +public class CompactionWithAllNullRowsTest extends AbstractTableModelCompactionTest { private final String performerType; private String threadName; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java index 6746f0a274697..877756b73d5b7 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java @@ -25,7 +25,6 @@ import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; -import org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -47,7 +46,7 @@ import java.util.concurrent.TimeUnit; @RunWith(Parameterized.class) -public class TableModelCompactionWithTTLTest extends AbstractCompactionTest { +public class TableModelCompactionWithTTLTest extends AbstractTableModelCompactionTest { private final String performerType; private String threadName; @@ -57,7 +56,7 @@ public void setUp() throws IOException, WriteProcessException, MetadataException, InterruptedException { this.threadName = Thread.currentThread().getName(); Thread.currentThread().setName("pool-1-IoTDB-Compaction-Worker-1"); - DataNodeTableCache.getInstance().invalid(this.COMPACTION_TEST_SG); + DataNodeTableCache.getInstance().invalid(TABLE_MODEL_TEST_SG); super.setUp(); } @@ -65,7 +64,7 @@ public void setUp() public void tearDown() throws IOException, StorageEngineException { super.tearDown(); Thread.currentThread().setName(threadName); - DataNodeTableCache.getInstance().invalid(this.COMPACTION_TEST_SG); + DataNodeTableCache.getInstance().invalid(TABLE_MODEL_TEST_SG); } public TableModelCompactionWithTTLTest(String performerType) { @@ -170,7 +169,7 @@ public void createTable(String tableName, long ttl) { tsTable.addColumnSchema( new FieldColumnSchema("s1", TSDataType.STRING, TSEncoding.PLAIN, CompressionType.LZ4)); tsTable.addProp(TsTable.TTL_PROPERTY, ttl + ""); - DataNodeTableCache.getInstance().preUpdateTable(this.COMPACTION_TEST_SG, tsTable, null); - DataNodeTableCache.getInstance().commitUpdateTable(this.COMPACTION_TEST_SG, tableName, null); + DataNodeTableCache.getInstance().preUpdateTable(TABLE_MODEL_TEST_SG, tsTable, null); + DataNodeTableCache.getInstance().commitUpdateTable(TABLE_MODEL_TEST_SG, tableName, null); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelFastCompactionPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelFastCompactionPerformerTest.java index 778220d826389..8096003b9c916 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelFastCompactionPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelFastCompactionPerformerTest.java @@ -21,7 +21,6 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.db.exception.StorageEngineException; -import org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CrossSpaceCompactionTask; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask; @@ -38,14 +37,12 @@ import org.junit.Before; import org.junit.Test; -import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.util.Arrays; import java.util.Collections; import java.util.List; -public class TableModelFastCompactionPerformerTest extends AbstractCompactionTest { +public class TableModelFastCompactionPerformerTest extends AbstractTableModelCompactionTest { @Before public void setUp() @@ -143,62 +140,6 @@ public void testCrossSpaceCompactionOfTwoTableModelWithFastCompactionPerformer() } } - @Test - public void testCompactionWithV3Tsfile() throws IOException { - String pathStr = - this.getClass().getClassLoader().getResource("v3tsfile/compaction-test-tsfile").getFile(); - File v3TsFile = new File(pathStr); - File v3TsFileResource = new File(pathStr + "-resource"); - TsFileResource resource1 = createEmptyFileAndResource(true); - Files.copy(v3TsFile.toPath(), resource1.getTsFile().toPath()); - Files.copy( - v3TsFileResource.toPath(), new File(resource1.getTsFilePath() + ".resource").toPath()); - resource1.deserialize(); - - TsFileResource resource2 = createEmptyFileAndResource(true); - try (CompactionTableModelTestFileWriter writer = - new CompactionTableModelTestFileWriter(resource2)) { - writer.registerTableSchema("db1.t1", Arrays.asList("id1", "id2")); - writer.startChunkGroup("db1.t1", Arrays.asList("id_field1", "id_field2")); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Collections.singletonList("s1"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - - writer.startChunkGroup("d1"); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Collections.singletonList("s1"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - - writer.startChunkGroup("node1.node2.device"); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Collections.singletonList("s1"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - - writer.endFile(); - } - - seqResources.add(resource1); - seqResources.add(resource2); - InnerSpaceCompactionTask task = - new InnerSpaceCompactionTask( - 0, tsFileManager, seqResources, true, new FastCompactionPerformer(false), 0); - Assert.assertTrue(task.start()); - try (TsFileSequenceReader reader = - new TsFileSequenceReader( - tsFileManager.getTsFileList(true).get(0).getTsFile().getAbsolutePath())) { - Assert.assertEquals(1, reader.getTableSchemaMap().size()); - } - } - @Test public void testCompactionWithMultiTableSchema() throws IOException { TsFileResource resource1 = createEmptyFileAndResource(true); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadChunkCompactionPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadChunkCompactionPerformerTest.java index f2721652f99c5..06fff63881799 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadChunkCompactionPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadChunkCompactionPerformerTest.java @@ -21,10 +21,8 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.db.exception.StorageEngineException; -import org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadChunkCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask; -import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionTestFileWriter; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -42,15 +40,13 @@ import org.junit.Before; import org.junit.Test; -import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; -public class TableModelReadChunkCompactionPerformerTest extends AbstractCompactionTest { +public class TableModelReadChunkCompactionPerformerTest extends AbstractTableModelCompactionTest { private final String oldThreadName = Thread.currentThread().getName(); @@ -120,92 +116,6 @@ public void testSequenceInnerSpaceCompactionOfTwoTableModel() throws IOException } } - @Test - public void testSequenceInnerSpaceCompactionOfTwoV4TreeModel() throws IOException { - TsFileResource resource1 = createEmptyFileAndResource(true); - try (CompactionTestFileWriter writer = new CompactionTestFileWriter(resource1)) { - writer.startChunkGroup("d1"); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Collections.singletonList("s1"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - writer.endFile(); - } - TsFileResource resource2 = createEmptyFileAndResource(true); - try (CompactionTestFileWriter writer = new CompactionTestFileWriter(resource2)) { - writer.startChunkGroup("d1"); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Arrays.asList("s1", "s2"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(20, 22)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - writer.endFile(); - } - seqResources.add(resource1); - seqResources.add(resource2); - tsFileManager.addAll(seqResources, true); - - InnerSpaceCompactionTask task = - new InnerSpaceCompactionTask( - 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); - Assert.assertTrue(task.start()); - TsFileResource targetResource = tsFileManager.getTsFileList(true).get(0); - try (TsFileSequenceReader reader = - new TsFileSequenceReader(targetResource.getTsFile().getAbsolutePath())) { - Assert.assertTrue(reader.getTableSchemaMap().isEmpty()); - Map> allTimeseriesMetadata = - reader.getAllTimeseriesMetadata(true); - for (Map.Entry> entry : - allTimeseriesMetadata.entrySet()) { - Assert.assertEquals(3, entry.getValue().size()); - } - } - } - - @Test - public void testSequenceInnerSpaceCompactionOfTableModelAndTreeModel() throws IOException { - TsFileResource resource1 = createEmptyFileAndResource(true); - try (CompactionTestFileWriter writer = new CompactionTestFileWriter(resource1)) { - writer.startChunkGroup("d1"); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Collections.singletonList("s1"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - writer.endFile(); - } - TsFileResource resource2 = createEmptyFileAndResource(true); - try (CompactionTableModelTestFileWriter writer = - new CompactionTableModelTestFileWriter(resource2)) { - writer.registerTableSchema("t1", Arrays.asList("id1", "id2")); - writer.startChunkGroup("t1", Arrays.asList("id_field1", "id_field2")); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Collections.singletonList("s1"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - writer.endFile(); - } - seqResources.add(resource1); - seqResources.add(resource2); - tsFileManager.addAll(seqResources, true); - - InnerSpaceCompactionTask task = - new InnerSpaceCompactionTask( - 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); - Assert.assertTrue(task.start()); - TsFileResource targetResource = tsFileManager.getTsFileList(true).get(0); - try (TsFileSequenceReader reader = - new TsFileSequenceReader(targetResource.getTsFile().getAbsolutePath())) { - Assert.assertEquals(1, reader.getTableSchemaMap().size()); - } - } - @Test public void testSequenceInnerSpaceCompactionOfTableModelCanNotMatchTableSchema() throws IOException { @@ -271,67 +181,4 @@ public void testSequenceInnerSpaceCompactionOfTableModelCanNotMatchTableSchema() Assert.assertEquals("s2", tableSchema.get(3).getMeasurementName()); } } - - @Test - public void testCompactionWithV3Tsfile() throws IOException { - String pathStr = - this.getClass().getClassLoader().getResource("v3tsfile/compaction-test-tsfile").getFile(); - File v3TsFile = new File(pathStr); - File v3TsFileResource = new File(pathStr + "-resource"); - TsFileResource resource1 = createEmptyFileAndResource(true); - Files.copy(v3TsFile.toPath(), resource1.getTsFile().toPath()); - Files.copy( - v3TsFileResource.toPath(), new File(resource1.getTsFilePath() + ".resource").toPath()); - resource1.deserialize(); - - TsFileResource resource2 = createEmptyFileAndResource(true); - try (CompactionTableModelTestFileWriter writer = - new CompactionTableModelTestFileWriter(resource2)) { - writer.registerTableSchema("db1.t1", Arrays.asList("id1", "id2")); - writer.startChunkGroup("db1.t1", Arrays.asList("id_field1", "id_field2")); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Collections.singletonList("s1"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - - writer.startChunkGroup("d3"); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Collections.singletonList("s1"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - - writer.startChunkGroup("node1.node2.device"); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Collections.singletonList("s1"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - - writer.startChunkGroup("node1.node2.node3.device"); - writer.generateSimpleAlignedSeriesToCurrentDevice( - Collections.singletonList("s1"), - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - writer.endFile(); - } - - seqResources.add(resource1); - seqResources.add(resource2); - InnerSpaceCompactionTask task = - new InnerSpaceCompactionTask( - 0, tsFileManager, seqResources, true, new ReadChunkCompactionPerformer(), 0); - Assert.assertTrue(task.start()); - try (TsFileSequenceReader reader = - new TsFileSequenceReader( - tsFileManager.getTsFileList(true).get(0).getTsFile().getAbsolutePath())) { - Assert.assertEquals(1, reader.getTableSchemaMap().size()); - } - } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadPointCompactionPerformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadPointCompactionPerformerTest.java index 740c2434fdb31..54882c30285b5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadPointCompactionPerformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelReadPointCompactionPerformerTest.java @@ -21,7 +21,6 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.db.exception.StorageEngineException; -import org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CrossSpaceCompactionTask; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask; @@ -38,12 +37,10 @@ import org.junit.Before; import org.junit.Test; -import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.util.Arrays; -public class TableModelReadPointCompactionPerformerTest extends AbstractCompactionTest { +public class TableModelReadPointCompactionPerformerTest extends AbstractTableModelCompactionTest { private final String oldThreadName = Thread.currentThread().getName(); @@ -167,62 +164,6 @@ public void testCrossSpaceCompactionOfTwoTableModelWithReadPointCompactionPerfor } } - @Test - public void testCompactionWithV3Tsfile() throws IOException { - String pathStr = - this.getClass().getClassLoader().getResource("v3tsfile/compaction-test-tsfile").getFile(); - File v3TsFile = new File(pathStr); - File v3TsFileResource = new File(pathStr + "-resource"); - TsFileResource resource1 = createEmptyFileAndResource(true); - Files.copy(v3TsFile.toPath(), resource1.getTsFile().toPath()); - Files.copy( - v3TsFileResource.toPath(), new File(resource1.getTsFilePath() + ".resource").toPath()); - resource1.deserialize(); - - TsFileResource resource2 = createEmptyFileAndResource(true); - try (CompactionTableModelTestFileWriter writer = - new CompactionTableModelTestFileWriter(resource2)) { - writer.registerTableSchema("db1.db1.t1", Arrays.asList("id1", "id2")); - writer.startChunkGroup("db1.db1.t1", Arrays.asList("id_field1", "id_field2")); - writer.generateSimpleNonAlignedSeriesToCurrentDevice( - "s1", - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - - writer.startChunkGroup("d1"); - writer.generateSimpleNonAlignedSeriesToCurrentDevice( - "s1", - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - - writer.startChunkGroup("node1.node2.device"); - writer.generateSimpleNonAlignedSeriesToCurrentDevice( - "s1", - new TimeRange[][][] {new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12)}}}, - TSEncoding.PLAIN, - CompressionType.LZ4); - writer.endChunkGroup(); - - writer.endFile(); - } - - seqResources.add(resource1); - seqResources.add(resource2); - InnerSpaceCompactionTask task = - new InnerSpaceCompactionTask( - 0, tsFileManager, seqResources, true, new ReadPointCompactionPerformer(), 0); - Assert.assertTrue(task.start()); - try (TsFileSequenceReader reader = - new TsFileSequenceReader( - tsFileManager.getTsFileList(true).get(0).getTsFile().getAbsolutePath())) { - Assert.assertEquals(1, reader.getTableSchemaMap().size()); - } - } - @Test public void testCompactionWithMultiTableSchema() throws IOException { TsFileResource resource1 = createEmptyFileAndResource(true); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionUtilsTest.java index 2a0890e627760..f7c0ab4105384 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionUtilsTest.java @@ -59,10 +59,11 @@ public void testCompactionPathUtilsForTableModelDevice() throws Exception { IDeviceID deviceID = new StringArrayDeviceID(new String[] {"db.table1", null, "tag1"}); Assert.assertArrayEquals( - new String[] {"db.table1", null, "tag1"}, CompactionPathUtils.getPath(deviceID).getNodes()); + new String[] {"db.table1", null, "tag1"}, + CompactionPathUtils.getPath(deviceID, true).getNodes()); Assert.assertArrayEquals( new String[] {"db.table1", null, "tag1", "s1"}, - CompactionPathUtils.getPath(deviceID, "s1").getNodes()); + CompactionPathUtils.getPath(deviceID, "s1", true).getNodes()); } @Test @@ -71,9 +72,9 @@ public void testCompactionPathUtilsForTwoLevelTreeDevice() throws Exception { IDeviceID deviceID = deletionPath.getIDeviceID(); Assert.assertArrayEquals( - new String[] {"root", "repro"}, CompactionPathUtils.getPath(deviceID).getNodes()); + new String[] {"root", "repro"}, CompactionPathUtils.getPath(deviceID, false).getNodes()); Assert.assertArrayEquals( - deletionPath.getNodes(), CompactionPathUtils.getPath(deviceID, "s1").getNodes()); + deletionPath.getNodes(), CompactionPathUtils.getPath(deviceID, "s1", false).getNodes()); TreeDeletionEntry deletion = new TreeDeletionEntry(deletionPath, Long.MIN_VALUE, Long.MAX_VALUE); @@ -82,7 +83,7 @@ public void testCompactionPathUtilsForTwoLevelTreeDevice() throws Exception { modifications.append(deletion.keyOfPatternTree(), deletion); List matchedModifications = - CompactionUtils.getMatchedModifications(modifications, deviceID, "s1", null); + CompactionUtils.getMatchedModifications(modifications, deviceID, "s1", null, false); Assert.assertEquals(1, matchedModifications.size()); Assert.assertEquals( deletionPath, ((TreeDeletionEntry) matchedModifications.get(0)).getPathPattern()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedBitmapMemoryAccountingPerformanceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedBitmapMemoryAccountingPerformanceTest.java index b18b543a2d53b..8da57414e4a74 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedBitmapMemoryAccountingPerformanceTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedBitmapMemoryAccountingPerformanceTest.java @@ -266,14 +266,14 @@ private static void runWrite(Scenario scenario, AlignedWritableMemChunk[] memChu private static AlignedWritableMemChunk[] createMemChunks(Scenario scenario, int count) { AlignedWritableMemChunk[] memChunks = new AlignedWritableMemChunk[count]; for (int i = 0; i < count; i++) { - memChunks[i] = new AlignedWritableMemChunk(new ArrayList<>(scenario.schemas), false); + memChunks[i] = new AlignedWritableMemChunk(new ArrayList<>(scenario.schemas)); } return memChunks; } private static AccountingTarget createAccountingTarget(Scenario scenario) { AlignedWritableMemChunk memChunk = - new AlignedWritableMemChunk(new ArrayList<>(scenario.schemas), false); + new AlignedWritableMemChunk(new ArrayList<>(scenario.schemas)); memChunk.writeAlignedTablet( scenario.times, scenario.columns, @@ -289,8 +289,7 @@ private static AccountingTarget createAccountingTarget(Scenario scenario) { "0", Collections.singletonMap( deviceId, - new AlignedWritableMemChunkGroup( - memChunk, new ArrayList<>(scenario.schemas), false))); + new AlignedWritableMemChunkGroup(memChunk, new ArrayList<>(scenario.schemas)))); return new AccountingTarget(memTable, deviceId); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkDeserializeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkDeserializeTest.java index 5595390752a39..682b4b287d289 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkDeserializeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemChunkDeserializeTest.java @@ -239,7 +239,7 @@ public void testAlignedSeries() throws IOException, QueryProcessException, Metad new MeasurementSchema("s4", TSDataType.FLOAT), new MeasurementSchema("s5", TSDataType.DOUBLE), new MeasurementSchema("s6", TSDataType.TEXT)); - AlignedWritableMemChunk series = new AlignedWritableMemChunk(schemaList, false); + AlignedWritableMemChunk series = new AlignedWritableMemChunk(schemaList); int count = 1000; for (int i = 0; i < count; i++) { @@ -261,7 +261,7 @@ public void testAlignedSeries() throws IOException, QueryProcessException, Metad series.serializeToWAL(walBuffer); DataInputStream inputStream = new DataInputStream(new ByteArrayInputStream(walBuffer.getBuffer().array())); - AlignedWritableMemChunk memChunk = AlignedWritableMemChunk.deserialize(inputStream, false); + AlignedWritableMemChunk memChunk = AlignedWritableMemChunk.deserialize(inputStream); AlignedReadOnlyMemChunk readableChunk = (AlignedReadOnlyMemChunk) getAlignedReadOnlyChunk(memChunk, schemaList, measurementList); @@ -293,7 +293,7 @@ public void testNonAlignedMemChunkGroupSerializedSizeWithNonAsciiMeasurement() series.writeNonAlignedPoint(1, 1); WritableMemChunkGroup group = new WritableMemChunkGroup(); - group.getMemChunkMap().put(measurement, series); + group.getMemChunkMap(true).put(measurement, series); WALByteBufferForTest walBuffer = new WALByteBufferForTest(ByteBuffer.allocate(group.serializedSize())); @@ -303,7 +303,7 @@ public void testNonAlignedMemChunkGroupSerializedSizeWithNonAsciiMeasurement() DataInputStream inputStream = new DataInputStream(new ByteArrayInputStream(walBuffer.getBuffer().array())); WritableMemChunkGroup deserialized = WritableMemChunkGroup.deserialize(inputStream); - Assert.assertTrue(deserialized.getMemChunkMap().containsKey(measurement)); + Assert.assertTrue(deserialized.getMemChunkMap(true).containsKey(measurement)); } private WritableMemChunk createWritableMemChunkFromBytes(WritableMemChunk series) @@ -319,7 +319,7 @@ private WritableMemChunk createWritableMemChunkFromBytes(WritableMemChunk series private ReadOnlyMemChunk getReadOnlyChunk(WritableMemChunk memChunk, TSDataType dataType) throws QueryProcessException, IOException, MetadataException { WritableMemChunkGroup memChunkGroup = new WritableMemChunkGroup(); - memChunkGroup.getMemChunkMap().put("s1", memChunk); + memChunkGroup.getMemChunkMap(true).put("s1", memChunk); Map memTableMap = new HashMap<>(); memTableMap.put(deviceID, memChunkGroup); IMemTable memTable = new PrimitiveMemTable(storageGroup, dataRegionId, memTableMap); @@ -343,7 +343,7 @@ private ReadOnlyMemChunk getAlignedReadOnlyChunk( List measurementList) throws QueryProcessException, IOException, MetadataException { AlignedWritableMemChunkGroup memChunkGroup = - new AlignedWritableMemChunkGroup(memChunk, schemaList, false); + new AlignedWritableMemChunkGroup(memChunk, schemaList); Map memTableMap = new HashMap<>(); memTableMap.put(deviceID, memChunkGroup); IMemTable memTable = new PrimitiveMemTable(storageGroup, dataRegionId, memTableMap); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemTableFlushTaskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemTableFlushTaskTest.java index afe0384839bcb..ef4c520b13554 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemTableFlushTaskTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/MemTableFlushTaskTest.java @@ -97,7 +97,7 @@ public void testFlushMemTable() MemTableTestUtils.measurementId0, MemTableTestUtils.dataType0); MemTableFlushTask memTableFlushTask = - new MemTableFlushTask(memTable, writer, storageGroup, dataRegionId); + new MemTableFlushTask(memTable, writer, storageGroup, dataRegionId, true); assertTrue( writer .getVisibleMetadataList( @@ -134,7 +134,7 @@ public void testFlushVectorMemTable() throws ExecutionException, InterruptedException, IllegalPathException, WriteProcessException { MemTableTestUtils.produceVectorData(memTable); MemTableFlushTask memTableFlushTask = - new MemTableFlushTask(memTable, writer, storageGroup, dataRegionId); + new MemTableFlushTask(memTable, writer, storageGroup, dataRegionId, true); assertTrue( writer .getVisibleMetadataList(MemTableTestUtils.deviceId0, "sensor0", TSDataType.BOOLEAN) @@ -162,7 +162,7 @@ public void testFlushNullableVectorMemTable() throws ExecutionException, InterruptedException, IllegalPathException, WriteProcessException { MemTableTestUtils.produceNullableVectorData(memTable); MemTableFlushTask memTableFlushTask = - new MemTableFlushTask(memTable, writer, storageGroup, dataRegionId); + new MemTableFlushTask(memTable, writer, storageGroup, dataRegionId, true); assertTrue( writer .getVisibleMetadataList(MemTableTestUtils.deviceId0, "sensor0", TSDataType.BOOLEAN) @@ -192,7 +192,7 @@ public void testAlignedFlushWithoutDeletedMeasurementsSkipsColumnMapping() { memChunk.sortTvListForFlush(); BlockingQueue ioTaskQueue = new LinkedBlockingQueue<>(); - memChunk.encodeWorkingAlignedTVList(ioTaskQueue, 100, 100); + memChunk.encodeWorkingAlignedTVList(ioTaskQueue, 100, 100, true); assertFalse(memChunk.isColumnMappingBuilt()); assertFalse(ioTaskQueue.isEmpty()); @@ -206,7 +206,7 @@ public void testAlignedFlushWithDeletedMeasurementsKeepsColumnMapping() { memChunk.sortTvListForFlush(); BlockingQueue ioTaskQueue = new LinkedBlockingQueue<>(); - memChunk.encodeWorkingAlignedTVList(ioTaskQueue, 100, 100); + memChunk.encodeWorkingAlignedTVList(ioTaskQueue, 100, 100, true); assertTrue(memChunk.isColumnMappingBuilt()); assertFalse(ioTaskQueue.isEmpty()); @@ -228,7 +228,7 @@ private void checkAlignedFlushMergesDuplicateTimestamps(boolean removeSecondMeas Arrays.asList( new MeasurementSchema("s0", TSDataType.INT64, TSEncoding.PLAIN), new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN)); - AlignedWritableMemChunk memChunk = new AlignedWritableMemChunk(schemas, true); + AlignedWritableMemChunk memChunk = new AlignedWritableMemChunk(schemas); String alignedFilePath = TestConstant.OUTPUT_DATA_DIR.concat( "duplicateAlignedRows-" + removeSecondMeasurement + ".tsfile"); @@ -245,7 +245,7 @@ private void checkAlignedFlushMergesDuplicateTimestamps(boolean removeSecondMeas memChunk.sortTvListForFlush(); BlockingQueue ioTaskQueue = new LinkedBlockingQueue<>(); - memChunk.encodeWorkingAlignedTVList(ioTaskQueue, 100, 100); + memChunk.encodeWorkingAlignedTVList(ioTaskQueue, 100, 100, false); try (TsFileIOWriter alignedWriter = new TsFileIOWriter(new File(alignedFilePath))) { alignedWriter.startChunkGroup(IDeviceID.Factory.DEFAULT_FACTORY.create("root.d")); Object task; @@ -304,7 +304,7 @@ public void testAlignedFastPathKeepsPagesAndValuesAlignedAfterPartialSegmentSort Arrays.asList( new MeasurementSchema("s0", TSDataType.INT64, TSEncoding.PLAIN), new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN)); - AlignedWritableMemChunk memChunk = new AlignedWritableMemChunk(schemas, false); + AlignedWritableMemChunk memChunk = new AlignedWritableMemChunk(schemas); String alignedFilePath = TestConstant.OUTPUT_DATA_DIR.concat("testAlignedFastPath.tsfile"); try { @@ -320,7 +320,7 @@ public void testAlignedFastPathKeepsPagesAndValuesAlignedAfterPartialSegmentSort memChunk.sortTvListForFlush(); BlockingQueue ioTaskQueue = new LinkedBlockingQueue<>(); - memChunk.encodeWorkingAlignedTVList(ioTaskQueue, rowCount, 1024); + memChunk.encodeWorkingAlignedTVList(ioTaskQueue, rowCount, 1024, true); try (TsFileIOWriter alignedWriter = new TsFileIOWriter(new File(alignedFilePath))) { alignedWriter.startChunkGroup(IDeviceID.Factory.DEFAULT_FACTORY.create("root.d")); Object task; @@ -373,7 +373,7 @@ private void checkAlignedFlushAfterTimeDeletion(boolean removeColumn) throws IOE Arrays.asList( new MeasurementSchema("s0", TSDataType.INT32, TSEncoding.PLAIN), new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.PLAIN)); - AlignedWritableMemChunk memChunk = new AlignedWritableMemChunk(schemas, false); + AlignedWritableMemChunk memChunk = new AlignedWritableMemChunk(schemas); String alignedFilePath = TestConstant.OUTPUT_DATA_DIR.concat("testAlignedTimeDeletion" + pageSize + ".tsfile"); try { @@ -389,7 +389,7 @@ private void checkAlignedFlushAfterTimeDeletion(boolean removeColumn) throws IOE BlockingQueue ioTaskQueue = new LinkedBlockingQueue<>(); // Cover a single page and boundaries between pages and chunks. - memChunk.encodeWorkingAlignedTVList(ioTaskQueue, pageSize + 1, pageSize); + memChunk.encodeWorkingAlignedTVList(ioTaskQueue, pageSize + 1, pageSize, true); try (TsFileIOWriter alignedWriter = new TsFileIOWriter(new File(alignedFilePath))) { alignedWriter.startChunkGroup(IDeviceID.Factory.DEFAULT_FACTORY.create("root.d")); Object task; @@ -464,7 +464,7 @@ private void checkUnmaterializedSegments(boolean moved) throws Exception { schemas.add(new MeasurementSchema("s" + column, types.get(column), TSEncoding.PLAIN)); } schemas.add(new MeasurementSchema("empty", TSDataType.INT64, TSEncoding.PLAIN)); - AlignedWritableMemChunk chunk = new AlignedWritableMemChunk(schemas, true); + AlignedWritableMemChunk chunk = new AlignedWritableMemChunk(schemas); Object[][] expected = new Object[rowCount][types.size()]; String path = TestConstant.OUTPUT_DATA_DIR.concat("unmaterialized-" + moved + ".tsfile"); try { @@ -502,7 +502,7 @@ private void checkUnmaterializedSegments(boolean moved) throws Exception { } chunk.sortTvListForFlush(); BlockingQueue queue = new LinkedBlockingQueue<>(); - chunk.encodeWorkingAlignedTVList(queue, ARRAY_SIZE * 5 + 3, ARRAY_SIZE + 3); + chunk.encodeWorkingAlignedTVList(queue, ARRAY_SIZE * 5 + 3, ARRAY_SIZE + 3, false); try (TsFileIOWriter fileWriter = new TsFileIOWriter(new File(path))) { fileWriter.startChunkGroup(IDeviceID.Factory.DEFAULT_FACTORY.create("root.d")); Object task; @@ -559,7 +559,7 @@ private static class TrackingAlignedWritableMemChunk extends AlignedWritableMemC private boolean columnMappingBuilt; private TrackingAlignedWritableMemChunk(List schemaList) { - super(schemaList, false); + super(schemaList); } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java index dffd4d941d057..5a0f8a1e800f3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/PrimitiveMemTableTest.java @@ -183,11 +183,14 @@ public void testWriteDuringPrepareTVListAndActualQueryExecution() new StringArrayDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i}); } memTable.delete( - new TreeDeletionEntry(new MeasurementPath("root.test.d1.s1", TSDataType.INT32), 150, 160)); + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s1", TSDataType.INT32), 150, 160), + true); memTable.delete( - new TreeDeletionEntry(new MeasurementPath("root.test.d1.s2", TSDataType.INT32), 150, 160)); + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s2", TSDataType.INT32), 150, 160), + true); memTable.delete( - new TreeDeletionEntry(new MeasurementPath("root.test.d1.s3", TSDataType.INT32), 150, 160)); + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s3", TSDataType.INT32), 150, 160), + true); ResourceByPathUtils resourcesByPathUtils = ResourceByPathUtils.getResourceInstance( new AlignedFullPath( @@ -230,11 +233,14 @@ public void testWriteAndFlushSortDuringQuerySortTVListAndActualQueryExecution() new StringArrayDeviceID("root.test.d1"), measurementSchemas, i, new Object[] {i, i, i}); } memTable.delete( - new TreeDeletionEntry(new MeasurementPath("root.test.d1.s1", TSDataType.INT32), 150, 160)); + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s1", TSDataType.INT32), 150, 160), + true); memTable.delete( - new TreeDeletionEntry(new MeasurementPath("root.test.d1.s2", TSDataType.INT32), 150, 160)); + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s2", TSDataType.INT32), 150, 160), + true); memTable.delete( - new TreeDeletionEntry(new MeasurementPath("root.test.d1.s3", TSDataType.INT32), 150, 160)); + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s3", TSDataType.INT32), 150, 160), + true); ResourceByPathUtils resourcesByPathUtils = ResourceByPathUtils.getResourceInstance( new AlignedFullPath( @@ -290,11 +296,14 @@ public void testFlushingQueryDoesNotSortWorkingTVListUsedByPreviousQuery() memTable.writeAlignedRow(deviceID, measurementSchemas, i, new Object[] {i, i, i}); } memTable.delete( - new TreeDeletionEntry(new MeasurementPath("root.test.d1.s1", TSDataType.INT32), 1, 10)); + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s1", TSDataType.INT32), 1, 10), + true); memTable.delete( - new TreeDeletionEntry(new MeasurementPath("root.test.d1.s2", TSDataType.INT32), 1, 10)); + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s2", TSDataType.INT32), 1, 10), + true); memTable.delete( - new TreeDeletionEntry(new MeasurementPath("root.test.d1.s3", TSDataType.INT32), 1, 10)); + new TreeDeletionEntry(new MeasurementPath("root.test.d1.s3", TSDataType.INT32), 1, 10), + true); Assert.assertFalse(originalWorkingList.isSorted()); AlignedReadOnlyMemChunk flushingQueryMemChunk = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java index a1ba4cee07463..7a4eedccf74c2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunkRegionScanTest.java @@ -266,7 +266,8 @@ public void testAlignedWritableMemChunkRegionScan2() throws IllegalPathException memTable.delete( new TreeDeletionEntry( new MeasurementPath(new StringArrayDeviceID("root.test.d1"), "s1"), - new TimeRange(1, 1500))); + new TimeRange(1, 1500)), + true); writableMemChunk = (AlignedWritableMemChunk) memTable.getWritableMemChunk(new StringArrayDeviceID("root.test.d1"), ""); @@ -295,8 +296,7 @@ public void testTableWritableMemChunkRegionScan() { new MeasurementSchema("s1", TSDataType.INT32), new MeasurementSchema("s2", TSDataType.INT32), new MeasurementSchema("s3", TSDataType.INT32)); - AlignedWritableMemChunk writableMemChunk = - new AlignedWritableMemChunk(measurementSchemas, true); + AlignedWritableMemChunk writableMemChunk = new AlignedWritableMemChunk(measurementSchemas); int size = 100000; for (int i = 0; i < size; i++) { if (i <= 10000) { @@ -429,7 +429,8 @@ public void testNonAlignedWritableMemChunkRegionScan2() throws IllegalPathExcept memTable.delete( new TreeDeletionEntry( new MeasurementPath(new StringArrayDeviceID("root.test.d1"), "s1"), - new TimeRange(1, 1500))); + new TimeRange(1, 1500)), + true); WritableMemChunk writableMemChunk = (WritableMemChunk) memTable.getWritableMemChunk(new StringArrayDeviceID("root.test.d1"), "s1"); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/ImportWALTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/ImportWALTest.java index aa71595d4439b..7e8c40acf3ac7 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/ImportWALTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/ImportWALTest.java @@ -304,6 +304,49 @@ public void testUnrecognizedDirectoryDoesNotInferFromAncestor() { assertTrue(databases.values().stream().allMatch(database -> database == null)); } + /** + * Every directory must declare a database before any replay starts, otherwise the data model of a + * WAL snapshot could only be guessed. + */ + @Test + public void testUnresolvedDirectoryRequiresExplicitDatabase() { + final List files = + Arrays.asList( + Paths.get("factory-3/one.wal"), + Paths.get("0/two.wal"), + Paths.get("root.sg-4/three.wal")); + ImportWAL.requireResolvedDatabases( + ImportWAL.resolveDirectoryDatabases(files, "target", true, null)); + + final Map unresolved = + ImportWAL.resolveDirectoryDatabases(files, null, true, null); + final IllegalArgumentException failure = + assertThrows( + IllegalArgumentException.class, () -> ImportWAL.requireResolvedDatabases(unresolved)); + // The inferable directory is not reported, the others are listed in directory order. + assertTrue(failure.getMessage().contains("0, root.sg-4")); + assertTrue(failure.getMessage().contains("-db/--database")); + } + + /** A directory whose database cannot be inferred must fail before any Session is opened. */ + @Test + public void testUnresolvedDatabaseFailsBeforeReplay() throws Exception { + final Path directory = temporaryFolder.newFolder("0").toPath(); + createWALFile(directory, 0); + final ByteArrayOutputStream error = new ByteArrayOutputStream(); + + assertEquals( + 1, + ImportWAL.run( + new String[] {"-f", directory.toString(), "-pw", "password"}, + new PrintStream(error), + new PrintStream(error))); + + assertTrue(error.toString().contains(directory.toString())); + assertTrue(error.toString().contains("-db/--database")); + assertTrue(error.toString().contains("ElasticStrategy")); + } + /** * One thread may visit several databases; each directory needs its own Session and schema cache. */ @@ -403,6 +446,8 @@ public void testHelpDescribesDeleteSourceOption() { assertTrue(output.toString().contains("--on_unsupported")); assertTrue(output.toString().contains("--on_corrupted")); assertTrue(output.toString().contains("--skip_db_confirmation")); + assertTrue(output.toString().contains("ElasticStrategy")); + assertTrue(output.toString().contains("RoundRobinStrategy")); } @Test @@ -1547,7 +1592,7 @@ public void testReplayNonAlignedMemTableSnapshotAsTablets() throws Exception { new Object[] {10, null, DateUtils.parseDateExpressionToInt(firstDate)}); final Session treeSession = mock(Session.class); - new ImportWAL.WALReplayer(treeSession, null, null).replay(new WALInfoEntry(1, memTable)); + new ImportWAL.WALReplayer(treeSession, null, "root.sg").replay(new WALInfoEntry(1, memTable)); final ArgumentCaptor tabletCaptor = ArgumentCaptor.forClass(Tablet.class); verify(treeSession, times(3)).insertTablet(tabletCaptor.capture()); @@ -1593,7 +1638,7 @@ public void testReplayAlignedMemTableSnapshotPreservesNulls() throws Exception { new Object[] {10, 100L, DateUtils.parseDateExpressionToInt(firstDate)}); final Session treeSession = mock(Session.class); - new ImportWAL.WALReplayer(treeSession, null, null).replay(new WALInfoEntry(1, memTable)); + new ImportWAL.WALReplayer(treeSession, null, "root.sg").replay(new WALInfoEntry(1, memTable)); final ArgumentCaptor tabletCaptor = ArgumentCaptor.forClass(Tablet.class); verify(treeSession).insertAlignedTablet(tabletCaptor.capture()); @@ -1623,7 +1668,7 @@ public void testReplaySerializedMemTableSnapshot() throws Exception { final ImportWAL.ReplayStatistics statistics = ImportWAL.replayWALFiles( Collections.singletonList(walFile.toPath()), - new ImportWAL.WALReplayer(treeSession, null, null)); + new ImportWAL.WALReplayer(treeSession, null, "root.sg")); assertEquals(1, statistics.getReplayedOperationCount()); final ArgumentCaptor tabletCaptor = ArgumentCaptor.forClass(Tablet.class); @@ -1644,7 +1689,7 @@ public void testReplayMemTableSnapshotSplitsLargeChunk() throws Exception { } final Session treeSession = mock(Session.class); - new ImportWAL.WALReplayer(treeSession, null, null).replay(new WALInfoEntry(1, memTable)); + new ImportWAL.WALReplayer(treeSession, null, "root.sg").replay(new WALInfoEntry(1, memTable)); final ArgumentCaptor tabletCaptor = ArgumentCaptor.forClass(Tablet.class); verify(treeSession, times(2)).insertTablet(tabletCaptor.capture()); diff --git a/pom.xml b/pom.xml index fd936192e878a..e088f9397f99d 100644 --- a/pom.xml +++ b/pom.xml @@ -145,7 +145,7 @@ 0.24.0 1.9 1.5.6-3 - 2.4.1-260915-SNAPSHOT + 2.4.1-261009-SNAPSHOT en