Skip to content

[Feature] Make DROP TABLE asynchronous and expose deletion progress #18806

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Caideyipi wants to merge 3 commits into
base: master
Choose a base branch
from
Open

[Feature] Make DROP TABLE asynchronous and expose deletion progress #18806

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c4869ad
[Feature] Make DROP TABLE asynchronous and expose deletion progress
Caideyipi Oct 9, 2026
7eb5c4b
[Test] Wait for asynchronous DROP TABLE completion in deletion IT
Caideyipi Oct 9, 2026
1590aed
[Fix] Address asynchronous DROP TABLE review comments
Caideyipi Oct 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions ...gration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.apache.iotdb.itbase.category.TableClusterIT;
import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
import org.apache.iotdb.itbase.env.BaseEnv;
import org.apache.iotdb.itbase.exception.InconsistentDataException;
import org.apache.iotdb.itbase.exception.ParallelRequestTimeoutException;
import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.rpc.StatementExecutionException;
Expand Down Expand Up @@ -1361,6 +1362,25 @@ public void testDeleteTable() throws SQLException {

statement.execute("DROP TABLE vehicle" + testNum);

// DROP TABLE acknowledges submission before the table has been fully removed.
Awaitility.await()
.atMost(60, TimeUnit.SECONDS)
.pollInSameThread()
// Progress snapshots can differ between cluster nodes while deletion is running.
.ignoreException(InconsistentDataException.class)
.untilAsserted(
() -> {
try (ResultSet resultSet =
statement.executeQuery(
"SELECT state, error_message FROM information_schema.drop_table_procedures "
+ "WHERE database = 'test' AND table_name = 'vehicle"
+ testNum
+ "' ORDER BY procedure_id DESC LIMIT 1")) {
assertTrue(resultSet.next());
assertEquals(resultSet.getString(2), "SUCCESS", resultSet.getString(1));
}
});

try (ResultSet ignored = statement.executeQuery("SELECT * FROM vehicle" + testNum)) {
fail("Exception expected");
} catch (SQLException e) {
Expand Down
4 changes: 3 additions & 1 deletion integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -532,6 +532,7 @@ public void testInformationSchema() throws SQLException {
"current_queries,INF,",
"data_nodes,INF,",
"databases,INF,",
"drop_table_procedures,INF,",
"functions,INF,",
"keywords,INF,",
"nodes,INF,",
Expand Down Expand Up @@ -828,6 +829,7 @@ public void testInformationSchema() throws SQLException {
Arrays.asList(
"information_schema,databases,INF,USING,null,SYSTEM VIEW,false,",
"information_schema,tables,INF,USING,null,SYSTEM VIEW,false,",
"information_schema,drop_table_procedures,INF,USING,null,SYSTEM VIEW,false,",
"information_schema,columns,INF,USING,null,SYSTEM VIEW,false,",
"information_schema,queries,INF,USING,null,SYSTEM VIEW,false,",
"information_schema,regions,INF,USING,null,SYSTEM VIEW,false,",
Expand All @@ -853,7 +855,7 @@ public void testInformationSchema() throws SQLException {
TestUtils.assertResultSetEqual(
statement.executeQuery("count devices from tables where status = 'USING'"),
"count(devices),",
Collections.singleton("24,"));
Collections.singleton("25,"));
TestUtils.assertResultSetEqual(
statement.executeQuery(
"select * from columns where table_name = 'queries' or database = 'test'"),
Expand Down
66 changes: 66 additions & 0 deletions integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -546,6 +546,7 @@ public void testManageTable() {
Collections.singleton("null,"));

statement.execute("drop table table2");
awaitDropProcedure(statement, "test2", "table2");
try {
statement.executeQuery("describe table2");
fail();
Expand Down Expand Up @@ -1229,4 +1230,69 @@ public void testTreeViewTable() throws Exception {
}
}
}

@Test
public void testAsyncDropTableProgress() throws Exception {
try (final Connection connection =
EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
final Statement statement = connection.createStatement()) {
statement.execute("create database async_drop_db");
statement.execute("use async_drop_db");
statement.execute("create table drop_target (device string tag, reading int32)");
statement.execute("drop table drop_target");

// The drop may finish before the first query. Its completed result remains visible until
// the procedure cleaner evicts it, so verify that result after waiting for completion.
awaitDropProcedure(statement, "async_drop_db", "drop_target");
TestUtils.assertResultSetEqual(
statement.executeQuery("show tables from async_drop_db"),
"TableName,TTL(ms),",
Collections.emptySet());
try (final ResultSet resultSet =
statement.executeQuery(
"select procedure_id, state, progress, error_message "
+ "from information_schema.drop_table_procedures "
+ "where database = 'async_drop_db' and table_name = 'drop_target'")) {
assertTrue(resultSet.next());
assertTrue(resultSet.getLong(1) >= 0);
assertEquals("SUCCESS", resultSet.getString(2));
assertEquals("SUCCESS", resultSet.getString(3));
Assert.assertNull(resultSet.getString(4));
assertFalse(resultSet.next());
}
Comment on lines +1251 to +1262

@jt2594838 jt2594838 Oct 10, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are the results stored even after the procedure finishes? How long and how many?

@Caideyipi Caideyipi Oct 10, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. The view includes both active drops and completed results retained by the executor's existing completed-procedure cleaner. Retention uses procedure_completed_evict_ttl (60 seconds by default, measured from the procedure's last update); the cleaner runs every procedure_completed_clean_interval (30 seconds by default), so an expired record disappears on a subsequent cleanup. There is no separate count limit: all drop results within that retention window are included. This PR reuses the existing procedure store and cleanup policy rather than adding a separate history store. I also documented the shared TTL retention on getDropTableProcedures().

}
}

private static void awaitDropProcedure(
final Statement statement, final String database, final String tableName)
throws SQLException {
final long deadline = System.currentTimeMillis() + 60_000;
while (System.currentTimeMillis() < deadline) {
try (final ResultSet resultSet =
statement.executeQuery(
"select state, error_message from information_schema.drop_table_procedures "
+ "where database = '"
+ database
+ "' and table_name = '"
+ tableName
+ "'")) {
if (resultSet.next()) {
final String state = resultSet.getString(1);
if ("SUCCESS".equals(state)) {
return;
}
if ("ROLLEDBACK".equals(state)) {
throw new SQLException(resultSet.getString(2));
}
}
}
try {
Thread.sleep(20);
} catch (final InterruptedException e) {
Thread.currentThread().interrupt();
throw new SQLException(e);
}
}
throw new SQLException("Timed out waiting for drop table procedure");
}
}
10 changes: 10 additions & 0 deletions iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,7 @@
import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodes4InformationSchemaResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowDropTableProceduresResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowPipePluginReq;
import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq;
import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp;
Expand Down Expand Up @@ -3401,6 +3402,15 @@ public TShowTable4InformationSchemaResp showTables4InformationSchema() {
: new TShowTable4InformationSchemaResp(status);
}

@Override
public TShowDropTableProceduresResp showDropTableProcedures() {
final TSStatus status = confirmLeader();
return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
? new TShowDropTableProceduresResp(status)
.setDropTableProcedureInfoList(procedureManager.getDropTableProcedures())
: new TShowDropTableProceduresResp(status);
}

@Override
public TDescTableResp describeTable(
final String database, final String tableName, final boolean isDetails) {
Expand Down
3 changes: 3 additions & 0 deletions iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@
import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodes4InformationSchemaResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowDropTableProceduresResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowPipePluginReq;
import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq;
import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp;
Expand Down Expand Up @@ -916,6 +917,8 @@ TPermissionInfoResp login(

TShowTable4InformationSchemaResp showTables4InformationSchema();

TShowDropTableProceduresResp showDropTableProcedures();

TDescTableResp describeTable(
final String database, final String tableName, final boolean isDetails);

Expand Down
99 changes: 90 additions & 9 deletions ...b-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,14 @@
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.commons.exception.table.TableNotExistsException;
import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PathDeserializeUtil;
import org.apache.iotdb.commons.path.PathPatternTree;
import org.apache.iotdb.commons.pipe.agent.plugin.meta.PipePluginMeta;
import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant;
import org.apache.iotdb.commons.schema.table.TableNodeStatus;
import org.apache.iotdb.commons.schema.table.TreeViewSchema;
import org.apache.iotdb.commons.schema.table.TsTable;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil;
Expand All @@ -59,6 +61,7 @@
import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan;
import org.apache.iotdb.confignode.i18n.ManagerMessages;
import org.apache.iotdb.confignode.manager.partition.PartitionManager;
import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager;
import org.apache.iotdb.confignode.manager.subscription.SubscriptionCoordinator;
import org.apache.iotdb.confignode.persistence.ProcedureInfo;
import org.apache.iotdb.confignode.procedure.PartitionTableAutoCleaner;
Expand Down Expand Up @@ -158,6 +161,7 @@
import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceReq;
import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceResp;
import org.apache.iotdb.confignode.rpc.thrift.TDropPipePluginReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropTableProcedureInfo;
import org.apache.iotdb.confignode.rpc.thrift.TExtendRegionReq;
import org.apache.iotdb.confignode.rpc.thrift.TMigrateRegionReq;
import org.apache.iotdb.confignode.rpc.thrift.TReconstructRegionReq;
Expand Down Expand Up @@ -2376,15 +2380,63 @@ public TSStatus alterTableColumnDataType(TAlterOrDropTableReq req) {

public TSStatus dropTable(final TAlterOrDropTableReq req) {
final boolean isView = req.isSetIsView() && req.isIsView();
return executeWithoutDuplicate(
req.database,
null,
req.tableName,
req.queryId,
isView ? ProcedureType.DROP_VIEW_PROCEDURE : ProcedureType.DROP_TABLE_PROCEDURE,
isView
? new DropViewProcedure(req.database, req.tableName, req.queryId, false)
: new DropTableProcedure(req.database, req.tableName, req.queryId, false));
if (isView) {
return executeWithoutDuplicate(
req.database,
null,
req.tableName,
req.queryId,
ProcedureType.DROP_VIEW_PROCEDURE,
new DropViewProcedure(req.database, req.tableName, req.queryId, false));
}
synchronized (this) {
if (!executor.isRunning()) {
return RpcUtils.getStatus(TSStatusCode.CONFIG_NODE_LEADER_WARMING_UP);
}
final Pair<Long, Boolean> duplicate =
checkDuplicateTableTask(
req.database,
null,
req.tableName,
null,
req.queryId,
ProcedureType.DROP_TABLE_PROCEDURE);
if (duplicate.getLeft() != -1) {
return StatusUtils.OK;
}
if (Boolean.TRUE.equals(duplicate.getRight())) {
return RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK);
}
try {
final Optional<Pair<TsTable, TableNodeStatus>> table =
configManager
.getClusterSchemaManager()
.getTableAndStatusIfExists(req.database, req.tableName);
if (table.isEmpty()) {
final TableNotExistsException e =
new TableNotExistsException(req.database, req.tableName);
return RpcUtils.getStatus(e.getErrorCode(), e.getMessage());
}
if (table.get().getRight() == TableNodeStatus.PRE_DELETE) {
return RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK);
}
final Optional<Pair<TSStatus, TsTable>> viewCheck =
ClusterSchemaManager.checkTable4View(req.database, table.get().getLeft(), false);
if (viewCheck.isPresent()) {
return viewCheck.get().getLeft();
}
} catch (final MetadataException e) {
return RpcUtils.getStatus(e.getErrorCode(), e.getMessage());
}
final Procedure<ConfigNodeProcedureEnv> procedure =
new DropTableProcedure(req.database, req.tableName, req.queryId, false);
try {
executor.submitProcedureAndPersist(procedure);
return StatusUtils.OK;
} catch (final Exception e) {
return RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR, e.getMessage());
}
}
}

public TSStatus renameTable(final TAlterOrDropTableReq req) {
Expand Down Expand Up @@ -2484,6 +2536,35 @@ public Map<String, List<String>> getAllExecutingTables() {
return result;
}

/** Returns running drops and completed results retained by the executor's shared TTL cleaner. */
public List<TDropTableProcedureInfo> getDropTableProcedures() {
final Map<Long, DropTableProcedure> dropProcedures = new HashMap<>();
Stream.concat(
executor.getProcedures().values().stream(), executor.getCompletedProcedures().stream())
.filter(
procedure ->
procedure instanceof DropTableProcedure
&& !(procedure instanceof DropViewProcedure))
.map(procedure -> (DropTableProcedure) procedure)
.forEach(procedure -> dropProcedures.put(procedure.getProcId(), procedure));
return dropProcedures.values().stream()
.map(
procedure -> {
final TDropTableProcedureInfo info =
new TDropTableProcedureInfo(
procedure.getDatabase(),
procedure.getTableName(),
procedure.getProcId(),
procedure.getState().name(),
procedure.getDropProgress());
if (procedure.hasException()) {
info.setErrorMessage(procedure.getException().getMessage());
}
return info;
})
.collect(Collectors.toList());
}

public TSStatus executeWithoutDuplicate(
final String database,
final TsTable table,
Expand Down
Loading
Loading