You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2024/12/04 10:31:11 UTC
(iotdb) branch rc/2.0.1 created (now d0b17c396b1)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch rc/2.0.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at d0b17c396b1 Optimize table cache leader implement 2 (#14278)
This branch includes the following new commits:
new 1e09fd89107 Disabled some unsupportted operations
new 873e0da80b4 Diasble delete-from sql
new 3d91a908415 Add table series number in schema quota
new 641fc5aab19 Fix the deserialization of table deletion in WAL
new 3aadc712052 Fixed the bug that drop table won't release device num in quota & table requests may block forever after ConfigNode restart
new 53e20c10284 Enhance procedure recover policy
new d0b17c396b1 Optimize table cache leader implement 2 (#14278)
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 05/07: Fixed the bug that drop table won't release device num in quota & table requests may block forever after ConfigNode restart
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/2.0.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 3aadc71205287561d0021f6bc7a843ff2aeea2b6
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Mon Dec 2 14:08:38 2024 +0800
Fixed the bug that drop table won't release device num in quota & table requests may block forever after ConfigNode restart
---
.../org/apache/iotdb/confignode/manager/ProcedureManager.java | 2 +-
.../main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java | 7 ++++---
.../iotdb/db/schemaengine/rescon/DataNodeSchemaQuotaManager.java | 4 ++--
.../iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java | 5 ++++-
.../schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 2 --
.../iotdb/db/schemaengine/schemaregion/tag/TagLogFile.java | 9 ++++++---
6 files changed, 17 insertions(+), 12 deletions(-)
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
index 2822e03b7a0..dfe6f14441b 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
@@ -1720,7 +1720,7 @@ public class ProcedureManager {
ProcedureType type;
for (final Procedure<?> procedure : executor.getProcedures().values()) {
type = ProcedureFactory.getProcedureType(procedure);
- if (type == null) {
+ if (type == null || procedure.isFinished()) {
continue;
}
// A table shall not be concurrently operated or else the dataNode cache
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java
index 4f73a8707c1..de336b0a9df 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java
@@ -418,7 +418,8 @@ public class SchemaEngine {
* @param req heartbeat request
* @param resp heartbeat response
*/
- public void updateAndFillSchemaCountMap(TDataNodeHeartbeatReq req, TDataNodeHeartbeatResp resp) {
+ public void updateAndFillSchemaCountMap(
+ final TDataNodeHeartbeatReq req, final TDataNodeHeartbeatResp resp) {
// update DataNodeSchemaQuotaManager
schemaQuotaManager.updateRemain(
req.getTimeSeriesQuotaRemain(),
@@ -427,7 +428,7 @@ public class SchemaEngine {
if (resp.getRegionDeviceUsageMap() == null) {
resp.setRegionDeviceUsageMap(new HashMap<>());
}
- Map<Integer, Long> tmp = resp.getRegionDeviceUsageMap();
+ final Map<Integer, Long> tmp = resp.getRegionDeviceUsageMap();
SchemaRegionConsensusImpl.getInstance().getAllConsensusGroupIds().stream()
.filter(
consensusGroupId ->
@@ -446,7 +447,7 @@ public class SchemaEngine {
if (resp.getRegionSeriesUsageMap() == null) {
resp.setRegionSeriesUsageMap(new HashMap<>());
}
- Map<Integer, Long> tmp = resp.getRegionSeriesUsageMap();
+ final Map<Integer, Long> tmp = resp.getRegionSeriesUsageMap();
SchemaRegionConsensusImpl.getInstance().getAllConsensusGroupIds().stream()
.filter(
consensusGroupId ->
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/DataNodeSchemaQuotaManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/DataNodeSchemaQuotaManager.java
index 962e68c7512..f1f2506007c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/DataNodeSchemaQuotaManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/DataNodeSchemaQuotaManager.java
@@ -72,7 +72,7 @@ public class DataNodeSchemaQuotaManager {
}
}
- public void check(long acquireMeasurementNumber, int acquireDeviceNumber)
+ public void check(final long acquireMeasurementNumber, final int acquireDeviceNumber)
throws SchemaQuotaExceededException {
if (acquireDeviceNumber > 0) {
checkDeviceLevel();
@@ -80,7 +80,7 @@ public class DataNodeSchemaQuotaManager {
// if pass device check, check measurement level
try {
checkMeasurementLevel(acquireMeasurementNumber);
- } catch (SchemaQuotaExceededException e) {
+ } catch (final SchemaQuotaExceededException e) {
// if measurement level check failed, roll back device remain
if (acquireDeviceNumber > 0) {
deviceRemain.addAndGet(1L);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java
index ad29546a78b..f373f33a341 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaRegionStatistics.java
@@ -117,8 +117,11 @@ public class MemSchemaRegionStatistics implements ISchemaRegionStatistics {
tableDeviceNumber.computeIfPresent(table, (tableName, num) -> num - decrease);
}
+ // Reset table device, will alter the schema statistics as well
public void resetTableDevice(final String table) {
- tableDeviceNumber.computeIfPresent(table, (tableName, num) -> 0L);
+ final long num = tableDeviceNumber.remove(table);
+ devicesNumber.addAndGet(-num);
+ schemaEngineStatistics.deleteDevice(num);
}
public void addDevice() {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
index d16ab74ba03..9423aa9815a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
@@ -1701,8 +1701,6 @@ public class MTreeBelowSGMemoryImpl {
}
}
- public void renameTableAttribute() {}
-
public boolean deleteTableDevice(final String tableName, final IntConsumer attributeDeleter)
throws MetadataException {
if (!store.hasChild(storageGroupMNode, tableName)) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagLogFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagLogFile.java
index 9cf9d51cf21..b63d5a68abb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagLogFile.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagLogFile.java
@@ -41,6 +41,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
public class TagLogFile implements AutoCloseable {
@@ -359,8 +360,10 @@ public class TagLogFile implements AutoCloseable {
@Override
public void close() throws IOException {
- fileChannel.force(true);
- fileChannel.close();
- fileChannel = null;
+ if (Objects.nonNull(fileChannel) && fileChannel.isOpen()) {
+ fileChannel.force(true);
+ fileChannel.close();
+ fileChannel = null;
+ }
}
}
(iotdb) 01/07: Disabled some unsupportted operations
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/2.0.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1e09fd89107fda8b4ca6f1139eb3b5024b84f389
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Fri Nov 29 17:17:49 2024 +0800
Disabled some unsupportted operations
---
.../iotdb/relational/it/schema/IoTDBDeviceIT.java | 23 ++--
.../iotdb/relational/it/schema/IoTDBTableIT.java | 122 +++++++++++----------
.../plan/relational/sql/ast/DeleteDevice.java | 2 +
.../plan/relational/sql/ast/DropColumn.java | 15 +--
4 files changed, 82 insertions(+), 80 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java
index 808d4cbdfbc..e7ac5eef74c 100644
--- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDeviceIT.java
@@ -247,18 +247,21 @@ public class IoTDBDeviceIT {
TestUtils.assertResultSetSize(
statement.executeQuery("show devices from table0 offset 1 limit 1"), 1);
- // Test delete devices
- statement.execute("delete devices from table0 where region_id = '1' and plant_id = '5'");
- TestUtils.assertResultSetSize(statement.executeQuery("show devices from table0"), 1);
+ // TODO: Reopen
+ if (false) {
+ // Test delete devices
+ statement.execute("delete devices from table0 where region_id = '1' and plant_id = '5'");
+ TestUtils.assertResultSetSize(statement.executeQuery("show devices from table0"), 1);
- // Test successfully invalidate cache
- statement.execute(
- "insert into table0(region_id, plant_id, device_id, model, temperature, humidity) values('1', '5', '3', 'A', 37.6, 111.1)");
- TestUtils.assertResultSetSize(statement.executeQuery("show devices from table0"), 2);
+ // Test successfully invalidate cache
+ statement.execute(
+ "insert into table0(region_id, plant_id, device_id, model, temperature, humidity) values('1', '5', '3', 'A', 37.6, 111.1)");
+ TestUtils.assertResultSetSize(statement.executeQuery("show devices from table0"), 2);
- // Test successfully delete data
- TestUtils.assertResultSetSize(
- statement.executeQuery("select * from table0 where region_id = '1'"), 1);
+ // Test successfully delete data
+ TestUtils.assertResultSetSize(
+ statement.executeQuery("select * from table0 where region_id = '1'"), 1);
+ }
}
}
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java
index 251dd81f238..981c1c7b22a 100644
--- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java
@@ -385,66 +385,69 @@ public class IoTDBTableIT {
statement.execute(
"insert into table2(region_id, plant_id, color, temperature, speed) values(1, 1, 1, 1, 1)");
- // Test drop column
- statement.execute("alter table table2 drop column color");
-
- columnNames = new String[] {"time", "region_id", "plant_id", "temperature", "speed"};
- dataTypes = new String[] {"TIMESTAMP", "STRING", "STRING", "FLOAT", "DOUBLE"};
- categories = new String[] {"TIME", "ID", "ID", "MEASUREMENT", "MEASUREMENT"};
- final String[] statuses = new String[] {"USING", "USING", "USING", "USING", "USING"};
- try (final ResultSet resultSet = statement.executeQuery("describe table2 details")) {
- int cnt = 0;
- ResultSetMetaData metaData = resultSet.getMetaData();
- assertEquals(describeTableDetailsColumnHeaders.size(), metaData.getColumnCount());
- for (int i = 0; i < describeTableDetailsColumnHeaders.size(); i++) {
- assertEquals(
- describeTableDetailsColumnHeaders.get(i).getColumnName(),
- metaData.getColumnName(i + 1));
- }
- while (resultSet.next()) {
- assertEquals(columnNames[cnt], resultSet.getString(1));
- assertEquals(dataTypes[cnt], resultSet.getString(2));
- assertEquals(categories[cnt], resultSet.getString(3));
- assertEquals(statuses[cnt], resultSet.getString(4));
- cnt++;
+ // TODO: Reopen
+ if (false) {
+ // Test drop column
+ statement.execute("alter table table2 drop column color");
+
+ columnNames = new String[] {"time", "region_id", "plant_id", "temperature", "speed"};
+ dataTypes = new String[] {"TIMESTAMP", "STRING", "STRING", "FLOAT", "DOUBLE"};
+ categories = new String[] {"TIME", "ID", "ID", "MEASUREMENT", "MEASUREMENT"};
+ final String[] statuses = new String[] {"USING", "USING", "USING", "USING", "USING"};
+ try (final ResultSet resultSet = statement.executeQuery("describe table2 details")) {
+ int cnt = 0;
+ ResultSetMetaData metaData = resultSet.getMetaData();
+ assertEquals(describeTableDetailsColumnHeaders.size(), metaData.getColumnCount());
+ for (int i = 0; i < describeTableDetailsColumnHeaders.size(); i++) {
+ assertEquals(
+ describeTableDetailsColumnHeaders.get(i).getColumnName(),
+ metaData.getColumnName(i + 1));
+ }
+ while (resultSet.next()) {
+ assertEquals(columnNames[cnt], resultSet.getString(1));
+ assertEquals(dataTypes[cnt], resultSet.getString(2));
+ assertEquals(categories[cnt], resultSet.getString(3));
+ assertEquals(statuses[cnt], resultSet.getString(4));
+ cnt++;
+ }
+ assertEquals(columnNames.length, cnt);
}
- assertEquals(columnNames.length, cnt);
- }
- statement.execute("alter table table2 drop column speed");
+ statement.execute("alter table table2 drop column speed");
- try {
- statement.executeQuery("select color from table2");
- fail();
- } catch (final SQLException e) {
- assertEquals("701: Column 'color' cannot be resolved", e.getMessage());
- }
+ try {
+ statement.executeQuery("select color from table2");
+ fail();
+ } catch (final SQLException e) {
+ assertEquals("701: Column 'color' cannot be resolved", e.getMessage());
+ }
- try {
- statement.executeQuery("select speed from table2");
- fail();
- } catch (final SQLException e) {
- assertEquals("701: Column 'speed' cannot be resolved", e.getMessage());
- }
+ try {
+ statement.executeQuery("select speed from table2");
+ fail();
+ } catch (final SQLException e) {
+ assertEquals("701: Column 'speed' cannot be resolved", e.getMessage());
+ }
- try {
- statement.execute("alter table table2 drop column speed");
- } catch (final SQLException e) {
- assertEquals("616: Column speed in table 'test2.table2' does not exist.", e.getMessage());
- }
+ try {
+ statement.execute("alter table table2 drop column speed");
+ } catch (final SQLException e) {
+ assertEquals("616: Column speed in table 'test2.table2' does not exist.", e.getMessage());
+ }
- try {
- statement.execute("alter table table2 drop column time");
- } catch (final SQLException e) {
- assertEquals("701: Dropping id or time column is not supported.", e.getMessage());
- }
+ try {
+ statement.execute("alter table table2 drop column time");
+ } catch (final SQLException e) {
+ assertEquals("701: Dropping id or time column is not supported.", e.getMessage());
+ }
- // test data deletion by drop column
- statement.execute("alter table table2 add column speed double");
- TestUtils.assertResultSetEqual(
- statement.executeQuery("select speed from table2"),
- "speed,",
- Collections.singleton("null,"));
+ // test data deletion by drop column
+ statement.execute("alter table table2 add column speed double");
+ TestUtils.assertResultSetEqual(
+ statement.executeQuery("select speed from table2"),
+ "speed,",
+ Collections.singleton("null,"));
+ }
statement.execute("drop table table2");
try {
@@ -496,11 +499,14 @@ public class IoTDBTableIT {
assertEquals("500: Unknown database test1", e.getMessage());
}
- try {
- statement.execute("alter table test1.test drop column a");
- fail();
- } catch (final SQLException e) {
- assertEquals("500: Unknown database test1", e.getMessage());
+ // TODO: Reopen
+ if (false) {
+ try {
+ statement.execute("alter table test1.test drop column a");
+ fail();
+ } catch (final SQLException e) {
+ assertEquals("500: Unknown database test1", e.getMessage());
+ }
}
try {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java
index c331b2306ed..caadbf31b75 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.schema.filter.SchemaFilter;
import org.apache.iotdb.commons.schema.table.TsTable;
+import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.queryengine.common.SessionInfo;
import org.apache.iotdb.db.queryengine.common.header.ColumnHeader;
import org.apache.iotdb.db.queryengine.execution.operator.schema.source.DeviceBlackListConstructor;
@@ -67,6 +68,7 @@ public class DeleteDevice extends AbstractTraverseDevice {
public DeleteDevice(final NodeLocation location, final Table table, final Expression where) {
super(location, table, where);
+ throw new SemanticException("Delete device is unsupported yet.");
}
public void parseModEntries(final TsTable table) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropColumn.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropColumn.java
index bd776614ab7..1cad3f8f3a3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropColumn.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropColumn.java
@@ -19,6 +19,8 @@
package org.apache.iotdb.db.queryengine.plan.relational.sql.ast;
+import org.apache.iotdb.db.exception.sql.SemanticException;
+
import com.google.common.collect.ImmutableList;
import java.util.List;
@@ -35,18 +37,6 @@ public class DropColumn extends Statement {
private final boolean tableIfExists;
private final boolean columnIfExists;
- public DropColumn(
- final QualifiedName table,
- final Identifier field,
- final boolean tableIfExists,
- final boolean columnIfExists) {
- super(null);
- this.table = requireNonNull(table, "table is null");
- this.field = requireNonNull(field, "field is null");
- this.tableIfExists = tableIfExists;
- this.columnIfExists = columnIfExists;
- }
-
public DropColumn(
final NodeLocation location,
final QualifiedName table,
@@ -58,6 +48,7 @@ public class DropColumn extends Statement {
this.field = requireNonNull(field, "field is null");
this.tableIfExists = tableIfExists;
this.columnIfExists = columnIfExists;
+ throw new SemanticException("Drop column is unsupported yet.");
}
public QualifiedName getTable() {
(iotdb) 07/07: Optimize table cache leader implement 2 (#14278)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/2.0.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d0b17c396b1bb27907acd6db9ac8e4908c2997ba
Author: Haonan <hh...@outlook.com>
AuthorDate: Tue Dec 3 11:46:31 2024 +0800
Optimize table cache leader implement 2 (#14278)
---
.../src/main/java/org/apache/iotdb/session/Session.java | 5 +++++
.../java/org/apache/iotdb/session/util/SessionUtils.java | 15 +++++++++++++++
2 files changed, 20 insertions(+)
diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java
index 63eb0b77701..3fbdb4214ff 100644
--- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java
+++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java
@@ -2729,6 +2729,9 @@ public class Session implements ISession {
*/
public void insertRelationalTablet(Tablet tablet)
throws IoTDBConnectionException, StatementExecutionException {
+ if (tablet.getRowSize() == 0) {
+ return;
+ }
if (enableRedirection) {
insertRelationalTabletWithLeaderCache(tablet);
} else {
@@ -2750,6 +2753,8 @@ public class Session implements ISession {
Map<SessionConnection, Tablet> relationalTabletGroup = new HashMap<>();
if (tableModelDeviceIdToEndpoint.isEmpty()) {
relationalTabletGroup.put(defaultSessionConnection, tablet);
+ } else if (SessionUtils.isTabletContainsSingleDevice(tablet)) {
+ relationalTabletGroup.put(getSessionConnection(tablet.getDeviceID(0)), tablet);
} else {
for (int i = 0; i < tablet.getRowSize(); i++) {
IDeviceID iDeviceID = tablet.getDeviceID(i);
diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java
index 5febc232f07..0ec1eb1621a 100644
--- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java
+++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.rpc.UrlUtils;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.utils.BytesUtils;
@@ -344,6 +345,20 @@ public class SessionUtils {
}
}
+ /* Used for table model insert only. */
+ public static boolean isTabletContainsSingleDevice(Tablet tablet) {
+ if (tablet.getRowSize() == 1) {
+ return true;
+ }
+ IDeviceID firstDeviceId = tablet.getDeviceID(0);
+ for (int i = 1; i < tablet.getRowSize(); ++i) {
+ if (!firstDeviceId.equals(tablet.getDeviceID(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
public static List<TEndPoint> parseSeedNodeUrls(List<String> nodeUrls) {
if (nodeUrls == null) {
throw new NumberFormatException("nodeUrls is null");
(iotdb) 04/07: Fix the deserialization of table deletion in WAL
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/2.0.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 641fc5aab199a4b5603cbdca2f324084b0f8a2f0
Author: Jiang Tian <jt...@163.com>
AuthorDate: Mon Dec 2 14:08:18 2024 +0800
Fix the deserialization of table deletion in WAL
---
.../plan/node/write/RelationalDeleteDataNode.java | 8 ++++++--
.../node/write/RelationalDeleteDataNodeTest.java | 24 +++++++++++++++++-----
2 files changed, 25 insertions(+), 7 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNode.java
index 3bda6d51443..05e46cf895d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNode.java
@@ -105,10 +105,14 @@ public class RelationalDeleteDataNode extends AbstractDeleteDataNode {
throws IOException {
long searchIndex = stream.readLong();
- TableDeletionEntry entry = ((TableDeletionEntry) ModEntry.createFrom(stream));
+ int entryNum = ReadWriteForEncodingUtils.readVarInt(stream);
+ List<TableDeletionEntry> modEntries = new ArrayList<>(entryNum);
+ for (int i = 0; i < entryNum; i++) {
+ modEntries.add((TableDeletionEntry) ModEntry.createFrom(stream));
+ }
RelationalDeleteDataNode deleteDataNode =
- new RelationalDeleteDataNode(new PlanNodeId(""), entry);
+ new RelationalDeleteDataNode(new PlanNodeId(""), modEntries);
deleteDataNode.setSearchIndex(searchIndex);
return deleteDataNode;
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNodeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNodeTest.java
index 7ba8d2c5544..9e3e2e5e433 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNodeTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNodeTest.java
@@ -35,7 +35,9 @@ import org.apache.tsfile.file.metadata.IDeviceID.Factory;
import org.apache.tsfile.read.common.TimeRange;
import org.junit.Test;
+import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -90,16 +92,28 @@ public class RelationalDeleteDataNodeTest {
WALByteBufferForTest walByteBufferForTest = new WALByteBufferForTest(buffer);
relationalDeleteDataNode.serializeToWAL(walByteBufferForTest);
buffer.flip();
- PlanNode planNode = PlanNodeType.deserializeFromWAL(buffer);
+ PlanNode deserialized = PlanNodeType.deserializeFromWAL(buffer);
// plan node id is not serialized to WAL, manually set it to pass comparison
- planNode.setPlanNodeId(relationalDeleteDataNode.getPlanNodeId());
- assertEquals(relationalDeleteDataNode, planNode);
+ deserialized.setPlanNodeId(relationalDeleteDataNode.getPlanNodeId());
+ assertEquals(relationalDeleteDataNode, deserialized);
+
+ buffer.flip();
+ DataInputStream dataInputStream =
+ new DataInputStream(
+ new ByteArrayInputStream(
+ buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining()));
+ deserialized = PlanNodeType.deserializeFromWAL(dataInputStream);
+ // plan node id is not serialized to WAL, manually set it to pass comparison
+ deserialized.setPlanNodeId(relationalDeleteDataNode.getPlanNodeId());
+ assertEquals(relationalDeleteDataNode, deserialized);
buffer = relationalDeleteDataNode.serializeToDAL();
- AbstractDeleteDataNode deserialized = DeleteNodeType.deserializeFromDAL(buffer);
+ deserialized = DeleteNodeType.deserializeFromDAL(buffer);
// plan node id is not serialized to DAL, manually set it to pass comparison
deserialized.setPlanNodeId(relationalDeleteDataNode.getPlanNodeId());
assertEquals(relationalDeleteDataNode, deserialized);
- assertEquals(relationalDeleteDataNode.getProgressIndex(), deserialized.getProgressIndex());
+ assertEquals(
+ relationalDeleteDataNode.getProgressIndex(),
+ ((AbstractDeleteDataNode) deserialized).getProgressIndex());
}
}
(iotdb) 06/07: Enhance procedure recover policy
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/2.0.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 53e20c10284709e7fc5efb8ef88c05f0f53680b7
Author: Potato <ta...@apache.org>
AuthorDate: Mon Dec 2 14:09:09 2024 +0800
Enhance procedure recover policy
---
.../java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java
index 4cf768027c6..b84daf9f045 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java
@@ -127,8 +127,8 @@ public class ProcedureExecutor<Env> {
if (!proc.hasParent()) {
rollbackStack.put(proc.getProcId(), new RootProcedureStack<>());
}
+ procedures.putIfAbsent(proc.getProcId(), proc);
}
- procedures.putIfAbsent(proc.getProcId(), proc);
}
List<Procedure<Env>> runnableList = new ArrayList<>();
List<Procedure<Env>> failedList = new ArrayList<>();
(iotdb) 02/07: Diasble delete-from sql
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/2.0.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 873e0da80b42015ad0fbca1003a4da138e4e4c20
Author: Jiang Tian <jt...@163.com>
AuthorDate: Fri Nov 29 19:44:37 2024 +0800
Diasble delete-from sql
---
.../relational/it/db/it/IoTDBDeletionTableIT.java | 1 +
.../it/db/it/IoTDBDisableDeletionTableIT.java | 120 +++++++++++++++++++++
.../relational/analyzer/StatementAnalyzer.java | 3 +
3 files changed, 124 insertions(+)
diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java
index e31dcd1aa6f..fd0a23c97b1 100644
--- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java
@@ -50,6 +50,7 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+@Ignore
@RunWith(IoTDBTestRunner.class)
@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
public class IoTDBDeletionTableIT {
diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDisableDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDisableDeletionTableIT.java
new file mode 100644
index 00000000000..0f852e57635
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDisableDeletionTableIT.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.relational.it.db.it;
+
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.TableClusterIT;
+import org.apache.iotdb.itbase.category.TableLocalStandaloneIT;
+import org.apache.iotdb.itbase.env.BaseEnv;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Locale;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({TableLocalStandaloneIT.class, TableClusterIT.class})
+public class IoTDBDisableDeletionTableIT {
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Locale.setDefault(Locale.ENGLISH);
+
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setPartitionInterval(1000)
+ .setMemtableSizeThreshold(10000);
+ // Adjust memstable threshold size to make it flush automatically
+ EnvFactory.getEnv().initClusterEnvironment();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void testDeletionDisabled() throws SQLException {
+ try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
+ Statement statement = connection.createStatement()) {
+ statement.execute("create database test");
+ statement.execute("use test");
+ statement.execute(
+ "CREATE TABLE vehicle1(deviceId STRING ID, s0 INT32 MEASUREMENT, s1 INT64 MEASUREMENT, s2 FLOAT MEASUREMENT, s3 TEXT MEASUREMENT, s4 BOOLEAN MEASUREMENT)");
+
+ statement.execute("insert into vehicle1(time, deviceId, s0) values (10, 'd0', 310)");
+ statement.execute("insert into vehicle1(time, deviceId, s3) values (10, 'd0','text')");
+ statement.execute("insert into vehicle1(time, deviceId, s4) values (10, 'd0',true)");
+
+ try {
+ statement.execute("DELETE FROM vehicle1 WHERE s0 <= 300 AND s0 > 0");
+ fail("should not reach here!");
+ } catch (SQLException e) {
+ assertEquals("701: Delete statement is not supported yet.", e.getMessage());
+ }
+
+ try {
+ statement.execute("DELETE FROM vehicle1 WHERE s3 = 'text'");
+ fail("should not reach here!");
+ } catch (SQLException e) {
+ assertEquals("701: Delete statement is not supported yet.", e.getMessage());
+ }
+
+ try {
+ statement.execute("DELETE FROM vehicle1 WHERE s4 != true");
+ fail("should not reach here!");
+ } catch (SQLException e) {
+ assertEquals("701: Delete statement is not supported yet.", e.getMessage());
+ }
+
+ try {
+ statement.execute("DELETE FROM vehicle1 WHERE time < 10 and deviceId > 'd0'");
+ fail("should not reach here!");
+ } catch (SQLException e) {
+ assertEquals("701: Delete statement is not supported yet.", e.getMessage());
+ }
+
+ try {
+ statement.execute("DELETE FROM vehicle1 WHERE time < 10 and deviceId is not null");
+ fail("should not reach here!");
+ } catch (SQLException e) {
+ assertEquals("701: Delete statement is not supported yet.", e.getMessage());
+ }
+
+ try {
+ statement.execute("DELETE FROM vehicle1 WHERE time < 10 and deviceId = null");
+ fail("should not reach here!");
+ } catch (SQLException e) {
+ assertEquals("701: Delete statement is not supported yet.", e.getMessage());
+ }
+ }
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
index d4c32235a75..928354dee10 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java
@@ -533,6 +533,9 @@ public class StatementAnalyzer {
@Override
protected Scope visitDelete(Delete node, Optional<Scope> scope) {
+ if (true) {
+ throw new SemanticException("Delete statement is not supported yet.");
+ }
final Scope ret = Scope.create();
AnalyzeUtils.analyzeDelete(node, queryContext);
analysis.setScope(node, ret);
(iotdb) 03/07: Add table series number in schema quota
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/2.0.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 3d91a9084157c258aa77cc138ab62fb7896e8d61
Author: Jackie Tien <ja...@gmail.com>
AuthorDate: Sat Nov 30 13:23:35 2024 +0800
Add table series number in schema quota
---
.../apache/iotdb/db/schemaengine/SchemaEngine.java | 45 ++++++++++------------
1 file changed, 20 insertions(+), 25 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java
index 76958eaf1d2..4f73a8707c1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/SchemaEngine.java
@@ -388,31 +388,28 @@ public class SchemaEngine {
&& SchemaRegionConsensusImpl.getInstance().isLeader(entry.getKey()))
.forEach(
entry ->
- timeSeriesNum.put(
- entry.getKey().getId(),
- entry.getValue().getSchemaRegionStatistics().getSeriesNumber(false)
- + entry
- .getValue()
- .getSchemaRegionStatistics()
- .getTable2DevicesNumMap()
- .entrySet()
- .stream()
- .map(
- tableEntry -> {
- final TsTable table =
- DataNodeTableCache.getInstance()
- .getTable(
- PathUtils.unQualifyDatabaseName(
- entry.getValue().getDatabaseFullPath()),
- tableEntry.getKey());
- return Objects.nonNull(table)
- ? table.getMeasurementNum() * tableEntry.getValue()
- : 0;
- })
- .reduce(0L, Long::sum)));
+ timeSeriesNum.put(entry.getKey().getId(), getTimeSeriesNumber(entry.getValue())));
return timeSeriesNum;
}
+ // not including view number
+ private long getTimeSeriesNumber(ISchemaRegion schemaRegion) {
+ return schemaRegion.getSchemaRegionStatistics().getSeriesNumber(false)
+ + schemaRegion.getSchemaRegionStatistics().getTable2DevicesNumMap().entrySet().stream()
+ .map(
+ tableEntry -> {
+ final TsTable table =
+ DataNodeTableCache.getInstance()
+ .getTable(
+ PathUtils.unQualifyDatabaseName(schemaRegion.getDatabaseFullPath()),
+ tableEntry.getKey());
+ return Objects.nonNull(table)
+ ? table.getMeasurementNum() * tableEntry.getValue()
+ : 0;
+ })
+ .reduce(0L, Long::sum);
+ }
+
/**
* Update total count in schema quota manager and generate local count map response. If limit is
* not -1 and deviceNumMap/timeSeriesNumMap is null, fill deviceNumMap/timeSeriesNumMap of the
@@ -459,9 +456,7 @@ public class SchemaEngine {
tmp.put(
consensusGroupId.getId(),
Optional.ofNullable(schemaRegionMap.get(consensusGroupId))
- .map(
- schemaRegion ->
- schemaRegion.getSchemaRegionStatistics().getSeriesNumber(false))
+ .map(this::getTimeSeriesNumber)
.orElse(0L)));
}
}