You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2025/12/18 04:08:24 UTC
(iotdb) branch rc/2.0.6 updated (914430555c4 -> 5854babbcbe)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch rc/2.0.6
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 914430555c4 Correct the dependencies.json
new 93efc0a078b Implemented the alter encoding compression function for tree model (#16672)
new 06ca819daca Changed the SQL of AlterEncodingCompressor statement & banned the "root" timeSeries & handled the empty intersection path & refactored the IT (#16725)
new 8763d9a7da2 Fixed the procedure has_more_state bug & Fixed the IT for encoding & compressor (#16729)
new 096d9eb8965 Optimized the lock for encoding & compressor's invalidate cache (#16733)
new 09d526ced1a Pipe: Improved the total performance by disable some useless logic
new 5854babbcbe Solved the compilation problem
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/daily-it.yml | 900 ---------------------
.github/workflows/pipe-it.yml | 872 --------------------
.../it/schema/IoTDBAlterEncodingCompressorIT.java | 199 +++++
.../manual/basic/IoTDBPipeProtocolIT.java | 16 +-
.../treemodel/auto/basic/IoTDBPipeProtocolIT.java | 16 +-
.../treemodel/manual/IoTDBPipeInclusionIT.java | 75 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 6 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 8 +
.../client/async/CnToDnAsyncRequestType.java | 2 +
.../CnToDnInternalServiceAsyncRequestManager.java | 6 +
.../rpc/DataNodeAsyncRequestRPCHandler.java | 1 +
.../consensus/request/ConfigPhysicalPlan.java | 4 +
.../consensus/request/ConfigPhysicalPlanType.java | 1 +
.../request/ConfigPhysicalPlanVisitor.java | 8 +
...n.java => PipeAlterEncodingCompressorPlan.java} | 71 +-
.../iotdb/confignode/manager/ConfigManager.java | 18 +
.../apache/iotdb/confignode/manager/IManager.java | 3 +
.../iotdb/confignode/manager/ProcedureManager.java | 41 +
.../plugin/PipeConfigRegionSinkConstructor.java | 6 -
.../receiver/protocol/IoTDBConfigNodeReceiver.java | 49 +-
.../PipeConfigPhysicalPlanTSStatusVisitor.java | 11 +
.../client/IoTDBConfigNodeSyncClientManager.java | 6 +-
.../sink/protocol/IoTDBConfigRegionAirGapSink.java | 2 +
.../pipe/sink/protocol/IoTDBConfigRegionSink.java | 6 +-
.../pipe/source/ConfigRegionListeningFilter.java | 3 +
...eConfigPhysicalPlanTreePatternParseVisitor.java | 25 +
.../persistence/executor/ConfigPlanExecutor.java | 1 +
.../schema/AlterEncodingCompressorProcedure.java | 336 ++++++++
.../impl/schema/DataNodeTSStatusTaskExecutor.java | 74 ++
.../impl/schema/DeactivateTemplateProcedure.java | 26 +-
.../impl/schema/DeleteLogicalViewProcedure.java | 26 +-
.../impl/schema/DeleteTimeSeriesProcedure.java | 49 +-
.../table/AbstractAlterOrDropTableProcedure.java | 29 +-
...tate.java => AlterEncodingCompressorState.java} | 7 +-
.../procedure/store/ProcedureFactory.java | 9 +
.../confignode/procedure/store/ProcedureType.java | 2 +
.../thrift/ConfigNodeRPCServiceProcessor.java | 6 +
.../pipe/receiver/PipeEnrichedProcedureTest.java | 28 +
...a => AlterEncodingCompressorProcedureTest.java} | 26 +-
.../schemaregion/SchemaExecutionVisitor.java | 15 +
.../PipeDataRegionProcessorConstructor.java | 30 -
.../dataregion/PipeDataRegionSinkConstructor.java | 12 -
.../PipeSchemaRegionSinkConstructor.java | 6 -
.../client/IoTDBDataNodeAsyncClientManager.java | 9 +-
.../client/IoTDBDataNodeSyncClientManager.java | 6 +-
.../protocol/airgap/IoTDBDataNodeAirGapSink.java | 2 +
.../thrift/async/IoTDBDataRegionAsyncSink.java | 6 +-
.../thrift/sync/IoTDBDataNodeSyncSink.java | 6 +-
.../source/dataregion/IoTDBDataRegionSource.java | 49 +-
.../realtime/PipeRealtimeDataRegionSource.java | 19 +-
.../iotdb/db/protocol/client/ConfigNodeClient.java | 7 +
.../impl/DataNodeInternalRPCServiceImpl.java | 47 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 2 +-
.../execution/config/TreeConfigTaskVisitor.java | 14 +-
.../config/executor/ClusterConfigTaskExecutor.java | 62 ++
.../config/executor/IConfigTaskExecutor.java | 4 +
...eTask.java => AlterEncodingCompressorTask.java} | 17 +-
.../db/queryengine/plan/parser/ASTVisitor.java | 71 +-
.../plan/planner/plan/node/PlanNodeType.java | 4 +
.../plan/planner/plan/node/PlanVisitor.java | 5 +
.../write/AlterEncodingCompressorNode.java | 146 ++++
.../queryengine/plan/statement/StatementType.java | 1 +
.../plan/statement/StatementVisitor.java | 10 +-
.../metadata/AlterEncodingCompressorStatement.java | 160 ++++
.../metadata/DeleteDatabaseStatement.java | 2 +-
.../statement/metadata/ShowDatabaseStatement.java | 2 +-
.../schemaengine/schemaregion/ISchemaRegion.java | 3 +
.../schemaregion/SchemaRegionPlanType.java | 2 +
.../schemaregion/SchemaRegionPlanVisitor.java | 6 +
.../schemaregion/impl/SchemaRegionMemoryImpl.java | 37 +-
.../schemaregion/impl/SchemaRegionPBTreeImpl.java | 8 +
.../visitor/SchemaRegionPlanDeserializer.java | 7 +
.../visitor/SchemaRegionPlanSerializer.java | 8 +
.../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 33 +
.../write/req/SchemaRegionWritePlanFactory.java | 3 +
.../iotdb/commons/exception/MetadataException.java | 2 +-
.../apache/iotdb/commons/path/PathPatternNode.java | 7 +
.../apache/iotdb/commons/path/PathPatternTree.java | 13 +-
.../agent/plugin/builtin/BuiltinPipePlugin.java | 40 -
.../commons/pipe/receiver/IoTDBFileReceiver.java | 6 +
.../pipe/sink/client/IoTDBClientManager.java | 5 +-
.../pipe/sink/client/IoTDBSyncClientManager.java | 9 +-
.../common/PipeTransferHandshakeConstant.java | 1 +
.../commons/pipe/sink/protocol/IoTDBSink.java | 3 +-
.../pipe/sink/protocol/IoTDBSslSyncSink.java | 6 +-
.../iotdb/commons/pipe/source/IoTDBSource.java | 19 +-
.../apache/iotdb/commons/utils/SerializeUtils.java | 18 +
.../src/main/thrift/confignode.thrift | 12 +
.../src/main/thrift/datanode.thrift | 14 +
89 files changed, 1788 insertions(+), 2162 deletions(-)
delete mode 100644 .github/workflows/pipe-it.yml
create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterEncodingCompressorIT.java
copy iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/payload/{PipeDeleteLogicalViewPlan.java => PipeAlterEncodingCompressorPlan.java} (56%)
create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterEncodingCompressorProcedure.java
create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DataNodeTSStatusTaskExecutor.java
copy iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/{RemoveAINodeState.java => AlterEncodingCompressorState.java} (91%)
copy iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/{table/view/DropViewProcedureTest.java => AlterEncodingCompressorProcedureTest.java} (60%)
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/{ShowDatabaseTask.java => AlterEncodingCompressorTask.java} (68%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/write/AlterEncodingCompressorNode.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/AlterEncodingCompressorStatement.java
(iotdb) 06/06: Solved the compilation problem
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/2.0.6
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5854babbcbeb3c71e38cbe61752f5dc460f889ec
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Fri Nov 7 15:19:33 2025 +0800
Solved the compilation problem
---
.../it/schema/IoTDBAlterEncodingCompressorIT.java | 10 -------
.../treemodel/manual/IoTDBPipeInclusionIT.java | 16 +++++-----
.../receiver/protocol/IoTDBConfigNodeReceiver.java | 11 -------
...eConfigPhysicalPlanTreePatternParseVisitor.java | 2 +-
.../schema/AlterEncodingCompressorProcedure.java | 14 +++++----
.../queryengine/plan/analyze/AnalyzeVisitor.java | 2 +-
.../metadata/AlterEncodingCompressorStatement.java | 34 ++++++++++++++++++++++
7 files changed, 51 insertions(+), 38 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterEncodingCompressorIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterEncodingCompressorIT.java
index 7c9495d16eb..9328808f60e 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterEncodingCompressorIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterEncodingCompressorIT.java
@@ -156,16 +156,6 @@ public class IoTDBAlterEncodingCompressorIT extends AbstractSchemaIT {
e.getMessage());
}
- try {
- statement.execute(
- "alter timeSeries root.vehicle.wind.a, root.__audit.** set STORAGE_PROPERTIES encoding=PLAIN, compressor=LZMA2");
- fail();
- } catch (final SQLException e) {
- Assert.assertEquals(
- "803: 'AUDIT' permission is needed to alter the encoding and compressor of database root.__audit",
- e.getMessage());
- }
-
try {
statement.execute(
"alter timeSeries if permitted root.vehicle.**, root.__audit.** set STORAGE_PROPERTIES encoding=GORILLA, compressor=GZIP");
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java
index c4b42553912..47ab3ca764d 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java
@@ -77,13 +77,12 @@ public class IoTDBPipeInclusionIT extends AbstractPipeDualTreeModelManualIT {
if (!TestUtils.tryExecuteNonQueriesWithRetry(
senderEnv,
Arrays.asList(
- // TODO: add database creation after the database auto creating on receiver can be
- // banned
+ "create database root.ln",
"create timeSeries root.ln.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN",
"ALTER timeSeries root.ln.wf01.wt01.status ADD TAGS tag3=v3",
"ALTER timeSeries root.ln.wf01.wt01.status ADD ATTRIBUTES attr4=v4"),
null)) {
- return;
+ return;
}
TestUtils.assertDataEventuallyOnEnv(
@@ -95,10 +94,10 @@ public class IoTDBPipeInclusionIT extends AbstractPipeDualTreeModelManualIT {
if (!TestUtils.tryExecuteNonQueriesWithRetry(
senderEnv,
- Arrays.asList(
- "ALTER timeSeries root.** set STORAGE_PROPERTIES compressor=ZSTD",
- "insert into root.ln.wf01.wt01(time, status) values(now(), false)",
- "flush"),
+ Arrays.asList(
+ "ALTER timeSeries root.** set STORAGE_PROPERTIES compressor=ZSTD",
+ "insert into root.ln.wf01.wt01(time, status) values(now(), false)",
+ "flush"),
null)) {
return;
}
@@ -150,8 +149,7 @@ public class IoTDBPipeInclusionIT extends AbstractPipeDualTreeModelManualIT {
if (!TestUtils.tryExecuteNonQueriesWithRetry(
senderEnv,
Arrays.asList(
- // TODO: add database creation after the database auto creating on receiver can be
- // banned
+ "create timeseries root.ln",
"create timeseries root.ln.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN",
"ALTER timeseries root.ln.wf01.wt01.status ADD TAGS tag3=v3",
"ALTER timeseries root.ln.wf01.wt01.status ADD ATTRIBUTES attr4=v4"),
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java
index d746ceb76a3..8a143aa83e0 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java
@@ -37,7 +37,6 @@ import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferCom
import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferFileSealReqV1;
import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferFileSealReqV2;
import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant;
-import org.apache.iotdb.commons.schema.table.Audit;
import org.apache.iotdb.commons.schema.table.TreeViewSchema;
import org.apache.iotdb.commons.schema.table.TsTable;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema;
@@ -345,13 +344,6 @@ public class IoTDBConfigNodeReceiver extends IoTDBFileReceiver {
.getStatus();
case PipeAlterEncodingCompressor:
// Judge here in the future
- if (configManager
- .checkUserPrivileges(username, new PrivilegeUnion(PrivilegeType.AUDIT))
- .getStatus()
- .getCode()
- != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- ((PipeAlterEncodingCompressorPlan) plan).setMayAlterAudit(false);
- }
if (skipIfNoPrivileges.get()) {
final PathPatternTree pathPatternTree =
PathPatternTree.deserialize(
@@ -359,9 +351,6 @@ public class IoTDBConfigNodeReceiver extends IoTDBFileReceiver {
configManager
.fetchAuthizedPatternTree(username, PrivilegeType.WRITE_SCHEMA.ordinal())
.getPathPatternTree()));
- if (((PipeAlterEncodingCompressorPlan) plan).isMayAlterAudit()) {
- pathPatternTree.appendPathPattern(Audit.TREE_MODEL_AUDIT_DATABASE_PATH_PATTERN, true);
- }
((PipeAlterEncodingCompressorPlan) plan)
.setPatternTreeBytes(
PathPatternTreeUtils.intersectWithFullPathPrefixTree(
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitor.java
index 8aa0ae42dab..c0e394d5a93 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitor.java
@@ -254,7 +254,7 @@ public class PipeConfigPhysicalPlanTreePatternParseVisitor
@Override
public Optional<ConfigPhysicalPlan> visitPipeAlterEncodingCompressor(
final PipeAlterEncodingCompressorPlan pipeAlterEncodingCompressorPlan,
- final UnionIoTDBTreePattern pattern) {
+ final IoTDBTreePattern pattern) {
try {
final PathPatternTree intersectedTree =
pattern.getIntersection(
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterEncodingCompressorProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterEncodingCompressorProcedure.java
index e0f75722c9d..31f1a4f40eb 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterEncodingCompressorProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterEncodingCompressorProcedure.java
@@ -69,7 +69,9 @@ public class AlterEncodingCompressorProcedure
private boolean ifExists;
private byte encoding;
private byte compressor;
- private boolean mayAlterAudit;
+
+ // Reserved for potential upgrading
+ private boolean mayAlterAudit = false;
private transient ByteBuffer patternTreeBytes;
private transient String requestMessage;
@@ -149,7 +151,7 @@ public class AlterEncodingCompressorProcedure
private boolean alterEncodingCompressorInSchemaRegion(final ConfigNodeProcedureEnv env) {
final Map<TConsensusGroupId, TRegionReplicaSet> relatedSchemaRegionGroup =
- env.getConfigManager().getRelatedSchemaRegionGroup(patternTree, mayAlterAudit);
+ env.getConfigManager().getRelatedSchemaRegionGroup(patternTree);
if (relatedSchemaRegionGroup.isEmpty()) {
if (!ifExists) {
@@ -167,7 +169,7 @@ public class AlterEncodingCompressorProcedure
final DataNodeTSStatusTaskExecutor<TAlterEncodingCompressorReq> alterEncodingCompressorTask =
new DataNodeTSStatusTaskExecutor<TAlterEncodingCompressorReq>(
env,
- env.getConfigManager().getRelatedSchemaRegionGroup(patternTree, mayAlterAudit),
+ env.getConfigManager().getRelatedSchemaRegionGroup(patternTree),
false,
CnToDnAsyncRequestType.ALTER_ENCODING_COMPRESSOR,
((dataNodeLocation, consensusGroupIdList) ->
@@ -237,9 +239,9 @@ public class AlterEncodingCompressorProcedure
isGeneratedByPipe
? new PipeEnrichedPlan(
new PipeAlterEncodingCompressorPlan(
- patternTreeBytes, encoding, compressor, mayAlterAudit))
+ patternTreeBytes, encoding, compressor, false))
: new PipeAlterEncodingCompressorPlan(
- patternTreeBytes, encoding, compressor, mayAlterAudit));
+ patternTreeBytes, encoding, compressor, false));
} catch (final ConsensusException e) {
LOGGER.warn(ClusterManager.CONSENSUS_WRITE_ERROR, e);
result = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
@@ -285,7 +287,7 @@ public class AlterEncodingCompressorProcedure
ReadWriteIOUtils.write(ifExists, stream);
ReadWriteIOUtils.write(encoding, stream);
ReadWriteIOUtils.write(compressor, stream);
- ReadWriteIOUtils.write(mayAlterAudit, stream);
+ ReadWriteIOUtils.write(false, stream);
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
index 5b7f653425c..fb4580a28ca 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
@@ -3172,7 +3172,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
}
@Override
- public Analysis visitShowStorageGroup(
+ public Analysis visitShowDatabase(
ShowDatabaseStatement showDatabaseStatement, MPPQueryContext context) {
Analysis analysis = new Analysis();
analysis.setRealStatement(showDatabaseStatement);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/AlterEncodingCompressorStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/AlterEncodingCompressorStatement.java
index a8dd88841a4..39fcdfba376 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/AlterEncodingCompressorStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/AlterEncodingCompressorStatement.java
@@ -19,13 +19,20 @@
package org.apache.iotdb.db.queryengine.plan.statement.metadata;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.auth.AuthException;
+import org.apache.iotdb.commons.auth.entity.PrivilegeType;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PathPatternTree;
+import org.apache.iotdb.commons.path.PathPatternTreeUtils;
+import org.apache.iotdb.commons.utils.StatusUtils;
+import org.apache.iotdb.db.auth.AuthorityChecker;
import org.apache.iotdb.db.queryengine.plan.analyze.QueryType;
import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement;
import org.apache.iotdb.db.queryengine.plan.statement.Statement;
import org.apache.iotdb.db.queryengine.plan.statement.StatementType;
import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
@@ -33,6 +40,8 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding;
import java.util.List;
import java.util.Objects;
+import static org.apache.iotdb.db.auth.AuthorityChecker.getAuthorizedPathTree;
+
public class AlterEncodingCompressorStatement extends Statement implements IConfigStatement {
private PathPatternTree patternTree;
@@ -40,6 +49,8 @@ public class AlterEncodingCompressorStatement extends Statement implements IConf
private final CompressionType compressor;
private final boolean ifExists;
private final boolean ifPermitted;
+
+ // Reserved for upgrading
private boolean withAudit = false;
public AlterEncodingCompressorStatement(
@@ -109,6 +120,29 @@ public class AlterEncodingCompressorStatement extends Statement implements IConf
&& Objects.equals(this.ifPermitted, that.ifPermitted);
}
+ @Override
+ public TSStatus checkPermissionBeforeProcess(String userName) {
+ if (AuthorityChecker.SUPER_USER.equals(userName)) {
+ return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+ }
+ if (ifPermitted) {
+ try {
+ final PathPatternTree authTree =
+ getAuthorizedPathTree(userName, PrivilegeType.WRITE_SCHEMA);
+ setPatternTree(
+ PathPatternTreeUtils.intersectWithFullPathPrefixTree(getPatternTree(), authTree));
+ return StatusUtils.OK;
+ } catch (final AuthException e) {
+ return new TSStatus(e.getCode().getStatusCode());
+ }
+ }
+ return AuthorityChecker.getTSStatus(
+ AuthorityChecker.checkFullPathOrPatternListPermission(
+ userName, getPaths(), PrivilegeType.WRITE_SCHEMA),
+ getPaths(),
+ PrivilegeType.WRITE_SCHEMA);
+ }
+
@Override
public int hashCode() {
return Objects.hash(patternTree, encoding, compressor, ifExists, ifPermitted);
(iotdb) 02/06: Changed the SQL of AlterEncodingCompressor statement & banned the "root" timeSeries & handled the empty intersection path & refactored the IT (#16725)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/2.0.6
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 06ca819daca2ca7baed87def81425470621b2b0e
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Mon Nov 10 19:28:00 2025 +0800
Changed the SQL of AlterEncodingCompressor statement & banned the "root" timeSeries & handled the empty intersection path & refactored the IT (#16725)
---
.../it/schema/IoTDBAlterEncodingCompressorIT.java | 44 ++++++++++++++++------
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 2 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 ++
.../config/executor/ClusterConfigTaskExecutor.java | 5 +++
.../db/queryengine/plan/parser/ASTVisitor.java | 3 ++
5 files changed, 46 insertions(+), 12 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterEncodingCompressorIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterEncodingCompressorIT.java
index 42c0275ab35..7c9495d16eb 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterEncodingCompressorIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterEncodingCompressorIT.java
@@ -72,7 +72,15 @@ public class IoTDBAlterEncodingCompressorIT extends AbstractSchemaIT {
statement.execute("create timeSeries root.vehicle.wind.a int32");
try {
- statement.execute("alter timeSeries root.nonExist.** set encoding=PLAIN");
+ statement.execute("alter timeSeries root set STORAGE_PROPERTIES encoding=PLAIN");
+ fail();
+ } catch (final SQLException e) {
+ Assert.assertEquals("701: The timeSeries shall not be root.", e.getMessage());
+ }
+
+ try {
+ statement.execute(
+ "alter timeSeries root.nonExist.** set STORAGE_PROPERTIES encoding=PLAIN");
fail();
} catch (final SQLException e) {
Assert.assertEquals(
@@ -81,41 +89,47 @@ public class IoTDBAlterEncodingCompressorIT extends AbstractSchemaIT {
}
try {
- statement.execute("alter timeSeries if exists root.nonExist.** set encoding=PLAIN");
+ statement.execute(
+ "alter timeSeries if exists root.nonExist.** set STORAGE_PROPERTIES encoding=PLAIN");
} catch (final SQLException e) {
fail(
"Alter encoding & compressor shall not fail when timeSeries not exists if set if exists");
}
try {
- statement.execute("alter timeSeries if exists root.vehicle.** set encoding=aaa");
+ statement.execute(
+ "alter timeSeries if exists root.vehicle.** set STORAGE_PROPERTIES encoding=aaa");
fail();
} catch (final SQLException e) {
Assert.assertEquals("701: Unsupported encoding: AAA", e.getMessage());
}
try {
- statement.execute("alter timeSeries if exists root.vehicle.** set compressor=aaa");
+ statement.execute(
+ "alter timeSeries if exists root.vehicle.** set STORAGE_PROPERTIES compressor=aaa");
fail();
} catch (final SQLException e) {
Assert.assertEquals("701: Unsupported compressor: AAA", e.getMessage());
}
try {
- statement.execute("alter timeSeries if exists root.vehicle.** set falseKey=aaa");
+ statement.execute(
+ "alter timeSeries if exists root.vehicle.** set STORAGE_PROPERTIES falseKey=aaa");
fail();
} catch (final SQLException e) {
Assert.assertEquals("701: property falsekey is unsupported yet.", e.getMessage());
}
try {
- statement.execute("alter timeSeries if exists root.vehicle.** set encoding=DICTIONARY");
+ statement.execute(
+ "alter timeSeries if exists root.vehicle.** set STORAGE_PROPERTIES encoding=DICTIONARY");
fail();
} catch (final SQLException e) {
Assert.assertTrue(e.getMessage().contains("encoding DICTIONARY does not support INT32"));
}
- statement.execute("alter timeSeries root.** set encoding=Plain, compressor=LZMA2");
+ statement.execute(
+ "alter timeSeries root.** set STORAGE_PROPERTIES encoding=Plain, compressor=LZMA2");
try (final ResultSet resultSet = statement.executeQuery("SHOW TIMESERIES")) {
while (resultSet.next()) {
@@ -133,17 +147,18 @@ public class IoTDBAlterEncodingCompressorIT extends AbstractSchemaIT {
EnvFactory.getEnv().getConnection("IoTDBUser", "!@#$!dfdfzvd343");
final Statement statement = connection.createStatement()) {
try {
- statement.execute("alter timeSeries root.vechile.** set encoding=PLAIN, compressor=LZMA2");
+ statement.execute(
+ "alter timeSeries root.vehicle.** set STORAGE_PROPERTIES encoding=PLAIN, compressor=LZMA2");
fail();
} catch (final SQLException e) {
Assert.assertEquals(
- "803: No permissions for this operation, please add privilege WRITE_SCHEMA on [root.vechile.**]",
+ "803: No permissions for this operation, please add privilege WRITE_SCHEMA on [root.vehicle.**]",
e.getMessage());
}
try {
statement.execute(
- "alter timeSeries root.vechile.wind.a, root.__audit.** set encoding=PLAIN, compressor=LZMA2");
+ "alter timeSeries root.vehicle.wind.a, root.__audit.** set STORAGE_PROPERTIES encoding=PLAIN, compressor=LZMA2");
fail();
} catch (final SQLException e) {
Assert.assertEquals(
@@ -153,10 +168,17 @@ public class IoTDBAlterEncodingCompressorIT extends AbstractSchemaIT {
try {
statement.execute(
- "alter timeSeries if permitted root.vehicle.**, root.__audit.** set encoding=GORILLA, compressor=GZIP");
+ "alter timeSeries if permitted root.vehicle.**, root.__audit.** set STORAGE_PROPERTIES encoding=GORILLA, compressor=GZIP");
} catch (final SQLException e) {
fail("Alter encoding & compressor shall not fail when no privileges if set if permitted");
}
+
+ try {
+ statement.execute(
+ "alter timeSeries if permitted root.nonExist.** set STORAGE_PROPERTIES encoding=GORILLA, compressor=GZIP");
+ } catch (final SQLException e) {
+ fail("Alter encoding & compressor shall not fail if the intersected paths are empty");
+ }
}
try (final Connection connection = EnvFactory.getEnv().getConnection();
diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
index 28523138e84..2ad18560796 100644
--- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
+++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
@@ -178,7 +178,7 @@ alterClause
;
alterEncodingCompressor
- : ALTER TIMESERIES (IF EXISTS)? (IF PERMITTED)? prefixPath (COMMA prefixPath)* SET attributePair (COMMA attributePair)*
+ : ALTER TIMESERIES (IF EXISTS)? (IF PERMITTED)? prefixPath (COMMA prefixPath)* SET STORAGE_PROPERTIES attributePair (COMMA attributePair)*
;
aliasClause
diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
index 353463fa52b..3c12d55b6e2 100644
--- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
+++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
@@ -842,6 +842,10 @@ STOP
: S T O P
;
+STORAGE_PROPERTIES
+ : S T O R A G E '_' P R O P E R T I E S
+ ;
+
SUBSCRIPTION
: S U B S C R I P T I O N
;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
index 7df5fa2fb60..1d9072f9401 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
@@ -2610,6 +2610,11 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor {
final String queryId,
final AlterEncodingCompressorStatement alterEncodingCompressorStatement) {
final SettableFuture<ConfigTaskResult> future = SettableFuture.create();
+ // Will only occur if no permission
+ if (alterEncodingCompressorStatement.getPatternTree().isEmpty()) {
+ future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS));
+ return future;
+ }
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream);
try {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
index 4fc0670dfaf..7d6d30ff321 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
@@ -694,6 +694,9 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
}
}
+ if (tree.isEmpty()) {
+ throw new SemanticException("The timeSeries shall not be root.");
+ }
return new AlterEncodingCompressorStatement(
tree,
encoding,
(iotdb) 04/06: Optimized the lock for encoding & compressor's invalidate cache (#16733)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/2.0.6
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 096d9eb8965cd41a8277da51692030e723377ab4
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Tue Nov 11 19:26:28 2025 +0800
Optimized the lock for encoding & compressor's invalidate cache (#16733)
* fix
* jr
---
.../impl/schema/AlterEncodingCompressorProcedure.java | 2 +-
.../procedure/impl/schema/DeleteTimeSeriesProcedure.java | 7 ++++---
.../protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java | 11 ++++++++---
.../thrift-datanode/src/main/thrift/datanode.thrift | 1 +
4 files changed, 14 insertions(+), 7 deletions(-)
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterEncodingCompressorProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterEncodingCompressorProcedure.java
index 6f6d1bc1fd9..e0f75722c9d 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterEncodingCompressorProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterEncodingCompressorProcedure.java
@@ -131,7 +131,7 @@ public class AlterEncodingCompressorProcedure
break;
case CLEAR_CACHE:
LOGGER.info("Invalidate cache of timeSeries {}", requestMessage);
- invalidateCache(env, patternTreeBytes, requestMessage, this::setFailure);
+ invalidateCache(env, patternTreeBytes, requestMessage, this::setFailure, false);
collectPayload4Pipe(env);
return Flow.NO_MORE_STATE;
default:
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java
index 59ca04eccc6..ed0f3f0698b 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java
@@ -117,7 +117,7 @@ public class DeleteTimeSeriesProcedure
}
case CLEAN_DATANODE_SCHEMA_CACHE:
LOGGER.info("Invalidate cache of timeSeries {}", requestMessage);
- invalidateCache(env, patternTreeBytes, requestMessage, this::setFailure);
+ invalidateCache(env, patternTreeBytes, requestMessage, this::setFailure, true);
setNextState(DeleteTimeSeriesState.DELETE_DATA);
break;
case DELETE_DATA:
@@ -197,13 +197,14 @@ public class DeleteTimeSeriesProcedure
final ConfigNodeProcedureEnv env,
final ByteBuffer patternTreeBytes,
final String requestMessage,
- final Consumer<ProcedureException> setFailure) {
+ final Consumer<ProcedureException> setFailure,
+ final boolean needLock) {
final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
env.getConfigManager().getNodeManager().getRegisteredDataNodeLocations();
final DataNodeAsyncRequestContext<TInvalidateMatchedSchemaCacheReq, TSStatus> clientHandler =
new DataNodeAsyncRequestContext<>(
CnToDnAsyncRequestType.INVALIDATE_MATCHED_SCHEMA_CACHE,
- new TInvalidateMatchedSchemaCacheReq(patternTreeBytes),
+ new TInvalidateMatchedSchemaCacheReq(patternTreeBytes).setNeedLock(needLock),
dataNodeLocationMap);
CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
final Map<Integer, TSStatus> statusMap = clientHandler.getResponseMap();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
index 8b66f50b933..fc3a2191400 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
@@ -699,7 +699,10 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
@Override
public TSStatus invalidateMatchedSchemaCache(final TInvalidateMatchedSchemaCacheReq req) {
final TreeDeviceSchemaCacheManager cache = TreeDeviceSchemaCacheManager.getInstance();
- DataNodeSchemaLockManager.getInstance().takeWriteLock(SchemaLockType.VALIDATE_VS_DELETION_TREE);
+ if (req.needLock || !req.isSetNeedLock()) {
+ DataNodeSchemaLockManager.getInstance()
+ .takeWriteLock(SchemaLockType.VALIDATE_VS_DELETION_TREE);
+ }
try {
cache.takeWriteLock();
try {
@@ -708,8 +711,10 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
cache.releaseWriteLock();
}
} finally {
- DataNodeSchemaLockManager.getInstance()
- .releaseWriteLock(SchemaLockType.VALIDATE_VS_DELETION_TREE);
+ if (req.needLock || !req.isSetNeedLock()) {
+ DataNodeSchemaLockManager.getInstance()
+ .releaseWriteLock(SchemaLockType.VALIDATE_VS_DELETION_TREE);
+ }
}
return RpcUtils.SUCCESS_STATUS;
}
diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift
index f210ad5bf9a..353ec310117 100644
--- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift
+++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift
@@ -436,6 +436,7 @@ struct TRollbackSchemaBlackListReq {
struct TInvalidateMatchedSchemaCacheReq {
1: required binary pathPatternTree
+ 2: optional bool needLock
}
struct TFetchSchemaBlackListReq {
(iotdb) 03/06: Fixed the procedure has_more_state bug & Fixed the IT for encoding & compressor (#16729)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/2.0.6
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8763d9a7da2bff5c5b2c8c68f34255a5d0e1dd1c
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Tue Nov 11 14:18:08 2025 +0800
Fixed the procedure has_more_state bug & Fixed the IT for encoding & compressor (#16729)
* IT-fix
* fix
---
.../dual/treemodel/manual/IoTDBPipeInclusionIT.java | 20 ++++++++++++++------
.../schema/AlterEncodingCompressorProcedure.java | 9 ++++++---
2 files changed, 20 insertions(+), 9 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java
index 972b85bd139..c4b42553912 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java
@@ -81,10 +81,9 @@ public class IoTDBPipeInclusionIT extends AbstractPipeDualTreeModelManualIT {
// banned
"create timeSeries root.ln.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN",
"ALTER timeSeries root.ln.wf01.wt01.status ADD TAGS tag3=v3",
- "ALTER timeSeries root.ln.wf01.wt01.status ADD ATTRIBUTES attr4=v4",
- "ALTER timeSeries root.** set compressor=ZSTD"),
+ "ALTER timeSeries root.ln.wf01.wt01.status ADD ATTRIBUTES attr4=v4"),
null)) {
- return;
+ return;
}
TestUtils.assertDataEventuallyOnEnv(
@@ -92,16 +91,25 @@ public class IoTDBPipeInclusionIT extends AbstractPipeDualTreeModelManualIT {
"show timeseries root.ln.**",
"Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,",
Collections.singleton(
- "root.ln.wf01.wt01.status,null,root.ln,BOOLEAN,PLAIN,ZSTD,{\"tag3\":\"v3\"},{\"attr4\":\"v4\"},null,null,BASE,"));
+ "root.ln.wf01.wt01.status,null,root.ln,BOOLEAN,PLAIN,LZ4,{\"tag3\":\"v3\"},{\"attr4\":\"v4\"},null,null,BASE,"));
if (!TestUtils.tryExecuteNonQueriesWithRetry(
senderEnv,
- Arrays.asList(
- "insert into root.ln.wf01.wt01(time, status) values(now(), false)", "flush"),
+ Arrays.asList(
+ "ALTER timeSeries root.** set STORAGE_PROPERTIES compressor=ZSTD",
+ "insert into root.ln.wf01.wt01(time, status) values(now(), false)",
+ "flush"),
null)) {
return;
}
+ TestUtils.assertDataEventuallyOnEnv(
+ receiverEnv,
+ "show timeseries root.ln.**",
+ "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,",
+ Collections.singleton(
+ "root.ln.wf01.wt01.status,null,root.ln,BOOLEAN,PLAIN,ZSTD,{\"tag3\":\"v3\"},{\"attr4\":\"v4\"},null,null,BASE,"));
+
TestUtils.assertDataAlwaysOnEnv(
receiverEnv, "select * from root.ln.**", "Time,", Collections.emptySet());
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterEncodingCompressorProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterEncodingCompressorProcedure.java
index a76e3494d05..6f6d1bc1fd9 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterEncodingCompressorProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterEncodingCompressorProcedure.java
@@ -125,7 +125,9 @@ public class AlterEncodingCompressorProcedure
SerializeUtils.deserializeCompressorNullable(compressor),
requestMessage);
}
- alterEncodingCompressorInSchemaRegion(env);
+ if (!alterEncodingCompressorInSchemaRegion(env)) {
+ return Flow.NO_MORE_STATE;
+ }
break;
case CLEAR_CACHE:
LOGGER.info("Invalidate cache of timeSeries {}", requestMessage);
@@ -145,7 +147,7 @@ public class AlterEncodingCompressorProcedure
}
}
- private void alterEncodingCompressorInSchemaRegion(final ConfigNodeProcedureEnv env) {
+ private boolean alterEncodingCompressorInSchemaRegion(final ConfigNodeProcedureEnv env) {
final Map<TConsensusGroupId, TRegionReplicaSet> relatedSchemaRegionGroup =
env.getConfigManager().getRelatedSchemaRegionGroup(patternTree, mayAlterAudit);
@@ -159,7 +161,7 @@ public class AlterEncodingCompressorProcedure
.collect(Collectors.toList()),
false)));
}
- return;
+ return false;
}
final DataNodeTSStatusTaskExecutor<TAlterEncodingCompressorReq> alterEncodingCompressorTask =
@@ -222,6 +224,7 @@ public class AlterEncodingCompressorProcedure
};
alterEncodingCompressorTask.execute();
setNextState(AlterEncodingCompressorState.CLEAR_CACHE);
+ return true;
}
private void collectPayload4Pipe(final ConfigNodeProcedureEnv env) {
(iotdb) 01/06: Implemented the alter encoding compression function for tree model (#16672)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/2.0.6
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 93efc0a078b265644b76e17353c4665b494a0b62
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Thu Nov 6 20:40:50 2025 +0800
Implemented the alter encoding compression function for tree model (#16672)
* ifpermitted
* reconstruct
* final-prev
* complete-dn
* partial
* very-partial
* fix
* partial-set
* fix
* test
* fix
* shop
* fix
* some
* partial
* bishop
* fix
* fix
* grasia
* fix
* main
* fix
* partial
* fix
* minor
* fix
* fix
* spotless
* test
* part
* fix
* bug-fix
* fix
* Revert "fix"
This reverts commit da7a080906737af14c6022294935e963bae88ad2.
* Reapply "fix"
This reverts commit 27de58a8ffb335b85e617c60cbbc6e753f7141e4.
---
.../it/schema/IoTDBAlterEncodingCompressorIT.java | 187 ++++++++++++
.../treemodel/manual/IoTDBPipeInclusionIT.java | 65 ++++
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 6 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 +
.../client/async/CnToDnAsyncRequestType.java | 2 +
.../CnToDnInternalServiceAsyncRequestManager.java | 6 +
.../rpc/DataNodeAsyncRequestRPCHandler.java | 1 +
.../consensus/request/ConfigPhysicalPlan.java | 4 +
.../consensus/request/ConfigPhysicalPlanType.java | 1 +
.../request/ConfigPhysicalPlanVisitor.java | 8 +
.../payload/PipeAlterEncodingCompressorPlan.java | 113 +++++++
.../iotdb/confignode/manager/ConfigManager.java | 18 ++
.../apache/iotdb/confignode/manager/IManager.java | 3 +
.../iotdb/confignode/manager/ProcedureManager.java | 41 +++
.../receiver/protocol/IoTDBConfigNodeReceiver.java | 60 +++-
.../PipeConfigPhysicalPlanTSStatusVisitor.java | 11 +
.../client/IoTDBConfigNodeSyncClientManager.java | 6 +-
.../sink/protocol/IoTDBConfigRegionAirGapSink.java | 2 +
.../pipe/sink/protocol/IoTDBConfigRegionSink.java | 6 +-
.../pipe/source/ConfigRegionListeningFilter.java | 3 +
...eConfigPhysicalPlanTreePatternParseVisitor.java | 25 ++
.../persistence/executor/ConfigPlanExecutor.java | 1 +
.../schema/AlterEncodingCompressorProcedure.java | 331 +++++++++++++++++++++
.../impl/schema/DataNodeTSStatusTaskExecutor.java | 74 +++++
.../impl/schema/DeactivateTemplateProcedure.java | 26 +-
.../impl/schema/DeleteLogicalViewProcedure.java | 26 +-
.../impl/schema/DeleteTimeSeriesProcedure.java | 46 +--
.../table/AbstractAlterOrDropTableProcedure.java | 29 +-
.../state/AlterEncodingCompressorState.java} | 19 +-
.../procedure/store/ProcedureFactory.java | 9 +
.../confignode/procedure/store/ProcedureType.java | 2 +
.../thrift/ConfigNodeRPCServiceProcessor.java | 6 +
.../pipe/receiver/PipeEnrichedProcedureTest.java | 28 ++
.../AlterEncodingCompressorProcedureTest.java | 62 ++++
.../schemaregion/SchemaExecutionVisitor.java | 15 +
.../client/IoTDBDataNodeAsyncClientManager.java | 9 +-
.../client/IoTDBDataNodeSyncClientManager.java | 6 +-
.../protocol/airgap/IoTDBDataNodeAirGapSink.java | 2 +
.../thrift/async/IoTDBDataRegionAsyncSink.java | 6 +-
.../thrift/sync/IoTDBDataNodeSyncSink.java | 6 +-
.../iotdb/db/protocol/client/ConfigNodeClient.java | 7 +
.../impl/DataNodeInternalRPCServiceImpl.java | 36 ++-
.../execution/config/TreeConfigTaskVisitor.java | 14 +-
.../config/executor/ClusterConfigTaskExecutor.java | 57 ++++
.../config/executor/IConfigTaskExecutor.java | 4 +
.../metadata/AlterEncodingCompressorTask.java | 47 +++
.../db/queryengine/plan/parser/ASTVisitor.java | 68 ++++-
.../plan/planner/plan/node/PlanNodeType.java | 4 +
.../plan/planner/plan/node/PlanVisitor.java | 5 +
.../write/AlterEncodingCompressorNode.java | 146 +++++++++
.../queryengine/plan/statement/StatementType.java | 1 +
.../plan/statement/StatementVisitor.java | 10 +-
.../metadata/AlterEncodingCompressorStatement.java | 126 ++++++++
.../metadata/DeleteDatabaseStatement.java | 2 +-
.../statement/metadata/ShowDatabaseStatement.java | 2 +-
.../schemaengine/schemaregion/ISchemaRegion.java | 3 +
.../schemaregion/SchemaRegionPlanType.java | 2 +
.../schemaregion/SchemaRegionPlanVisitor.java | 6 +
.../schemaregion/impl/SchemaRegionMemoryImpl.java | 37 ++-
.../schemaregion/impl/SchemaRegionPBTreeImpl.java | 8 +
.../visitor/SchemaRegionPlanDeserializer.java | 7 +
.../visitor/SchemaRegionPlanSerializer.java | 8 +
.../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 33 ++
.../write/req/SchemaRegionWritePlanFactory.java | 3 +
.../iotdb/commons/exception/MetadataException.java | 2 +-
.../apache/iotdb/commons/path/PathPatternNode.java | 7 +
.../apache/iotdb/commons/path/PathPatternTree.java | 13 +-
.../commons/pipe/receiver/IoTDBFileReceiver.java | 6 +
.../pipe/sink/client/IoTDBClientManager.java | 5 +-
.../pipe/sink/client/IoTDBSyncClientManager.java | 9 +-
.../common/PipeTransferHandshakeConstant.java | 1 +
.../commons/pipe/sink/protocol/IoTDBSink.java | 3 +-
.../pipe/sink/protocol/IoTDBSslSyncSink.java | 6 +-
.../apache/iotdb/commons/utils/SerializeUtils.java | 18 ++
.../src/main/thrift/confignode.thrift | 12 +
.../src/main/thrift/datanode.thrift | 13 +
76 files changed, 1833 insertions(+), 173 deletions(-)
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterEncodingCompressorIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterEncodingCompressorIT.java
new file mode 100644
index 00000000000..42c0275ab35
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/IoTDBAlterEncodingCompressorIT.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.it.schema;
+
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.itbase.category.LocalStandaloneIT;
+import org.apache.iotdb.util.AbstractSchemaIT;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runners.Parameterized;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.jupiter.api.Assertions.fail;
+
+@Category({LocalStandaloneIT.class, ClusterIT.class})
+public class IoTDBAlterEncodingCompressorIT extends AbstractSchemaIT {
+
+ public IoTDBAlterEncodingCompressorIT(SchemaTestMode schemaTestMode) {
+ super(schemaTestMode);
+ }
+
+ @Parameterized.BeforeParam
+ public static void before() throws Exception {
+ setUpEnvironment();
+ EnvFactory.getEnv().initClusterEnvironment();
+ }
+
+ @Parameterized.AfterParam
+ public static void after() throws Exception {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ tearDownEnvironment();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ clearSchema();
+ }
+
+ @Test
+ public void alterEncodingAndCompressorTest() throws Exception {
+ if (schemaTestMode.equals(SchemaTestMode.PBTree)) {
+ return;
+ }
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
+ statement.execute("create timeSeries root.vehicle.wind.a int32");
+
+ try {
+ statement.execute("alter timeSeries root.nonExist.** set encoding=PLAIN");
+ fail();
+ } catch (final SQLException e) {
+ Assert.assertEquals(
+ "508: Timeseries [root.nonExist.**] does not exist or is represented by device template",
+ e.getMessage());
+ }
+
+ try {
+ statement.execute("alter timeSeries if exists root.nonExist.** set encoding=PLAIN");
+ } catch (final SQLException e) {
+ fail(
+ "Alter encoding & compressor shall not fail when timeSeries not exists if set if exists");
+ }
+
+ try {
+ statement.execute("alter timeSeries if exists root.vehicle.** set encoding=aaa");
+ fail();
+ } catch (final SQLException e) {
+ Assert.assertEquals("701: Unsupported encoding: AAA", e.getMessage());
+ }
+
+ try {
+ statement.execute("alter timeSeries if exists root.vehicle.** set compressor=aaa");
+ fail();
+ } catch (final SQLException e) {
+ Assert.assertEquals("701: Unsupported compressor: AAA", e.getMessage());
+ }
+
+ try {
+ statement.execute("alter timeSeries if exists root.vehicle.** set falseKey=aaa");
+ fail();
+ } catch (final SQLException e) {
+ Assert.assertEquals("701: property falsekey is unsupported yet.", e.getMessage());
+ }
+
+ try {
+ statement.execute("alter timeSeries if exists root.vehicle.** set encoding=DICTIONARY");
+ fail();
+ } catch (final SQLException e) {
+ Assert.assertTrue(e.getMessage().contains("encoding DICTIONARY does not support INT32"));
+ }
+
+ statement.execute("alter timeSeries root.** set encoding=Plain, compressor=LZMA2");
+
+ try (final ResultSet resultSet = statement.executeQuery("SHOW TIMESERIES")) {
+ while (resultSet.next()) {
+ assertEquals("PLAIN", resultSet.getString(5));
+ assertEquals("LZMA2", resultSet.getString(6));
+ }
+ }
+
+ statement.execute("create user IoTDBUser '!@#$!dfdfzvd343'");
+ statement.execute("grant write on root.vehicle.wind.a to user IoTDBUser");
+ statement.execute("create timeSeries root.vehicle.wind.b int32");
+ }
+
+ try (final Connection connection =
+ EnvFactory.getEnv().getConnection("IoTDBUser", "!@#$!dfdfzvd343");
+ final Statement statement = connection.createStatement()) {
+ try {
+ statement.execute("alter timeSeries root.vechile.** set encoding=PLAIN, compressor=LZMA2");
+ fail();
+ } catch (final SQLException e) {
+ Assert.assertEquals(
+ "803: No permissions for this operation, please add privilege WRITE_SCHEMA on [root.vechile.**]",
+ e.getMessage());
+ }
+
+ try {
+ statement.execute(
+ "alter timeSeries root.vechile.wind.a, root.__audit.** set encoding=PLAIN, compressor=LZMA2");
+ fail();
+ } catch (final SQLException e) {
+ Assert.assertEquals(
+ "803: 'AUDIT' permission is needed to alter the encoding and compressor of database root.__audit",
+ e.getMessage());
+ }
+
+ try {
+ statement.execute(
+ "alter timeSeries if permitted root.vehicle.**, root.__audit.** set encoding=GORILLA, compressor=GZIP");
+ } catch (final SQLException e) {
+ fail("Alter encoding & compressor shall not fail when no privileges if set if permitted");
+ }
+ }
+
+ try (final Connection connection = EnvFactory.getEnv().getConnection();
+ final Statement statement = connection.createStatement()) {
+ try (final ResultSet resultSet =
+ statement.executeQuery("SHOW TIMESERIES root.__audit.**._0.password")) {
+ while (resultSet.next()) {
+ assertEquals("PLAIN", resultSet.getString(5));
+ assertEquals("LZMA2", resultSet.getString(6));
+ }
+ }
+
+ try (final ResultSet resultSet =
+ statement.executeQuery("SHOW TIMESERIES root.vehicle.wind.b")) {
+ resultSet.next();
+ assertEquals("TS_2DIFF", resultSet.getString(5));
+ assertEquals("LZ4", resultSet.getString(6));
+ }
+
+ try (final ResultSet resultSet =
+ statement.executeQuery("SHOW TIMESERIES root.vehicle.wind.a")) {
+ resultSet.next();
+ assertEquals("GORILLA", resultSet.getString(5));
+ assertEquals("GZIP", resultSet.getString(6));
+ }
+ }
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java
index 42073c27bc0..972b85bd139 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/manual/IoTDBPipeInclusionIT.java
@@ -49,6 +49,71 @@ public class IoTDBPipeInclusionIT extends AbstractPipeDualTreeModelManualIT {
final String receiverIp = receiverDataNode.getIp();
final int receiverPort = receiverDataNode.getPort();
+ try (final SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) {
+ final Map<String, String> sourceAttributes = new HashMap<>();
+ final Map<String, String> processorAttributes = new HashMap<>();
+ final Map<String, String> sinkAttributes = new HashMap<>();
+
+ sourceAttributes.put("source.inclusion", "schema");
+
+ sinkAttributes.put("sink", "iotdb-thrift-sink");
+ sinkAttributes.put("sink.ip", receiverIp);
+ sinkAttributes.put("sink.port", Integer.toString(receiverPort));
+
+ final TSStatus status =
+ client.createPipe(
+ new TCreatePipeReq("testPipe", sinkAttributes)
+ .setExtractorAttributes(sourceAttributes)
+ .setProcessorAttributes(processorAttributes));
+
+ Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode());
+
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.startPipe("testPipe").getCode());
+
+ // Do not fail if the failure has nothing to do with pipe
+ // Because the failures will randomly generate due to resource limitation
+ if (!TestUtils.tryExecuteNonQueriesWithRetry(
+ senderEnv,
+ Arrays.asList(
+ // TODO: add database creation after the database auto creating on receiver can be
+ // banned
+ "create timeSeries root.ln.wf01.wt01.status with datatype=BOOLEAN,encoding=PLAIN",
+ "ALTER timeSeries root.ln.wf01.wt01.status ADD TAGS tag3=v3",
+ "ALTER timeSeries root.ln.wf01.wt01.status ADD ATTRIBUTES attr4=v4",
+ "ALTER timeSeries root.** set compressor=ZSTD"),
+ null)) {
+ return;
+ }
+
+ TestUtils.assertDataEventuallyOnEnv(
+ receiverEnv,
+ "show timeseries root.ln.**",
+ "Timeseries,Alias,Database,DataType,Encoding,Compression,Tags,Attributes,Deadband,DeadbandParameters,ViewType,",
+ Collections.singleton(
+ "root.ln.wf01.wt01.status,null,root.ln,BOOLEAN,PLAIN,ZSTD,{\"tag3\":\"v3\"},{\"attr4\":\"v4\"},null,null,BASE,"));
+
+ if (!TestUtils.tryExecuteNonQueriesWithRetry(
+ senderEnv,
+ Arrays.asList(
+ "insert into root.ln.wf01.wt01(time, status) values(now(), false)", "flush"),
+ null)) {
+ return;
+ }
+
+ TestUtils.assertDataAlwaysOnEnv(
+ receiverEnv, "select * from root.ln.**", "Time,", Collections.emptySet());
+ }
+ }
+
+ @Test
+ public void testPureSchemaInclusionWithMultiplePattern() throws Exception {
+ final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0);
+
+ final String receiverIp = receiverDataNode.getIp();
+ final int receiverPort = receiverDataNode.getPort();
+
try (final SyncConfigNodeIServiceClient client =
(SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) {
final Map<String, String> extractorAttributes = new HashMap<>();
diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
index 7816d9db9da..28523138e84 100644
--- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
+++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
@@ -39,7 +39,7 @@ ddlStatement
// Database
: createDatabase | dropDatabase | dropPartition | alterDatabase | showDatabases | countDatabases
// Timeseries & Path
- | createTimeseries | dropTimeseries | alterTimeseries
+ | createTimeseries | dropTimeseries | alterTimeseries | alterEncodingCompressor
| showDevices | showTimeseries | showChildPaths | showChildNodes | countDevices | countTimeseries | countNodes
// Device Template
| createSchemaTemplate | createTimeseriesUsingSchemaTemplate | dropSchemaTemplate | dropTimeseriesOfSchemaTemplate
@@ -177,6 +177,10 @@ alterClause
| UPSERT aliasClause? tagClause? attributeClause?
;
+alterEncodingCompressor
+ : ALTER TIMESERIES (IF EXISTS)? (IF PERMITTED)? prefixPath (COMMA prefixPath)* SET attributePair (COMMA attributePair)*
+ ;
+
aliasClause
: ALIAS operator_eq alias
;
diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
index 72b4001390d..353463fa52b 100644
--- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
+++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
@@ -606,6 +606,10 @@ PATHS
: P A T H S
;
+PERMITTED
+ : P E R M I T T E D
+ ;
+
PIPE
: P I P E
;
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java
index badad8a1fe7..9aa4acb392b 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java
@@ -90,6 +90,8 @@ public enum CnToDnAsyncRequestType {
DELETE_DATA_FOR_DELETE_SCHEMA,
DELETE_TIMESERIES,
+ ALTER_ENCODING_COMPRESSOR,
+
CONSTRUCT_SCHEMA_BLACK_LIST_WITH_TEMPLATE,
ROLLBACK_SCHEMA_BLACK_LIST_WITH_TEMPLATE,
DEACTIVATE_TEMPLATE,
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java
index f683b9d8e42..cf2096753c4 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java
@@ -48,6 +48,7 @@ import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.CheckS
import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.ConsumerGroupPushMetaRPCHandler;
import org.apache.iotdb.confignode.client.async.handlers.rpc.subscription.TopicPushMetaRPCHandler;
import org.apache.iotdb.mpp.rpc.thrift.TActiveTriggerInstanceReq;
+import org.apache.iotdb.mpp.rpc.thrift.TAlterEncodingCompressorReq;
import org.apache.iotdb.mpp.rpc.thrift.TAlterViewReq;
import org.apache.iotdb.mpp.rpc.thrift.TCheckSchemaRegionUsingTemplateReq;
import org.apache.iotdb.mpp.rpc.thrift.TCheckTimeSeriesExistenceReq;
@@ -308,6 +309,11 @@ public class CnToDnInternalServiceAsyncRequestManager
CnToDnAsyncRequestType.DELETE_TIMESERIES,
(req, client, handler) ->
client.deleteTimeSeries((TDeleteTimeSeriesReq) req, (SchemaUpdateRPCHandler) handler));
+ actionMapBuilder.put(
+ CnToDnAsyncRequestType.ALTER_ENCODING_COMPRESSOR,
+ (req, client, handler) ->
+ client.alterEncodingCompressor(
+ (TAlterEncodingCompressorReq) req, (SchemaUpdateRPCHandler) handler));
actionMapBuilder.put(
CnToDnAsyncRequestType.CONSTRUCT_SCHEMA_BLACK_LIST_WITH_TEMPLATE,
(req, client, handler) ->
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java
index 46b9f44d8cd..301bfc71f97 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java
@@ -83,6 +83,7 @@ public abstract class DataNodeAsyncRequestRPCHandler<Response>
case ROLLBACK_SCHEMA_BLACK_LIST:
case DELETE_DATA_FOR_DELETE_SCHEMA:
case DELETE_TIMESERIES:
+ case ALTER_ENCODING_COMPRESSOR:
case CONSTRUCT_SCHEMA_BLACK_LIST_WITH_TEMPLATE:
case ROLLBACK_SCHEMA_BLACK_LIST_WITH_TEMPLATE:
case DEACTIVATE_TEMPLATE:
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java
index a3c6dc3b97b..fbefcbacc89 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java
@@ -62,6 +62,7 @@ import org.apache.iotdb.confignode.consensus.request.write.partition.CreateDataP
import org.apache.iotdb.confignode.consensus.request.write.partition.CreateSchemaPartitionPlan;
import org.apache.iotdb.confignode.consensus.request.write.partition.RemoveRegionLocationPlan;
import org.apache.iotdb.confignode.consensus.request.write.partition.UpdateRegionLocationPlan;
+import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeAlterEncodingCompressorPlan;
import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeCreateTableOrViewPlan;
import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeactivateTemplatePlan;
import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteDevicesPlan;
@@ -546,6 +547,9 @@ public abstract class ConfigPhysicalPlan implements IConsensusRequest {
case PipeDeleteDevices:
plan = new PipeDeleteDevicesPlan();
break;
+ case PipeAlterEncodingCompressor:
+ plan = new PipeAlterEncodingCompressorPlan();
+ break;
case UpdateTriggersOnTransferNodes:
plan = new UpdateTriggersOnTransferNodesPlan();
break;
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java
index 2997fc06936..ed6d0fdc3b2 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java
@@ -301,6 +301,7 @@ public enum ConfigPhysicalPlanType {
PipeSetTTL((short) 1705),
PipeCreateTableOrView((short) 1706),
PipeDeleteDevices((short) 1707),
+ PipeAlterEncodingCompressor((short) 1708),
/** Subscription */
CreateTopic((short) 1800),
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java
index 774da4917fe..d8f0ecccd37 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanVisitor.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan;
import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan;
import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan;
import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan;
+import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeAlterEncodingCompressorPlan;
import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeCreateTableOrViewPlan;
import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeactivateTemplatePlan;
import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteDevicesPlan;
@@ -189,6 +190,8 @@ public abstract class ConfigPhysicalPlanVisitor<R, C> {
return visitRenameTable((RenameTablePlan) plan, context);
case RenameView:
return visitRenameView((RenameViewPlan) plan, context);
+ case PipeAlterEncodingCompressor:
+ return visitPipeAlterEncodingCompressor((PipeAlterEncodingCompressorPlan) plan, context);
default:
return visitPlan(plan, context);
}
@@ -500,4 +503,9 @@ public abstract class ConfigPhysicalPlanVisitor<R, C> {
public R visitRenameView(final RenameViewPlan renameViewPlan, final C context) {
return visitRenameTable(renameViewPlan, context);
}
+
+ public R visitPipeAlterEncodingCompressor(
+ final PipeAlterEncodingCompressorPlan pipeAlterEncodingCompressorPlan, final C context) {
+ return visitPlan(pipeAlterEncodingCompressorPlan, context);
+ }
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/payload/PipeAlterEncodingCompressorPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/payload/PipeAlterEncodingCompressorPlan.java
new file mode 100644
index 00000000000..bd31a7c57b2
--- /dev/null
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/payload/PipeAlterEncodingCompressorPlan.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.consensus.request.write.pipe.payload;
+
+import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan;
+import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType;
+
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import javax.annotation.Nonnull;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public class PipeAlterEncodingCompressorPlan extends ConfigPhysicalPlan {
+
+ private ByteBuffer patternTreeBytes;
+ private byte encoding;
+ private byte compressor;
+ private boolean mayAlterAudit;
+
+ public PipeAlterEncodingCompressorPlan() {
+ super(ConfigPhysicalPlanType.PipeAlterEncodingCompressor);
+ }
+
+ public PipeAlterEncodingCompressorPlan(
+ final @Nonnull ByteBuffer patternTreeBytes,
+ final byte encoding,
+ final byte compressor,
+ final boolean mayAlterAudit) {
+ super(ConfigPhysicalPlanType.PipeAlterEncodingCompressor);
+ this.patternTreeBytes = patternTreeBytes;
+ this.encoding = encoding;
+ this.compressor = compressor;
+ this.mayAlterAudit = mayAlterAudit;
+ }
+
+ public void setPatternTreeBytes(ByteBuffer patternTreeBytes) {
+ this.patternTreeBytes = patternTreeBytes;
+ }
+
+ public ByteBuffer getPatternTreeBytes() {
+ patternTreeBytes.rewind();
+ return patternTreeBytes;
+ }
+
+ public byte getEncoding() {
+ return encoding;
+ }
+
+ public byte getCompressor() {
+ return compressor;
+ }
+
+ public void setMayAlterAudit(final boolean mayAlterAudit) {
+ this.mayAlterAudit = mayAlterAudit;
+ }
+
+ public boolean isMayAlterAudit() {
+ return mayAlterAudit;
+ }
+
+ @Override
+ protected void serializeImpl(final DataOutputStream stream) throws IOException {
+ stream.writeShort(getType().getPlanType());
+ ReadWriteIOUtils.write(patternTreeBytes, stream);
+ ReadWriteIOUtils.write(encoding, stream);
+ ReadWriteIOUtils.write(compressor, stream);
+ }
+
+ @Override
+ protected void deserializeImpl(final ByteBuffer buffer) throws IOException {
+ patternTreeBytes = ByteBuffer.wrap(ReadWriteIOUtils.readBinary(buffer).getValues());
+ encoding = ReadWriteIOUtils.readByte(buffer);
+ compressor = ReadWriteIOUtils.readByte(buffer);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(patternTreeBytes, encoding, compressor);
+ }
+
+ @Override
+ public String toString() {
+ return "PipeAlterEncodingCompressorPlan{"
+ + "patternTreeBytes="
+ + patternTreeBytes
+ + ", encoding="
+ + encoding
+ + ", compressor="
+ + compressor
+ + '}';
+ }
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
index 2b90eace388..2942f39feee 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
@@ -146,6 +146,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAINodeInfo;
import org.apache.iotdb.confignode.rpc.thrift.TAINodeRegisterReq;
import org.apache.iotdb.confignode.rpc.thrift.TAINodeRestartReq;
import org.apache.iotdb.confignode.rpc.thrift.TAINodeRestartResp;
+import org.apache.iotdb.confignode.rpc.thrift.TAlterEncodingCompressorReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq;
@@ -2168,6 +2169,23 @@ public class ConfigManager implements IManager {
}
}
+ @Override
+ public TSStatus alterEncodingCompressor(final TAlterEncodingCompressorReq req) {
+ TSStatus status = confirmLeader();
+ if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ return procedureManager.alterEncodingCompressor(
+ req.getQueryId(),
+ PathPatternTree.deserialize(req.pathPatternTree),
+ req.getEncoding(),
+ req.getCompressor(),
+ req.isIfExists(),
+ req.isIsGeneratedByPipe(),
+ req.isMayAlterAudit());
+ } else {
+ return status;
+ }
+ }
+
@Override
public TSStatus deleteTimeSeries(TDeleteTimeSeriesReq req) {
TSStatus status = confirmLeader();
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
index 2e7936f48b7..d438ded0b57 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
@@ -66,6 +66,7 @@ import org.apache.iotdb.confignode.manager.subscription.SubscriptionManager;
import org.apache.iotdb.confignode.rpc.thrift.TAINodeRegisterReq;
import org.apache.iotdb.confignode.rpc.thrift.TAINodeRestartReq;
import org.apache.iotdb.confignode.rpc.thrift.TAINodeRestartResp;
+import org.apache.iotdb.confignode.rpc.thrift.TAlterEncodingCompressorReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq;
@@ -700,6 +701,8 @@ public interface IManager {
TSStatus alterSchemaTemplate(TAlterSchemaTemplateReq req);
+ TSStatus alterEncodingCompressor(TAlterEncodingCompressorReq req);
+
/** Delete timeseries. */
TSStatus deleteTimeSeries(TDeleteTimeSeriesReq req);
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
index 9a5feb0b407..5b341c41b65 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
@@ -84,6 +84,7 @@ import org.apache.iotdb.confignode.procedure.impl.region.RegionMigrateProcedure;
import org.apache.iotdb.confignode.procedure.impl.region.RegionMigrationPlan;
import org.apache.iotdb.confignode.procedure.impl.region.RegionOperationProcedure;
import org.apache.iotdb.confignode.procedure.impl.region.RemoveRegionPeerProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.AlterEncodingCompressorProcedure;
import org.apache.iotdb.confignode.procedure.impl.schema.AlterLogicalViewProcedure;
import org.apache.iotdb.confignode.procedure.impl.schema.DeactivateTemplateProcedure;
import org.apache.iotdb.confignode.procedure.impl.schema.DeleteDatabaseProcedure;
@@ -306,6 +307,46 @@ public class ProcedureManager {
}
}
+ public TSStatus alterEncodingCompressor(
+ final String queryId,
+ final PathPatternTree patternTree,
+ final byte encoding,
+ final byte compressor,
+ final boolean ifExists,
+ final boolean isGeneratedByPipe,
+ final boolean mayAlterAudit) {
+ AlterEncodingCompressorProcedure procedure = null;
+ synchronized (this) {
+ ProcedureType type;
+ AlterEncodingCompressorProcedure alterEncodingCompressorProcedure;
+ for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
+ type = ProcedureFactory.getProcedureType(runningProcedure);
+ if (type == null || !type.equals(ProcedureType.ALTER_ENCODING_COMPRESSOR_PROCEDURE)) {
+ continue;
+ }
+ alterEncodingCompressorProcedure = ((AlterEncodingCompressorProcedure) runningProcedure);
+ if (queryId.equals(alterEncodingCompressorProcedure.getQueryId())) {
+ procedure = alterEncodingCompressorProcedure;
+ break;
+ }
+ }
+
+ if (procedure == null) {
+ procedure =
+ new AlterEncodingCompressorProcedure(
+ isGeneratedByPipe,
+ queryId,
+ patternTree,
+ ifExists,
+ encoding,
+ compressor,
+ mayAlterAudit);
+ this.executor.submitProcedure(procedure);
+ }
+ }
+ return waitingProcedureFinished(procedure);
+ }
+
public TSStatus deleteTimeSeries(
String queryId, PathPatternTree patternTree, boolean isGeneratedByPipe) {
DeleteTimeSeriesProcedure procedure = null;
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java
index d52ad5cd7e8..d746ceb76a3 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/protocol/IoTDBConfigNodeReceiver.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PathPatternTree;
+import org.apache.iotdb.commons.path.PathPatternTreeUtils;
import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern;
import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern;
import org.apache.iotdb.commons.pipe.receiver.IoTDBFileReceiver;
@@ -36,6 +37,7 @@ import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferCom
import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferFileSealReqV1;
import org.apache.iotdb.commons.pipe.sink.payload.thrift.request.PipeTransferFileSealReqV2;
import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant;
+import org.apache.iotdb.commons.schema.table.Audit;
import org.apache.iotdb.commons.schema.table.TreeViewSchema;
import org.apache.iotdb.commons.schema.table.TsTable;
import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema;
@@ -51,6 +53,7 @@ import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan;
import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan;
import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan;
import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan;
+import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeAlterEncodingCompressorPlan;
import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeCreateTableOrViewPlan;
import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeactivateTemplatePlan;
import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteDevicesPlan;
@@ -279,7 +282,7 @@ public class IoTDBConfigNodeReceiver extends IoTDBFileReceiver {
return result;
}
- private TSStatus checkPermission(final ConfigPhysicalPlan plan) {
+ private TSStatus checkPermission(final ConfigPhysicalPlan plan) throws IOException {
TSStatus status = loginIfNecessary();
if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
return status;
@@ -340,6 +343,45 @@ public class IoTDBConfigNodeReceiver extends IoTDBFileReceiver {
.getAllPathPatterns()),
PrivilegeType.WRITE_SCHEMA))
.getStatus();
+ case PipeAlterEncodingCompressor:
+ // Judge here in the future
+ if (configManager
+ .checkUserPrivileges(username, new PrivilegeUnion(PrivilegeType.AUDIT))
+ .getStatus()
+ .getCode()
+ != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ ((PipeAlterEncodingCompressorPlan) plan).setMayAlterAudit(false);
+ }
+ if (skipIfNoPrivileges.get()) {
+ final PathPatternTree pathPatternTree =
+ PathPatternTree.deserialize(
+ ByteBuffer.wrap(
+ configManager
+ .fetchAuthizedPatternTree(username, PrivilegeType.WRITE_SCHEMA.ordinal())
+ .getPathPatternTree()));
+ if (((PipeAlterEncodingCompressorPlan) plan).isMayAlterAudit()) {
+ pathPatternTree.appendPathPattern(Audit.TREE_MODEL_AUDIT_DATABASE_PATH_PATTERN, true);
+ }
+ ((PipeAlterEncodingCompressorPlan) plan)
+ .setPatternTreeBytes(
+ PathPatternTreeUtils.intersectWithFullPathPrefixTree(
+ PathPatternTree.deserialize(
+ ((PipeAlterEncodingCompressorPlan) plan).getPatternTreeBytes()),
+ pathPatternTree)
+ .serialize());
+ return StatusUtils.OK;
+ } else {
+ return configManager
+ .checkUserPrivileges(
+ username,
+ new PrivilegeUnion(
+ new ArrayList<>(
+ PathPatternTree.deserialize(
+ ((PipeAlterEncodingCompressorPlan) plan).getPatternTreeBytes())
+ .getAllPathPatterns()),
+ PrivilegeType.WRITE_SCHEMA))
+ .getStatus();
+ }
case PipeDeleteLogicalView:
return configManager
.checkUserPrivileges(
@@ -635,7 +677,21 @@ public class IoTDBConfigNodeReceiver extends IoTDBFileReceiver {
return configManager
.getProcedureManager()
.deactivateTemplate(
- queryId, ((PipeDeactivateTemplatePlan) plan).getTemplateSetInfo(), true);
+ queryId,
+ ((PipeDeactivateTemplatePlan) plan).getTemplateSetInfo(),
+ shouldMarkAsPipeRequest.get());
+ case PipeAlterEncodingCompressor:
+ return configManager
+ .getProcedureManager()
+ .alterEncodingCompressor(
+ queryId,
+ PathPatternTree.deserialize(
+ ((PipeAlterEncodingCompressorPlan) plan).getPatternTreeBytes()),
+ ((PipeAlterEncodingCompressorPlan) plan).getEncoding(),
+ ((PipeAlterEncodingCompressorPlan) plan).getCompressor(),
+ true,
+ shouldMarkAsPipeRequest.get(),
+ ((PipeAlterEncodingCompressorPlan) plan).isMayAlterAudit());
case UpdateTriggerStateInTable:
// TODO: Record complete message in trigger
return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/visitor/PipeConfigPhysicalPlanTSStatusVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/visitor/PipeConfigPhysicalPlanTSStatusVisitor.java
index e1b6ce7c707..cc8b57deb8e 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/visitor/PipeConfigPhysicalPlanTSStatusVisitor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/receiver/visitor/PipeConfigPhysicalPlanTSStatusVisitor.java
@@ -27,6 +27,7 @@ import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan;
import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan;
import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan;
import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan;
+import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeAlterEncodingCompressorPlan;
import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeCreateTableOrViewPlan;
import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeactivateTemplatePlan;
import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteDevicesPlan;
@@ -579,4 +580,14 @@ public class PipeConfigPhysicalPlanTSStatusVisitor
}
return visitPlan(plan, context);
}
+
+ @Override
+ public TSStatus visitPipeAlterEncodingCompressor(
+ final PipeAlterEncodingCompressorPlan plan, final TSStatus context) {
+ if (context.getCode() == TSStatusCode.PATH_NOT_EXIST.getStatusCode()) {
+ return new TSStatus(TSStatusCode.PIPE_RECEIVER_IDEMPOTENT_CONFLICT_EXCEPTION.getStatusCode())
+ .setMessage(context.getMessage());
+ }
+ return visitPlan(plan, context);
+ }
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/client/IoTDBConfigNodeSyncClientManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/client/IoTDBConfigNodeSyncClientManager.java
index 8b6fb03f047..c58f2186056 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/client/IoTDBConfigNodeSyncClientManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/client/IoTDBConfigNodeSyncClientManager.java
@@ -46,7 +46,8 @@ public class IoTDBConfigNodeSyncClientManager extends IoTDBSyncClientManager {
boolean shouldReceiverConvertOnTypeMismatch,
String loadTsFileStrategy,
boolean validateTsFile,
- boolean shouldMarkAsPipeRequest) {
+ boolean shouldMarkAsPipeRequest,
+ final boolean skipIfNoPrivileges) {
super(
endPoints,
useSSL,
@@ -59,7 +60,8 @@ public class IoTDBConfigNodeSyncClientManager extends IoTDBSyncClientManager {
shouldReceiverConvertOnTypeMismatch,
loadTsFileStrategy,
validateTsFile,
- shouldMarkAsPipeRequest);
+ shouldMarkAsPipeRequest,
+ skipIfNoPrivileges);
}
@Override
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/IoTDBConfigRegionAirGapSink.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/IoTDBConfigRegionAirGapSink.java
index 0ca96e021e3..20d6558da75 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/IoTDBConfigRegionAirGapSink.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/IoTDBConfigRegionAirGapSink.java
@@ -84,6 +84,8 @@ public class IoTDBConfigRegionAirGapSink extends IoTDBAirGapSink {
params.put(
PipeTransferHandshakeConstant.HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST,
Boolean.toString(shouldMarkAsPipeRequest));
+ params.put(
+ PipeTransferHandshakeConstant.HANDSHAKE_KEY_SKIP_IF, Boolean.toString(skipIfNoPrivileges));
return PipeTransferConfigNodeHandshakeV2Req.toTPipeTransferBytes(params);
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/IoTDBConfigRegionSink.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/IoTDBConfigRegionSink.java
index 7738ddbca12..e93af4c6989 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/IoTDBConfigRegionSink.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/sink/protocol/IoTDBConfigRegionSink.java
@@ -75,7 +75,8 @@ public class IoTDBConfigRegionSink extends IoTDBSslSyncSink {
final boolean shouldReceiverConvertOnTypeMismatch,
final String loadTsFileStrategy,
final boolean validateTsFile,
- final boolean shouldMarkAsPipeRequest) {
+ final boolean shouldMarkAsPipeRequest,
+ final boolean skipIfNoPrivileges) {
return new IoTDBConfigNodeSyncClientManager(
nodeUrls,
useSSL,
@@ -87,7 +88,8 @@ public class IoTDBConfigRegionSink extends IoTDBSslSyncSink {
shouldReceiverConvertOnTypeMismatch,
loadTsFileStrategy,
validateTsFile,
- shouldMarkAsPipeRequest);
+ shouldMarkAsPipeRequest,
+ skipIfNoPrivileges);
}
@Override
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/ConfigRegionListeningFilter.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/ConfigRegionListeningFilter.java
index d251705aca9..f475bc62591 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/ConfigRegionListeningFilter.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/ConfigRegionListeningFilter.java
@@ -88,6 +88,9 @@ public class ConfigRegionListeningFilter {
Arrays.asList(
ConfigPhysicalPlanType.UnsetTemplate, ConfigPhysicalPlanType.PipeUnsetTemplate)));
+ OPTION_PLAN_MAP.put(
+ new PartialPath("schema.timeseries.ordinary.alter"),
+ Collections.singletonList(ConfigPhysicalPlanType.PipeAlterEncodingCompressor));
OPTION_PLAN_MAP.put(
new PartialPath("schema.timeseries.ordinary.drop"),
Collections.singletonList(ConfigPhysicalPlanType.PipeDeleteTimeSeries));
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitor.java
index 992b74ca0e5..8aa0ae42dab 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/source/PipeConfigPhysicalPlanTreePatternParseVisitor.java
@@ -29,6 +29,7 @@ import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan;
import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan;
import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan;
import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan;
+import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeAlterEncodingCompressorPlan;
import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeactivateTemplatePlan;
import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteLogicalViewPlan;
import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeDeleteTimeSeriesPlan;
@@ -250,6 +251,30 @@ public class PipeConfigPhysicalPlanTreePatternParseVisitor
}
}
+ @Override
+ public Optional<ConfigPhysicalPlan> visitPipeAlterEncodingCompressor(
+ final PipeAlterEncodingCompressorPlan pipeAlterEncodingCompressorPlan,
+ final UnionIoTDBTreePattern pattern) {
+ try {
+ final PathPatternTree intersectedTree =
+ pattern.getIntersection(
+ PathPatternTree.deserialize(pipeAlterEncodingCompressorPlan.getPatternTreeBytes()));
+ return !intersectedTree.isEmpty()
+ ? Optional.of(
+ new PipeAlterEncodingCompressorPlan(
+ intersectedTree.serialize(),
+ pipeAlterEncodingCompressorPlan.getEncoding(),
+ pipeAlterEncodingCompressorPlan.getCompressor(),
+ pipeAlterEncodingCompressorPlan.isMayAlterAudit()))
+ : Optional.empty();
+ } catch (final IOException e) {
+ LOGGER.warn(
+ "Serialization failed for the alter encoding time series plan in pipe transmission, skip transfer",
+ e);
+ return Optional.empty();
+ }
+ }
+
@Override
public Optional<ConfigPhysicalPlan> visitPipeDeactivateTemplate(
final PipeDeactivateTemplatePlan pipeDeactivateTemplatePlan, final IoTDBTreePattern pattern) {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
index 7a22e443d10..8624accf64e 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
@@ -671,6 +671,7 @@ public class ConfigPlanExecutor {
case PipeDeleteLogicalView:
case PipeDeactivateTemplate:
case PipeDeleteDevices:
+ case PipeAlterEncodingCompressor:
// Pipe payload, used to trigger plan extraction.
// Will not be actually executed.
return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterEncodingCompressorProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterEncodingCompressorProcedure.java
new file mode 100644
index 00000000000..a76e3494d05
--- /dev/null
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterEncodingCompressorProcedure.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.procedure.impl.schema;
+
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.exception.MetadataException;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.path.PathPatternTree;
+import org.apache.iotdb.commons.utils.SerializeUtils;
+import org.apache.iotdb.commons.utils.TestOnly;
+import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType;
+import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeAlterEncodingCompressorPlan;
+import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeEnrichedPlan;
+import org.apache.iotdb.confignode.manager.ClusterManager;
+import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
+import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
+import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure;
+import org.apache.iotdb.confignode.procedure.state.AlterEncodingCompressorState;
+import org.apache.iotdb.confignode.procedure.store.ProcedureType;
+import org.apache.iotdb.consensus.exception.ConsensusException;
+import org.apache.iotdb.db.exception.metadata.PathNotExistException;
+import org.apache.iotdb.mpp.rpc.thrift.TAlterEncodingCompressorReq;
+import org.apache.iotdb.pipe.api.exception.PipeException;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.apache.iotdb.confignode.procedure.impl.schema.DeleteTimeSeriesProcedure.invalidateCache;
+import static org.apache.iotdb.confignode.procedure.impl.schema.DeleteTimeSeriesProcedure.preparePatternTreeBytesData;
+
+public class AlterEncodingCompressorProcedure
+ extends StateMachineProcedure<ConfigNodeProcedureEnv, AlterEncodingCompressorState> {
+ private static final Logger LOGGER = LoggerFactory.getLogger(AlterEncodingCompressorState.class);
+ private String queryId;
+ private PathPatternTree patternTree;
+ private boolean ifExists;
+ private byte encoding;
+ private byte compressor;
+ private boolean mayAlterAudit;
+
+ private transient ByteBuffer patternTreeBytes;
+ private transient String requestMessage;
+
+ public AlterEncodingCompressorProcedure(final boolean isGeneratedByPipe) {
+ super(isGeneratedByPipe);
+ }
+
+ public AlterEncodingCompressorProcedure(
+ final boolean isGeneratedByPipe,
+ final String queryId,
+ final PathPatternTree pathPatternTree,
+ final boolean ifExists,
+ final byte encoding,
+ final byte compressor,
+ final boolean mayAlterAudit) {
+ super(isGeneratedByPipe);
+ this.queryId = queryId;
+ setPatternTree(pathPatternTree);
+ this.ifExists = ifExists;
+ this.encoding = encoding;
+ this.compressor = compressor;
+ this.mayAlterAudit = mayAlterAudit;
+ }
+
+ public String getQueryId() {
+ return queryId;
+ }
+
+ @TestOnly
+ public PathPatternTree getPatternTree() {
+ return patternTree;
+ }
+
+ public void setPatternTree(final PathPatternTree patternTree) {
+ this.patternTree = patternTree;
+ requestMessage = patternTree.getAllPathPatterns().toString();
+ patternTreeBytes = preparePatternTreeBytesData(patternTree);
+ }
+
+ @Override
+ protected Flow executeFromState(
+ final ConfigNodeProcedureEnv env, final AlterEncodingCompressorState state)
+ throws InterruptedException {
+ final long startTime = System.currentTimeMillis();
+ try {
+ switch (state) {
+ case ALTER_SCHEMA_REGION:
+ if (LOGGER.isInfoEnabled()) {
+ LOGGER.info(
+ "Alter encoding {} & compressor {} in schema region for timeSeries {}",
+ SerializeUtils.deserializeEncodingNullable(encoding),
+ SerializeUtils.deserializeCompressorNullable(compressor),
+ requestMessage);
+ }
+ alterEncodingCompressorInSchemaRegion(env);
+ break;
+ case CLEAR_CACHE:
+ LOGGER.info("Invalidate cache of timeSeries {}", requestMessage);
+ invalidateCache(env, patternTreeBytes, requestMessage, this::setFailure);
+ collectPayload4Pipe(env);
+ return Flow.NO_MORE_STATE;
+ default:
+ setFailure(new ProcedureException("Unrecognized state " + state));
+ return Flow.NO_MORE_STATE;
+ }
+ return Flow.HAS_MORE_STATE;
+ } finally {
+ LOGGER.info(
+ "AlterEncodingCompressor-[{}] costs {}ms",
+ state,
+ (System.currentTimeMillis() - startTime));
+ }
+ }
+
+ private void alterEncodingCompressorInSchemaRegion(final ConfigNodeProcedureEnv env) {
+ final Map<TConsensusGroupId, TRegionReplicaSet> relatedSchemaRegionGroup =
+ env.getConfigManager().getRelatedSchemaRegionGroup(patternTree, mayAlterAudit);
+
+ if (relatedSchemaRegionGroup.isEmpty()) {
+ if (!ifExists) {
+ setFailure(
+ new ProcedureException(
+ new PathNotExistException(
+ patternTree.getAllPathPatterns().stream()
+ .map(PartialPath::getFullPath)
+ .collect(Collectors.toList()),
+ false)));
+ }
+ return;
+ }
+
+ final DataNodeTSStatusTaskExecutor<TAlterEncodingCompressorReq> alterEncodingCompressorTask =
+ new DataNodeTSStatusTaskExecutor<TAlterEncodingCompressorReq>(
+ env,
+ env.getConfigManager().getRelatedSchemaRegionGroup(patternTree, mayAlterAudit),
+ false,
+ CnToDnAsyncRequestType.ALTER_ENCODING_COMPRESSOR,
+ ((dataNodeLocation, consensusGroupIdList) ->
+ new TAlterEncodingCompressorReq(consensusGroupIdList, patternTreeBytes, ifExists)
+ .setCompressor(compressor)
+ .setEncoding(encoding))) {
+
+ private final Map<TDataNodeLocation, TSStatus> failureMap = new HashMap<>();
+
+ @Override
+ protected List<TConsensusGroupId> processResponseOfOneDataNode(
+ final TDataNodeLocation dataNodeLocation,
+ final List<TConsensusGroupId> consensusGroupIdList,
+ final TSStatus response) {
+ final List<TConsensusGroupId> failedRegionList = new ArrayList<>();
+ if (response.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ failureMap.remove(dataNodeLocation);
+ return failedRegionList;
+ }
+
+ if (response.getCode() == TSStatusCode.MULTIPLE_ERROR.getStatusCode()) {
+ final List<TSStatus> subStatus = response.getSubStatus();
+ for (int i = 0; i < subStatus.size(); i++) {
+ if (subStatus.get(i).getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()
+ && !(subStatus.get(i).getCode() == TSStatusCode.PATH_NOT_EXIST.getStatusCode()
+ && ifExists)) {
+ failedRegionList.add(consensusGroupIdList.get(i));
+ }
+ }
+ } else if (!(response.getCode() == TSStatusCode.PATH_NOT_EXIST.getStatusCode()
+ && ifExists)) {
+ failedRegionList.addAll(consensusGroupIdList);
+ }
+ if (!failedRegionList.isEmpty()) {
+ failureMap.put(dataNodeLocation, response);
+ } else {
+ failureMap.remove(dataNodeLocation);
+ }
+ return failedRegionList;
+ }
+
+ @Override
+ protected void onAllReplicasetFailure(
+ final TConsensusGroupId consensusGroupId,
+ final Set<TDataNodeLocation> dataNodeLocationSet) {
+ setFailure(
+ new ProcedureException(
+ new MetadataException(
+ String.format(
+ "Alter encoding compressor %s in schema regions failed. Failures: %s",
+ requestMessage, failureMap))));
+ interruptTask();
+ }
+ };
+ alterEncodingCompressorTask.execute();
+ setNextState(AlterEncodingCompressorState.CLEAR_CACHE);
+ }
+
+ private void collectPayload4Pipe(final ConfigNodeProcedureEnv env) {
+ TSStatus result;
+ try {
+ result =
+ env.getConfigManager()
+ .getConsensusManager()
+ .write(
+ isGeneratedByPipe
+ ? new PipeEnrichedPlan(
+ new PipeAlterEncodingCompressorPlan(
+ patternTreeBytes, encoding, compressor, mayAlterAudit))
+ : new PipeAlterEncodingCompressorPlan(
+ patternTreeBytes, encoding, compressor, mayAlterAudit));
+ } catch (final ConsensusException e) {
+ LOGGER.warn(ClusterManager.CONSENSUS_WRITE_ERROR, e);
+ result = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
+ result.setMessage(e.getMessage());
+ }
+ if (result.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ throw new PipeException(result.getMessage());
+ }
+ }
+
+ @Override
+ protected void rollbackState(
+ final ConfigNodeProcedureEnv env,
+ final AlterEncodingCompressorState alterEncodingCompressorState)
+ throws IOException, InterruptedException, ProcedureException {
+ // Not supported now
+ }
+
+ @Override
+ protected AlterEncodingCompressorState getState(final int stateId) {
+ return AlterEncodingCompressorState.values()[stateId];
+ }
+
+ @Override
+ protected int getStateId(final AlterEncodingCompressorState alterEncodingCompressorState) {
+ return alterEncodingCompressorState.ordinal();
+ }
+
+ @Override
+ protected AlterEncodingCompressorState getInitialState() {
+ return AlterEncodingCompressorState.ALTER_SCHEMA_REGION;
+ }
+
+ @Override
+ public void serialize(final DataOutputStream stream) throws IOException {
+ stream.writeShort(
+ isGeneratedByPipe
+ ? ProcedureType.PIPE_ENRICHED_ALTER_ENCODING_COMPRESSOR_PROCEDURE.getTypeCode()
+ : ProcedureType.ALTER_ENCODING_COMPRESSOR_PROCEDURE.getTypeCode());
+ super.serialize(stream);
+ ReadWriteIOUtils.write(queryId, stream);
+ patternTree.serialize(stream);
+ ReadWriteIOUtils.write(ifExists, stream);
+ ReadWriteIOUtils.write(encoding, stream);
+ ReadWriteIOUtils.write(compressor, stream);
+ ReadWriteIOUtils.write(mayAlterAudit, stream);
+ }
+
+ @Override
+ public void deserialize(final ByteBuffer byteBuffer) {
+ super.deserialize(byteBuffer);
+ queryId = ReadWriteIOUtils.readString(byteBuffer);
+ setPatternTree(PathPatternTree.deserialize(byteBuffer));
+ ifExists = ReadWriteIOUtils.readBoolean(byteBuffer);
+ encoding = ReadWriteIOUtils.readByte(byteBuffer);
+ compressor = ReadWriteIOUtils.readByte(byteBuffer);
+ mayAlterAudit = ReadWriteIOUtils.readBoolean(byteBuffer);
+ }
+
+ @Override
+ public boolean equals(final Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ final AlterEncodingCompressorProcedure that = (AlterEncodingCompressorProcedure) o;
+ return this.getProcId() == that.getProcId()
+ && this.getCurrentState().equals(that.getCurrentState())
+ && this.getCycles() == getCycles()
+ && Objects.equals(this.queryId, that.queryId)
+ && this.isGeneratedByPipe == that.isGeneratedByPipe
+ && this.patternTree.equals(that.patternTree)
+ && this.encoding == that.encoding
+ && this.compressor == that.compressor
+ && this.mayAlterAudit == that.mayAlterAudit;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ getProcId(),
+ getCurrentState(),
+ getCycles(),
+ queryId,
+ isGeneratedByPipe,
+ patternTree,
+ ifExists,
+ encoding,
+ compressor,
+ mayAlterAudit);
+ }
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DataNodeTSStatusTaskExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DataNodeTSStatusTaskExecutor.java
new file mode 100644
index 00000000000..700cb3a9a49
--- /dev/null
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DataNodeTSStatusTaskExecutor.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.procedure.impl.schema;
+
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType;
+import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.function.BiFunction;
+
+public abstract class DataNodeTSStatusTaskExecutor<Q>
+ extends DataNodeRegionTaskExecutor<Q, TSStatus> {
+
+ protected DataNodeTSStatusTaskExecutor(
+ final ConfigNodeProcedureEnv env,
+ final Map<TConsensusGroupId, TRegionReplicaSet> targetRegionGroup,
+ final boolean executeOnAllReplicaset,
+ final CnToDnAsyncRequestType dataNodeRequestType,
+ final BiFunction<TDataNodeLocation, List<TConsensusGroupId>, Q> dataNodeRequestGenerator) {
+ super(
+ env,
+ targetRegionGroup,
+ executeOnAllReplicaset,
+ dataNodeRequestType,
+ dataNodeRequestGenerator);
+ }
+
+ @Override
+ protected List<TConsensusGroupId> processResponseOfOneDataNode(
+ final TDataNodeLocation dataNodeLocation,
+ final List<TConsensusGroupId> consensusGroupIdList,
+ final TSStatus response) {
+ final List<TConsensusGroupId> failedRegionList = new ArrayList<>();
+ if (response.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ return failedRegionList;
+ }
+
+ if (response.getCode() == TSStatusCode.MULTIPLE_ERROR.getStatusCode()) {
+ final List<TSStatus> subStatus = response.getSubStatus();
+ for (int i = 0; i < subStatus.size(); i++) {
+ if (subStatus.get(i).getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ failedRegionList.add(consensusGroupIdList.get(i));
+ }
+ }
+ } else {
+ failedRegionList.addAll(consensusGroupIdList);
+ }
+ return failedRegionList;
+ }
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeactivateTemplateProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeactivateTemplateProcedure.java
index 1df9da1509d..7e1ca07086a 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeactivateTemplateProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeactivateTemplateProcedure.java
@@ -428,8 +428,7 @@ public class DeactivateTemplateProcedure
getProcId(), getCurrentState(), getCycles(), isGeneratedByPipe, queryId, templateSetInfo);
}
- private class DeactivateTemplateRegionTaskExecutor<Q>
- extends DataNodeRegionTaskExecutor<Q, TSStatus> {
+ private class DeactivateTemplateRegionTaskExecutor<Q> extends DataNodeTSStatusTaskExecutor<Q> {
private final String taskName;
@@ -459,29 +458,6 @@ public class DeactivateTemplateProcedure
this.taskName = taskName;
}
- @Override
- protected List<TConsensusGroupId> processResponseOfOneDataNode(
- TDataNodeLocation dataNodeLocation,
- List<TConsensusGroupId> consensusGroupIdList,
- TSStatus response) {
- List<TConsensusGroupId> failedRegionList = new ArrayList<>();
- if (response.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- return failedRegionList;
- }
-
- if (response.getCode() == TSStatusCode.MULTIPLE_ERROR.getStatusCode()) {
- List<TSStatus> subStatus = response.getSubStatus();
- for (int i = 0; i < subStatus.size(); i++) {
- if (subStatus.get(i).getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- failedRegionList.add(consensusGroupIdList.get(i));
- }
- }
- } else {
- failedRegionList.addAll(consensusGroupIdList);
- }
- return failedRegionList;
- }
-
@Override
protected void onAllReplicasetFailure(
TConsensusGroupId consensusGroupId, Set<TDataNodeLocation> dataNodeLocationSet) {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteLogicalViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteLogicalViewProcedure.java
index 6b448e32579..1cfead14fcf 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteLogicalViewProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteLogicalViewProcedure.java
@@ -335,8 +335,7 @@ public class DeleteLogicalViewProcedure
getProcId(), getCurrentState(), getCycles(), isGeneratedByPipe, patternTree);
}
- private class DeleteLogicalViewRegionTaskExecutor<Q>
- extends DataNodeRegionTaskExecutor<Q, TSStatus> {
+ private class DeleteLogicalViewRegionTaskExecutor<Q> extends DataNodeTSStatusTaskExecutor<Q> {
private final String taskName;
@@ -350,29 +349,6 @@ public class DeleteLogicalViewProcedure
this.taskName = taskName;
}
- @Override
- protected List<TConsensusGroupId> processResponseOfOneDataNode(
- final TDataNodeLocation dataNodeLocation,
- final List<TConsensusGroupId> consensusGroupIdList,
- final TSStatus response) {
- final List<TConsensusGroupId> failedRegionList = new ArrayList<>();
- if (response.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- return failedRegionList;
- }
-
- if (response.getCode() == TSStatusCode.MULTIPLE_ERROR.getStatusCode()) {
- final List<TSStatus> subStatus = response.getSubStatus();
- for (int i = 0; i < subStatus.size(); i++) {
- if (subStatus.get(i).getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- failedRegionList.add(consensusGroupIdList.get(i));
- }
- }
- } else {
- failedRegionList.addAll(consensusGroupIdList);
- }
- return failedRegionList;
- }
-
@Override
protected void onAllReplicasetFailure(
final TConsensusGroupId consensusGroupId,
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java
index b49834abaa2..59ca04eccc6 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java
@@ -60,6 +60,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.BiFunction;
+import java.util.function.Consumer;
import java.util.stream.Collectors;
public class DeleteTimeSeriesProcedure
@@ -116,7 +117,8 @@ public class DeleteTimeSeriesProcedure
}
case CLEAN_DATANODE_SCHEMA_CACHE:
LOGGER.info("Invalidate cache of timeSeries {}", requestMessage);
- invalidateCache(env);
+ invalidateCache(env, patternTreeBytes, requestMessage, this::setFailure);
+ setNextState(DeleteTimeSeriesState.DELETE_DATA);
break;
case DELETE_DATA:
LOGGER.info("Delete data of timeSeries {}", requestMessage);
@@ -191,7 +193,11 @@ public class DeleteTimeSeriesProcedure
: 0;
}
- private void invalidateCache(final ConfigNodeProcedureEnv env) {
+ public static void invalidateCache(
+ final ConfigNodeProcedureEnv env,
+ final ByteBuffer patternTreeBytes,
+ final String requestMessage,
+ final Consumer<ProcedureException> setFailure) {
final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
env.getConfigManager().getNodeManager().getRegisteredDataNodeLocations();
final DataNodeAsyncRequestContext<TInvalidateMatchedSchemaCacheReq, TSStatus> clientHandler =
@@ -205,13 +211,11 @@ public class DeleteTimeSeriesProcedure
// All dataNodes must clear the related schemaEngine cache
if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
LOGGER.error("Failed to invalidate schemaEngine cache of timeSeries {}", requestMessage);
- setFailure(
+ setFailure.accept(
new ProcedureException(new MetadataException("Invalidate schemaEngine cache failed")));
return;
}
}
-
- setNextState(DeleteTimeSeriesState.DELETE_DATA);
}
private void deleteData(final ConfigNodeProcedureEnv env) {
@@ -339,7 +343,7 @@ public class DeleteTimeSeriesProcedure
patternTreeBytes = preparePatternTreeBytesData(patternTree);
}
- private ByteBuffer preparePatternTreeBytesData(final PathPatternTree patternTree) {
+ public static ByteBuffer preparePatternTreeBytesData(final PathPatternTree patternTree) {
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream);
try {
@@ -374,7 +378,9 @@ public class DeleteTimeSeriesProcedure
@Override
public boolean equals(final Object o) {
- if (this == o) return true;
+ if (this == o) {
+ return true;
+ }
if (o == null || getClass() != o.getClass()) {
return false;
}
@@ -392,8 +398,7 @@ public class DeleteTimeSeriesProcedure
getProcId(), getCurrentState(), getCycles(), isGeneratedByPipe, patternTree);
}
- private class DeleteTimeSeriesRegionTaskExecutor<Q>
- extends DataNodeRegionTaskExecutor<Q, TSStatus> {
+ private class DeleteTimeSeriesRegionTaskExecutor<Q> extends DataNodeTSStatusTaskExecutor<Q> {
private final String taskName;
@@ -423,29 +428,6 @@ public class DeleteTimeSeriesProcedure
this.taskName = taskName;
}
- @Override
- protected List<TConsensusGroupId> processResponseOfOneDataNode(
- final TDataNodeLocation dataNodeLocation,
- final List<TConsensusGroupId> consensusGroupIdList,
- final TSStatus response) {
- final List<TConsensusGroupId> failedRegionList = new ArrayList<>();
- if (response.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- return failedRegionList;
- }
-
- if (response.getCode() == TSStatusCode.MULTIPLE_ERROR.getStatusCode()) {
- final List<TSStatus> subStatus = response.getSubStatus();
- for (int i = 0; i < subStatus.size(); i++) {
- if (subStatus.get(i).getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- failedRegionList.add(consensusGroupIdList.get(i));
- }
- }
- } else {
- failedRegionList.addAll(consensusGroupIdList);
- }
- return failedRegionList;
- }
-
@Override
protected void onAllReplicasetFailure(
final TConsensusGroupId consensusGroupId,
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java
index d818300ae75..fe92c802ce6 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java
@@ -29,9 +29,8 @@ import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType;
import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.DataNodeRegionTaskExecutor;
+import org.apache.iotdb.confignode.procedure.impl.schema.DataNodeTSStatusTaskExecutor;
import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils;
-import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.slf4j.Logger;
@@ -42,7 +41,6 @@ import javax.annotation.Nullable;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -186,7 +184,7 @@ public abstract class AbstractAlterOrDropTableProcedure<T>
}
}
- protected class TableRegionTaskExecutor<Q> extends DataNodeRegionTaskExecutor<Q, TSStatus> {
+ protected class TableRegionTaskExecutor<Q> extends DataNodeTSStatusTaskExecutor<Q> {
private final String taskName;
@@ -200,29 +198,6 @@ public abstract class AbstractAlterOrDropTableProcedure<T>
this.taskName = taskName;
}
- @Override
- protected List<TConsensusGroupId> processResponseOfOneDataNode(
- final TDataNodeLocation dataNodeLocation,
- final List<TConsensusGroupId> consensusGroupIdList,
- final TSStatus response) {
- final List<TConsensusGroupId> failedRegionList = new ArrayList<>();
- if (response.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- return failedRegionList;
- }
-
- if (response.getCode() == TSStatusCode.MULTIPLE_ERROR.getStatusCode()) {
- final List<TSStatus> subStatus = response.getSubStatus();
- for (int i = 0; i < subStatus.size(); i++) {
- if (subStatus.get(i).getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- failedRegionList.add(consensusGroupIdList.get(i));
- }
- }
- } else {
- failedRegionList.addAll(consensusGroupIdList);
- }
- return failedRegionList;
- }
-
@Override
protected void onAllReplicasetFailure(
final TConsensusGroupId consensusGroupId,
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferHandshakeConstant.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/AlterEncodingCompressorState.java
similarity index 50%
copy from iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferHandshakeConstant.java
copy to iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/AlterEncodingCompressorState.java
index 8976a25c5cc..4045971d2f1 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferHandshakeConstant.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/AlterEncodingCompressorState.java
@@ -17,20 +17,9 @@
* under the License.
*/
-package org.apache.iotdb.commons.pipe.sink.payload.thrift.common;
+package org.apache.iotdb.confignode.procedure.state;
-public class PipeTransferHandshakeConstant {
-
- public static final String HANDSHAKE_KEY_TIME_PRECISION = "timestampPrecision";
- public static final String HANDSHAKE_KEY_CLUSTER_ID = "clusterID";
- public static final String HANDSHAKE_KEY_CONVERT_ON_TYPE_MISMATCH = "convertOnTypeMismatch";
- public static final String HANDSHAKE_KEY_LOAD_TSFILE_STRATEGY = "loadTsFileStrategy";
- public static final String HANDSHAKE_KEY_USERNAME = "username";
- public static final String HANDSHAKE_KEY_PASSWORD = "password";
- public static final String HANDSHAKE_KEY_VALIDATE_TSFILE = "validateTsFile";
- public static final String HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST = "markAsPipeRequest";
-
- private PipeTransferHandshakeConstant() {
- // Utility class
- }
+public enum AlterEncodingCompressorState {
+ ALTER_SCHEMA_REGION,
+ CLEAR_CACHE
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java
index 6ea7796a00d..e023171f4fa 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java
@@ -44,6 +44,7 @@ import org.apache.iotdb.confignode.procedure.impl.region.NotifyRegionMigrationPr
import org.apache.iotdb.confignode.procedure.impl.region.ReconstructRegionProcedure;
import org.apache.iotdb.confignode.procedure.impl.region.RegionMigrateProcedure;
import org.apache.iotdb.confignode.procedure.impl.region.RemoveRegionPeerProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.AlterEncodingCompressorProcedure;
import org.apache.iotdb.confignode.procedure.impl.schema.AlterLogicalViewProcedure;
import org.apache.iotdb.confignode.procedure.impl.schema.DeactivateTemplateProcedure;
import org.apache.iotdb.confignode.procedure.impl.schema.DeleteDatabaseProcedure;
@@ -140,6 +141,9 @@ public class ProcedureFactory implements IProcedureFactory {
case NOTIFY_REGION_MIGRATION_PROCEDURE:
procedure = new NotifyRegionMigrationProcedure();
break;
+ case ALTER_ENCODING_COMPRESSOR_PROCEDURE:
+ procedure = new AlterEncodingCompressorProcedure(false);
+ break;
case DELETE_TIMESERIES_PROCEDURE:
procedure = new DeleteTimeSeriesProcedure(false);
break;
@@ -343,6 +347,9 @@ public class ProcedureFactory implements IProcedureFactory {
case PIPE_ENRICHED_RENAME_VIEW_PROCEDURE:
procedure = new RenameViewProcedure(true);
break;
+ case PIPE_ENRICHED_ALTER_ENCODING_COMPRESSOR_PROCEDURE:
+ procedure = new AlterEncodingCompressorProcedure(true);
+ break;
case REMOVE_AI_NODE_PROCEDURE:
procedure = new RemoveAINodeProcedure();
break;
@@ -425,6 +432,8 @@ public class ProcedureFactory implements IProcedureFactory {
return ProcedureType.REMOVE_REGION_PEER_PROCEDURE;
} else if (procedure instanceof CreateRegionGroupsProcedure) {
return ProcedureType.CREATE_REGION_GROUPS;
+ } else if (procedure instanceof AlterEncodingCompressorProcedure) {
+ return ProcedureType.ALTER_ENCODING_COMPRESSOR_PROCEDURE;
} else if (procedure instanceof DeleteTimeSeriesProcedure) {
return ProcedureType.DELETE_TIMESERIES_PROCEDURE;
} else if (procedure instanceof ReconstructRegionProcedure) {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java
index 76386dde139..dd4936e7ac3 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java
@@ -46,6 +46,7 @@ public enum ProcedureType {
/** Timeseries */
DELETE_TIMESERIES_PROCEDURE((short) 300),
+ ALTER_ENCODING_COMPRESSOR_PROCEDURE((short) 301),
/** Trigger */
CREATE_TRIGGER_PROCEDURE((short) 400),
@@ -144,6 +145,7 @@ public enum ProcedureType {
PIPE_ENRICHED_SET_VIEW_PROPERTIES_PROCEDURE((short) 145),
PIPE_ENRICHED_RENAME_VIEW_COLUMN_PROCEDURE((short) 146),
PIPE_ENRICHED_RENAME_VIEW_PROCEDURE((short) 147),
+ PIPE_ENRICHED_ALTER_ENCODING_COMPRESSOR_PROCEDURE((short) 148),
/** Subscription */
CREATE_TOPIC_PROCEDURE((short) 1500),
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
index 77192cd5f34..9fbd7ff9543 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
@@ -92,6 +92,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAINodeRemoveReq;
import org.apache.iotdb.confignode.rpc.thrift.TAINodeRestartReq;
import org.apache.iotdb.confignode.rpc.thrift.TAINodeRestartResp;
import org.apache.iotdb.confignode.rpc.thrift.TAddConsensusGroupReq;
+import org.apache.iotdb.confignode.rpc.thrift.TAlterEncodingCompressorReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq;
@@ -1151,6 +1152,11 @@ public class ConfigNodeRPCServiceProcessor implements IConfigNodeRPCService.Ifac
return configManager.alterSchemaTemplate(req);
}
+ @Override
+ public TSStatus alterEncodingCompressor(final TAlterEncodingCompressorReq req) throws TException {
+ return configManager.alterEncodingCompressor(req);
+ }
+
@Override
public TSStatus deleteTimeSeries(TDeleteTimeSeriesReq req) {
return configManager.deleteTimeSeries(req);
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/receiver/PipeEnrichedProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/receiver/PipeEnrichedProcedureTest.java
index 53d025d6b2f..4ee972e4910 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/receiver/PipeEnrichedProcedureTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/pipe/receiver/PipeEnrichedProcedureTest.java
@@ -33,6 +33,7 @@ import org.apache.iotdb.commons.trigger.TriggerInformation;
import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType;
import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan;
import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan;
+import org.apache.iotdb.confignode.procedure.impl.schema.AlterEncodingCompressorProcedure;
import org.apache.iotdb.confignode.procedure.impl.schema.AlterLogicalViewProcedure;
import org.apache.iotdb.confignode.procedure.impl.schema.DeactivateTemplateProcedure;
import org.apache.iotdb.confignode.procedure.impl.schema.DeleteDatabaseProcedure;
@@ -760,4 +761,31 @@ public class PipeEnrichedProcedureTest {
Assert.assertEquals(setViewPropertiesProcedure, deserializedProcedure);
}
+
+ @Test
+ public void alterEncodingCompressorTest() throws IllegalPathException, IOException {
+ final String queryId = "1";
+ final PathPatternTree patternTree = new PathPatternTree();
+ patternTree.appendPathPattern(new PartialPath("root.sg1.**"));
+ patternTree.appendPathPattern(new PartialPath("root.sg2.*.s1"));
+ patternTree.constructTree();
+ final AlterEncodingCompressorProcedure alterEncodingCompressorProcedure =
+ new AlterEncodingCompressorProcedure(
+ false, queryId, patternTree, false, (byte) 0, (byte) 0, false);
+
+ final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream);
+ alterEncodingCompressorProcedure.serialize(dataOutputStream);
+
+ final ByteBuffer byteBuffer = ByteBuffer.wrap(byteArrayOutputStream.toByteArray());
+
+ Assert.assertEquals(
+ ProcedureType.ALTER_ENCODING_COMPRESSOR_PROCEDURE.getTypeCode(), byteBuffer.getShort());
+
+ final AlterEncodingCompressorProcedure deserializedProcedure =
+ new AlterEncodingCompressorProcedure(false);
+ deserializedProcedure.deserialize(byteBuffer);
+
+ Assert.assertEquals(alterEncodingCompressorProcedure, deserializedProcedure);
+ }
}
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterEncodingCompressorProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterEncodingCompressorProcedureTest.java
new file mode 100644
index 00000000000..b6f2541b215
--- /dev/null
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/AlterEncodingCompressorProcedureTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.procedure.impl.schema;
+
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.path.PathPatternTree;
+import org.apache.iotdb.confignode.procedure.store.ProcedureType;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class AlterEncodingCompressorProcedureTest {
+ @Test
+ public void serializeDeserializeTest() throws IllegalPathException, IOException {
+ final String queryId = "1";
+ final PathPatternTree patternTree = new PathPatternTree();
+ patternTree.appendPathPattern(new PartialPath("root.sg1.**"));
+ patternTree.appendPathPattern(new PartialPath("root.sg2.*.s1"));
+ patternTree.constructTree();
+ final AlterEncodingCompressorProcedure alterEncodingCompressorProcedure =
+ new AlterEncodingCompressorProcedure(
+ false, queryId, patternTree, false, (byte) 0, (byte) 0, false);
+
+ final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream);
+ alterEncodingCompressorProcedure.serialize(dataOutputStream);
+
+ final ByteBuffer byteBuffer = ByteBuffer.wrap(byteArrayOutputStream.toByteArray());
+
+ Assert.assertEquals(
+ ProcedureType.ALTER_ENCODING_COMPRESSOR_PROCEDURE.getTypeCode(), byteBuffer.getShort());
+
+ final AlterEncodingCompressorProcedure deserializedProcedure =
+ new AlterEncodingCompressorProcedure(false);
+ deserializedProcedure.deserialize(byteBuffer);
+
+ Assert.assertEquals(alterEncodingCompressorProcedure, deserializedProcedure);
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java
index 6b9ba0a2612..8d3c2155838 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java
@@ -34,6 +34,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.ActivateTemplateNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.AlterEncodingCompressorNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.AlterTimeSeriesNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.BatchActivateTemplateNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.ConstructSchemaBlackListNode;
@@ -554,6 +555,20 @@ public class SchemaExecutionVisitor extends PlanVisitor<TSStatus, ISchemaRegion>
}
}
+ @Override
+ public TSStatus visitAlterEncodingCompressor(
+ final AlterEncodingCompressorNode node, final ISchemaRegion schemaRegion) {
+ try {
+ schemaRegion.alterEncodingCompressor(node);
+ return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS);
+ } catch (final MetadataException e) {
+ if (e.getErrorCode() != TSStatusCode.PATH_NOT_EXIST.getStatusCode()) {
+ logger.error(e.getMessage(), e);
+ }
+ return RpcUtils.getStatus(e.getErrorCode(), e.getMessage());
+ }
+ }
+
@Override
public TSStatus visitPreDeactivateTemplate(
final PreDeactivateTemplateNode node, final ISchemaRegion schemaRegion) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/client/IoTDBDataNodeAsyncClientManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/client/IoTDBDataNodeAsyncClientManager.java
index ff244215ef6..39e9017deeb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/client/IoTDBDataNodeAsyncClientManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/client/IoTDBDataNodeAsyncClientManager.java
@@ -100,7 +100,8 @@ public class IoTDBDataNodeAsyncClientManager extends IoTDBClientManager
final String loadTsFileStrategy,
final boolean validateTsFile,
final boolean shouldMarkAsPipeRequest,
- final boolean isTSFileUsed) {
+ final boolean isTSFileUsed,
+ final boolean skipIfNoPrivileges) {
super(
endPoints,
useLeaderCache,
@@ -109,7 +110,8 @@ public class IoTDBDataNodeAsyncClientManager extends IoTDBClientManager
shouldReceiverConvertOnTypeMismatch,
loadTsFileStrategy,
validateTsFile,
- shouldMarkAsPipeRequest);
+ shouldMarkAsPipeRequest,
+ skipIfNoPrivileges);
endPointSet = new HashSet<>(endPoints);
@@ -297,6 +299,9 @@ public class IoTDBDataNodeAsyncClientManager extends IoTDBClientManager
params.put(
PipeTransferHandshakeConstant.HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST,
Boolean.toString(shouldMarkAsPipeRequest));
+ params.put(
+ PipeTransferHandshakeConstant.HANDSHAKE_KEY_SKIP_IF,
+ Boolean.toString(skipIfNoPrivileges));
client.setTimeoutDynamically(PipeConfig.getInstance().getPipeConnectorHandshakeTimeoutMs());
client.pipeTransfer(PipeTransferDataNodeHandshakeV2Req.toTPipeTransferReq(params), callback);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/client/IoTDBDataNodeSyncClientManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/client/IoTDBDataNodeSyncClientManager.java
index 98fcb30c8c7..f4d911b1fc9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/client/IoTDBDataNodeSyncClientManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/client/IoTDBDataNodeSyncClientManager.java
@@ -56,7 +56,8 @@ public class IoTDBDataNodeSyncClientManager extends IoTDBSyncClientManager
final boolean shouldReceiverConvertOnTypeMismatch,
final String loadTsFileStrategy,
final boolean validateTsFile,
- final boolean shouldMarkAsPipeRequest) {
+ final boolean shouldMarkAsPipeRequest,
+ final boolean skipIfNoPrivileges) {
super(
endPoints,
useSSL,
@@ -69,7 +70,8 @@ public class IoTDBDataNodeSyncClientManager extends IoTDBSyncClientManager
shouldReceiverConvertOnTypeMismatch,
loadTsFileStrategy,
validateTsFile,
- shouldMarkAsPipeRequest);
+ shouldMarkAsPipeRequest,
+ skipIfNoPrivileges);
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataNodeAirGapSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataNodeAirGapSink.java
index d2fc67a4c6f..c4a53794472 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataNodeAirGapSink.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/airgap/IoTDBDataNodeAirGapSink.java
@@ -68,6 +68,8 @@ public abstract class IoTDBDataNodeAirGapSink extends IoTDBAirGapSink {
params.put(
PipeTransferHandshakeConstant.HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST,
Boolean.toString(shouldMarkAsPipeRequest));
+ params.put(
+ PipeTransferHandshakeConstant.HANDSHAKE_KEY_SKIP_IF, Boolean.toString(skipIfNoPrivileges));
return PipeTransferDataNodeHandshakeV2Req.toTPipeTransferBytes(params);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java
index ff35329ffb9..dd396b8514b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/async/IoTDBDataRegionAsyncSink.java
@@ -156,7 +156,8 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink {
loadTsFileStrategy,
loadTsFileValidation,
shouldMarkAsPipeRequest,
- false);
+ false,
+ skipIfNoPrivileges);
transferTsFileClientManager =
new IoTDBDataNodeAsyncClientManager(
@@ -171,7 +172,8 @@ public class IoTDBDataRegionAsyncSink extends IoTDBSink {
loadTsFileStrategy,
loadTsFileValidation,
shouldMarkAsPipeRequest,
- isSplitTSFileBatchModeEnabled);
+ isSplitTSFileBatchModeEnabled,
+ skipIfNoPrivileges);
if (isTabletBatchModeEnabled) {
tabletBatchBuilder = new PipeTransferBatchReqBuilder(parameters);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/sync/IoTDBDataNodeSyncSink.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/sync/IoTDBDataNodeSyncSink.java
index ec26a613ef4..287bd385651 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/sync/IoTDBDataNodeSyncSink.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/thrift/sync/IoTDBDataNodeSyncSink.java
@@ -51,7 +51,8 @@ public abstract class IoTDBDataNodeSyncSink extends IoTDBSslSyncSink {
final boolean shouldReceiverConvertOnTypeMismatch,
final String loadTsFileStrategy,
final boolean validateTsFile,
- final boolean shouldMarkAsPipeRequest) {
+ final boolean shouldMarkAsPipeRequest,
+ final boolean skipIfNoPrivileges) {
clientManager =
new IoTDBDataNodeSyncClientManager(
nodeUrls,
@@ -65,7 +66,8 @@ public abstract class IoTDBDataNodeSyncSink extends IoTDBSslSyncSink {
shouldReceiverConvertOnTypeMismatch,
loadTsFileStrategy,
validateTsFile,
- shouldMarkAsPipeRequest);
+ shouldMarkAsPipeRequest,
+ skipIfNoPrivileges);
return clientManager;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java
index b02e2b57de7..b5e3d8f0bdc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java
@@ -49,6 +49,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TAINodeRemoveReq;
import org.apache.iotdb.confignode.rpc.thrift.TAINodeRestartReq;
import org.apache.iotdb.confignode.rpc.thrift.TAINodeRestartResp;
import org.apache.iotdb.confignode.rpc.thrift.TAddConsensusGroupReq;
+import org.apache.iotdb.confignode.rpc.thrift.TAlterEncodingCompressorReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq;
@@ -1070,6 +1071,12 @@ public class ConfigNodeClient implements IConfigNodeRPCService.Iface, ThriftClie
() -> client.alterSchemaTemplate(req), status -> !updateConfigNodeLeader(status));
}
+ @Override
+ public TSStatus alterEncodingCompressor(TAlterEncodingCompressorReq req) throws TException {
+ return executeRemoteCallWithRetry(
+ () -> client.alterEncodingCompressor(req), status -> !updateConfigNodeLeader(status));
+ }
+
@Override
public TSStatus deleteTimeSeries(TDeleteTimeSeriesReq req) throws TException {
return executeRemoteCallWithRetry(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
index d48379ad693..8b66f50b933 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
@@ -79,6 +79,7 @@ import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta;
import org.apache.iotdb.commons.trigger.TriggerInformation;
import org.apache.iotdb.commons.udf.UDFInformation;
import org.apache.iotdb.commons.utils.PathUtils;
+import org.apache.iotdb.commons.utils.SerializeUtils;
import org.apache.iotdb.commons.utils.StatusUtils;
import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.consensus.common.Peer;
@@ -138,6 +139,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.AlterEncodingCompressorNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.ConstructSchemaBlackListNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.DeactivateTemplateNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.DeleteTimeSeriesNode;
@@ -198,6 +200,7 @@ import org.apache.iotdb.metrics.utils.MetricLevel;
import org.apache.iotdb.metrics.utils.SystemMetric;
import org.apache.iotdb.mpp.rpc.thrift.IDataNodeRPCService;
import org.apache.iotdb.mpp.rpc.thrift.TActiveTriggerInstanceReq;
+import org.apache.iotdb.mpp.rpc.thrift.TAlterEncodingCompressorReq;
import org.apache.iotdb.mpp.rpc.thrift.TAlterViewReq;
import org.apache.iotdb.mpp.rpc.thrift.TAttributeUpdateReq;
import org.apache.iotdb.mpp.rpc.thrift.TCancelFragmentInstanceReq;
@@ -777,11 +780,11 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
return executeInternalSchemaTask(
req.getSchemaRegionIdList(),
consensusGroupId -> {
- String storageGroup =
+ String database =
schemaEngine
.getSchemaRegion(new SchemaRegionId(consensusGroupId.getId()))
.getDatabaseFullPath();
- PathPatternTree filteredPatternTree = filterPathPatternTree(patternTree, storageGroup);
+ PathPatternTree filteredPatternTree = filterPathPatternTree(patternTree, database);
if (filteredPatternTree.isEmpty()) {
return RpcUtils.SUCCESS_STATUS;
}
@@ -797,6 +800,35 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
});
}
+ @Override
+ public TSStatus alterEncodingCompressor(final TAlterEncodingCompressorReq req) throws TException {
+ final PathPatternTree patternTree =
+ PathPatternTree.deserialize(ByteBuffer.wrap(req.getPathPatternTree()));
+ return executeInternalSchemaTask(
+ req.getSchemaRegionIdList(),
+ consensusGroupId -> {
+ final String database =
+ schemaEngine
+ .getSchemaRegion(new SchemaRegionId(consensusGroupId.getId()))
+ .getDatabaseFullPath();
+ final PathPatternTree filteredPatternTree = filterPathPatternTree(patternTree, database);
+ if (filteredPatternTree.isEmpty()) {
+ return RpcUtils.SUCCESS_STATUS;
+ }
+ final RegionWriteExecutor executor = new RegionWriteExecutor();
+ return executor
+ .execute(
+ new SchemaRegionId(consensusGroupId.getId()),
+ new AlterEncodingCompressorNode(
+ new PlanNodeId(""),
+ filteredPatternTree,
+ req.isIfExists(),
+ SerializeUtils.deserializeEncodingNullable(req.getEncoding()),
+ SerializeUtils.deserializeCompressorNullable(req.getCompressor())))
+ .getStatus();
+ });
+ }
+
@Override
public TSStatus constructSchemaBlackListWithTemplate(TConstructSchemaBlackListWithTemplateReq req)
throws TException {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java
index 04ccc59580c..7433c4863d9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java
@@ -27,6 +27,7 @@ import org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
import org.apache.iotdb.db.auth.AuthorityChecker;
import org.apache.iotdb.db.exception.sql.SemanticException;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
+import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.AlterEncodingCompressorTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.CountDatabaseTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.CountTimeSlotListTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.CreateContinuousQueryTask;
@@ -119,6 +120,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.AuthorType;
import org.apache.iotdb.db.queryengine.plan.statement.Statement;
import org.apache.iotdb.db.queryengine.plan.statement.StatementNode;
import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor;
+import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterEncodingCompressorStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDatabaseStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountTimeSlotListStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.CreateContinuousQueryStatement;
@@ -244,14 +246,13 @@ public class TreeConfigTaskVisitor extends StatementVisitor<IConfigTask, MPPQuer
}
@Override
- public IConfigTask visitDeleteStorageGroup(
+ public IConfigTask visitDeleteDatabase(
DeleteDatabaseStatement statement, MPPQueryContext context) {
return new DeleteStorageGroupTask(statement);
}
@Override
- public IConfigTask visitShowStorageGroup(
- ShowDatabaseStatement statement, MPPQueryContext context) {
+ public IConfigTask visitShowDatabase(ShowDatabaseStatement statement, MPPQueryContext context) {
return new ShowDatabaseTask(statement);
}
@@ -655,6 +656,13 @@ public class TreeConfigTaskVisitor extends StatementVisitor<IConfigTask, MPPQuer
return new DropSubscriptionTask(dropSubscriptionStatement);
}
+ @Override
+ public IConfigTask visitAlterEncodingCompressor(
+ AlterEncodingCompressorStatement alterEncodingCompressorStatement, MPPQueryContext context) {
+ return new AlterEncodingCompressorTask(
+ context.getQueryId().getId(), alterEncodingCompressorStatement);
+ }
+
@Override
public IConfigTask visitDeleteTimeSeries(
DeleteTimeSeriesStatement deleteTimeSeriesStatement, MPPQueryContext context) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
index 6a1d7fb5c10..7df5fa2fb60 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
@@ -81,8 +81,10 @@ import org.apache.iotdb.commons.udf.service.UDFClassLoader;
import org.apache.iotdb.commons.udf.service.UDFExecutableManager;
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
import org.apache.iotdb.commons.utils.PathUtils;
+import org.apache.iotdb.commons.utils.SerializeUtils;
import org.apache.iotdb.commons.utils.TimePartitionUtils;
import org.apache.iotdb.confignode.rpc.thrift.TAINodeRemoveReq;
+import org.apache.iotdb.confignode.rpc.thrift.TAlterEncodingCompressorReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq;
import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq;
@@ -241,6 +243,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCluster;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDB;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Use;
+import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterEncodingCompressorStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDatabaseStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountTimeSlotListStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.CreateContinuousQueryStatement;
@@ -2602,6 +2605,60 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor {
return future;
}
+ @Override
+ public SettableFuture<ConfigTaskResult> alterEncodingCompressor(
+ final String queryId,
+ final AlterEncodingCompressorStatement alterEncodingCompressorStatement) {
+ final SettableFuture<ConfigTaskResult> future = SettableFuture.create();
+ final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream);
+ try {
+ alterEncodingCompressorStatement.getPatternTree().serialize(dataOutputStream);
+ } catch (final IOException ignored) {
+ // memory operation, won't happen
+ }
+ final TAlterEncodingCompressorReq req =
+ new TAlterEncodingCompressorReq(
+ queryId,
+ ByteBuffer.wrap(byteArrayOutputStream.toByteArray()),
+ SerializeUtils.serializeNullable(alterEncodingCompressorStatement.getEncoding()),
+ SerializeUtils.serializeNullable(alterEncodingCompressorStatement.getCompressor()),
+ alterEncodingCompressorStatement.ifExists(),
+ alterEncodingCompressorStatement.isWithAudit());
+ try (final ConfigNodeClient client =
+ CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
+ TSStatus tsStatus;
+ do {
+ try {
+ tsStatus = client.alterEncodingCompressor(req);
+ } catch (final TTransportException e) {
+ if (e.getType() == TTransportException.TIMED_OUT
+ || e.getCause() instanceof SocketTimeoutException) {
+ // time out mainly caused by slow execution, wait until
+ tsStatus = RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK);
+ } else {
+ throw e;
+ }
+ }
+ // keep waiting until task ends
+ } while (TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode() == tsStatus.getCode());
+
+ if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) {
+ if (tsStatus.getCode() == TSStatusCode.MULTIPLE_ERROR.getStatusCode()) {
+ future.setException(
+ new BatchProcessException(tsStatus.subStatus.toArray(new TSStatus[0])));
+ } else {
+ future.setException(new IoTDBException(tsStatus));
+ }
+ } else {
+ future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS));
+ }
+ } catch (final ClientManagerException | TException e) {
+ future.setException(e);
+ }
+ return future;
+ }
+
@Override
public SettableFuture<ConfigTaskResult> deleteTimeSeries(
final String queryId, final DeleteTimeSeriesStatement deleteTimeSeriesStatement) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java
index 69ee3a06f89..82e7ab20199 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java
@@ -45,6 +45,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCluster;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDB;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Use;
+import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterEncodingCompressorStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDatabaseStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountTimeSlotListStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.CreateContinuousQueryStatement;
@@ -233,6 +234,9 @@ public interface IConfigTaskExecutor {
SettableFuture<ConfigTaskResult> showTopics(ShowTopicsStatement showTopicsStatement);
+ SettableFuture<ConfigTaskResult> alterEncodingCompressor(
+ String queryId, AlterEncodingCompressorStatement alterEncodingCompressorStatement);
+
SettableFuture<ConfigTaskResult> deleteTimeSeries(
String queryId, DeleteTimeSeriesStatement deleteTimeSeriesStatement);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/AlterEncodingCompressorTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/AlterEncodingCompressorTask.java
new file mode 100644
index 00000000000..21a878397bf
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/AlterEncodingCompressorTask.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.execution.config.metadata;
+
+import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult;
+import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask;
+import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor;
+import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterEncodingCompressorStatement;
+
+import com.google.common.util.concurrent.ListenableFuture;
+
+public class AlterEncodingCompressorTask implements IConfigTask {
+
+ private final String queryId;
+
+ private final AlterEncodingCompressorStatement alterEncodingCompressorStatement;
+
+ public AlterEncodingCompressorTask(
+ final String queryId,
+ final AlterEncodingCompressorStatement alterEncodingCompressorStatement) {
+ this.queryId = queryId;
+ this.alterEncodingCompressorStatement = alterEncodingCompressorStatement;
+ }
+
+ @Override
+ public ListenableFuture<ConfigTaskResult> execute(final IConfigTaskExecutor configTaskExecutor)
+ throws InterruptedException {
+ return configTaskExecutor.alterEncodingCompressor(queryId, alterEncodingCompressorStatement);
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
index 73f1bb0ffbe..4fc0670dfaf 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
@@ -30,6 +30,7 @@ import org.apache.iotdb.commons.cq.TimeoutPolicy;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.path.PathPatternTree;
import org.apache.iotdb.commons.schema.cache.CacheClearOptions;
import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant;
import org.apache.iotdb.commons.schema.filter.SchemaFilter;
@@ -143,6 +144,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.literal.DoubleLiteral;
import org.apache.iotdb.db.queryengine.plan.statement.literal.Literal;
import org.apache.iotdb.db.queryengine.plan.statement.literal.LongLiteral;
import org.apache.iotdb.db.queryengine.plan.statement.literal.StringLiteral;
+import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterEncodingCompressorStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDatabaseStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDevicesStatement;
@@ -285,6 +287,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
@@ -295,6 +298,9 @@ import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static com.google.common.collect.ImmutableList.toImmutableList;
+import static org.apache.iotdb.commons.conf.IoTDBConstant.COLUMN_TIMESERIES_COMPRESSION;
+import static org.apache.iotdb.commons.conf.IoTDBConstant.COLUMN_TIMESERIES_COMPRESSOR;
+import static org.apache.iotdb.commons.conf.IoTDBConstant.COLUMN_TIMESERIES_ENCODING;
import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_RESULT_NODES;
import static org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.FIELD;
import static org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory.TAG;
@@ -456,13 +462,11 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
final IoTDBDescriptor ioTDBDescriptor = IoTDBDescriptor.getInstance();
createTimeSeriesStatement.setEncoding(
ioTDBDescriptor.getDefaultEncodingByType(createTimeSeriesStatement.getDataType()));
- if (props != null
- && props.containsKey(IoTDBConstant.COLUMN_TIMESERIES_ENCODING.toLowerCase())) {
- String encodingString =
- props.get(IoTDBConstant.COLUMN_TIMESERIES_ENCODING.toLowerCase()).toUpperCase();
+ if (props != null && props.containsKey(COLUMN_TIMESERIES_ENCODING.toLowerCase())) {
+ String encodingString = props.get(COLUMN_TIMESERIES_ENCODING.toLowerCase()).toUpperCase();
try {
createTimeSeriesStatement.setEncoding(TSEncoding.valueOf(encodingString));
- props.remove(IoTDBConstant.COLUMN_TIMESERIES_ENCODING.toLowerCase());
+ props.remove(COLUMN_TIMESERIES_ENCODING.toLowerCase());
} catch (Exception e) {
throw new SemanticException(String.format("Unsupported encoding: %s", encodingString));
}
@@ -519,13 +523,12 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
}
TSEncoding encoding = IoTDBDescriptor.getInstance().getDefaultEncodingByType(dataType);
- if (props.containsKey(IoTDBConstant.COLUMN_TIMESERIES_ENCODING.toLowerCase())) {
- String encodingString =
- props.get(IoTDBConstant.COLUMN_TIMESERIES_ENCODING.toLowerCase()).toUpperCase();
+ if (props.containsKey(COLUMN_TIMESERIES_ENCODING.toLowerCase())) {
+ String encodingString = props.get(COLUMN_TIMESERIES_ENCODING.toLowerCase()).toUpperCase();
try {
encoding = TSEncoding.valueOf(encodingString);
createAlignedTimeSeriesStatement.addEncoding(encoding);
- props.remove(IoTDBConstant.COLUMN_TIMESERIES_ENCODING.toLowerCase());
+ props.remove(COLUMN_TIMESERIES_ENCODING.toLowerCase());
} catch (Exception e) {
throw new SemanticException(String.format("unsupported encoding: %s", encodingString));
}
@@ -659,6 +662,46 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
}
}
+ @Override
+ public Statement visitAlterEncodingCompressor(
+ final IoTDBSqlParser.AlterEncodingCompressorContext ctx) {
+ final PathPatternTree tree = new PathPatternTree();
+ ctx.prefixPath().forEach(path -> tree.appendPathPattern(parsePrefixPath(path)));
+ tree.constructTree();
+ TSEncoding encoding = null;
+ CompressionType compressor = null;
+ for (final IoTDBSqlParser.AttributePairContext pair : ctx.attributePair()) {
+ final String key = parseAttributeKey(pair.key).toLowerCase(Locale.ENGLISH);
+ final String value = parseAttributeValue(pair.value).toUpperCase(Locale.ENGLISH);
+ switch (key) {
+ case COLUMN_TIMESERIES_ENCODING:
+ try {
+ encoding = TSEncoding.valueOf(value);
+ } catch (final Exception e) {
+ throw new SemanticException(String.format("Unsupported encoding: %s", value));
+ }
+ break;
+ case COLUMN_TIMESERIES_COMPRESSOR:
+ case COLUMN_TIMESERIES_COMPRESSION:
+ try {
+ compressor = CompressionType.valueOf(value);
+ } catch (final Exception e) {
+ throw new SemanticException(String.format("Unsupported compressor: %s", value));
+ }
+ break;
+ default:
+ throw new SemanticException(String.format("property %s is unsupported yet.", key));
+ }
+ }
+
+ return new AlterEncodingCompressorStatement(
+ tree,
+ encoding,
+ compressor,
+ Objects.nonNull(ctx.EXISTS()),
+ Objects.nonNull(ctx.PERMITTED()));
+ }
+
// Drop Timeseries ======================================================================
@Override
@@ -3788,13 +3831,12 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
}
TSEncoding encoding = IoTDBDescriptor.getInstance().getDefaultEncodingByType(dataType);
- if (props.containsKey(IoTDBConstant.COLUMN_TIMESERIES_ENCODING.toLowerCase())) {
- String encodingString =
- props.get(IoTDBConstant.COLUMN_TIMESERIES_ENCODING.toLowerCase()).toUpperCase();
+ if (props.containsKey(COLUMN_TIMESERIES_ENCODING.toLowerCase())) {
+ String encodingString = props.get(COLUMN_TIMESERIES_ENCODING.toLowerCase()).toUpperCase();
try {
encoding = TSEncoding.valueOf(encodingString);
encodings.add(encoding);
- props.remove(IoTDBConstant.COLUMN_TIMESERIES_ENCODING.toLowerCase());
+ props.remove(COLUMN_TIMESERIES_ENCODING.toLowerCase());
} catch (Exception e) {
throw new SemanticException(String.format("Unsupported encoding: %s", encodingString));
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java
index f7459a0eb57..e848d87e66d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java
@@ -39,6 +39,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.Seri
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TimeSeriesCountNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TimeSeriesSchemaScanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.ActivateTemplateNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.AlterEncodingCompressorNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.AlterTimeSeriesNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.BatchActivateTemplateNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.ConstructSchemaBlackListNode;
@@ -260,6 +261,7 @@ public enum PlanNodeType {
CONTINUOUS_SAME_SEARCH_INDEX_SEPARATOR((short) 97),
LAST_QUERY_SCAN((short) 98),
+ ALTER_ENCODING_COMPRESSOR((short) 99),
CREATE_OR_UPDATE_TABLE_DEVICE((short) 902),
TABLE_DEVICE_QUERY_SCAN((short) 903),
@@ -585,6 +587,8 @@ public enum PlanNodeType {
"You should never see ContinuousSameSearchIndexSeparatorNode in this function, because ContinuousSameSearchIndexSeparatorNode should never be used in network transmission.");
case 98:
return LastQueryScanNode.deserialize(buffer);
+ case 99:
+ return AlterEncodingCompressorNode.deserialize(buffer);
case 902:
return CreateOrUpdateTableDeviceNode.deserialize(buffer);
case 903:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
index 9ac845015e3..f3648e7ea00 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
@@ -36,6 +36,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.Seri
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TimeSeriesCountNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TimeSeriesSchemaScanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.ActivateTemplateNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.AlterEncodingCompressorNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.AlterTimeSeriesNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.BatchActivateTemplateNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.ConstructSchemaBlackListNode;
@@ -495,6 +496,10 @@ public abstract class PlanVisitor<R, C> {
return visitPlan(node, context);
}
+ public R visitAlterEncodingCompressor(AlterEncodingCompressorNode node, C context) {
+ return visitPlan(node, context);
+ }
+
public R visitConstructSchemaBlackList(ConstructSchemaBlackListNode node, C context) {
return visitPlan(node, context);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/write/AlterEncodingCompressorNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/write/AlterEncodingCompressorNode.java
new file mode 100644
index 00000000000..2e4c711483e
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metadata/write/AlterEncodingCompressorNode.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write;
+
+import org.apache.iotdb.commons.path.PathPatternTree;
+import org.apache.iotdb.commons.utils.SerializeUtils;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
+import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan;
+import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanType;
+import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanVisitor;
+
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+public class AlterEncodingCompressorNode extends PlanNode implements ISchemaRegionPlan {
+
+ public static final AlterEncodingCompressorNode MOCK_INSTANCE =
+ new AlterEncodingCompressorNode(new PlanNodeId(""), null, false, null, null);
+
+ private final PathPatternTree patternTree;
+ private final boolean ifExists;
+ private final TSEncoding encoding;
+ private final CompressionType compressionType;
+
+ public AlterEncodingCompressorNode(
+ final PlanNodeId id,
+ final PathPatternTree patternTree,
+ final boolean ifExists,
+ final TSEncoding encoding,
+ final CompressionType compressionType) {
+ super(id);
+ this.patternTree = patternTree;
+ this.ifExists = ifExists;
+ this.encoding = encoding;
+ this.compressionType = compressionType;
+ }
+
+ public PathPatternTree getPatternTree() {
+ return patternTree;
+ }
+
+ public boolean isIfExists() {
+ return ifExists;
+ }
+
+ public TSEncoding getEncoding() {
+ return encoding;
+ }
+
+ public CompressionType getCompressionType() {
+ return compressionType;
+ }
+
+ @Override
+ public List<PlanNode> getChildren() {
+ return null;
+ }
+
+ @Override
+ public void addChild(final PlanNode child) {}
+
+ @Override
+ public PlanNode clone() {
+ return null;
+ }
+
+ @Override
+ public int allowedChildCount() {
+ return 0;
+ }
+
+ @Override
+ public List<String> getOutputColumnNames() {
+ return null;
+ }
+
+ @Override
+ public <R, C> R accept(final PlanVisitor<R, C> visitor, final C context) {
+ return visitor.visitAlterEncodingCompressor(this, context);
+ }
+
+ @Override
+ protected void serializeAttributes(final ByteBuffer byteBuffer) {
+ PlanNodeType.ALTER_ENCODING_COMPRESSOR.serialize(byteBuffer);
+ patternTree.serialize(byteBuffer);
+ ReadWriteIOUtils.write(ifExists, byteBuffer);
+ ReadWriteIOUtils.write(SerializeUtils.serializeNullable(encoding), byteBuffer);
+ ReadWriteIOUtils.write(SerializeUtils.serializeNullable(compressionType), byteBuffer);
+ }
+
+ @Override
+ protected void serializeAttributes(final DataOutputStream stream) throws IOException {
+ PlanNodeType.ALTER_ENCODING_COMPRESSOR.serialize(stream);
+ patternTree.serialize(stream);
+ ReadWriteIOUtils.write(ifExists, stream);
+ ReadWriteIOUtils.write(SerializeUtils.serializeNullable(encoding), stream);
+ ReadWriteIOUtils.write(SerializeUtils.serializeNullable(compressionType), stream);
+ }
+
+ public static AlterEncodingCompressorNode deserialize(final ByteBuffer byteBuffer) {
+ final PathPatternTree patternTree = PathPatternTree.deserialize(byteBuffer);
+ final boolean ifExists = ReadWriteIOUtils.readBoolean(byteBuffer);
+ final TSEncoding encoding =
+ SerializeUtils.deserializeEncodingNullable(ReadWriteIOUtils.readByte(byteBuffer));
+ final CompressionType compressor =
+ SerializeUtils.deserializeCompressorNullable(ReadWriteIOUtils.readByte(byteBuffer));
+ final PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer);
+ return new AlterEncodingCompressorNode(planNodeId, patternTree, ifExists, encoding, compressor);
+ }
+
+ @Override
+ public SchemaRegionPlanType getPlanType() {
+ return SchemaRegionPlanType.ALTER_ENCODING_COMPRESSOR;
+ }
+
+ @Override
+ public <R, C> R accept(final SchemaRegionPlanVisitor<R, C> visitor, final C context) {
+ return visitor.visitAlterEncodingCompressor(this, context);
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java
index 748241d189b..df01dfe0f07 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java
@@ -56,6 +56,7 @@ public enum StatementType {
CREATE_MULTI_TIME_SERIES,
DELETE_TIME_SERIES,
ALTER_TIME_SERIES,
+ ALTER_ENCODING_COMPRESSOR,
CHANGE_ALIAS,
CHANGE_TAG_OFFSET,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java
index d2da4814541..e28ab4d464d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java
@@ -33,6 +33,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalBatchActi
import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateMultiTimeSeriesStatement;
import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateTimeSeriesStatement;
import org.apache.iotdb.db.queryengine.plan.statement.internal.SeriesSchemaFetchStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterEncodingCompressorStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDatabaseStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDevicesStatement;
@@ -194,11 +195,16 @@ public abstract class StatementVisitor<R, C> {
return visitStatement(alterTimeSeriesStatement, context);
}
+ public R visitAlterEncodingCompressor(
+ AlterEncodingCompressorStatement alterEncodingCompressorStatement, C context) {
+ return visitStatement(alterEncodingCompressorStatement, context);
+ }
+
public R visitDeleteTimeSeries(DeleteTimeSeriesStatement deleteTimeSeriesStatement, C context) {
return visitStatement(deleteTimeSeriesStatement, context);
}
- public R visitDeleteStorageGroup(DeleteDatabaseStatement deleteDatabaseStatement, C context) {
+ public R visitDeleteDatabase(DeleteDatabaseStatement deleteDatabaseStatement, C context) {
return visitStatement(deleteDatabaseStatement, context);
}
@@ -378,7 +384,7 @@ public abstract class StatementVisitor<R, C> {
return visitStatement(authorStatement, context);
}
- public R visitShowStorageGroup(ShowDatabaseStatement showDatabaseStatement, C context) {
+ public R visitShowDatabase(ShowDatabaseStatement showDatabaseStatement, C context) {
return visitStatement(showDatabaseStatement, context);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/AlterEncodingCompressorStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/AlterEncodingCompressorStatement.java
new file mode 100644
index 00000000000..a8dd88841a4
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/AlterEncodingCompressorStatement.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.statement.metadata;
+
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.path.PathPatternTree;
+import org.apache.iotdb.db.queryengine.plan.analyze.QueryType;
+import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement;
+import org.apache.iotdb.db.queryengine.plan.statement.Statement;
+import org.apache.iotdb.db.queryengine.plan.statement.StatementType;
+import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor;
+
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+
+import java.util.List;
+import java.util.Objects;
+
+public class AlterEncodingCompressorStatement extends Statement implements IConfigStatement {
+
+ private PathPatternTree patternTree;
+ private final TSEncoding encoding;
+ private final CompressionType compressor;
+ private final boolean ifExists;
+ private final boolean ifPermitted;
+ private boolean withAudit = false;
+
+ public AlterEncodingCompressorStatement(
+ final PathPatternTree pathPatternTree,
+ final TSEncoding encoding,
+ final CompressionType compressor,
+ final boolean ifExists,
+ final boolean ifPermitted) {
+ statementType = StatementType.ALTER_ENCODING_COMPRESSOR;
+ this.patternTree = pathPatternTree;
+ this.encoding = encoding;
+ this.compressor = compressor;
+ this.ifExists = ifExists;
+ this.ifPermitted = ifPermitted;
+ }
+
+ public TSEncoding getEncoding() {
+ return encoding;
+ }
+
+ public CompressionType getCompressor() {
+ return compressor;
+ }
+
+ public PathPatternTree getPatternTree() {
+ return patternTree;
+ }
+
+ public void setPatternTree(final PathPatternTree patternTree) {
+ this.patternTree = patternTree;
+ }
+
+ public boolean ifExists() {
+ return ifExists;
+ }
+
+ public boolean ifPermitted() {
+ return ifPermitted;
+ }
+
+ public void setWithAudit(final boolean withAudit) {
+ this.withAudit = withAudit;
+ }
+
+ public boolean isWithAudit() {
+ return withAudit;
+ }
+
+ @Override
+ public List<? extends PartialPath> getPaths() {
+ return patternTree.getAllPathPatterns();
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ final AlterEncodingCompressorStatement that = (AlterEncodingCompressorStatement) obj;
+ return Objects.equals(this.patternTree, that.patternTree)
+ && Objects.equals(this.encoding, that.encoding)
+ && Objects.equals(this.compressor, that.compressor)
+ && Objects.equals(this.ifExists, that.ifExists)
+ && Objects.equals(this.ifPermitted, that.ifPermitted);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(patternTree, encoding, compressor, ifExists, ifPermitted);
+ }
+
+ @Override
+ public <R, C> R accept(final StatementVisitor<R, C> visitor, final C context) {
+ return visitor.visitAlterEncodingCompressor(this, context);
+ }
+
+ @Override
+ public QueryType getQueryType() {
+ return QueryType.WRITE;
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DeleteDatabaseStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DeleteDatabaseStatement.java
index 33786815e16..494f9274635 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DeleteDatabaseStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/DeleteDatabaseStatement.java
@@ -77,7 +77,7 @@ public class DeleteDatabaseStatement extends Statement implements IConfigStateme
@Override
public <R, C> R accept(StatementVisitor<R, C> visitor, C context) {
- return visitor.visitDeleteStorageGroup(this, context);
+ return visitor.visitDeleteDatabase(this, context);
}
public void setPrefixPath(List<String> prefixPathList) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java
index d91e1f65f0d..e1a53662242 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java
@@ -116,7 +116,7 @@ public class ShowDatabaseStatement extends ShowStatement implements IConfigState
@Override
public <R, C> R accept(final StatementVisitor<R, C> visitor, C context) {
- return visitor.visitShowStorageGroup(this, context);
+ return visitor.visitShowDatabase(this, context);
}
@Override
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java
index a9b002bbd6d..916e52b4aca 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java
@@ -27,6 +27,7 @@ import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PathPatternTree;
import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException;
import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.AlterEncodingCompressorNode;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableId;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.ConstructTableDevicesBlackListNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode;
@@ -208,6 +209,8 @@ public interface ISchemaRegion {
*/
void deleteTimeseriesInBlackList(final PathPatternTree patternTree) throws MetadataException;
+ void alterEncodingCompressor(final AlterEncodingCompressorNode node) throws MetadataException;
+
// endregion
// region Interfaces for Logical View
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java
index 9808a19143b..b0d41c725ee 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java
@@ -39,6 +39,8 @@ public enum SchemaRegionPlanType {
ACTIVATE_TEMPLATE_IN_CLUSTER((byte) 63),
PRE_DELETE_TIMESERIES_IN_CLUSTER((byte) 64),
ROLLBACK_PRE_DELETE_TIMESERIES((byte) 65),
+ ALTER_ENCODING_COMPRESSOR((byte) 66),
+
// endregion
PRE_DEACTIVATE_TEMPLATE((byte) 0),
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanVisitor.java
index b0514d7d542..0cf087a40e1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanVisitor.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.schemaengine.schemaregion;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.AlterEncodingCompressorNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.ConstructTableDevicesBlackListNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.DeleteTableDeviceNode;
@@ -179,4 +180,9 @@ public abstract class SchemaRegionPlanVisitor<R, C> {
final TableAttributeColumnDropNode dropTableAttributePlan, final C context) {
return visitSchemaRegionPlan(dropTableAttributePlan, context);
}
+
+ public R visitAlterEncodingCompressor(
+ final AlterEncodingCompressorNode alterEncodingCompressorNode, final C context) {
+ return visitSchemaRegionPlan(alterEncodingCompressorNode, context);
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
index 3f49dfec93d..37f1b44f342 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
@@ -38,6 +38,7 @@ import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.metadata.PathNotExistException;
import org.apache.iotdb.db.exception.metadata.SchemaDirCreationFailureException;
import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException;
import org.apache.iotdb.db.exception.metadata.SeriesOverflowException;
@@ -49,6 +50,7 @@ import org.apache.iotdb.db.queryengine.execution.operator.schema.source.TableDev
import org.apache.iotdb.db.queryengine.execution.relational.ColumnTransformerBuilder;
import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider;
import org.apache.iotdb.db.queryengine.plan.planner.LocalExecutionPlanner;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.AlterEncodingCompressorNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.CreateTimeSeriesNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.InputLocation;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata;
@@ -942,21 +944,36 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion {
}
@Override
- public void deleteTimeseriesInBlackList(PathPatternTree patternTree) throws MetadataException {
- for (PartialPath pathPattern : patternTree.getAllPathPatterns()) {
- for (PartialPath path : mTree.getPreDeletedTimeSeries(pathPattern)) {
+ public void deleteTimeseriesInBlackList(final PathPatternTree patternTree)
+ throws MetadataException {
+ for (final PartialPath pathPattern : patternTree.getAllPathPatterns()) {
+ for (final PartialPath path : mTree.getPreDeletedTimeSeries(pathPattern)) {
try {
deleteSingleTimeseriesInBlackList(path);
writeToMLog(
SchemaRegionWritePlanFactory.getDeleteTimeSeriesPlan(
Collections.singletonList(path)));
- } catch (IOException e) {
+ } catch (final IOException e) {
throw new MetadataException(e);
}
}
}
}
+ @Override
+ public void alterEncodingCompressor(final AlterEncodingCompressorNode node)
+ throws MetadataException {
+ boolean exist = false;
+ for (final PartialPath pathPattern : node.getPatternTree().getAllPathPatterns()) {
+ exist |=
+ mTree.alterEncodingCompressor(pathPattern, node.getEncoding(), node.getCompressionType());
+ }
+ if (!exist) {
+ throw new PathNotExistException(node.getPatternTree().getAllPathPatterns().toString(), false);
+ }
+ writeToMLog(node);
+ }
+
@Override
public void createLogicalView(final ICreateLogicalViewPlan plan) throws MetadataException {
if (!regionStatistics.isAllowToCreateNewSeries()) {
@@ -2059,5 +2076,17 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion {
return new RecoverOperationResult(e);
}
}
+
+ @Override
+ public RecoverOperationResult visitAlterEncodingCompressor(
+ final AlterEncodingCompressorNode alterEncodingCompressorNode,
+ final SchemaRegionMemoryImpl context) {
+ try {
+ alterEncodingCompressor(alterEncodingCompressorNode);
+ return RecoverOperationResult.SUCCESS;
+ } catch (final MetadataException e) {
+ return new RecoverOperationResult(e);
+ }
+ }
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java
index 6b8000ecc59..bf5e650de75 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java
@@ -39,6 +39,7 @@ import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException;
import org.apache.iotdb.db.exception.metadata.SchemaDirCreationFailureException;
import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException;
import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.AlterEncodingCompressorNode;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableId;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.ConstructTableDevicesBlackListNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode;
@@ -941,6 +942,13 @@ public class SchemaRegionPBTreeImpl implements ISchemaRegion {
}
}
+ @Override
+ public void alterEncodingCompressor(final AlterEncodingCompressorNode node)
+ throws MetadataException {
+ throw new UnsupportedOperationException(
+ "PBTree does not support altering encoding and compressor yet.");
+ }
+
@Override
public void createLogicalView(ICreateLogicalViewPlan plan) throws MetadataException {
while (!regionStatistics.isAllowToCreateNewSeries()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanDeserializer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanDeserializer.java
index 258f271cc9d..163ccb4e59d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanDeserializer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanDeserializer.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PathDeserializeUtil;
import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.AlterEncodingCompressorNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.ConstructTableDevicesBlackListNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.DeleteTableDeviceNode;
@@ -475,5 +476,11 @@ public class SchemaRegionPlanDeserializer implements IDeserializer<ISchemaRegion
final TableAttributeColumnDropNode dropTableAttributePlan, final ByteBuffer buffer) {
return (TableAttributeColumnDropNode) PlanNodeType.deserialize(buffer);
}
+
+ @Override
+ public ISchemaRegionPlan visitAlterEncodingCompressor(
+ final AlterEncodingCompressorNode alterEncodingCompressorNode, final ByteBuffer buffer) {
+ return (AlterEncodingCompressorNode) PlanNodeType.deserialize(buffer);
+ }
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanSerializer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanSerializer.java
index 63854468b34..b7b7d9758ca 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanSerializer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanSerializer.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion.logfile.visitor;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.AlterEncodingCompressorNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.ConstructTableDevicesBlackListNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.DeleteTableDeviceNode;
@@ -545,6 +546,13 @@ public class SchemaRegionPlanSerializer implements ISerializer<ISchemaRegionPlan
return visitPlanNode(dropTableAttributePlan, outputStream);
}
+ @Override
+ public SchemaRegionPlanSerializationResult visitAlterEncodingCompressor(
+ final AlterEncodingCompressorNode alterEncodingCompressorNode,
+ final DataOutputStream outputStream) {
+ return visitPlanNode(alterEncodingCompressorNode, outputStream);
+ }
+
private SchemaRegionPlanSerializationResult visitPlanNode(
final PlanNode planNode, final DataOutputStream outputStream) {
try {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
index 605dd3b7a28..dfd19c91f94 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java
@@ -74,6 +74,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.utils.MetaFormatUtils;
import org.apache.iotdb.db.schemaengine.schemaregion.utils.filter.DeviceFilterVisitor;
import org.apache.iotdb.db.schemaengine.template.Template;
import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeSpaceQuotaManager;
+import org.apache.iotdb.db.utils.SchemaUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import com.google.common.util.concurrent.ListenableFuture;
@@ -693,6 +694,38 @@ public class MTreeBelowSGMemoryImpl {
return result;
}
+ public boolean alterEncodingCompressor(
+ final PartialPath pathPattern, final TSEncoding encoding, final CompressionType compressor)
+ throws MetadataException {
+ final boolean[] exist = {false};
+ try (final MeasurementUpdater<IMemMNode> collector =
+ new MeasurementUpdater<IMemMNode>(
+ rootNode, pathPattern, store, false, SchemaConstant.ALL_MATCH_SCOPE) {
+ @Override
+ protected void updateMeasurement(final IMeasurementMNode<IMemMNode> node)
+ throws MetadataException {
+ if (node.isLogicalView()) {
+ return;
+ }
+ exist[0] = true;
+ final IMeasurementSchema schema = node.getSchema();
+ if (Objects.nonNull(encoding)) {
+ SchemaUtils.checkDataTypeWithEncoding(node.getDataType(), encoding);
+ }
+ node.setSchema(
+ new MeasurementSchema(
+ schema.getMeasurementName(),
+ schema.getType(),
+ Objects.nonNull(encoding) ? encoding : schema.getEncodingType(),
+ Objects.nonNull(compressor) ? compressor : schema.getCompressor(),
+ schema.getProps()));
+ }
+ }) {
+ collector.traverse();
+ }
+ return exist[0];
+ }
+
// TODO: seems useless
public Set<PartialPath> getDevicesOfPreDeletedTimeSeries(final PartialPath pathPattern)
throws MetadataException {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/write/req/SchemaRegionWritePlanFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/write/req/SchemaRegionWritePlanFactory.java
index 5fe5ec993cd..8a87a6f3829 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/write/req/SchemaRegionWritePlanFactory.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/write/req/SchemaRegionWritePlanFactory.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion.write.req;
import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.AlterEncodingCompressorNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.ConstructTableDevicesBlackListNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode;
import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.DeleteTableDeviceNode;
@@ -87,6 +88,8 @@ public class SchemaRegionWritePlanFactory {
return new PreDeleteTimeSeriesPlanImpl();
case ROLLBACK_PRE_DELETE_TIMESERIES:
return new RollbackPreDeleteTimeSeriesPlanImpl();
+ case ALTER_ENCODING_COMPRESSOR:
+ return AlterEncodingCompressorNode.MOCK_INSTANCE;
case PRE_DEACTIVATE_TEMPLATE:
return new PreDeactivateTemplatePlanImpl();
case ROLLBACK_PRE_DEACTIVATE_TEMPLATE:
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/MetadataException.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/MetadataException.java
index 3a83a6310f5..5229c971630 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/MetadataException.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/MetadataException.java
@@ -45,7 +45,7 @@ public class MetadataException extends IoTDBException {
}
public MetadataException(String message, Throwable cause) {
- super(message + cause.getMessage(), TSStatusCode.METADATA_ERROR.getStatusCode());
+ super(message, cause, TSStatusCode.METADATA_ERROR.getStatusCode());
}
public MetadataException(IoTDBException exception) {
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternNode.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternNode.java
index b5da17d92bd..46bc4d2a28d 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternNode.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternNode.java
@@ -247,6 +247,13 @@ public class PathPatternNode<V, S extends PathPatternNode.Serializer<V>> {
}
}
+ void clear() {
+ if (Objects.nonNull(valueSet)) {
+ valueSet.clear();
+ }
+ children.clear();
+ }
+
public static <V, T extends PathPatternNode.Serializer<V>> PathPatternNode<V, T> deserializeNode(
final ByteBuffer buffer, final T serializer, final Consumer<String> nodeNameProcessor) {
final PathPatternNode<V, T> node =
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTree.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTree.java
index f7703dd6d98..9f16d306b43 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTree.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTree.java
@@ -97,9 +97,17 @@ public class PathPatternTree {
appendBranchWithoutPrune(root, pathNodes, 0);
}
+ public void appendPathPattern(final PartialPath pathPattern) {
+ appendPathPattern(pathPattern, false);
+ }
+
/** Add a pathPattern (may contain wildcards) to pathPatternList. */
- public void appendPathPattern(PartialPath pathPattern) {
+ public void appendPathPattern(final PartialPath pathPattern, final boolean isReload) {
if (useWildcard) {
+ // This does not guarantee multi-thread safety
+ if (isReload && (pathPatternList == null || pathPatternList.isEmpty())) {
+ pathPatternList = getAllPathPatterns();
+ }
boolean isExist = false;
for (PartialPath path : pathPatternList) {
if (path.include(pathPattern)) {
@@ -113,6 +121,9 @@ public class PathPatternTree {
pathPatternList.removeIf(pathPattern::include);
pathPatternList.add(pathPattern);
}
+ if (isReload) {
+ root.clear();
+ }
} else {
appendBranchWithoutPrune(root, pathPattern.getNodes(), 0);
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java
index b6316a42ddc..85a14fe5345 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/receiver/IoTDBFileReceiver.java
@@ -90,6 +90,7 @@ public abstract class IoTDBFileReceiver implements IoTDBReceiver {
protected final AtomicBoolean validateTsFile = new AtomicBoolean(true);
protected final AtomicBoolean shouldMarkAsPipeRequest = new AtomicBoolean(true);
+ protected final AtomicBoolean skipIfNoPrivileges = new AtomicBoolean(false);
@Override
public IoTDBSinkRequestVersion getVersion() {
@@ -329,6 +330,11 @@ public abstract class IoTDBFileReceiver implements IoTDBReceiver {
.getOrDefault(
PipeTransferHandshakeConstant.HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST, "true")));
+ skipIfNoPrivileges.set(
+ Boolean.parseBoolean(
+ req.getParams()
+ .getOrDefault(PipeTransferHandshakeConstant.HANDSHAKE_KEY_SKIP_IF, "false")));
+
// Handle the handshake request as a v1 request.
// Here we construct a fake "dataNode" request to valid from v1 validation logic, though
// it may not require the actual type of the v1 request.
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/client/IoTDBClientManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/client/IoTDBClientManager.java
index 8cb118a7d90..aab7697e3d4 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/client/IoTDBClientManager.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/client/IoTDBClientManager.java
@@ -48,6 +48,7 @@ public abstract class IoTDBClientManager {
protected final String loadTsFileStrategy;
protected final boolean shouldMarkAsPipeRequest;
+ protected final boolean skipIfNoPrivileges;
// This flag indicates whether the receiver supports mods transferring if
// it is a DataNode receiver. The flag is useless for configNode receiver.
@@ -68,7 +69,8 @@ public abstract class IoTDBClientManager {
final boolean shouldReceiverConvertOnTypeMismatch,
final String loadTsFileStrategy,
final boolean validateTsFile,
- final boolean shouldMarkAsPipeRequest) {
+ final boolean shouldMarkAsPipeRequest,
+ final boolean skipIfNoPrivileges) {
this.endPointList = endPointList;
this.useLeaderCache = useLeaderCache;
@@ -79,6 +81,7 @@ public abstract class IoTDBClientManager {
this.loadTsFileStrategy = loadTsFileStrategy;
this.validateTsFile = validateTsFile;
this.shouldMarkAsPipeRequest = shouldMarkAsPipeRequest;
+ this.skipIfNoPrivileges = skipIfNoPrivileges;
}
public boolean supportModsIfIsDataNodeReceiver() {
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/client/IoTDBSyncClientManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/client/IoTDBSyncClientManager.java
index 754e8a81775..e16e107d635 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/client/IoTDBSyncClientManager.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/client/IoTDBSyncClientManager.java
@@ -78,7 +78,8 @@ public abstract class IoTDBSyncClientManager extends IoTDBClientManager implemen
boolean shouldReceiverConvertOnTypeMismatch,
String loadTsFileStrategy,
boolean validateTsFile,
- boolean shouldMarkAsPipeRequest) {
+ boolean shouldMarkAsPipeRequest,
+ final boolean skipIfNoPrivileges) {
super(
endPoints,
useLeaderCache,
@@ -87,7 +88,8 @@ public abstract class IoTDBSyncClientManager extends IoTDBClientManager implemen
shouldReceiverConvertOnTypeMismatch,
loadTsFileStrategy,
validateTsFile,
- shouldMarkAsPipeRequest);
+ shouldMarkAsPipeRequest,
+ skipIfNoPrivileges);
this.useSSL = useSSL;
this.trustStorePath = trustStorePath;
@@ -239,6 +241,9 @@ public abstract class IoTDBSyncClientManager extends IoTDBClientManager implemen
params.put(
PipeTransferHandshakeConstant.HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST,
Boolean.toString(shouldMarkAsPipeRequest));
+ params.put(
+ PipeTransferHandshakeConstant.HANDSHAKE_KEY_SKIP_IF,
+ Boolean.toString(skipIfNoPrivileges));
// Try to handshake by PipeTransferHandshakeV2Req.
TPipeTransferResp resp = client.pipeTransfer(buildHandshakeV2Req(params));
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferHandshakeConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferHandshakeConstant.java
index 8976a25c5cc..cc3dd79e33b 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferHandshakeConstant.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/payload/thrift/common/PipeTransferHandshakeConstant.java
@@ -29,6 +29,7 @@ public class PipeTransferHandshakeConstant {
public static final String HANDSHAKE_KEY_PASSWORD = "password";
public static final String HANDSHAKE_KEY_VALIDATE_TSFILE = "validateTsFile";
public static final String HANDSHAKE_KEY_MARK_AS_PIPE_REQUEST = "markAsPipeRequest";
+ public static final String HANDSHAKE_KEY_SKIP_IF = "skipIf";
private PipeTransferHandshakeConstant() {
// Utility class
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java
index 0babe802639..aa7c9bc5f1d 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSink.java
@@ -163,6 +163,7 @@ public abstract class IoTDBSink implements PipeConnector {
protected boolean loadTsFileValidation;
protected boolean shouldMarkAsPipeRequest;
+ protected boolean skipIfNoPrivileges;
private boolean isRpcCompressionEnabled;
private final List<PipeCompressor> compressors = new ArrayList<>();
@@ -424,7 +425,7 @@ public abstract class IoTDBSink implements PipeConnector {
.filter(s -> !s.isEmpty())
.map(String::toLowerCase)
.collect(Collectors.toSet());
- boolean skipIfNoPrivileges = skipIfOptionSet.remove(CONNECTOR_IOTDB_SKIP_IF_NO_PRIVILEGES);
+ skipIfNoPrivileges = skipIfOptionSet.remove(CONNECTOR_IOTDB_SKIP_IF_NO_PRIVILEGES);
if (!skipIfOptionSet.isEmpty()) {
throw new PipeParameterNotValidException(
String.format("Parameters in set %s are not allowed in 'skipif'", skipIfOptionSet));
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSslSyncSink.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSslSyncSink.java
index 66d84ed7d2c..3b0f2899224 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSslSyncSink.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/sink/protocol/IoTDBSslSyncSink.java
@@ -139,7 +139,8 @@ public abstract class IoTDBSslSyncSink extends IoTDBSink {
shouldReceiverConvertOnTypeMismatch,
loadTsFileStrategy,
loadTsFileValidation,
- shouldMarkAsPipeRequest);
+ shouldMarkAsPipeRequest,
+ skipIfNoPrivileges);
}
protected abstract IoTDBSyncClientManager constructClient(
@@ -156,7 +157,8 @@ public abstract class IoTDBSslSyncSink extends IoTDBSink {
final boolean shouldReceiverConvertOnTypeMismatch,
final String loadTsFileStrategy,
final boolean validateTsFile,
- final boolean shouldMarkAsPipeRequest);
+ final boolean shouldMarkAsPipeRequest,
+ final boolean skipIfNoPrivileges);
@Override
public void handshake() throws Exception {
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/SerializeUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/SerializeUtils.java
index 7b34111aa1f..d0b57328dcb 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/SerializeUtils.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/SerializeUtils.java
@@ -23,6 +23,8 @@ import org.apache.iotdb.commons.auth.entity.PrivilegeType;
import org.apache.tsfile.common.conf.TSFileConfig;
import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.read.TimeValuePair;
import org.apache.tsfile.read.common.BatchData;
import org.apache.tsfile.read.common.BatchData.BatchDataType;
@@ -46,6 +48,22 @@ public class SerializeUtils {
// util class
}
+ public static byte serializeNullable(final TSEncoding encoding) {
+ return encoding == null ? -1 : encoding.serialize();
+ }
+
+ public static TSEncoding deserializeEncodingNullable(final byte encoding) {
+ return encoding == -1 ? null : TSEncoding.deserialize(encoding);
+ }
+
+ public static byte serializeNullable(final CompressionType compressor) {
+ return compressor == null ? -1 : compressor.serialize();
+ }
+
+ public static CompressionType deserializeCompressorNullable(final byte compressor) {
+ return compressor == -1 ? null : CompressionType.deserialize(compressor);
+ }
+
public static void serialize(String str, DataOutputStream dataOutputStream) {
try {
byte[] strBytes = str.getBytes(TSFileConfig.STRING_CHARSET);
diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
index aa3c312bdd0..8f1ba916257 100644
--- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
+++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
@@ -905,6 +905,16 @@ struct TPipeConfigTransferResp {
2: optional binary body
}
+struct TAlterEncodingCompressorReq {
+ 1: required string queryId
+ 2: required binary pathPatternTree
+ 3: required byte encoding
+ 4: required byte compressor
+ 5: required bool ifExists
+ 6: required bool mayAlterAudit
+ 7: optional bool isGeneratedByPipe
+}
+
struct TDeleteTimeSeriesReq {
1: required string queryId
2: required binary pathPatternTree
@@ -1833,6 +1843,8 @@ service IConfigNodeRPCService {
common.TSStatus alterSchemaTemplate(TAlterSchemaTemplateReq req)
+ common.TSStatus alterEncodingCompressor(TAlterEncodingCompressorReq req)
+
/**
* Generate a set of DeleteTimeSeriesProcedure to delete some specific TimeSeries
*
diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift
index 5be8808db50..f210ad5bf9a 100644
--- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift
+++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift
@@ -460,6 +460,14 @@ struct TDeleteTimeSeriesReq {
3: optional bool isGeneratedByPipe
}
+struct TAlterEncodingCompressorReq {
+ 1: required list<common.TConsensusGroupId> schemaRegionIdList
+ 2: required binary pathPatternTree
+ 3: required bool ifExists
+ 4: optional byte encoding
+ 5: optional byte compressor
+}
+
struct TConstructSchemaBlackListWithTemplateReq {
1: required list<common.TConsensusGroupId> schemaRegionIdList
2: required map<string, list<i32>> templateSetInfo
@@ -1041,6 +1049,11 @@ service IDataNodeRPCService {
*/
common.TSStatus deleteTimeSeries(TDeleteTimeSeriesReq req)
+ /**
+ * Alter matched timeseries to specific encoding and compressor in target schemaRegions
+ */
+ common.TSStatus alterEncodingCompressor(TAlterEncodingCompressorReq req)
+
/**
* Construct schema black list in target schemaRegion to block R/W on matched timeseries represent by template
*/
(iotdb) 05/06: Pipe: Improved the total performance by disable some useless logic
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rc/2.0.6
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 09d526ced1ad675308506b623ca24cc7c771f536
Author: Caideyipi <87...@users.noreply.github.com>
AuthorDate: Fri Aug 15 14:21:01 2025 +0800
Pipe: Improved the total performance by disable some useless logic
---
.github/workflows/daily-it.yml | 900 ---------------------
.github/workflows/pipe-it.yml | 872 --------------------
.../manual/basic/IoTDBPipeProtocolIT.java | 16 +-
.../treemodel/auto/basic/IoTDBPipeProtocolIT.java | 16 +-
.../plugin/PipeConfigRegionSinkConstructor.java | 6 -
.../PipeDataRegionProcessorConstructor.java | 30 -
.../dataregion/PipeDataRegionSinkConstructor.java | 12 -
.../PipeSchemaRegionSinkConstructor.java | 6 -
.../source/dataregion/IoTDBDataRegionSource.java | 49 +-
.../realtime/PipeRealtimeDataRegionSource.java | 19 +-
.../agent/plugin/builtin/BuiltinPipePlugin.java | 40 -
.../iotdb/commons/pipe/source/IoTDBSource.java | 19 +-
12 files changed, 29 insertions(+), 1956 deletions(-)
diff --git a/.github/workflows/daily-it.yml b/.github/workflows/daily-it.yml
index 8f28c54e56a..1ab0d3bb857 100644
--- a/.github/workflows/daily-it.yml
+++ b/.github/workflows/daily-it.yml
@@ -81,903 +81,3 @@ jobs:
name: table-standalone-log-java${{ matrix.java }}-${{ runner.os }}
path: integration-test/target/cluster-logs
retention-days: 3
- PipeSingle:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster1: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode]
- cluster2: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode]
- os: [ ubuntu-latest ]
- exclude:
- - cluster1: LightWeightStandaloneMode
- cluster2: LightWeightStandaloneMode
- - cluster1: LightWeightStandaloneMode
- cluster2: ScalableSingleNodeMode
- - cluster1: ScalableSingleNodeMode
- cluster2: LightWeightStandaloneMode
- - cluster1: ScalableSingleNodeMode
- cluster2: HighPerformanceMode
- - cluster1: HighPerformanceMode
- cluster2: LightWeightStandaloneMode
- - cluster1: HighPerformanceMode
- cluster2: HighPerformanceMode
- - cluster1: PipeConsensusBatchMode
- cluster2: LightWeightStandaloneMode
- - cluster1: PipeConsensusBatchMode
- cluster2: HighPerformanceMode
- - cluster1: PipeConsensusStreamMode
- cluster2: LightWeightStandaloneMode
- - cluster1: PipeConsensusStreamMode
- cluster2: HighPerformanceMode
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: liberica
- java-version: ${{ matrix.java }}
- - name: Cache Maven packages
- uses: actions/cache@v4
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in client.yml
- # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
- -pl integration-test \
- -am -PMultiClusterIT1 \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log ======================"
-
- if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact"; then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to artifact transfer issue."
- echo "Treating this as a success because the issue is likely transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: cluster-log-single-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- PipeDualTreeAutoBasic:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode]
- os: [ ubuntu-latest ]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: liberica
- java-version: ${{ matrix.java }}
- - name: Cache Maven packages
- uses: actions/cache@v4
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in client.yml
- # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \
- -pl integration-test \
- -am -PMultiClusterIT2DualTreeAutoBasic \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log ======================"
-
- if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact"; then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to artifact transfer issue."
- echo "Treating this as a success because the issue is likely transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: cluster-log-dual-tree-auto-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- PipeDualTreeAutoEnhanced:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster1: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode]
- cluster2: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode]
- os: [ ubuntu-latest ]
- exclude:
- - cluster1: LightWeightStandaloneMode
- cluster2: LightWeightStandaloneMode
- - cluster1: LightWeightStandaloneMode
- cluster2: ScalableSingleNodeMode
- - cluster1: ScalableSingleNodeMode
- cluster2: LightWeightStandaloneMode
- - cluster1: ScalableSingleNodeMode
- cluster2: HighPerformanceMode
- - cluster1: HighPerformanceMode
- cluster2: LightWeightStandaloneMode
- - cluster1: HighPerformanceMode
- cluster2: HighPerformanceMode
- - cluster1: PipeConsensusBatchMode
- cluster2: LightWeightStandaloneMode
- - cluster1: PipeConsensusBatchMode
- cluster2: HighPerformanceMode
- - cluster1: PipeConsensusStreamMode
- cluster2: LightWeightStandaloneMode
- - cluster1: PipeConsensusStreamMode
- cluster2: HighPerformanceMode
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: liberica
- java-version: ${{ matrix.java }}
- - name: Cache Maven packages
- uses: actions/cache@v4
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in client.yml
- # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
- -pl integration-test \
- -am -PMultiClusterIT2DualTreeAutoEnhanced \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log ======================"
-
- if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact"; then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to artifact transfer issue."
- echo "Treating this as a success because the issue is likely transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: cluster-log-dual-tree-auto-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- PipeDualTreeManual:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster1: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode]
- cluster2: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode]
- os: [ ubuntu-latest ]
- exclude:
- - cluster1: LightWeightStandaloneMode
- cluster2: LightWeightStandaloneMode
- - cluster1: LightWeightStandaloneMode
- cluster2: ScalableSingleNodeMode
- - cluster1: ScalableSingleNodeMode
- cluster2: LightWeightStandaloneMode
- - cluster1: ScalableSingleNodeMode
- cluster2: HighPerformanceMode
- - cluster1: HighPerformanceMode
- cluster2: LightWeightStandaloneMode
- - cluster1: HighPerformanceMode
- cluster2: HighPerformanceMode
- - cluster1: PipeConsensusBatchMode
- cluster2: LightWeightStandaloneMode
- - cluster1: PipeConsensusBatchMode
- cluster2: HighPerformanceMode
- - cluster1: PipeConsensusStreamMode
- cluster2: LightWeightStandaloneMode
- - cluster1: PipeConsensusStreamMode
- cluster2: HighPerformanceMode
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: liberica
- java-version: ${{ matrix.java }}
- - name: Cache Maven packages
- uses: actions/cache@v4
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in client.yml
- # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
- -pl integration-test \
- -am -PMultiClusterIT2DualTreeManual \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log ======================"
-
- if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact"; then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to artifact transfer issue."
- echo "Treating this as a success because the issue is likely transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: cluster-log-dual-tree-manual-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- SubscriptionTreeArchVerification:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [ 17 ]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster1: [ ScalableSingleNodeMode, PipeConsensusBatchMode, PipeConsensusStreamMode ]
- cluster2: [ ScalableSingleNodeMode ]
- os: [ ubuntu-latest ]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: liberica
- java-version: ${{ matrix.java }}
- - name: Cache Maven packages
- uses: actions/cache@v4
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in client.yml
- # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
- -pl integration-test \
- -am -PMultiClusterIT2SubscriptionTreeArchVerification \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log ======================"
-
- if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact"; then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to artifact transfer issue."
- echo "Treating this as a success because the issue is likely transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: cluster-log-subscription-tree-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- SubscriptionTableArchVerification:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [ 17 ]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster1: [ ScalableSingleNodeMode ]
- cluster2: [ ScalableSingleNodeMode ]
- os: [ ubuntu-latest ]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: liberica
- java-version: ${{ matrix.java }}
- - name: Cache Maven packages
- uses: actions/cache@v4
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in client.yml
- # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
- -pl integration-test \
- -am -PMultiClusterIT2SubscriptionTableArchVerification \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log ======================"
-
- if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact"; then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to artifact transfer issue."
- echo "Treating this as a success because the issue is likely transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: cluster-log-subscription-table-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- SubscriptionTreeRegressionConsumer:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [ 17 ]
- # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal
- cluster1: [ ScalableSingleNodeMode, PipeConsensusBatchMode, PipeConsensusStreamMode ]
- cluster2: [ ScalableSingleNodeMode ]
- os: [ ubuntu-latest ]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: liberica
- java-version: ${{ matrix.java }}
- - name: Cache Maven packages
- uses: actions/cache@v4
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in client.yml
- # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
- -pl integration-test \
- -am -PMultiClusterIT2SubscriptionTreeRegressionConsumer \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log ======================"
-
- if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact"; then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to artifact transfer issue."
- echo "Treating this as a success because the issue is likely transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: cluster-log-subscription-tree-regression-consumer-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- SubscriptionTreeRegressionMisc:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [ 17 ]
- # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal
- cluster1: [ ScalableSingleNodeMode, PipeConsensusBatchMode, PipeConsensusStreamMode ]
- cluster2: [ ScalableSingleNodeMode ]
- os: [ ubuntu-latest ]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: liberica
- java-version: ${{ matrix.java }}
- - name: Cache Maven packages
- uses: actions/cache@v4
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in client.yml
- # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
- -pl integration-test \
- -am -PMultiClusterIT2SubscriptionTreeRegressionMisc \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log ======================"
-
- if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact"; then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to artifact transfer issue."
- echo "Treating this as a success because the issue is likely transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: cluster-log-subscription-tree-regression-misc-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- PipeDualTableManualBasic:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode]
- os: [ ubuntu-latest ]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: liberica
- java-version: ${{ matrix.java }}
- - name: Cache Maven packages
- uses: actions/cache@v4
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in client.yml
- # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \
- -pl integration-test \
- -am -PMultiClusterIT2DualTableManualBasic \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log ======================"
-
- if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact"; then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to artifact transfer issue."
- echo "Treating this as a success because the issue is likely transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: cluster-log-dual-table-manual-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- PipeDualTableManualEnhanced:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster: [LightWeightStandaloneMode, ScalableSingleNodeMode, HighPerformanceMode, PipeConsensusBatchMode, PipeConsensusStreamMode]
- os: [ ubuntu-latest ]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: liberica
- java-version: ${{ matrix.java }}
- - name: Cache Maven packages
- uses: actions/cache@v4
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in client.yml
- # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \
- -pl integration-test \
- -am -PMultiClusterIT2DualTableManualEnhanced \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log ======================"
-
- if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact"; then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to artifact transfer issue."
- echo "Treating this as a success because the issue is likely transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: cluster-log-dual-table-manual-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
- path: integration-test/target/cluster-logs
- retention-days: 30
\ No newline at end of file
diff --git a/.github/workflows/pipe-it.yml b/.github/workflows/pipe-it.yml
deleted file mode 100644
index 97e6fad1903..00000000000
--- a/.github/workflows/pipe-it.yml
+++ /dev/null
@@ -1,872 +0,0 @@
-name: Multi-Cluster IT
-
-on:
- push:
- branches:
- - master
- - 'rel/*'
- - 'rc/*'
- paths-ignore:
- - 'docs/**'
- - 'site/**'
- - 'iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/**' #queryengine
- pull_request:
- branches:
- - master
- - 'rel/*'
- - 'rc/*'
- - 'force_ci/**'
- paths-ignore:
- - 'docs/**'
- - 'site/**'
- - 'iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/**' #queryengine
- # allow manually run the action:
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
- MAVEN_ARGS: --batch-mode --no-transfer-progress
- DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
-
-jobs:
- single:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster1: [HighPerformanceMode]
- cluster2: [HighPerformanceMode]
- os: [ ubuntu-latest ]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: liberica
- java-version: ${{ matrix.java }}
- - name: Cache Maven packages
- uses: actions/cache@v4
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in client.yml
- # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
- -pl integration-test \
- -am -PMultiClusterIT1 \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log ======================"
-
- if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact"; then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to artifact transfer issue."
- echo "Treating this as a success because the issue is likely transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: cluster-log-single-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- dual-tree-auto-basic:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster: [HighPerformanceMode]
- os: [ ubuntu-latest ]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: liberica
- java-version: ${{ matrix.java }}
- - name: Cache Maven packages
- uses: actions/cache@v4
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in client.yml
- # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \
- -pl integration-test \
- -am -PMultiClusterIT2DualTreeAutoBasic \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log ======================"
-
- if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact"; then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to artifact transfer issue."
- echo "Treating this as a success because the issue is likely transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: cluster-log-dual-tree-auto-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- dual-tree-auto-enhanced:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster1: [HighPerformanceMode]
- cluster2: [HighPerformanceMode]
- os: [ ubuntu-latest ]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: liberica
- java-version: ${{ matrix.java }}
- - name: Cache Maven packages
- uses: actions/cache@v4
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in client.yml
- # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
- -pl integration-test \
- -am -PMultiClusterIT2DualTreeAutoEnhanced \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log ======================"
-
- if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact"; then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to artifact transfer issue."
- echo "Treating this as a success because the issue is likely transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: cluster-log-dual-tree-auto-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- dual-tree-manual:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster1: [HighPerformanceMode]
- cluster2: [HighPerformanceMode]
- os: [ ubuntu-latest ]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: liberica
- java-version: ${{ matrix.java }}
- - name: Cache Maven packages
- uses: actions/cache@v4
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in client.yml
- # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
- -pl integration-test \
- -am -PMultiClusterIT2DualTreeManual \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log ======================"
-
- if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact"; then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to artifact transfer issue."
- echo "Treating this as a success because the issue is likely transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: cluster-log-dual-tree-manual-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- subscription-tree-arch-verification:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [ 17 ]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster1: [ ScalableSingleNodeMode ]
- cluster2: [ ScalableSingleNodeMode ]
- os: [ ubuntu-latest ]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: liberica
- java-version: ${{ matrix.java }}
- - name: Cache Maven packages
- uses: actions/cache@v4
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in client.yml
- # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
- -pl integration-test \
- -am -PMultiClusterIT2SubscriptionTreeArchVerification \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log ======================"
-
- if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact"; then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to artifact transfer issue."
- echo "Treating this as a success because the issue is likely transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: cluster-log-subscription-tree-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- subscription-table-arch-verification:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [ 17 ]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster1: [ ScalableSingleNodeMode ]
- cluster2: [ ScalableSingleNodeMode ]
- os: [ ubuntu-latest ]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: liberica
- java-version: ${{ matrix.java }}
- - name: Cache Maven packages
- uses: actions/cache@v4
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in client.yml
- # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
- -pl integration-test \
- -am -PMultiClusterIT2SubscriptionTableArchVerification \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log ======================"
-
- if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact"; then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to artifact transfer issue."
- echo "Treating this as a success because the issue is likely transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: cluster-log-subscription-table-arch-verification-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- subscription-tree-regression-consumer:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [ 17 ]
- # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal
- cluster1: [ ScalableSingleNodeMode ]
- cluster2: [ ScalableSingleNodeMode ]
- os: [ ubuntu-latest ]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: liberica
- java-version: ${{ matrix.java }}
- - name: Cache Maven packages
- uses: actions/cache@v4
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in client.yml
- # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
- -pl integration-test \
- -am -PMultiClusterIT2SubscriptionTreeRegressionConsumer \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log ======================"
-
- if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact"; then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to artifact transfer issue."
- echo "Treating this as a success because the issue is likely transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: cluster-log-subscription-tree-regression-consumer-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- subscription-tree-regression-misc:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [ 17 ]
- # do not use HighPerformanceMode here, otherwise some tests will cause the GH runner to receive a shutdown signal
- cluster1: [ ScalableSingleNodeMode ]
- cluster2: [ ScalableSingleNodeMode ]
- os: [ ubuntu-latest ]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: liberica
- java-version: ${{ matrix.java }}
- - name: Cache Maven packages
- uses: actions/cache@v4
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in client.yml
- # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster1 }},${{ matrix.cluster2 }} \
- -pl integration-test \
- -am -PMultiClusterIT2SubscriptionTreeRegressionMisc \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log ======================"
-
- if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact"; then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to artifact transfer issue."
- echo "Treating this as a success because the issue is likely transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: cluster-log-subscription-tree-regression-misc-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster1 }}-${{ matrix.cluster2 }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- dual-table-manual-basic:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster: [HighPerformanceMode]
- os: [ ubuntu-latest ]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: liberica
- java-version: ${{ matrix.java }}
- - name: Cache Maven packages
- uses: actions/cache@v4
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in client.yml
- # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \
- -pl integration-test \
- -am -PMultiClusterIT2DualTableManualBasic \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log ======================"
-
- if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact"; then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to artifact transfer issue."
- echo "Treating this as a success because the issue is likely transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: cluster-log-dual-table-manual-basic-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
- path: integration-test/target/cluster-logs
- retention-days: 30
- dual-table-manual-enhanced:
- strategy:
- fail-fast: false
- max-parallel: 15
- matrix:
- java: [17]
- # StrongConsistencyClusterMode is ignored now because RatisConsensus has not been supported yet.
- cluster: [HighPerformanceMode]
- os: [ ubuntu-latest ]
- runs-on: ${{ matrix.os }}
- steps:
- - uses: actions/checkout@v4
- - name: Set up JDK ${{ matrix.java }}
- uses: actions/setup-java@v4
- with:
- distribution: liberica
- java-version: ${{ matrix.java }}
- - name: Cache Maven packages
- uses: actions/cache@v4
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Sleep for a random duration between 0 and 10000 milliseconds
- run: |
- sleep $(( $(( RANDOM % 10000 + 1 )) / 1000))
- - name: IT Test
- shell: bash
- # we do not compile client-cpp for saving time, it is tested in client.yml
- # we can skip influxdb-protocol because it has been tested separately in influxdb-protocol.yml
- run: |
- retry() {
- local -i max_attempts=3
- local -i attempt=1
- local -i retry_sleep=5
- local test_output
-
- while [ $attempt -le $max_attempts ]; do
- mvn clean verify \
- -P with-integration-tests \
- -DskipUTs \
- -DintegrationTest.forkCount=1 -DConfigNodeMaxHeapSize=256 -DDataNodeMaxHeapSize=1024 -DDataNodeMaxDirectMemorySize=768 \
- -DClusterConfigurations=${{ matrix.cluster }},${{ matrix.cluster }} \
- -pl integration-test \
- -am -PMultiClusterIT2DualTableManualEnhanced \
- -ntp >> ~/run-tests-$attempt.log && return 0
- test_output=$(cat ~/run-tests-$attempt.log)
-
- echo "==================== BEGIN: ~/run-tests-$attempt.log ===================="
- echo "$test_output"
- echo "==================== END: ~/run-tests-$attempt.log ======================"
-
- if ! mv ~/run-tests-$attempt.log integration-test/target/cluster-logs/ 2>/dev/null; then
- echo "Failed to move log file ~/run-tests-$attempt.log to integration-test/target/cluster-logs/. Skipping..."
- fi
-
- if echo "$test_output" | grep -q "Could not transfer artifact"; then
- if [ $attempt -lt $max_attempts ]; then
- echo "Test failed with artifact transfer issue, attempt $attempt. Retrying in $retry_sleep seconds..."
- sleep $retry_sleep
- attempt=$((attempt + 1))
- else
- echo "Test failed after $max_attempts attempts due to artifact transfer issue."
- echo "Treating this as a success because the issue is likely transient."
- return 0
- fi
- elif [ $? -ne 0 ]; then
- echo "Test failed with a different error."
- return 1
- else
- echo "Tests passed"
- return 0
- fi
- done
- }
- retry
- - name: Upload Artifact
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: cluster-log-dual-table-manual-enhanced-java${{ matrix.java }}-${{ runner.os }}-${{ matrix.cluster }}-${{ matrix.cluster }}
- path: integration-test/target/cluster-logs
- retention-days: 30
\ No newline at end of file
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java
index 939e84b9f49..869eebeefbc 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/manual/basic/IoTDBPipeProtocolIT.java
@@ -365,11 +365,6 @@ public class IoTDBPipeProtocolIT extends AbstractPipeTableModelDualManualIT {
doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName());
}
- @Test
- public void testAirGapConnectorUseNodeUrls() throws Exception {
- doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName());
- }
-
private void doTestUseNodeUrls(String connectorName) throws Exception {
senderEnv
.getConfig()
@@ -413,16 +408,7 @@ public class IoTDBPipeProtocolIT extends AbstractPipeTableModelDualManualIT {
boolean insertResult = true;
for (final DataNodeWrapper wrapper : receiverEnv.getDataNodeWrapperList()) {
- if (connectorName.equals(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName())) {
- // Use default port for convenience
- nodeUrlsBuilder
- .append(wrapper.getIp())
- .append(":")
- .append(wrapper.getPipeAirGapReceiverPort())
- .append(",");
- } else {
- nodeUrlsBuilder.append(wrapper.getIpAndPortString()).append(",");
- }
+ nodeUrlsBuilder.append(wrapper.getIpAndPortString()).append(",");
}
try (final SyncConfigNodeIServiceClient client =
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java
index 1249bf90e78..9c81fe049a6 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/treemodel/auto/basic/IoTDBPipeProtocolIT.java
@@ -352,11 +352,6 @@ public class IoTDBPipeProtocolIT extends AbstractPipeDualTreeModelAutoIT {
doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName());
}
- @Test
- public void testAirGapConnectorUseNodeUrls() throws Exception {
- doTestUseNodeUrls(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName());
- }
-
private void doTestUseNodeUrls(String connectorName) throws Exception {
senderEnv
.getConfig()
@@ -391,16 +386,7 @@ public class IoTDBPipeProtocolIT extends AbstractPipeDualTreeModelAutoIT {
final StringBuilder nodeUrlsBuilder = new StringBuilder();
for (final DataNodeWrapper wrapper : receiverEnv.getDataNodeWrapperList()) {
- if (connectorName.equals(BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName())) {
- // Use default port for convenience
- nodeUrlsBuilder
- .append(wrapper.getIp())
- .append(":")
- .append(wrapper.getPipeAirGapReceiverPort())
- .append(",");
- } else {
- nodeUrlsBuilder.append(wrapper.getIpAndPortString()).append(",");
- }
+ nodeUrlsBuilder.append(wrapper.getIpAndPortString()).append(",");
}
try (final SyncConfigNodeIServiceClient client =
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructor.java
index c7222e9a1bc..82c394c7fb5 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/plugin/PipeConfigRegionSinkConstructor.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.confignode.manager.pipe.agent.plugin;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.donothing.DoNothingSink;
import org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeSinkConstructor;
-import org.apache.iotdb.confignode.manager.pipe.sink.protocol.IoTDBConfigRegionAirGapSink;
import org.apache.iotdb.confignode.manager.pipe.sink.protocol.IoTDBConfigRegionSink;
import org.apache.iotdb.pipe.api.PipeConnector;
@@ -41,9 +40,6 @@ class PipeConfigRegionSinkConstructor extends PipeSinkConstructor {
pluginConstructors.put(
BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName(),
IoTDBConfigRegionSink::new);
- pluginConstructors.put(
- BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName(),
- IoTDBConfigRegionAirGapSink::new);
pluginConstructors.put(
BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(), DoNothingSink::new);
@@ -55,8 +51,6 @@ class PipeConfigRegionSinkConstructor extends PipeSinkConstructor {
BuiltinPipePlugin.IOTDB_THRIFT_SYNC_SINK.getPipePluginName(), IoTDBConfigRegionSink::new);
pluginConstructors.put(
BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_SINK.getPipePluginName(), IoTDBConfigRegionSink::new);
- pluginConstructors.put(
- BuiltinPipePlugin.IOTDB_AIR_GAP_SINK.getPipePluginName(), IoTDBConfigRegionAirGapSink::new);
pluginConstructors.put(
BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(), DoNothingSink::new);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionProcessorConstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionProcessorConstructor.java
index 31cc8250ebf..33143bdc27e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionProcessorConstructor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionProcessorConstructor.java
@@ -21,18 +21,10 @@ package org.apache.iotdb.db.pipe.agent.plugin.dataregion;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.donothing.DoNothingProcessor;
-import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.throwing.ThrowingExceptionProcessor;
import org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeProcessorConstructor;
import org.apache.iotdb.commons.pipe.agent.plugin.meta.DataNodePipePluginMetaKeeper;
-import org.apache.iotdb.db.pipe.processor.aggregate.AggregateProcessor;
-import org.apache.iotdb.db.pipe.processor.aggregate.operator.processor.StandardStatisticsOperatorProcessor;
-import org.apache.iotdb.db.pipe.processor.aggregate.window.processor.TumblingWindowingProcessor;
-import org.apache.iotdb.db.pipe.processor.downsampling.changing.ChangingValueSamplingProcessor;
-import org.apache.iotdb.db.pipe.processor.downsampling.sdt.SwingingDoorTrendingSamplingProcessor;
-import org.apache.iotdb.db.pipe.processor.downsampling.tumbling.TumblingTimeSamplingProcessor;
import org.apache.iotdb.db.pipe.processor.pipeconsensus.PipeConsensusProcessor;
import org.apache.iotdb.db.pipe.processor.schemachange.RenameDatabaseProcessor;
-import org.apache.iotdb.db.pipe.processor.twostage.plugin.TwoStageCountProcessor;
class PipeDataRegionProcessorConstructor extends PipeProcessorConstructor {
@@ -44,28 +36,6 @@ class PipeDataRegionProcessorConstructor extends PipeProcessorConstructor {
protected void initConstructors() {
pluginConstructors.put(
BuiltinPipePlugin.DO_NOTHING_PROCESSOR.getPipePluginName(), DoNothingProcessor::new);
- pluginConstructors.put(
- BuiltinPipePlugin.TUMBLING_TIME_SAMPLING_PROCESSOR.getPipePluginName(),
- TumblingTimeSamplingProcessor::new);
- pluginConstructors.put(
- BuiltinPipePlugin.SDT_SAMPLING_PROCESSOR.getPipePluginName(),
- SwingingDoorTrendingSamplingProcessor::new);
- pluginConstructors.put(
- BuiltinPipePlugin.CHANGING_VALUE_SAMPLING_PROCESSOR.getPipePluginName(),
- ChangingValueSamplingProcessor::new);
- pluginConstructors.put(
- BuiltinPipePlugin.THROWING_EXCEPTION_PROCESSOR.getPipePluginName(),
- ThrowingExceptionProcessor::new);
- pluginConstructors.put(
- BuiltinPipePlugin.AGGREGATE_PROCESSOR.getPipePluginName(), AggregateProcessor::new);
- pluginConstructors.put(
- BuiltinPipePlugin.STANDARD_STATISTICS_PROCESSOR.getPipePluginName(),
- StandardStatisticsOperatorProcessor::new);
- pluginConstructors.put(
- BuiltinPipePlugin.TUMBLING_WINDOWING_PROCESSOR.getPipePluginName(),
- TumblingWindowingProcessor::new);
- pluginConstructors.put(
- BuiltinPipePlugin.COUNT_POINT_PROCESSOR.getPipePluginName(), TwoStageCountProcessor::new);
pluginConstructors.put(
BuiltinPipePlugin.PIPE_CONSENSUS_PROCESSOR.getPipePluginName(),
PipeConsensusProcessor::new);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java
index 536cf71cdb8..09773d0cad5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/dataregion/PipeDataRegionSinkConstructor.java
@@ -23,10 +23,7 @@ import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.donothing.DoNothingSink;
import org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeSinkConstructor;
import org.apache.iotdb.commons.pipe.agent.plugin.meta.DataNodePipePluginMetaKeeper;
-import org.apache.iotdb.db.pipe.sink.protocol.airgap.IoTDBDataRegionAirGapSink;
import org.apache.iotdb.db.pipe.sink.protocol.legacy.IoTDBLegacyPipeSink;
-import org.apache.iotdb.db.pipe.sink.protocol.opcda.OpcDaSink;
-import org.apache.iotdb.db.pipe.sink.protocol.opcua.OpcUaSink;
import org.apache.iotdb.db.pipe.sink.protocol.pipeconsensus.PipeConsensusAsyncSink;
import org.apache.iotdb.db.pipe.sink.protocol.thrift.async.IoTDBDataRegionAsyncSink;
import org.apache.iotdb.db.pipe.sink.protocol.thrift.sync.IoTDBDataRegionSyncSink;
@@ -59,13 +56,8 @@ class PipeDataRegionSinkConstructor extends PipeSinkConstructor {
pluginConstructors.put(
BuiltinPipePlugin.IOTDB_LEGACY_PIPE_CONNECTOR.getPipePluginName(),
IoTDBLegacyPipeSink::new);
- pluginConstructors.put(
- BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName(),
- IoTDBDataRegionAirGapSink::new);
pluginConstructors.put(
BuiltinPipePlugin.WEBSOCKET_CONNECTOR.getPipePluginName(), WebSocketSink::new);
- pluginConstructors.put(BuiltinPipePlugin.OPC_UA_CONNECTOR.getPipePluginName(), OpcUaSink::new);
- pluginConstructors.put(BuiltinPipePlugin.OPC_DA_CONNECTOR.getPipePluginName(), OpcDaSink::new);
pluginConstructors.put(
BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(), DoNothingSink::new);
pluginConstructors.put(
@@ -82,12 +74,8 @@ class PipeDataRegionSinkConstructor extends PipeSinkConstructor {
IoTDBDataRegionAsyncSink::new);
pluginConstructors.put(
BuiltinPipePlugin.IOTDB_LEGACY_PIPE_SINK.getPipePluginName(), IoTDBLegacyPipeSink::new);
- pluginConstructors.put(
- BuiltinPipePlugin.IOTDB_AIR_GAP_SINK.getPipePluginName(), IoTDBDataRegionAirGapSink::new);
pluginConstructors.put(
BuiltinPipePlugin.WEBSOCKET_SINK.getPipePluginName(), WebSocketSink::new);
- pluginConstructors.put(BuiltinPipePlugin.OPC_UA_SINK.getPipePluginName(), OpcUaSink::new);
- pluginConstructors.put(BuiltinPipePlugin.OPC_DA_SINK.getPipePluginName(), OpcDaSink::new);
pluginConstructors.put(
BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(), DoNothingSink::new);
pluginConstructors.put(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java
index 160ecf54c01..a7752994ee2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/schemaregion/PipeSchemaRegionSinkConstructor.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.db.pipe.agent.plugin.schemaregion;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.BuiltinPipePlugin;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.donothing.DoNothingSink;
import org.apache.iotdb.commons.pipe.agent.plugin.constructor.PipeSinkConstructor;
-import org.apache.iotdb.db.pipe.sink.protocol.airgap.IoTDBSchemaRegionAirGapSink;
import org.apache.iotdb.db.pipe.sink.protocol.thrift.sync.IoTDBSchemaRegionSink;
import org.apache.iotdb.pipe.api.PipeConnector;
@@ -41,9 +40,6 @@ class PipeSchemaRegionSinkConstructor extends PipeSinkConstructor {
pluginConstructors.put(
BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName(),
IoTDBSchemaRegionSink::new);
- pluginConstructors.put(
- BuiltinPipePlugin.IOTDB_AIR_GAP_CONNECTOR.getPipePluginName(),
- IoTDBSchemaRegionAirGapSink::new);
pluginConstructors.put(
BuiltinPipePlugin.DO_NOTHING_CONNECTOR.getPipePluginName(), DoNothingSink::new);
@@ -55,8 +51,6 @@ class PipeSchemaRegionSinkConstructor extends PipeSinkConstructor {
BuiltinPipePlugin.IOTDB_THRIFT_SYNC_SINK.getPipePluginName(), IoTDBSchemaRegionSink::new);
pluginConstructors.put(
BuiltinPipePlugin.IOTDB_THRIFT_ASYNC_SINK.getPipePluginName(), IoTDBSchemaRegionSink::new);
- pluginConstructors.put(
- BuiltinPipePlugin.IOTDB_AIR_GAP_SINK.getPipePluginName(), IoTDBSchemaRegionAirGapSink::new);
pluginConstructors.put(
BuiltinPipePlugin.DO_NOTHING_SINK.getPipePluginName(), DoNothingSink::new);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java
index 4dc5fef859f..a310785cdc3 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.pipe.source.dataregion;
import org.apache.iotdb.commons.consensus.DataRegionId;
+import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta;
import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant;
import org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
import org.apache.iotdb.commons.pipe.datastructure.pattern.IoTDBTreePattern;
@@ -141,6 +142,21 @@ public class IoTDBDataRegionSource extends IoTDBSource {
public void validate(final PipeParameterValidator validator) throws Exception {
super.validate(validator);
+ final boolean forwardingPipeRequests =
+ validator
+ .getParameters()
+ .getBooleanOrDefault(
+ Arrays.asList(
+ PipeSourceConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY,
+ PipeSourceConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY),
+ PipeSourceConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_DEFAULT_VALUE);
+ if (!forwardingPipeRequests) {
+ throw new PipeParameterNotValidException(
+ String.format(
+ "The parameter %s cannot be set to false.",
+ PipeSourceConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY));
+ }
+
final boolean isTreeDialect =
validator
.getParameters()
@@ -262,32 +278,6 @@ public class IoTDBDataRegionSource extends IoTDBSource {
Arrays.asList(EXTRACTOR_REALTIME_ENABLE_KEY, SOURCE_REALTIME_ENABLE_KEY),
EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE));
- // Validate extractor.realtime.mode
- if (validator
- .getParameters()
- .getBooleanOrDefault(
- Arrays.asList(EXTRACTOR_REALTIME_ENABLE_KEY, SOURCE_REALTIME_ENABLE_KEY),
- EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE)
- || validator
- .getParameters()
- .hasAnyAttributes(
- SOURCE_START_TIME_KEY,
- EXTRACTOR_START_TIME_KEY,
- SOURCE_END_TIME_KEY,
- EXTRACTOR_END_TIME_KEY)) {
- validator.validateAttributeValueRange(
- validator.getParameters().hasAttribute(EXTRACTOR_REALTIME_MODE_KEY)
- ? EXTRACTOR_REALTIME_MODE_KEY
- : SOURCE_REALTIME_MODE_KEY,
- true,
- EXTRACTOR_REALTIME_MODE_FILE_VALUE,
- EXTRACTOR_REALTIME_MODE_HYBRID_VALUE,
- EXTRACTOR_REALTIME_MODE_LOG_VALUE,
- EXTRACTOR_REALTIME_MODE_FORCED_LOG_VALUE,
- EXTRACTOR_REALTIME_MODE_STREAM_MODE_VALUE,
- EXTRACTOR_REALTIME_MODE_BATCH_MODE_VALUE);
- }
-
checkInvalidParameters(validator);
constructHistoricalExtractor();
@@ -447,6 +437,13 @@ public class IoTDBDataRegionSource extends IoTDBSource {
return;
}
+ if (!(pipeName != null
+ && (pipeName.startsWith(PipeStaticMeta.SUBSCRIPTION_PIPE_PREFIX)
+ || pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)))) {
+ realtimeExtractor = new PipeRealtimeDataRegionTsFileSource();
+ return;
+ }
+
// Use hybrid mode by default
if (!parameters.hasAnyAttributes(EXTRACTOR_MODE_STREAMING_KEY, SOURCE_MODE_STREAMING_KEY)
&& !parameters.hasAnyAttributes(EXTRACTOR_REALTIME_MODE_KEY, SOURCE_REALTIME_MODE_KEY)) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java
index 8f0874330cf..076021dde3e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/realtime/PipeRealtimeDataRegionSource.java
@@ -114,7 +114,7 @@ public abstract class PipeRealtimeDataRegionSource implements PipeExtractor {
private final AtomicReference<Pair<Long, Long>> dataRegionTimePartitionIdBound =
new AtomicReference<>();
- protected boolean isForwardingPipeRequests;
+ protected boolean isForwardingPipeRequests = true;
private boolean shouldTransferModFile; // Whether to transfer mods
@@ -246,22 +246,7 @@ public abstract class PipeRealtimeDataRegionSource implements PipeExtractor {
? TimePartitionUtils.getTimePartitionId(realtimeDataExtractionEndTime)
: TimePartitionUtils.getTimePartitionId(realtimeDataExtractionEndTime) - 1;
- final boolean isDoubleLiving =
- parameters.getBooleanOrDefault(
- Arrays.asList(
- PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY,
- PipeSourceConstant.SOURCE_MODE_DOUBLE_LIVING_KEY),
- PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE);
- if (isDoubleLiving) {
- isForwardingPipeRequests = false;
- } else {
- isForwardingPipeRequests =
- parameters.getBooleanOrDefault(
- Arrays.asList(
- PipeSourceConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY,
- PipeSourceConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY),
- PipeSourceConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_DEFAULT_VALUE);
- }
+ isForwardingPipeRequests = true;
if (parameters.hasAnyAttributes(EXTRACTOR_MODS_KEY, SOURCE_MODS_KEY)) {
shouldTransferModFile =
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java
index 8da1e50a4d5..daab48bbe23 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java
@@ -19,27 +19,16 @@
package org.apache.iotdb.commons.pipe.agent.plugin.builtin;
-import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.aggregate.AggregateProcessor;
-import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.aggregate.StandardStatisticsProcessor;
-import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.aggregate.TumblingWindowingProcessor;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.donothing.DoNothingProcessor;
-import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.downsampling.ChangingValueSamplingProcessor;
-import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.downsampling.SwingingDoorTrendingSamplingProcessor;
-import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.downsampling.TumblingTimeSamplingProcessor;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.pipeconsensus.PipeConsensusProcessor;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.schemachange.RenameDatabaseProcessor;
-import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.throwing.ThrowingExceptionProcessor;
-import org.apache.iotdb.commons.pipe.agent.plugin.builtin.processor.twostage.TwoStageCountProcessor;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.donothing.DoNothingSink;
-import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.airgap.IoTDBAirGapSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.consensus.PipeConsensusAsyncSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBLegacyPipeSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftAsyncSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSslSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.iotdb.thrift.IoTDBThriftSyncSink;
-import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcda.OpcDaSink;
-import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.opcua.OpcUaSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.websocket.WebSocketSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.sink.writeback.WriteBackSink;
import org.apache.iotdb.commons.pipe.agent.plugin.builtin.source.donothing.DoNothingSource;
@@ -64,18 +53,8 @@ public enum BuiltinPipePlugin {
// processors
DO_NOTHING_PROCESSOR("do-nothing-processor", DoNothingProcessor.class),
- TUMBLING_TIME_SAMPLING_PROCESSOR(
- "tumbling-time-sampling-processor", TumblingTimeSamplingProcessor.class),
- SDT_SAMPLING_PROCESSOR("sdt-sampling-processor", SwingingDoorTrendingSamplingProcessor.class),
- CHANGING_VALUE_SAMPLING_PROCESSOR(
- "changing-value-sampling-processor", ChangingValueSamplingProcessor.class),
- THROWING_EXCEPTION_PROCESSOR("throwing-exception-processor", ThrowingExceptionProcessor.class),
- AGGREGATE_PROCESSOR("aggregate-processor", AggregateProcessor.class),
- COUNT_POINT_PROCESSOR("count-point-processor", TwoStageCountProcessor.class),
// Hidden-processors, which are plugins of the processors
- STANDARD_STATISTICS_PROCESSOR("standard-statistics-processor", StandardStatisticsProcessor.class),
- TUMBLING_WINDOWING_PROCESSOR("tumbling-windowing-processor", TumblingWindowingProcessor.class),
PIPE_CONSENSUS_PROCESSOR("pipe-consensus-processor", PipeConsensusProcessor.class),
RENAME_DATABASE_PROCESSOR("rename-database-processor", RenameDatabaseProcessor.class),
@@ -86,12 +65,9 @@ public enum BuiltinPipePlugin {
IOTDB_THRIFT_SYNC_CONNECTOR("iotdb-thrift-sync-connector", IoTDBThriftSyncSink.class),
IOTDB_THRIFT_ASYNC_CONNECTOR("iotdb-thrift-async-connector", IoTDBThriftAsyncSink.class),
IOTDB_LEGACY_PIPE_CONNECTOR("iotdb-legacy-pipe-connector", IoTDBLegacyPipeSink.class),
- IOTDB_AIR_GAP_CONNECTOR("iotdb-air-gap-connector", IoTDBAirGapSink.class),
PIPE_CONSENSUS_ASYNC_CONNECTOR("pipe-consensus-async-connector", PipeConsensusAsyncSink.class),
WEBSOCKET_CONNECTOR("websocket-connector", WebSocketSink.class),
- OPC_UA_CONNECTOR("opc-ua-connector", OpcUaSink.class),
- OPC_DA_CONNECTOR("opc-da-connector", OpcDaSink.class),
WRITE_BACK_CONNECTOR("write-back-connector", WriteBackSink.class),
DO_NOTHING_SINK("do-nothing-sink", DoNothingSink.class),
@@ -100,10 +76,7 @@ public enum BuiltinPipePlugin {
IOTDB_THRIFT_SYNC_SINK("iotdb-thrift-sync-sink", IoTDBThriftSyncSink.class),
IOTDB_THRIFT_ASYNC_SINK("iotdb-thrift-async-sink", IoTDBThriftAsyncSink.class),
IOTDB_LEGACY_PIPE_SINK("iotdb-legacy-pipe-sink", IoTDBLegacyPipeSink.class),
- IOTDB_AIR_GAP_SINK("iotdb-air-gap-sink", IoTDBAirGapSink.class),
WEBSOCKET_SINK("websocket-sink", WebSocketSink.class),
- OPC_UA_SINK("opc-ua-sink", OpcUaSink.class),
- OPC_DA_SINK("opc-da-sink", OpcDaSink.class),
WRITE_BACK_SINK("write-back-sink", WriteBackSink.class),
SUBSCRIPTION_SINK("subscription-sink", DoNothingSink.class),
PIPE_CONSENSUS_ASYNC_SINK("pipe-consensus-async-sink", PipeConsensusAsyncSink.class),
@@ -151,14 +124,6 @@ public enum BuiltinPipePlugin {
// Sources
DO_NOTHING_SOURCE.getPipePluginName().toUpperCase(),
// Processors
- TUMBLING_TIME_SAMPLING_PROCESSOR.getPipePluginName().toUpperCase(),
- SDT_SAMPLING_PROCESSOR.getPipePluginName().toUpperCase(),
- CHANGING_VALUE_SAMPLING_PROCESSOR.getPipePluginName().toUpperCase(),
- THROWING_EXCEPTION_PROCESSOR.getPipePluginName().toUpperCase(),
- AGGREGATE_PROCESSOR.getPipePluginName().toUpperCase(),
- COUNT_POINT_PROCESSOR.getPipePluginName().toUpperCase(),
- STANDARD_STATISTICS_PROCESSOR.getPipePluginName().toUpperCase(),
- TUMBLING_WINDOWING_PROCESSOR.getPipePluginName().toUpperCase(),
PIPE_CONSENSUS_PROCESSOR.getPipePluginName().toUpperCase(),
RENAME_DATABASE_PROCESSOR.getPipePluginName().toUpperCase(),
// Connectors
@@ -168,10 +133,7 @@ public enum BuiltinPipePlugin {
IOTDB_THRIFT_SYNC_CONNECTOR.getPipePluginName().toUpperCase(),
IOTDB_THRIFT_ASYNC_CONNECTOR.getPipePluginName().toUpperCase(),
IOTDB_LEGACY_PIPE_CONNECTOR.getPipePluginName().toUpperCase(),
- IOTDB_AIR_GAP_CONNECTOR.getPipePluginName().toUpperCase(),
WEBSOCKET_CONNECTOR.getPipePluginName().toUpperCase(),
- OPC_UA_CONNECTOR.getPipePluginName().toUpperCase(),
- OPC_DA_CONNECTOR.getPipePluginName().toUpperCase(),
WRITE_BACK_CONNECTOR.getPipePluginName().toUpperCase(),
PIPE_CONSENSUS_ASYNC_CONNECTOR.getPipePluginName().toUpperCase(),
// Sinks
@@ -179,8 +141,6 @@ public enum BuiltinPipePlugin {
IOTDB_THRIFT_ASYNC_SINK.getPipePluginName().toUpperCase(),
IOTDB_LEGACY_PIPE_SINK.getPipePluginName().toUpperCase(),
WEBSOCKET_SINK.getPipePluginName().toUpperCase(),
- OPC_UA_SINK.getPipePluginName().toUpperCase(),
- OPC_DA_SINK.getPipePluginName().toUpperCase(),
SUBSCRIPTION_SINK.getPipePluginName().toUpperCase(),
PIPE_CONSENSUS_ASYNC_SINK.getPipePluginName().toUpperCase())));
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBSource.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBSource.java
index c16f2310629..fcb6a5e1238 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBSource.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/source/IoTDBSource.java
@@ -61,7 +61,7 @@ public abstract class IoTDBSource implements PipeExtractor {
protected int regionId;
protected PipeTaskMeta pipeTaskMeta;
- protected boolean isForwardingPipeRequests;
+ protected boolean isForwardingPipeRequests = true;
// The value is always true after the first start even the extractor is closed
protected final AtomicBoolean hasBeenStarted = new AtomicBoolean(false);
@@ -157,22 +157,7 @@ public abstract class IoTDBSource implements PipeExtractor {
taskID = pipeName + "_" + regionId + "_" + creationTime;
pipeTaskMeta = environment.getPipeTaskMeta();
- final boolean isDoubleLiving =
- parameters.getBooleanOrDefault(
- Arrays.asList(
- PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY,
- PipeSourceConstant.SOURCE_MODE_DOUBLE_LIVING_KEY),
- PipeSourceConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE);
- if (isDoubleLiving) {
- isForwardingPipeRequests = false;
- } else {
- isForwardingPipeRequests =
- parameters.getBooleanOrDefault(
- Arrays.asList(
- PipeSourceConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY,
- PipeSourceConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY),
- PipeSourceConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_DEFAULT_VALUE);
- }
+ isForwardingPipeRequests = true;
userName =
parameters.getStringByKeys(