You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2022/07/07 12:34:29 UTC
[iotdb] branch multileader_restart_test updated (62cdf9adb7 -> 2509fc7d17)
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a change to branch multileader_restart_test
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 62cdf9adb7 fix the issues and add step tracker
add 3f596c676f [IOTDB-3550]Fix show child nodes (#6598)
add 2343a33b88 remove duplicate IT (#6602)
add 05ffd8893b [IOTDB-3711] Update RegionRouteMap based on heartbeat sample (#6590)
add a0e30f5157 [IOTDB-3745]Deduplicate mods of target files in compaction (#6591)
add d7f26ab019 [IOTDB-3387]Print all bad files in Validation Tool (#6489)
add f7464253fb [IOTDB-3637] Optimize getOrCreatePartition process (#6596)
add c609bd01d8 [IOTDB-3313] Mappable UDTF, Reduce boxing/unboxing overhead in UDF InputLayer / OutputLayer (#6565)
add 73db1ee9e4 [IOTDB-3628][IOTDB-3630][IOTDB-3764] Fix some faild ITs of UDF
add bd0bab3d20 Change fill doc (#6605)
add 8fea934fd7 finish (#6610)
add 9c244b7f4f [IOTDB-3747] Fix log bug (#6612)
new 283bd3a5f9 Merge branch 'master' into multileader_restart_test
new 2509fc7d17 add step tracker
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../confignode/client/AsyncDataNodeClientPool.java | 19 +
.../handlers/ConfigNodeHeartbeatHandler.java | 6 +-
.../client/handlers/DataNodeHeartbeatHandler.java | 24 +-
...ndler.java => UpdateRegionRouteMapHandler.java} | 12 +-
.../request/read/GetDataPartitionPlan.java | 46 +-
.../request/read/GetOrCreateDataPartitionPlan.java | 24 +
.../read/GetOrCreateSchemaPartitionPlan.java | 9 +
.../request/read/GetSchemaPartitionPlan.java | 5 +-
.../response/DataNodeConfigurationResp.java | 6 +-
.../consensus/response/DataPartitionResp.java | 30 +
.../consensus/response/SchemaPartitionResp.java | 19 +
.../iotdb/confignode/manager/ConfigManager.java | 167 ++-
.../apache/iotdb/confignode/manager/IManager.java | 20 +-
.../iotdb/confignode/manager/NodeManager.java | 44 +-
.../iotdb/confignode/manager/PartitionManager.java | 343 ++----
.../iotdb/confignode/manager/load/LoadManager.java | 98 +-
.../load/heartbeat/ConfigNodeHeartbeatCache.java | 40 +-
.../load/heartbeat/DataNodeHeartbeatCache.java | 32 +-
.../{IHeartbeatStatistic.java => INodeCache.java} | 16 +-
.../manager/load/heartbeat/IRegionGroupCache.java | 16 +-
...rtbeatPackage.java => NodeHeartbeatSample.java} | 6 +-
.../manager/load/heartbeat/RegionGroupCache.java | 59 +-
...beatPackage.java => RegionHeartbeatSample.java} | 21 +-
.../iotdb/confignode/persistence/NodeInfo.java | 11 +-
.../persistence/partition/PartitionInfo.java | 31 -
.../partition/StorageGroupPartitionTable.java | 54 -
.../thrift/ConfigNodeRPCServiceProcessor.java | 61 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 12 +-
.../multileader/client/DispatchLogHandler.java | 5 +
.../service/MultiLeaderRPCServiceProcessor.java | 4 +
docs/UserGuide/Query-Data/Fill-Null-Value.md | 389 ++----
docs/zh/UserGuide/Query-Data/Fill-Null-Value.md | 399 ++-----
.../org/apache/iotdb/db/it/IoTDBNestedQueryIT.java | 5 +-
.../db/it/udf/IoTDBUDTFBuiltinFunctionIT.java | 7 -
.../db/integration/IoTDBAutoCreateSchemaIT.java | 208 ----
.../IoTDBCreateAlignedTimeseriesIT.java | 27 -
.../db/integration/IoTDBCreateStorageGroupIT.java | 131 --
.../db/integration/IoTDBDeleteStorageGroupIT.java | 183 ---
.../iotdb/db/integration/IoTDBDeletionIT.java | 529 --------
.../iotdb/db/integration/IoTDBMetadataFetchIT.java | 660 ----------
.../iotdb/db/integration/IoTDBMultiDeviceIT.java | 297 -----
.../iotdb/db/integration/IoTDBNestedQueryIT.java | 681 -----------
.../IoTDBSameMeasurementsDifferentTypesIT.java | 182 ---
.../iotdb/db/integration/IoTDBSimpleQueryIT.java | 1258 --------------------
.../IoTDBSyntaxConventionIdentifierIT.java | 998 ----------------
.../IoTDBSyntaxConventionStringLiteralIT.java | 764 ------------
.../iotdb/db/integration/IoTDBTagAlterIT.java | 625 ----------
.../apache/iotdb/db/integration/IoTDBTagIT.java | 887 +-------------
.../db/integration/IoTDBUDFWindowQueryIT.java | 35 +-
.../integration/IoTDBUDTFAlignByTimeQueryIT.java | 628 ----------
.../db/integration/aligned/IoTDBDeletionIT.java | 554 ---------
.../commons/partition/DataPartitionTable.java | 24 +-
.../commons/udf/builtin/String/UDTFEndsWith.java | 13 +-
.../commons/udf/builtin/String/UDTFLower.java | 14 +-
.../commons/udf/builtin/String/UDTFUpper.java | 14 +-
.../apache/iotdb/commons/udf/builtin/UDTFAbs.java | 27 +-
.../apache/iotdb/commons/udf/builtin/UDTFMath.java | 29 +-
.../iotdb/commons/udf/builtin/UDTFOnOff.java | 34 +-
.../apache/iotdb/db/client/ConfigNodeClient.java | 69 ++
.../db/engine/compaction/CompactionUtils.java | 49 +-
.../engine/storagegroup/TsFileNameGenerator.java | 23 +
.../operator/schema/NodePathsConvertOperator.java | 3 +
.../operator/schema/SchemaQueryMergeOperator.java | 10 +-
.../plan/expression/multi/FunctionExpression.java | 7 +
.../db/mpp/plan/statement/literal/LongLiteral.java | 11 +-
.../multi/MappableUDFQueryRowTransformer.java | 122 ++
.../transformer/multi/UDFQueryRowTransformer.java | 2 +-
.../multi/UDFQueryRowWindowTransformer.java | 2 +-
.../dag/transformer/multi/UDFQueryTransformer.java | 83 +-
...rmer.java => UniversalUDFQueryTransformer.java} | 48 +-
.../mpp/transformation/dag/udf/UDTFExecutor.java | 13 +
.../db/tools/validate/TsFileValidationTool.java | 181 ++-
.../cross/RewriteCrossSpaceCompactionTest.java | 8 +-
.../compaction/inner/InnerSeqCompactionTest.java | 90 ++
.../src/main/thrift/confignode.thrift | 26 +
.../main/java/org/apache/iotdb/udf/api/UDTF.java | 14 +
.../api/customizer/strategy/AccessStrategy.java | 3 +
...gy.java => MappableRowByRowAccessStrategy.java} | 12 +-
78 files changed, 1675 insertions(+), 9943 deletions(-)
copy confignode/src/main/java/org/apache/iotdb/confignode/client/handlers/{SetTTLHandler.java => UpdateRegionRouteMapHandler.java} (74%)
rename confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/{IHeartbeatStatistic.java => INodeCache.java} (77%)
copy confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/{HeartbeatPackage.java => NodeHeartbeatSample.java} (89%)
rename confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/{HeartbeatPackage.java => RegionHeartbeatSample.java} (71%)
delete mode 100644 integration/src/test/java/org/apache/iotdb/db/integration/IoTDBAutoCreateSchemaIT.java
delete mode 100644 integration/src/test/java/org/apache/iotdb/db/integration/IoTDBCreateStorageGroupIT.java
delete mode 100644 integration/src/test/java/org/apache/iotdb/db/integration/IoTDBDeleteStorageGroupIT.java
delete mode 100644 integration/src/test/java/org/apache/iotdb/db/integration/IoTDBDeletionIT.java
delete mode 100644 integration/src/test/java/org/apache/iotdb/db/integration/IoTDBMetadataFetchIT.java
delete mode 100644 integration/src/test/java/org/apache/iotdb/db/integration/IoTDBMultiDeviceIT.java
delete mode 100644 integration/src/test/java/org/apache/iotdb/db/integration/IoTDBNestedQueryIT.java
delete mode 100644 integration/src/test/java/org/apache/iotdb/db/integration/IoTDBSameMeasurementsDifferentTypesIT.java
delete mode 100644 integration/src/test/java/org/apache/iotdb/db/integration/IoTDBSimpleQueryIT.java
delete mode 100644 integration/src/test/java/org/apache/iotdb/db/integration/IoTDBSyntaxConventionIdentifierIT.java
delete mode 100644 integration/src/test/java/org/apache/iotdb/db/integration/IoTDBSyntaxConventionStringLiteralIT.java
delete mode 100644 integration/src/test/java/org/apache/iotdb/db/integration/IoTDBTagAlterIT.java
delete mode 100644 integration/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBDeletionIT.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/transformation/dag/transformer/multi/MappableUDFQueryRowTransformer.java
copy server/src/main/java/org/apache/iotdb/db/mpp/transformation/dag/transformer/multi/{UDFQueryTransformer.java => UniversalUDFQueryTransformer.java} (77%)
copy udf-api/src/main/java/org/apache/iotdb/udf/api/customizer/strategy/{RowByRowAccessStrategy.java => MappableRowByRowAccessStrategy.java} (81%)
[iotdb] 02/02: add step tracker
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch multileader_restart_test
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2509fc7d1743f266d185bee4e26bc03d15a7cac5
Author: Jinrui.Zhang <xi...@gmail.com>
AuthorDate: Thu Jul 7 20:34:19 2022 +0800
add step tracker
---
.../iotdb/consensus/multileader/client/DispatchLogHandler.java | 5 +++++
.../multileader/service/MultiLeaderRPCServiceProcessor.java | 4 ++++
2 files changed, 9 insertions(+)
diff --git a/consensus/src/main/java/org/apache/iotdb/consensus/multileader/client/DispatchLogHandler.java b/consensus/src/main/java/org/apache/iotdb/consensus/multileader/client/DispatchLogHandler.java
index 14a2b4ad58..d10263b540 100644
--- a/consensus/src/main/java/org/apache/iotdb/consensus/multileader/client/DispatchLogHandler.java
+++ b/consensus/src/main/java/org/apache/iotdb/consensus/multileader/client/DispatchLogHandler.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.consensus.multileader.client;
+import org.apache.iotdb.commons.StepTracker;
import org.apache.iotdb.consensus.multileader.logdispatcher.LogDispatcher.LogDispatcherThread;
import org.apache.iotdb.consensus.multileader.logdispatcher.PendingBatch;
import org.apache.iotdb.consensus.multileader.thrift.TSyncLogRes;
@@ -37,10 +38,12 @@ public class DispatchLogHandler implements AsyncMethodCallback<TSyncLogRes> {
private final LogDispatcherThread thread;
private final PendingBatch batch;
private int retryCount;
+ private final long startTime;
public DispatchLogHandler(LogDispatcherThread thread, PendingBatch batch) {
this.thread = thread;
this.batch = batch;
+ this.startTime = System.nanoTime();
}
@Override
@@ -58,6 +61,8 @@ public class DispatchLogHandler implements AsyncMethodCallback<TSyncLogRes> {
} else {
thread.getSyncStatus().removeBatch(batch);
}
+ StepTracker.trace("dispatcherSendBatch", 10, startTime, System.nanoTime());
+ StepTracker.trace("dispatcherBatchSize", 10, 0, this.batch.getBatches().size() * 1000L);
}
@Override
diff --git a/consensus/src/main/java/org/apache/iotdb/consensus/multileader/service/MultiLeaderRPCServiceProcessor.java b/consensus/src/main/java/org/apache/iotdb/consensus/multileader/service/MultiLeaderRPCServiceProcessor.java
index 196bf120ff..c8fe9fbfe2 100644
--- a/consensus/src/main/java/org/apache/iotdb/consensus/multileader/service/MultiLeaderRPCServiceProcessor.java
+++ b/consensus/src/main/java/org/apache/iotdb/consensus/multileader/service/MultiLeaderRPCServiceProcessor.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.consensus.multileader.service;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.StepTracker;
import org.apache.iotdb.commons.consensus.ConsensusGroupId;
import org.apache.iotdb.consensus.common.request.ByteBufferConsensusRequest;
import org.apache.iotdb.consensus.multileader.MultiLeaderConsensus;
@@ -52,6 +53,7 @@ public class MultiLeaderRPCServiceProcessor implements MultiLeaderConsensusIServ
@Override
public void syncLog(TSyncLogReq req, AsyncMethodCallback<TSyncLogRes> resultHandler)
throws TException {
+ long startTime = System.nanoTime();
try {
ConsensusGroupId groupId =
ConsensusGroupId.Factory.createFromTConsensusGroupId(req.getConsensusGroupId());
@@ -82,6 +84,8 @@ public class MultiLeaderRPCServiceProcessor implements MultiLeaderConsensusIServ
resultHandler.onComplete(new TSyncLogRes(statuses));
} catch (Exception e) {
resultHandler.onError(e);
+ } finally {
+ StepTracker.trace("ProcessSyncLog", 10, startTime, System.nanoTime());
}
}
[iotdb] 01/02: Merge branch 'master' into multileader_restart_test
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch multileader_restart_test
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 283bd3a5f9a5b792e9d39f31df8ae51963961fed
Merge: 62cdf9adb7 9c244b7f4f
Author: Jinrui.Zhang <xi...@gmail.com>
AuthorDate: Thu Jul 7 20:03:11 2022 +0800
Merge branch 'master' into multileader_restart_test
.../confignode/client/AsyncDataNodeClientPool.java | 19 +
.../handlers/ConfigNodeHeartbeatHandler.java | 6 +-
.../client/handlers/DataNodeHeartbeatHandler.java | 24 +-
.../handlers/UpdateRegionRouteMapHandler.java | 58 +
.../request/read/GetDataPartitionPlan.java | 46 +-
.../request/read/GetOrCreateDataPartitionPlan.java | 24 +
.../read/GetOrCreateSchemaPartitionPlan.java | 9 +
.../request/read/GetSchemaPartitionPlan.java | 5 +-
.../response/DataNodeConfigurationResp.java | 6 +-
.../consensus/response/DataPartitionResp.java | 30 +
.../consensus/response/SchemaPartitionResp.java | 19 +
.../iotdb/confignode/manager/ConfigManager.java | 167 ++-
.../apache/iotdb/confignode/manager/IManager.java | 20 +-
.../iotdb/confignode/manager/NodeManager.java | 44 +-
.../iotdb/confignode/manager/PartitionManager.java | 343 ++----
.../iotdb/confignode/manager/load/LoadManager.java | 98 +-
.../load/heartbeat/ConfigNodeHeartbeatCache.java | 40 +-
.../load/heartbeat/DataNodeHeartbeatCache.java | 32 +-
.../{IHeartbeatStatistic.java => INodeCache.java} | 16 +-
.../manager/load/heartbeat/IRegionGroupCache.java | 16 +-
...rtbeatPackage.java => NodeHeartbeatSample.java} | 6 +-
.../manager/load/heartbeat/RegionGroupCache.java | 59 +-
...beatPackage.java => RegionHeartbeatSample.java} | 21 +-
.../iotdb/confignode/persistence/NodeInfo.java | 11 +-
.../persistence/partition/PartitionInfo.java | 31 -
.../partition/StorageGroupPartitionTable.java | 54 -
.../thrift/ConfigNodeRPCServiceProcessor.java | 61 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 12 +-
docs/UserGuide/Query-Data/Fill-Null-Value.md | 389 ++----
docs/zh/UserGuide/Query-Data/Fill-Null-Value.md | 399 ++-----
.../org/apache/iotdb/db/it/IoTDBNestedQueryIT.java | 5 +-
.../db/it/udf/IoTDBUDTFBuiltinFunctionIT.java | 7 -
.../db/integration/IoTDBAutoCreateSchemaIT.java | 208 ----
.../IoTDBCreateAlignedTimeseriesIT.java | 27 -
.../db/integration/IoTDBCreateStorageGroupIT.java | 131 --
.../db/integration/IoTDBDeleteStorageGroupIT.java | 183 ---
.../iotdb/db/integration/IoTDBDeletionIT.java | 529 --------
.../iotdb/db/integration/IoTDBMetadataFetchIT.java | 660 ----------
.../iotdb/db/integration/IoTDBMultiDeviceIT.java | 297 -----
.../iotdb/db/integration/IoTDBNestedQueryIT.java | 681 -----------
.../IoTDBSameMeasurementsDifferentTypesIT.java | 182 ---
.../iotdb/db/integration/IoTDBSimpleQueryIT.java | 1258 --------------------
.../IoTDBSyntaxConventionIdentifierIT.java | 998 ----------------
.../IoTDBSyntaxConventionStringLiteralIT.java | 764 ------------
.../iotdb/db/integration/IoTDBTagAlterIT.java | 625 ----------
.../apache/iotdb/db/integration/IoTDBTagIT.java | 887 +-------------
.../db/integration/IoTDBUDFWindowQueryIT.java | 35 +-
.../integration/IoTDBUDTFAlignByTimeQueryIT.java | 628 ----------
.../db/integration/aligned/IoTDBDeletionIT.java | 554 ---------
.../commons/partition/DataPartitionTable.java | 24 +-
.../commons/udf/builtin/String/UDTFEndsWith.java | 13 +-
.../commons/udf/builtin/String/UDTFLower.java | 14 +-
.../commons/udf/builtin/String/UDTFUpper.java | 14 +-
.../apache/iotdb/commons/udf/builtin/UDTFAbs.java | 27 +-
.../apache/iotdb/commons/udf/builtin/UDTFMath.java | 29 +-
.../iotdb/commons/udf/builtin/UDTFOnOff.java | 34 +-
.../apache/iotdb/db/client/ConfigNodeClient.java | 69 ++
.../db/engine/compaction/CompactionUtils.java | 49 +-
.../engine/storagegroup/TsFileNameGenerator.java | 23 +
.../operator/schema/NodePathsConvertOperator.java | 3 +
.../operator/schema/SchemaQueryMergeOperator.java | 10 +-
.../plan/expression/multi/FunctionExpression.java | 7 +
.../db/mpp/plan/statement/literal/LongLiteral.java | 11 +-
.../multi/MappableUDFQueryRowTransformer.java | 122 ++
.../transformer/multi/UDFQueryRowTransformer.java | 2 +-
.../multi/UDFQueryRowWindowTransformer.java | 2 +-
.../dag/transformer/multi/UDFQueryTransformer.java | 83 +-
...rmer.java => UniversalUDFQueryTransformer.java} | 48 +-
.../mpp/transformation/dag/udf/UDTFExecutor.java | 13 +
.../db/tools/validate/TsFileValidationTool.java | 181 ++-
.../cross/RewriteCrossSpaceCompactionTest.java | 8 +-
.../compaction/inner/InnerSeqCompactionTest.java | 90 ++
.../src/main/thrift/confignode.thrift | 26 +
.../main/java/org/apache/iotdb/udf/api/UDTF.java | 14 +
.../api/customizer/strategy/AccessStrategy.java | 3 +
.../strategy/MappableRowByRowAccessStrategy.java | 59 +
76 files changed, 1772 insertions(+), 9930 deletions(-)