You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2022/04/19 07:15:44 UTC
[iotdb] 01/02: Merge branch 'master' into xingtanzjr/mpp_issues
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch xingtanzjr/mpp_issues
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit bc62d7eb24482345bc30545025b39754d87b0f8d
Merge: fb7bf77ccc 8a00e48876
Author: Jinrui.Zhang <xi...@gmail.com>
AuthorDate: Tue Apr 19 15:12:54 2022 +0800
Merge branch 'master' into xingtanzjr/mpp_issues
.github/workflows/grafana-plugin.yml | 7 +-
.../iotdb/confignode/cli/TemporaryClient.java | 22 +-
.../consensus/response/PermissionInfoDataSet.java | 32 +-
.../iotdb/confignode/manager/ConfigManager.java | 19 +-
.../apache/iotdb/confignode/manager/Manager.java | 14 +-
.../confignode/manager/PermissionManager.java | 5 +
.../persistence/AuthorInfoPersistence.java | 129 +-
.../confignode/service/executor/PlanExecutor.java | 4 +-
.../server/ConfigNodeRPCServerProcessor.java | 36 +-
.../apache/iotdb/confignode/auth/AuthorTest.java | 44 -
.../server/ConfigNodeRPCServerProcessorTest.java | 375 ++
docs/Community/About.md | 79 +
grafana-plugin/.gitignore | 1 +
grafana-plugin/Magefile.go | 32 +
grafana-plugin/README.md | 38 +-
grafana-plugin/backend-compile.sh | 36 +
grafana-plugin/go.mod | 23 +
grafana-plugin/go.sum | 659 +++
grafana-plugin/package.json | 7 +-
grafana-plugin/pkg/main.go | 40 +
grafana-plugin/pkg/plugin/plugin.go | 291 ++
grafana-plugin/pom.xml | 21 +
grafana-plugin/src/QueryEditor.tsx | 2 +-
grafana-plugin/src/datasource.ts | 130 +-
grafana-plugin/src/plugin.json | 3 +
grafana-plugin/yarn.lock | 4726 ++++++++++----------
.../db/integration/IoTDBTriggerExecutionIT.java | 188 +-
.../resources/conf/iotdb-engine.properties | 6 -
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 -
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 8 -
.../db/engine/trigger/executor/TriggerEngine.java | 20 +-
.../engine/trigger/executor/TriggerExecutor.java | 29 +-
.../service/TriggerRegistrationService.java | 52 +-
.../iotdb/db/metadata/LocalSchemaProcessor.java | 11 +-
.../idtable/entry/InsertMeasurementMNode.java | 20 +
.../org/apache/iotdb/db/metadata/mnode/IMNode.java | 8 +
.../org/apache/iotdb/db/metadata/mnode/MNode.java | 29 +
.../iotdb/db/metadata/mnode/MeasurementMNode.java | 2 -
.../mnode/estimator/BasicMNodSizeEstimator.java | 6 +-
.../db/metadata/schemaregion/ISchemaRegion.java | 5 +-
.../db/metadata/schemaregion/SchemaRegion.java | 11 +-
.../schemaregion/rocksdb/RSchemaRegion.java | 6 +-
.../schemaregion/rocksdb/mnode/RMNode.java | 20 +
.../iotdb/db/mpp/buffer/DataBlockManager.java | 22 +-
.../org/apache/iotdb/db/mpp/buffer/SinkHandle.java | 17 +-
.../apache/iotdb/db/mpp/buffer/SourceHandle.java | 16 +-
.../org/apache/iotdb/db/mpp/common/QueryId.java | 2 +-
.../apache/iotdb/db/mpp/execution/Coordinator.java | 2 +-
.../mpp/execution/config/AuthorizerConfigTask.java | 95 +
.../scheduler/AbstractFragInsStateTracker.java | 5 +-
.../scheduler/InternalServiceClientFactory.java | 7 +-
.../scheduler/SimpleFragInstanceDispatcher.java | 4 +-
.../execution/scheduler/SimpleQueryTerminator.java | 5 +-
.../db/mpp/operator/process/TransformOperator.java | 152 +-
.../operator/schema/DevicesSchemaScanOperator.java | 12 +-
.../db/mpp/operator/schema/SchemaScanOperator.java | 17 +-
.../schema/TimeSeriesSchemaScanOperator.java | 12 +-
.../apache/iotdb/db/mpp/sql/analyze/Analyzer.java | 8 +-
.../db/mpp/sql/planner/LocalExecutionPlanner.java | 8 +-
.../iotdb/db/mpp/sql/planner/QueryPlanBuilder.java | 9 +-
.../sql/planner/plan/node/PlanGraphPrinter.java | 309 ++
.../db/mpp/sql/planner/plan/node/PlanNodeType.java | 12 +-
.../node/metedata/read/DevicesSchemaScanNode.java | 47 +-
.../plan/node/metedata/read/SchemaFetchNode.java | 6 -
.../plan/node/metedata/read/SchemaScanNode.java | 27 +-
.../plan/node/metedata/read/ShowDevicesNode.java | 67 -
.../planner/plan/node/metedata/read/ShowNode.java | 36 -
.../metedata/read/TimeSeriesSchemaScanNode.java | 51 +-
.../plan/node/metedata/write/AuthorNode.java | 2 +-
.../metadata/ShowStorageGroupStatement.java | 7 +
.../java/org/apache/iotdb/db/service/DataNode.java | 6 +-
.../iotdb/db/service/DataNodeManagementServer.java | 102 -
.../db/service/DataNodeManagementServerMBean.java | 22 -
.../apache/iotdb/db/service/InternalService.java | 4 +-
.../iotdb/db/service/InternalServiceImpl.java | 112 -
.../handler/DataNodeManagementServiceHandler.java | 58 -
.../handler}/InternalServiceThriftHandler.java | 2 +-
...ntServiceImpl.java => InternalServiceImpl.java} | 105 +-
.../apache/iotdb/db/mpp/buffer/SinkHandleTest.java | 16 +-
.../iotdb/db/mpp/buffer/SourceHandleTest.java | 30 +-
.../{ => schema}/SchemaScanOperatorTest.java | 20 +-
.../db/mpp/sql/plan/DistributionPlannerTest.java | 1 -
.../read/DeviceSchemaScanNodeSerdeTest.java | 76 +
.../metadata/read/ShowDevicesNodeSerdeTest.java | 42 -
.../read/TimeSeriesSchemaScanNodeSerdeTest.java | 79 +
.../plan/node/process/AggregateNodeSerdeTest.java | 15 +-
.../node/process/DeviceMergeNodeSerdeTest.java | 17 +-
.../plan/node/process/ExchangeNodeSerdeTest.java | 19 +-
.../sql/plan/node/process/FillNodeSerdeTest.java | 17 +-
.../sql/plan/node/process/FilterNodeSerdeTest.java | 17 +-
.../plan/node/process/FilterNullNodeSerdeTest.java | 17 +-
.../node/process/GroupByLevelNodeSerdeTest.java | 17 +-
.../sql/plan/node/process/LimitNodeSerdeTest.java | 17 +-
.../sql/plan/node/process/OffsetNodeSerdeTest.java | 15 +-
.../sql/plan/node/process/SortNodeSerdeTest.java | 18 +-
.../plan/node/process/TimeJoinNodeSerdeTest.java | 19 +-
.../plan/node/sink/FragmentSinkNodeSerdeTest.java | 7 +-
.../iotdb/db/service/InternalServiceImplTest.java | 1 +
.../src/main/thrift/confignode.thrift | 7 +
thrift/src/main/thrift/management.thrift | 77 -
thrift/src/main/thrift/mpp.thrift | 92 +-
.../tsfile/read/common/block/TsBlockBuilder.java | 15 +-
.../iotdb/tsfile/utils/ReadWriteIOUtils.java | 12 +
103 files changed, 5652 insertions(+), 3586 deletions(-)
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/buffer/SinkHandle.java
index fc3cd825fe,af4ecf089d..c569c8d04e
--- a/server/src/main/java/org/apache/iotdb/db/mpp/buffer/SinkHandle.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/buffer/SinkHandle.java
@@@ -160,10 -160,14 +160,10 @@@ public class SinkHandle implements ISin
}
private void sendEndOfDataBlockEvent() throws TException {
- logger.debug(
- "Send end of data block event to plan node {} of {}. {}",
- remotePlanNodeId,
- remoteFragmentInstanceId,
- Thread.currentThread().getName());
+ logger.info("[SinkHandle {}]: send end of data block event.", this.getRemotePlanNodeId());
int attempt = 0;
- EndOfDataBlockEvent endOfDataBlockEvent =
- new EndOfDataBlockEvent(
+ TEndOfDataBlockEvent endOfDataBlockEvent =
+ new TEndOfDataBlockEvent(
remoteFragmentInstanceId,
remotePlanNodeId,
localFragmentInstanceId,
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/buffer/SourceHandle.java
index 01dc2f0fb1,aece366b82..d044be172e
--- a/server/src/main/java/org/apache/iotdb/db/mpp/buffer/SourceHandle.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/buffer/SourceHandle.java
@@@ -316,14 -307,15 +316,14 @@@ public class SourceHandle implements IS
@Override
public void run() {
- logger.debug(
- "Get data blocks [{}, {}) from {} for plan node {} of {}.",
+ logger.info(
+ "[SourceHandle-{}]: Get data blocks [{}, {}) from {}",
+ localPlanNodeId,
startSequenceId,
endSequenceId,
- remoteFragmentInstanceId,
- localPlanNodeId,
- localFragmentInstanceId);
+ remoteFragmentInstanceId);
- GetDataBlockRequest req =
- new GetDataBlockRequest(remoteFragmentInstanceId, startSequenceId, endSequenceId);
+ TGetDataBlockRequest req =
+ new TGetDataBlockRequest(remoteFragmentInstanceId, startSequenceId, endSequenceId);
int attempt = 0;
while (attempt < MAX_ATTEMPT_TIMES) {
attempt += 1;
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/execution/scheduler/SimpleFragInstanceDispatcher.java
index b39ec04406,17a193d9b4..40b4df4a63
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/scheduler/SimpleFragInstanceDispatcher.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/scheduler/SimpleFragInstanceDispatcher.java
@@@ -48,29 -46,33 +48,29 @@@ public class SimpleFragInstanceDispatch
return executor.submit(
() -> {
TSendFragmentInstanceResp resp = new TSendFragmentInstanceResp(false);
- try {
- for (FragmentInstance instance : instances) {
- // TODO: (jackie tien) change the port
- InternalService.Iface client =
- InternalServiceClientFactory.getInternalServiceClient(
- new Endpoint(
- instance.getHostEndpoint().getIp(),
- IoTDBDescriptor.getInstance().getConfig().getInternalPort()));
- // TODO: (xingtanzjr) consider how to handle the buffer here
- ByteBuffer buffer = ByteBuffer.allocate(1024 * 1024);
- instance.serializeRequest(buffer);
- buffer.flip();
- TConsensusGroupId groupId =
- new TConsensusGroupId(
- instance.getRegionReplicaSet().getConsensusGroupId().getId(),
- instance.getRegionReplicaSet().getConsensusGroupId().getType().toString());
- TSendFragmentInstanceReq req =
- new TSendFragmentInstanceReq(
- new TFragmentInstance(buffer), groupId, instance.getType().toString());
- resp = client.sendFragmentInstance(req);
- if (!resp.accepted) {
- break;
- }
+ for (FragmentInstance instance : instances) {
+ InternalService.Iface client =
- InternalServiceClientFactory.getMppServiceClient(
++ InternalServiceClientFactory.getInternalServiceClient(
+ new Endpoint(
+ instance.getHostEndpoint().getIp(),
- IoTDBDescriptor.getInstance().getConfig().getMppPort()));
++ IoTDBDescriptor.getInstance().getConfig().getInternalPort()));
+ // TODO: (xingtanzjr) consider how to handle the buffer here
+ ByteBuffer buffer = ByteBuffer.allocate(1024 * 1024);
+ instance.serializeRequest(buffer);
+ buffer.flip();
+ TConsensusGroupId groupId =
+ new TConsensusGroupId(
+ instance.getRegionReplicaSet().getConsensusGroupId().getId(),
+ instance.getRegionReplicaSet().getConsensusGroupId().getType().toString());
+ TSendFragmentInstanceReq req =
+ new TSendFragmentInstanceReq(
+ new TFragmentInstance(buffer), groupId, instance.getType().toString());
+ LOGGER.info("start to dispatch fragment instance: {}", instance.getId());
+ resp = client.sendFragmentInstance(req);
+ LOGGER.info("dispatch complete: {}", instance.getId());
+ if (!resp.accepted) {
+ break;
}
- } catch (Exception e) {
- // TODO: (xingtanzjr) add more details
- return new FragInstanceDispatchResult(false);
}
return new FragInstanceDispatchResult(resp.accepted);
});