You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2022/04/20 02:42:30 UTC
[iotdb] branch xingtanzjr/mpp_issues updated (200e66c70b -> 755237895b)
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a change to branch xingtanzjr/mpp_issues
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 200e66c70b fix partition field
add a448999829 [IOTDB-2898] Improve deletion of wal files (#5524)
add 0b61882ab4 Add failed method for Driver (#5589)
add 3d93a7764f Optimize dependencies management to reduce the lib size (#5582)
add f52beecbde Add input location for TimeJoinNode (#5585)
add ac51b0802a [IOTDB-2873] Apache IoTDB Dashboard: Grafana Metrics Template (#5587)
add 4d34f363af [IOTDB-2925]Add Schema File sketch tool (#5570)
add 2be90fe96d Rename thrift EndPoint to TEndPoint (#5590)
add e891a68c13 [IOTDB-2917] Fix flush operation has one residual wal file (#5560)
add eba44b18df [IOTDB-2954] fix influxdb protocol bug found in benchmark test of verificationQueryMode (#5594)
add 79b22a2799 Add UTs for new insert framework and fix insertRowNode memControl (#5576)
add 3bc4955429 [IOTDB-2887][IOTDB-2930][IOTDB-2932] Fix recover test error in CI & Fix NPE while concurrent createTimeseries & Add config check for schema engine mode (#5601)
add 1d0b0ec288 Fix some issues in MPP framework (#5596)
new 15fe08abda Merge branch 'master' into xingtanzjr/mpp_issues
new 755237895b spotless
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
LICENSE | 1 +
.../iotdb/cluster/coordinator/Coordinator.java | 6 +-
.../cluster/server/member/DataGroupMember.java | 11 +-
.../cluster/server/member/MetaGroupMember.java | 6 +-
.../cluster/server/service/BaseAsyncService.java | 4 +-
.../apache/iotdb/cluster/utils/StatusUtils.java | 6 +-
.../consensus/response/DataNodesInfoDataSet.java | 4 +-
.../confignode/consensus/RatisConsensusDemo.java | 6 +-
.../manager/ConfigManagerManualTest.java | 6 +-
.../server/ConfigNodeRPCServerProcessorTest.java | 26 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 4 +-
cross-tests/pom.xml | 2 +
distribution/src/assembly/all.xml | 4 +
distribution/src/assembly/server.xml | 4 +
docs/UserGuide/Maintenance-Tools/Metric-Tool.md | 12 +-
...MLogParser-Tool.md => SchemaFileSketch-Tool.md} | 12 +-
docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md | 12 +-
...MLogParser-Tool.md => SchemaFileSketch-Tool.md} | 14 +-
.../Apache IoTDB Dashboard v0.13.1.json | 1527 ++++++++++++++++++++
.../Apache IoTDB Dashboard v0.14.0.json | 1527 ++++++++++++++++++++
influxdb-protocol/pom.xml | 2 +-
.../iotdb/influxdb/protocol/dto/SessionPoint.java | 8 +-
.../iotdb/influxdb/session/InfluxDBSession.java | 6 +-
.../influxdb/integration/IoTDBInfluxDBIT.java | 4 +-
.../iotdb/commons/partition/RegionReplicaSet.java | 6 +-
server/pom.xml | 1 +
.../resources/conf/iotdb-engine.properties | 36 +-
.../SchemaFileSketcher.bat} | 4 +-
.../SchemaFileSketcher.sh} | 4 +-
.../tools/{mlog => schema}/mLogParser.bat | 2 +-
.../resources/tools/{mlog => schema}/mLogParser.sh | 0
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 22 +-
.../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 8 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 40 +-
.../db/engine/storagegroup/TsFileProcessor.java | 214 ++-
.../apache/iotdb/db/metadata/LocalConfigNode.java | 2 +-
.../db/metadata/LocalSchemaPartitionTable.java | 6 +
.../mtree/store/disk/MTreeFlushTaskManager.java | 2 +-
.../mtree/store/disk/MTreeReleaseTaskManager.java | 2 +-
.../mtree/store/disk/schemafile/ISegment.java | 2 +
.../mtree/store/disk/schemafile/RecordUtils.java | 24 +-
.../mtree/store/disk/schemafile/SchemaFile.java | 37 +-
.../mtree/store/disk/schemafile/SchemaPage.java | 9 +-
.../mtree/store/disk/schemafile/Segment.java | 51 +
.../apache/iotdb/db/mpp/buffer/SourceHandle.java | 19 +-
.../apache/iotdb/db/mpp/execution/DataDriver.java | 5 +
.../org/apache/iotdb/db/mpp/execution/Driver.java | 11 +-
.../db/mpp/execution/FragmentInstanceContext.java | 6 +
.../iotdb/db/mpp/execution/SchemaDriver.java | 5 +
.../db/mpp/schedule/task/FragmentInstanceTask.java | 3 +
.../{IOutputPlanNode.java => InputLocation.java} | 24 +-
.../db/mpp/sql/planner/plan/OutputColumn.java | 55 +
.../planner/plan/node/process/TimeJoinNode.java | 10 +
.../db/protocol/influxdb/handler/QueryHandler.java | 4 +-
.../util/{CommonUtils.java => JacksonUtils.java} | 38 +-
.../db/protocol/influxdb/util/StringUtils.java | 3 +-
.../db/query/dataset/AlignByDeviceDataSet.java | 4 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 4 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 2 +-
.../service/thrift/impl/InternalServiceImpl.java | 6 +-
.../db/service/thrift/impl/TSServiceImpl.java | 47 +-
.../db/tools/{mlog => schema}/MLogParser.java | 2 +-
.../db/tools/schema/SchemaFileSketchTool.java | 165 +++
.../java/org/apache/iotdb/db/wal/WALManager.java | 33 +-
.../iotdb/db/wal/buffer/AbstractWALBuffer.java | 5 -
.../org/apache/iotdb/db/wal/buffer/WALBuffer.java | 54 +-
.../iotdb/db/wal/checkpoint/CheckpointManager.java | 28 +
.../java/org/apache/iotdb/db/wal/node/WALNode.java | 85 +-
.../iotdb/db/wal/recover/WALNodeRecoverTask.java | 4 +
.../iotdb/db/wal/recover/WALRecoverManager.java | 4 +
...GroupProcessorTest.java => DataRegionTest.java} | 439 +++---
.../engine/storagegroup/TsFileProcessorTest.java | 17 +
...ocessorTest.java => TsFileProcessorV2Test.java} | 104 +-
.../org/apache/iotdb/db/tools/MLogParserTest.java | 2 +-
.../iotdb/db/tools/SchemaFileSketchTest.java | 158 ++
.../apache/iotdb/db/utils/EnvironmentUtils.java | 0
.../org/apache/iotdb/db/utils/SchemaTestUtils.java | 0
.../org/apache/iotdb/db/wal/WALManagerTest.java | 8 +-
.../iotdb/db/wal/buffer/WALBufferCommonTest.java | 3 +-
.../db/wal/checkpoint/CheckpointManagerTest.java | 3 +-
.../apache/iotdb/db/wal/io/CheckpointFileTest.java | 4 +-
.../org/apache/iotdb/db/wal/io/WALFileTest.java | 3 +-
.../apache/iotdb/db/wal/node/WALFakeNodeTest.java | 18 +-
.../org/apache/iotdb/db/wal/node/WALNodeTest.java | 19 +-
.../db/wal/recover/WALRecoverManagerTest.java | 8 +-
.../org/apache/iotdb/rpc/RedirectException.java | 14 +-
.../main/java/org/apache/iotdb/rpc/RpcUtils.java | 4 +-
.../java/org/apache/iotdb/session/Session.java | 52 +-
.../apache/iotdb/session/SessionConnection.java | 16 +-
.../apache/iotdb/session/util/SessionUtils.java | 12 +-
.../apache/iotdb/session/SessionCacheLeaderUT.java | 28 +-
.../src/main/thrift/confignode.thrift | 4 +-
thrift/src/main/thrift/common.thrift | 6 +-
.../iotdb/tsfile/encoding/decoder/FreqDecoder.java | 26 +-
.../iotdb/tsfile/encoding/encoder/FreqEncoder.java | 26 +-
.../apache/iotdb/tsfile/utils/BitConstructor.java | 30 +-
96 files changed, 4589 insertions(+), 727 deletions(-)
copy docs/UserGuide/Maintenance-Tools/{MLogParser-Tool.md => SchemaFileSketch-Tool.md} (66%)
copy docs/zh/UserGuide/Maintenance-Tools/{MLogParser-Tool.md => SchemaFileSketch-Tool.md} (67%)
create mode 100644 grafana-metrics-example/Apache IoTDB Dashboard v0.13.1.json
create mode 100644 grafana-metrics-example/Apache IoTDB Dashboard v0.14.0.json
copy server/src/assembly/resources/tools/{mlog/mLogParser.bat => schema/SchemaFileSketcher.bat} (95%)
copy server/src/assembly/resources/tools/{tsfileToolSet/print-tsfile-sketch.sh => schema/SchemaFileSketcher.sh} (91%)
mode change 100755 => 100644
rename server/src/assembly/resources/tools/{mlog => schema}/mLogParser.bat (98%)
rename server/src/assembly/resources/tools/{mlog => schema}/mLogParser.sh (100%)
copy server/src/main/java/org/apache/iotdb/db/mpp/sql/planner/plan/{IOutputPlanNode.java => InputLocation.java} (65%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/sql/planner/plan/OutputColumn.java
copy server/src/main/java/org/apache/iotdb/db/protocol/influxdb/util/{CommonUtils.java => JacksonUtils.java} (59%)
rename server/src/main/java/org/apache/iotdb/db/tools/{mlog => schema}/MLogParser.java (99%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/tools/schema/SchemaFileSketchTool.java
copy server/src/test/java/org/apache/iotdb/db/engine/storagegroup/{StorageGroupProcessorTest.java => DataRegionTest.java} (68%)
copy server/src/test/java/org/apache/iotdb/db/engine/storagegroup/{TsFileProcessorTest.java => TsFileProcessorV2Test.java} (84%)
create mode 100644 server/src/test/java/org/apache/iotdb/db/tools/SchemaFileSketchTest.java
rename server/src/{main => test}/java/org/apache/iotdb/db/utils/EnvironmentUtils.java (100%)
rename server/src/{main => test}/java/org/apache/iotdb/db/utils/SchemaTestUtils.java (100%)
[iotdb] 02/02: spotless
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch xingtanzjr/mpp_issues
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 755237895b588b4ed5b8e0b7fd8616962ae5ded9
Author: Jinrui.Zhang <xi...@gmail.com>
AuthorDate: Wed Apr 20 10:42:16 2022 +0800
spotless
---
.../main/java/org/apache/iotdb/db/mpp/sql/statement/ConfigStatement.java | 1 -
1 file changed, 1 deletion(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/sql/statement/ConfigStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/sql/statement/ConfigStatement.java
index 19e6f2afe2..304401bc0c 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/sql/statement/ConfigStatement.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/sql/statement/ConfigStatement.java
@@ -24,4 +24,3 @@ package org.apache.iotdb.db.mpp.sql.statement;
* statements which need to be transformed into IConfigTask should extend this class
*/
public abstract class ConfigStatement extends Statement {}
-
[iotdb] 01/02: Merge branch 'master' into xingtanzjr/mpp_issues
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch xingtanzjr/mpp_issues
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 15fe08abda7303ce1e064de13d39d2898627f270
Merge: 200e66c70b 1d0b0ec288
Author: Jinrui.Zhang <xi...@gmail.com>
AuthorDate: Wed Apr 20 10:37:27 2022 +0800
Merge branch 'master' into xingtanzjr/mpp_issues
LICENSE | 1 +
.../iotdb/cluster/coordinator/Coordinator.java | 6 +-
.../cluster/server/member/DataGroupMember.java | 11 +-
.../cluster/server/member/MetaGroupMember.java | 6 +-
.../cluster/server/service/BaseAsyncService.java | 4 +-
.../apache/iotdb/cluster/utils/StatusUtils.java | 6 +-
.../consensus/response/DataNodesInfoDataSet.java | 4 +-
.../confignode/consensus/RatisConsensusDemo.java | 6 +-
.../manager/ConfigManagerManualTest.java | 6 +-
.../server/ConfigNodeRPCServerProcessorTest.java | 26 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 4 +-
cross-tests/pom.xml | 2 +
distribution/src/assembly/all.xml | 4 +
distribution/src/assembly/server.xml | 4 +
docs/UserGuide/Maintenance-Tools/Metric-Tool.md | 12 +-
.../Maintenance-Tools/SchemaFileSketch-Tool.md | 38 +
docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md | 12 +-
.../Maintenance-Tools/SchemaFileSketch-Tool.md | 35 +
.../Apache IoTDB Dashboard v0.13.1.json | 1527 ++++++++++++++++++++
.../Apache IoTDB Dashboard v0.14.0.json | 1527 ++++++++++++++++++++
influxdb-protocol/pom.xml | 2 +-
.../iotdb/influxdb/protocol/dto/SessionPoint.java | 8 +-
.../iotdb/influxdb/session/InfluxDBSession.java | 6 +-
.../influxdb/integration/IoTDBInfluxDBIT.java | 4 +-
.../iotdb/commons/partition/RegionReplicaSet.java | 6 +-
server/pom.xml | 1 +
.../resources/conf/iotdb-engine.properties | 36 +-
.../SchemaFileSketcher.bat} | 4 +-
.../mLogParser.sh => schema/SchemaFileSketcher.sh} | 4 +-
.../tools/{mlog => schema}/mLogParser.bat | 2 +-
.../resources/tools/{mlog => schema}/mLogParser.sh | 0
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 22 +-
.../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 8 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 40 +-
.../db/engine/storagegroup/TsFileProcessor.java | 214 ++-
.../apache/iotdb/db/metadata/LocalConfigNode.java | 2 +-
.../db/metadata/LocalSchemaPartitionTable.java | 6 +
.../mtree/store/disk/MTreeFlushTaskManager.java | 2 +-
.../mtree/store/disk/MTreeReleaseTaskManager.java | 2 +-
.../mtree/store/disk/schemafile/ISegment.java | 2 +
.../mtree/store/disk/schemafile/RecordUtils.java | 24 +-
.../mtree/store/disk/schemafile/SchemaFile.java | 37 +-
.../mtree/store/disk/schemafile/SchemaPage.java | 9 +-
.../mtree/store/disk/schemafile/Segment.java | 51 +
.../apache/iotdb/db/mpp/buffer/SourceHandle.java | 19 +-
.../apache/iotdb/db/mpp/execution/DataDriver.java | 5 +
.../org/apache/iotdb/db/mpp/execution/Driver.java | 11 +-
.../db/mpp/execution/FragmentInstanceContext.java | 6 +
.../iotdb/db/mpp/execution/SchemaDriver.java | 5 +
.../db/mpp/schedule/task/FragmentInstanceTask.java | 3 +
.../plan/InputLocation.java} | 26 +-
.../db/mpp/sql/planner/plan/OutputColumn.java | 55 +
.../planner/plan/node/process/TimeJoinNode.java | 10 +
.../db/mpp/sql/statement/ConfigStatement.java | 1 +
.../db/protocol/influxdb/handler/QueryHandler.java | 4 +-
.../db/protocol/influxdb/util/JacksonUtils.java | 43 +-
.../db/protocol/influxdb/util/StringUtils.java | 3 +-
.../db/query/dataset/AlignByDeviceDataSet.java | 4 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 4 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 2 +-
.../service/thrift/impl/InternalServiceImpl.java | 6 +-
.../db/service/thrift/impl/TSServiceImpl.java | 47 +-
.../db/tools/{mlog => schema}/MLogParser.java | 2 +-
.../db/tools/schema/SchemaFileSketchTool.java | 165 +++
.../java/org/apache/iotdb/db/wal/WALManager.java | 33 +-
.../iotdb/db/wal/buffer/AbstractWALBuffer.java | 5 -
.../org/apache/iotdb/db/wal/buffer/WALBuffer.java | 54 +-
.../iotdb/db/wal/checkpoint/CheckpointManager.java | 28 +
.../java/org/apache/iotdb/db/wal/node/WALNode.java | 85 +-
.../iotdb/db/wal/recover/WALNodeRecoverTask.java | 4 +
.../iotdb/db/wal/recover/WALRecoverManager.java | 4 +
.../db/engine/storagegroup/DataRegionTest.java | 910 ++++++++++++
.../engine/storagegroup/TsFileProcessorTest.java | 17 +
...ocessorTest.java => TsFileProcessorV2Test.java} | 104 +-
.../org/apache/iotdb/db/tools/MLogParserTest.java | 2 +-
.../iotdb/db/tools/SchemaFileSketchTest.java | 158 ++
.../apache/iotdb/db/utils/EnvironmentUtils.java | 0
.../org/apache/iotdb/db/utils/SchemaTestUtils.java | 0
.../org/apache/iotdb/db/wal/WALManagerTest.java | 8 +-
.../iotdb/db/wal/buffer/WALBufferCommonTest.java | 3 +-
.../db/wal/checkpoint/CheckpointManagerTest.java | 3 +-
.../apache/iotdb/db/wal/io/CheckpointFileTest.java | 4 +-
.../org/apache/iotdb/db/wal/io/WALFileTest.java | 3 +-
.../apache/iotdb/db/wal/node/WALFakeNodeTest.java | 18 +-
.../org/apache/iotdb/db/wal/node/WALNodeTest.java | 19 +-
.../db/wal/recover/WALRecoverManagerTest.java | 8 +-
.../org/apache/iotdb/rpc/RedirectException.java | 14 +-
.../main/java/org/apache/iotdb/rpc/RpcUtils.java | 4 +-
.../java/org/apache/iotdb/session/Session.java | 52 +-
.../apache/iotdb/session/SessionConnection.java | 16 +-
.../apache/iotdb/session/util/SessionUtils.java | 12 +-
.../apache/iotdb/session/SessionCacheLeaderUT.java | 28 +-
.../src/main/thrift/confignode.thrift | 4 +-
thrift/src/main/thrift/common.thrift | 6 +-
.../iotdb/tsfile/encoding/decoder/FreqDecoder.java | 26 +-
.../iotdb/tsfile/encoding/encoder/FreqEncoder.java | 26 +-
.../apache/iotdb/tsfile/utils/BitConstructor.java | 30 +-
97 files changed, 5325 insertions(+), 517 deletions(-)
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/buffer/SourceHandle.java
index e33265a6a2,37a5bce1b0..c6e534aba1
--- a/server/src/main/java/org/apache/iotdb/db/mpp/buffer/SourceHandle.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/buffer/SourceHandle.java
@@@ -206,19 -205,9 +206,11 @@@ public class SourceHandle implements IS
}
synchronized void setNoMoreTsBlocks(int lastSequenceId) {
+ logger.info(
+ "[SourceHandle {}]: No more TsBlock. {} ", localPlanNodeId, remoteFragmentInstanceId);
this.lastSequenceId = lastSequenceId;
- if (!blocked.isDone() && currSequenceId - 1 == lastSequenceId) {
- logger.info(
- "[SourceHandle {}]: all blocks are consumed. set blocked to null.", localPlanNodeId);
+ if (!blocked.isDone() && remoteTsBlockedConsumedUp()) {
blocked.set(null);
- } else {
- logger.info(
- "[SourceHandle {}]: No need to set blocked. Blocked: {}, Consumed: {} ",
- localPlanNodeId,
- blocked.isDone(),
- currSequenceId - 1 == lastSequenceId);
}
}
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/sql/planner/plan/InputLocation.java
index 304401bc0c,705d8cbace..d173149a85
--- a/server/src/main/java/org/apache/iotdb/db/mpp/sql/planner/plan/InputLocation.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/sql/planner/plan/InputLocation.java
@@@ -16,11 -16,24 +16,25 @@@
* specific language governing permissions and limitations
* under the License.
*/
+
- package org.apache.iotdb.db.mpp.sql.statement;
+ package org.apache.iotdb.db.mpp.sql.planner.plan;
- /**
- * ConfigStatement represents the statement which should be executed by ConfigNode All the
- * statements which need to be transformed into IConfigTask should extend this class
- */
- public abstract class ConfigStatement extends Statement {}
+ public class InputLocation {
+ // which input tsblock
+ private final int tsBlockIndex;
+ // which value column of that tsblock
+ private final int valueColumnIndex;
+
+ public InputLocation(int tsBlockIndex, int valueColumnIndex) {
+ this.tsBlockIndex = tsBlockIndex;
+ this.valueColumnIndex = valueColumnIndex;
+ }
+
+ public int getTsBlockIndex() {
+ return tsBlockIndex;
+ }
+
+ public int getValueColumnIndex() {
+ return valueColumnIndex;
+ }
+ }
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/sql/statement/ConfigStatement.java
index 304401bc0c,304401bc0c..19e6f2afe2
--- a/server/src/main/java/org/apache/iotdb/db/mpp/sql/statement/ConfigStatement.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/sql/statement/ConfigStatement.java
@@@ -24,3 -24,3 +24,4 @@@ package org.apache.iotdb.db.mpp.sql.sta
* statements which need to be transformed into IConfigTask should extend this class
*/
public abstract class ConfigStatement extends Statement {}
++