You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2022/11/30 07:08:14 UTC
[iotdb] 01/01: resolve conflicts
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch QueryMetrics
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5d654bdbbd3b6a2b5fc7a614b870ae8443c3c0ae
Merge: 069dc241bc a7a1738c1e
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Wed Nov 30 15:07:53 2022 +0800
resolve conflicts
.github/workflows/cluster-it-1c1d.yml | 2 +
.github/workflows/cluster-it-1c3d.yml | 2 +
.github/workflows/site-build.yml | 4 +-
Jenkinsfile | 8 +-
RELEASE_NOTES.md | 101 ++---
antlr/pom.xml | 2 +-
cli/pom.xml | 2 +-
client-cpp/pom.xml | 6 +-
client-py/pom.xml | 2 +-
client-py/setup.py | 2 +-
code-coverage/pom.xml | 2 +-
compile-tools/pom.xml | 8 +-
compile-tools/thrift/pom.xml | 2 +-
confignode/pom.xml | 4 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 66 ++-
.../confignode/conf/ConfigNodeDescriptor.java | 46 +-
.../read/partition/GetDataPartitionPlan.java | 34 +-
.../partition/GetOrCreateDataPartitionPlan.java | 5 +-
.../confignode/manager/ClusterSchemaManager.java | 16 +-
.../iotdb/confignode/manager/ConfigManager.java | 13 +-
.../iotdb/confignode/manager/ConsensusManager.java | 2 +
.../iotdb/confignode/manager/load/LoadManager.java | 7 +-
.../manager/load/balancer/PartitionBalancer.java | 5 +-
.../manager/load/balancer/RouteBalancer.java | 2 +-
.../partition/GreedyPartitionAllocator.java | 13 +-
.../balancer/partition/IPartitionAllocator.java | 5 +-
.../load/balancer/router/RegionRouteMap.java | 5 +
.../iotdb/confignode/manager/node/NodeManager.java | 26 +-
.../manager/partition/PartitionManager.java | 33 +-
...Policy.java => RegionGroupExtensionPolicy.java} | 8 +-
.../persistence/partition/PartitionInfo.java | 16 +-
.../partition/StorageGroupPartitionTable.java | 8 +-
.../impl/schema/DeleteStorageGroupProcedure.java | 5 +
.../request/ConfigPhysicalPlanSerDeTest.java | 27 +-
consensus/pom.xml | 4 +-
.../org/apache/iotdb/consensus/common/Utils.java | 33 ++
.../apache/iotdb/consensus/config/RatisConfig.java | 36 +-
.../ratis/ApplicationStateMachineProxy.java | 4 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 48 +++
.../iotdb/consensus/ratis/SnapshotStorage.java | 19 +-
.../iotdb/consensus/ratis/RatisConsensusTest.java | 11 +-
cross-tests/pom.xml | 4 +-
distribution/pom.xml | 4 +-
docs/UserGuide/API/InfluxDB-Protocol.md | 2 +-
docs/UserGuide/API/Programming-Java-Native-API.md | 4 +-
docs/UserGuide/API/Programming-TsFile-API.md | 4 +-
docs/UserGuide/Cluster/Cluster-Concept.md | 16 +-
docs/UserGuide/Cluster/Cluster-Setup.md | 21 +-
.../UserGuide/Ecosystem-Integration/Hive-TsFile.md | 8 +-
.../Ecosystem-Integration/MapReduce-TsFile.md | 2 +-
.../Ecosystem-Integration/Spark-TsFile.md | 2 +-
docs/UserGuide/Monitor-Alert/Metric-Tool.md | 220 +++++-----
.../Operators-Functions/User-Defined-Function.md | 15 +-
.../UserGuide/QuickStart/Command-Line-Interface.md | 51 +--
docs/UserGuide/QuickStart/QuickStart.md | 2 -
docs/UserGuide/QuickStart/WayToGetIoTDB.md | 8 +-
docs/UserGuide/Reference/SQL-Reference.md | 2 +-
docs/UserGuide/Trigger/Implement-Trigger.md | 2 +-
docs/zh/UserGuide/API/InfluxDB-Protocol.md | 2 +-
docs/zh/UserGuide/API/Programming-JDBC.md | 2 +-
.../UserGuide/API/Programming-Java-Native-API.md | 4 +-
docs/zh/UserGuide/API/Programming-TsFile-API.md | 4 +-
docs/zh/UserGuide/Cluster/Cluster-Setup.md | 1 -
.../UserGuide/Ecosystem-Integration/Hive-TsFile.md | 8 +-
.../Ecosystem-Integration/MapReduce-TsFile.md | 2 +-
.../Ecosystem-Integration/Spark-TsFile.md | 2 +-
docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md | 139 +++---
.../Operators-Functions/User-Defined-Function.md | 16 +-
.../UserGuide/QuickStart/Command-Line-Interface.md | 35 +-
docs/zh/UserGuide/QuickStart/QuickStart.md | 1 -
docs/zh/UserGuide/QuickStart/WayToGetIoTDB.md | 2 +-
docs/zh/UserGuide/Reference/SQL-Reference.md | 2 +-
docs/zh/UserGuide/Trigger/Implement-Trigger.md | 2 +-
example/client-cpp-example/pom.xml | 4 +-
example/ext-pipe-plugin-example/README.md | 2 +-
example/ext-pipe-plugin-example/pom.xml | 4 +-
example/flink/pom.xml | 2 +-
example/hadoop/pom.xml | 2 +-
example/influxdb-protocol-example/pom.xml | 2 +-
example/jdbc/pom.xml | 2 +-
example/kafka/pom.xml | 4 +-
example/kafka/readme.md | 4 +-
example/mqtt-customize/pom.xml | 2 +-
example/mqtt/pom.xml | 2 +-
example/pom.xml | 2 +-
example/pulsar/pom.xml | 2 +-
example/rabbitmq/pom.xml | 2 +-
example/rest-java-example/pom.xml | 2 +-
example/rocketmq/pom.xml | 2 +-
example/rocketmq/readme.md | 4 +-
example/session/pom.xml | 2 +-
example/trigger/pom.xml | 4 +-
example/tsfile/pom.xml | 2 +-
example/tsfile/readme.md | 2 +-
example/udf/pom.xml | 4 +-
.../java/org/apache/iotdb/udf/UDTFExample.java | 4 +-
external-api/pom.xml | 2 +-
external-pipe-api/pom.xml | 2 +-
flink-iotdb-connector/pom.xml | 2 +-
flink-tsfile-connector/pom.xml | 2 +-
grafana-connector/pom.xml | 4 +-
.../Apache IoTDB ConfigNode Dashboard.json | 113 ++++-
.../Apache IoTDB DataNode Dashboard.json | 123 +++++-
grafana-plugin/pom.xml | 10 +-
hadoop/README.md | 2 +-
hadoop/pom.xml | 2 +-
hive-connector/pom.xml | 2 +-
influxdb-protocol/pom.xml | 2 +-
integration-test/pom.xml | 218 +++++-----
.../java/org/apache/iotdb/it/env/MppConfig.java | 14 +
.../org/apache/iotdb/itbase/env/BaseConfig.java | 16 +
.../it/IoTDBClusterRegionLeaderBalancingIT.java | 20 +-
.../confignode/it/IoTDBConfigNodeSnapshotIT.java | 8 +-
.../it/IoTDBConfigNodeSwitchLeaderIT.java | 18 +-
...T.java => IoTDBAutoRegionGroupExtensionIT.java} | 133 ++++--
...java => IoTDBCustomRegionGroupExtensionIT.java} | 24 +-
.../it/partition/IoTDBPartitionDurableIT.java | 7 +-
.../it/partition/IoTDBPartitionGetterIT.java | 24 +-
.../partition/IoTDBPartitionInheritPolicyIT.java | 5 +-
.../confignode/it/utils/ConfigNodeTestUtils.java | 28 +-
integration/pom.xml | 28 +-
jdbc/README.md | 4 +-
jdbc/pom.xml | 4 +-
jdbc/src/main/feature/feature.xml | 4 +-
library-udf/pom.xml | 2 +-
metrics/dropwizard-metrics/pom.xml | 2 +-
metrics/interface/pom.xml | 2 +-
metrics/micrometer-metrics/pom.xml | 4 +-
metrics/pom.xml | 2 +-
node-commons/pom.xml | 2 +-
.../resources/conf/iotdb-common.properties | 20 +-
.../iotdb/commons/partition/DataPartition.java | 21 +-
.../commons/partition/DataPartitionQueryParam.java | 35 ++
.../commons/partition/DataPartitionTable.java | 23 +-
.../commons/partition/SeriesPartitionTable.java | 34 +-
.../commons/utils/ThriftCommonsSerDeUtils.java | 20 +
openapi/pom.xml | 2 +-
pom.xml | 12 +-
rewrite-tsfile-tool/pom.xml | 2 +-
schema-engine-rocksdb/pom.xml | 2 +-
.../schemaregion/rocksdb/RSchemaRegion.java | 8 +-
schema-engine-tag/README.md | 4 +-
schema-engine-tag/pom.xml | 2 +-
server/pom.xml | 4 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 19 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 3 +
.../db/consensus/DataRegionConsensusImpl.java | 2 +
.../db/consensus/SchemaRegionConsensusImpl.java | 2 +
.../statemachine/DataRegionStateMachine.java | 7 +-
.../apache/iotdb/db/engine/StorageEngineV2.java | 2 +-
.../iotdb/db/engine/snapshot/SnapshotLoader.java | 8 +
.../iotdb/db/engine/snapshot/SnapshotTaker.java | 4 +
.../iotdb/db/engine/storagegroup/DataRegion.java | 11 +-
.../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 35 +-
.../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 35 +-
.../metadata/schemaregion/RSchemaRegionLoader.java | 14 +-
.../db/metadata/schemaregion/SchemaEngine.java | 48 +--
.../schemaregion/SchemaRegionMemoryImpl.java | 12 +-
.../schemaregion/SchemaRegionSchemaFileImpl.java | 8 +-
.../execution/exchange/MPPDataExchangeManager.java | 6 +-
.../db/mpp/execution/exchange/SinkHandle.java | 6 +-
.../db/mpp/execution/exchange/SourceHandle.java | 4 +-
.../execution/executor/RegionWriteExecutor.java | 2 +-
.../fragment/FragmentInstanceManager.java | 4 +-
.../operator/process/TagAggregationOperator.java | 41 +-
.../execution/schedule/AbstractDriverThread.java | 2 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 146 +++++--
.../mpp/plan/analyze/ClusterPartitionFetcher.java | 93 +++--
.../db/mpp/plan/analyze/IPartitionFetcher.java | 9 +
.../plan/analyze/StandalonePartitionFetcher.java | 7 +
.../db/mpp/plan/analyze/cache/PartitionCache.java | 2 +-
.../db/mpp/plan/execution/QueryExecution.java | 2 +-
.../mpp/plan/execution/config/ConfigExecution.java | 2 +-
.../config/executor/ClusterConfigTaskExecutor.java | 64 +--
.../plan/planner/plan/node/write/InsertNode.java | 36 +-
.../scheduler/FixedRateFragInsStateTracker.java | 2 +-
.../scheduler/FragmentInstanceDispatcherImpl.java | 16 +-
.../scheduler/load/LoadTsFileDispatcherImpl.java | 18 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 67 +--
.../service/thrift/impl/ClientRPCServiceImpl.java | 2 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 28 +-
.../engine/compaction/CompactionSchedulerTest.java | 1 -
.../db/engine/snapshot/IoTDBSnapshotTest.java | 8 +-
.../iotdb/db/metadata/mtree/MTreeBelowSGTest.java | 6 +-
.../mpp/plan/analyze/FakePartitionFetcherImpl.java | 6 +
.../mpp/plan/analyze/QueryTimePartitionTest.java | 73 +++-
.../iotdb/db/mpp/plan/plan/distribution/Util.java | 6 +
.../plan/node/write/WritePlanNodeSplitTest.java | 18 +-
service-rpc/pom.xml | 2 +-
session/pom.xml | 2 +-
site/pom.xml | 66 ++-
site/src/main/.vuepress/config.js | 464 +++++++++++++++++++++
spark-iotdb-connector/pom.xml | 2 +-
spark-tsfile/pom.xml | 2 +-
testcontainer/pom.xml | 7 +-
thrift-commons/pom.xml | 2 +-
thrift-confignode/pom.xml | 2 +-
.../src/main/thrift/confignode.thrift | 14 +-
thrift-influxdb/pom.xml | 2 +-
thrift-iot-consensus/pom.xml | 2 +-
thrift/pom.xml | 2 +-
trigger-api/pom.xml | 2 +-
tsfile/README.md | 2 +-
tsfile/pom.xml | 4 +-
udf-api/pom.xml | 2 +-
zeppelin-interpreter/pom.xml | 2 +-
206 files changed, 2607 insertions(+), 1180 deletions(-)
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
index fdcba680c6,5d27adb255..2afefb046e
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
@@@ -364,22 -361,36 +367,41 @@@ public class AnalyzeVisitor extends Sta
Set<String> deviceSet =
allSelectedPath.stream().map(MeasurementPath::getDevice).collect(Collectors.toSet());
- List<TTimePartitionSlot> timePartitionSlotList =
++ long startTime = System.nanoTime();
++
+ Pair<List<TTimePartitionSlot>, Pair<Boolean, Boolean>> res =
getTimePartitionSlotList(analysis.getGlobalTimeFilter());
- Map<String, List<DataPartitionQueryParam>> sgNameToQueryParamsMap = new HashMap<>();
- for (String devicePath : deviceSet) {
- DataPartitionQueryParam queryParam =
- new DataPartitionQueryParam(devicePath, timePartitionSlotList);
- sgNameToQueryParamsMap
- .computeIfAbsent(schemaTree.getBelongedDatabase(devicePath), key -> new ArrayList<>())
- .add(queryParam);
+ DataPartition dataPartition;
+
+ // there is no satisfied time range
+ if (res.left.isEmpty() && !res.right.left) {
+ dataPartition =
+ new DataPartition(
+ Collections.emptyMap(),
+ CONFIG.getSeriesPartitionExecutorClass(),
+ CONFIG.getSeriesPartitionSlotNum());
+ } else {
+ Map<String, List<DataPartitionQueryParam>> sgNameToQueryParamsMap = new HashMap<>();
+ for (String devicePath : deviceSet) {
+ DataPartitionQueryParam queryParam =
+ new DataPartitionQueryParam(devicePath, res.left, res.right.left, res.right.right);
+ sgNameToQueryParamsMap
+ .computeIfAbsent(schemaTree.getBelongedDatabase(devicePath), key -> new ArrayList<>())
+ .add(queryParam);
+ }
+
+ if (res.right.left || res.right.right) {
+ dataPartition =
+ partitionFetcher.getDataPartitionWithUnclosedTimeRange(sgNameToQueryParamsMap);
+ } else {
+ dataPartition = partitionFetcher.getDataPartition(sgNameToQueryParamsMap);
+ }
}
- long startTime = System.nanoTime();
- DataPartition dataPartition = partitionFetcher.getDataPartition(sgNameToQueryParamsMap);
+ QueryStatistics.getInstance()
+ .addCost(QueryStatistics.PARTITION_FETCHER, System.nanoTime() - startTime);
++
analysis.setDataPartitionInfo(dataPartition);
return analysis;
@@@ -1117,6 -1128,6 +1139,7 @@@
private void analyzeDataPartition(
Analysis analysis, QueryStatement queryStatement, ISchemaTree schemaTree) {
++ long startTime = System.nanoTime();
Set<String> deviceSet = new HashSet<>();
if (queryStatement.isAlignByDevice()) {
deviceSet = analysis.getDeviceToSourceExpressions().keySet();
@@@ -1128,6 -1139,6 +1151,8 @@@
DataPartition dataPartition =
fetchDataPartitionByDevices(deviceSet, schemaTree, analysis.getGlobalTimeFilter());
analysis.setDataPartitionInfo(dataPartition);
++ QueryStatistics.getInstance()
++ .addCost(QueryStatistics.PARTITION_FETCHER, System.nanoTime() - startTime);
}
private DataPartition fetchDataPartitionByDevices(