You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hu...@apache.org on 2022/12/06 15:15:59 UTC
[iotdb] branch QueryMetrics updated (5d654bdbbd -> 72020ef96b)
This is an automated email from the ASF dual-hosted git repository.
hui pushed a change to branch QueryMetrics
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 5d654bdbbd resolve conflicts
add 7bc76e6d40 add html auto-rediect for M4 under-review paper temporarily (#8252)
add ca2d255485 add back the M4 doc temporarily (#8273)
add 8553d9e08d [To rel/1.0] [IOTDB-5047] Fix iotdb.jmx.port missing from data-env.sh (#8240)
add 06bc2c3d9c fix logs in compaction writer (#8262)
add 41ba0f2896 [To rel/1.0][IOTDB-5093]Fix compaction selector bug (#8267)
add 4f233bf3b8 [To rel/1.0][IOTDB-4730] Database name check when passed in Session (#8266)
add 4c77331807 set cross_compaction to true and add compaction validation (#8268)
add 701072af05 Revert "[maven-release-plugin] prepare for next development iteration"
add 88db5b214f Revert "[maven-release-plugin] prepare release v1.0.0"
add 4d0b3af94e [maven-release-plugin] prepare release v1.0.0
add 7a1415ecec [maven-release-plugin] prepare for next development iteration
add 7522b82520 [To rel/1.0] Enable tests after v1.0.0 released (#8272)
add c621ca87ed [To rel/1.0] [IOTDB-4414] Modify openId related documents and configuration files (#8270)
add 739bf205f4 [To rel/1.0] fix compaction ci (#8277)
add 942bedaca8 Revert "[To rel/1.0] Enable tests after v1.0.0 released (#8272)"
add 2cd5b43de1 Revert "[maven-release-plugin] prepare for next development iteration"
add d92e5fd9c1 Revert "[maven-release-plugin] prepare release v1.0.0"
add 3ed60f37a6 [maven-release-plugin] prepare release v1.0.0
add 7db3d5654a [maven-release-plugin] prepare for next development iteration
add 539b02732b [To rel/1.0] Enable tests after v1.0.0 released (#8283)
add d2d1142835 Merge branch 'rel/1.0' of https://github.com/apache/iotdb
add b411c1ef28 Fix site compile OOM (#8284)
add f21edc60f1 Add v1.0.x in site navigation bar (#8288)
add cdf7d6dab2 [To rel/1.0][IOTDB-4730] Add IT for DB name check passed in by Session (#8281)
add 1e3665193f [To rel/1.0] add IT for IoTDBCustomRegionGroupExtension (#8264)
add 84c01ae81c [To rel/1.0] [IOTDB-4433][IOTDB-4497] Add Cluster Maintenance doc (#8286)
add 1484436e2e fix (#8285)
add 59cc2f2932 [To rel/1.0][IOTDB-5105] Remove least_data_region_group_num check when register Non-Seed-ConfigNode (#8289)
add 7dfd121668 Revert "[To rel/1.0] Enable tests after v1.0.0 released (#8283)"
add 49baf307b7 Revert "[maven-release-plugin] prepare for next development iteration"
add 7e05e78d97 Revert "[maven-release-plugin] prepare release v1.0.0"
add 50d7863f17 update testcontainer version
add ea2fdef0c0 [maven-release-plugin] prepare release v1.0.0
add dc67933e98 [maven-release-plugin] prepare for next development iteration
add bc1e4ea189 Revert "Revert "[To rel/1.0] Enable tests after v1.0.0 released (#8283)""
add 77e9071692 Merge branch 'rel/1.0' of https://github.com/apache/iotdb
add 93e0209fa8 [To rel/1.0] Fix snapshot bug (#8294)
add 3f1d7d04e8 Revert "[To rel/1.0] Fix snapshot bug (#8294)" (#8295)
add 8b29938117 Add 1.0.0 download links (#8293)
add 8e262dc538 [To rel/1.0] Fix snapshot bug (#8296)
add cf9545b244 [To rel/1.0] add auth link in quick start doc (#8299)
add f9360c53a6 [To rel/1.0] [IOTDB-5109] Fix the load of role snapshot (#8302)
add ff953d36c3 add more logs for snapshot (#8298)
add 586c241967 Revert "Revert "Revert "[To rel/1.0] Enable tests after v1.0.0 released (#8283)"""
add a92372d92a Revert "[maven-release-plugin] prepare for next development iteration"
add 0b9275f8dc Revert "[maven-release-plugin] prepare release v1.0.0"
add fbbca3ffa3 [maven-release-plugin] prepare release v1.0.0
add 9f3600d165 [maven-release-plugin] prepare for next development iteration
add 774806fce3 update testcontainer version
add b140acb1a0 Revert "Revert "Revert "Revert "[To rel/1.0] Enable tests after v1.0.0 released (#8283)""""
add 1d06983a94 [To rel/1.0] Update doc on DataRegion using RatisConsensus (#8308)
add 759fcaa8f0 add confignode notice (#8309)
add ca7f58e16e [To rel/1.0] Add Deploy recommendation in docs (#8306)
add 8a443e8e70 Merge branch 'rel/1.0' of https://github.com/apache/iotdb
add 22457f5527 add ratis log doc (#8312)
add 393a8d673f fix cluster doc (#8314)
add 36b96fd078 Fix readme of start-server to start-standalone (#8316)
add 5510c79fbf Fix linux CI (#8310)
add c4b153d3de [IOTDB-5026] Improve last query on aligned timeseries
add fe3c779b8c [To rel/1.0] Rename StorageEngineV2 to StorageEngine (#8238)
add 985b1bb16d [IOTDB-5102] Support Align by deivce in VisitExplain (#8287)
add aabae384bf [IOTDB-5090] Add npe check in DataNode (#8328)
add e2e69bb730 Bump maven-bundle-plugin from 5.1.7 to 5.1.8 (#8276)
add 1d89d1ab60 Rename SeriesSlotId to SeriesSlotNum in show regions (#8326)
add dc69fee679 [IOTDB-4978] Fix deadLock caused by blocked operation in IntoOperator (#8269)
add 4f0b5751b3 [IOTDB-5066] Upgraded the GetSlots SQLs (#8333)
add ff5d9c04e9 Update Site,Add 'Edit this page on GitHub' (#8335)
add ac2ee40cfb [IOTDB-4902]Optimize process logic for aggregation when there is only one data region
add 9a72a2aa20 [IOTDB-5122] forward exception when Ratis request failed (#8338)
add 1384d892e3 Fix RatisConsensus Param errors related to LogPurgePreservation (#8340)
add 9e2dc233f1 Update Page.vue：change editLinkText size to 16px (#8343)
add 9ae87ffb93 [IOTDB-5123] fix ratis disk control param mismatch (#8339)
add 11f369181a Add an interface for query engine which storage engine must provide (#8345)
add 64ed75f500 add uts of fast compaction performer (#8346)
add 58e2d17e5d Update iotdb-doap info and python readme (#8355)
add 997f2e4ef6 document bug fix about 1.0 version (#8330)
add d8e4901054 Bump mockito-core from 2.23.0 to 2.23.4 (#8274)
add d6671b3662 [IOTDB-5120] update doc about operator-function
new d209479eda Merge remote-tracking branch 'origin/master' into QueryMetrics
new 72020ef96b add io metrics
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/main-unix.yml | 2 +-
README.md | 16 +-
README_ZH.md | 14 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 5 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 +
client-py/README.md | 7 +-
client-py/requirements_dev.txt | 2 +-
confignode/pom.xml | 2 +-
.../src/assembly/resources/conf/confignode-env.bat | 2 +
.../src/assembly/resources/conf/confignode-env.sh | 2 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 32 +-
.../confignode/conf/ConfigNodeDescriptor.java | 52 +-
.../request/read/region/GetRegionIdPlan.java | 2 +-
.../iotdb/confignode/manager/ConfigManager.java | 9 +-
.../iotdb/confignode/manager/ConsensusManager.java | 5 +-
.../apache/iotdb/confignode/manager/IManager.java | 4 +-
.../iotdb/confignode/manager/node/NodeManager.java | 6 +-
.../manager/partition/PartitionManager.java | 16 +-
.../persistence/executor/ConfigPlanExecutor.java | 14 +-
.../partition/StorageGroupPartitionTable.java | 2 +
.../iotdb/confignode/service/ConfigNode.java | 4 +
.../confignode/service/ConfigNodeCommandLine.java | 3 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 11 +-
consensus/pom.xml | 2 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 11 +-
.../iotdb/consensus/ratis/RatisConsensusTest.java | 5 +
cross-tests/pom.xml | 2 +-
docs/Download/README.md | 46 +-
docs/UserGuide/Cluster/Cluster-Concept.md | 5 +-
.../Cluster-Maintenance.md} | 274 +--
docs/UserGuide/Cluster/Cluster-Setup.md | 13 +-
.../UserGuide/Cluster/Deployment-Recommendation.md | 181 ++
docs/UserGuide/Data-Concept/Data-Type.md | 2 +-
.../Maintenance-Tools/Maintenance-Command.md | 383 ----
.../Operators-Functions/Data-Profiling.md | 4 +-
docs/UserGuide/Operators-Functions/String.md | 4 +-
.../Operators-Functions/User-Defined-Function.md | 2 +-
.../UserGuide/QuickStart/Command-Line-Interface.md | 7 +-
docs/UserGuide/QuickStart/QuickStart.md | 15 +-
docs/UserGuide/Reference/Common-Config-Manual.md | 23 +-
.../{Blog/index.md => UserGuide/UDF-Library/M4.md} | 6 +-
docs/zh/Download/README.md | 54 +-
docs/zh/UserGuide/Cluster/Cluster-Concept.md | 4 +-
.../Cluster-Maintenance.md} | 223 +--
docs/zh/UserGuide/Cluster/Cluster-Setup.md | 13 +-
.../UserGuide/Cluster/Deployment-Recommendation.md | 179 ++
docs/zh/UserGuide/Data-Concept/Data-Type.md | 2 +-
.../Maintenance-Tools/Maintenance-Command.md | 363 ----
.../Operators-Functions/Data-Profiling.md | 4 +-
docs/zh/UserGuide/Operators-Functions/Overview.md | 2 +-
docs/zh/UserGuide/Operators-Functions/String.md | 4 +-
.../Operators-Functions/User-Defined-Function.md | 2 +-
.../UserGuide/QuickStart/Command-Line-Interface.md | 4 +-
docs/zh/UserGuide/QuickStart/QuickStart.md | 14 +-
.../zh/UserGuide/Reference/Common-Config-Manual.md | 22 +-
.../Reference/ConfigNode-Config-Manual.md | 2 +-
integration-test/pom.xml | 212 +-
.../java/org/apache/iotdb/it/env/MppConfig.java | 9 +
.../org/apache/iotdb/itbase/env/BaseConfig.java | 8 +
.../IoTDBCustomRegionGroupExtensionIT.java | 39 +-
.../it/partition/IoTDBPartitionGetterIT.java | 4 +-
.../it/aggregation/IoTDBAggregationOptimizeIT.java | 181 ++
.../iotdb/db/it/env/StandaloneEnvConfig.java | 14 +
.../iotdb/db/it/selectinto/IoTDBSelectInto2IT.java | 3 +
.../iotdb/db/it/selectinto/IoTDBSelectInto3IT.java | 3 +
.../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 5 +
.../iotdb/session/it/IoTDBSessionSimpleIT.java | 31 +
integration/pom.xml | 20 +-
.../db/integration/IoTDBLoadExternalTsfileIT.java | 1032 ----------
.../IoTDBLoadExternalTsfileWithVirtualSGIT.java | 137 --
.../db/integration/IoTDBNewTsFileCompactionIT.java | 36 +-
.../db/integration/IoTDBRewriteTsFileToolIT.java | 59 +-
.../iotdb/db/integration/IoTDBSnapshotIT.java | 285 ---
.../aligned/IoTDBLoadExternalAlignedTsFileIT.java | 2027 ++++++++++----------
.../resources/conf/iotdb-common.properties | 13 +-
.../assembly/resources/sbin/start-standalone.bat | 3 +
.../assembly/resources/sbin/stop-standalone.bat | 5 +-
.../commons/auth/role/LocalFileRoleAccessor.java | 23 +-
.../apache/iotdb/commons/conf/IoTDBConstant.java | 8 +-
.../commons/partition/DataPartitionTable.java | 6 +-
.../commons/partition/SchemaPartitionTable.java | 6 +-
.../commons/partition/SeriesPartitionTable.java | 12 +-
.../org/apache/iotdb/commons/path/AlignedPath.java | 18 +
.../org/apache/iotdb/commons/path/PartialPath.java | 3 +
.../iotdb/commons/service/StartupChecks.java | 44 +-
pom.xml | 6 +-
server/pom.xml | 2 +-
.../src/assembly/resources/conf/datanode-env.bat | 2 +
server/src/assembly/resources/conf/datanode-env.sh | 2 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 43 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 22 +-
.../db/consensus/DataRegionConsensusImpl.java | 7 +-
.../db/consensus/SchemaRegionConsensusImpl.java | 3 +-
.../statemachine/DataRegionStateMachine.java | 16 +-
.../org/apache/iotdb/db/engine/StorageEngine.java | 777 +++++++-
.../apache/iotdb/db/engine/StorageEngineV2.java | 820 --------
.../apache/iotdb/db/engine/cache/ChunkCache.java | 5 +
.../db/engine/cache/TimeSeriesMetadataCache.java | 6 +
.../db/engine/compaction/CompactionUtils.java | 40 +
.../compaction/cross/CrossSpaceCompactionTask.java | 12 +
.../rewrite/CrossSpaceCompactionResource.java | 13 +-
.../RewriteCrossSpaceCompactionSelector.java | 5 +-
.../compaction/inner/InnerSpaceCompactionTask.java | 11 +
.../writer/AbstractCompactionWriter.java | 6 +-
.../writer/AbstractCrossCompactionWriter.java | 2 +-
.../iotdb/db/engine/load/LoadTsFileManager.java | 2 +-
.../iotdb/db/engine/snapshot/SnapshotLoader.java | 4 +-
.../iotdb/db/engine/snapshot/SnapshotTaker.java | 8 +-
.../iotdb/db/engine/storagegroup/DataRegion.java | 173 +-
.../db/engine/storagegroup/DataRegionMetrics.java | 2 +-
.../storagegroup/IDataRegionForQuery.java} | 40 +-
.../engine/storagegroup/TimePartitionManager.java | 4 +-
.../db/engine/storagegroup/TsFileProcessor.java | 4 +-
.../engine/storagegroup/TsFileProcessorInfo.java | 2 +-
.../db/engine/storagegroup/TsFileResourceList.java | 4 +-
.../db/engine/storagegroup/VirtualDataRegion.java | 78 +
.../storagegroup/timeindex/DeviceTimeIndex.java | 9 +-
.../storagegroup/timeindex/FileTimeIndex.java | 6 +-
.../iotdb/db/localconfignode/LocalConfigNode.java | 4 +-
.../db/mpp/aggregation/ExtremeAccumulator.java | 20 +-
.../db/mpp/aggregation/FirstValueAccumulator.java | 28 +-
.../db/mpp/aggregation/LastValueAccumulator.java | 27 +-
.../db/mpp/aggregation/MaxValueAccumulator.java | 21 +-
.../db/mpp/aggregation/MinTimeAccumulator.java | 1 +
.../db/mpp/aggregation/MinValueAccumulator.java | 21 +-
.../iotdb/db/mpp/aggregation/SumAccumulator.java | 1 +
.../db/mpp/common/header/ColumnHeaderConstant.java | 12 +-
.../iotdb/db/mpp/execution/driver/DataDriver.java | 4 +-
.../db/mpp/execution/driver/DataDriverContext.java | 8 +-
.../fragment/FragmentInstanceContext.java | 22 +-
.../fragment/FragmentInstanceManager.java | 21 +-
.../operator/process/AbstractIntoOperator.java | 215 ++-
.../operator/process/DeviceViewIntoOperator.java | 105 +-
.../execution/operator/process/IntoOperator.java | 48 +-
.../process/join/VerticallyConcatOperator.java | 207 ++
...r.java => AbstractUpdateLastCacheOperator.java} | 74 +-
.../last/AlignedUpdateLastCacheOperator.java | 89 +
.../operator/process/last/LastQueryOperator.java | 6 +-
.../process/last/LastQuerySortOperator.java | 11 +-
.../operator/process/last/LastQueryUtil.java | 18 +
.../process/last/UpdateLastCacheOperator.java | 84 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 1 -
.../plan/analyze/StandalonePartitionFetcher.java | 4 +-
.../config/executor/ClusterConfigTaskExecutor.java | 11 +-
.../config/metadata/GetTimeSlotListTask.java | 7 +-
.../memory/StatementMemorySourceVisitor.java | 10 +-
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 17 +-
.../db/mpp/plan/parser/StatementGenerator.java | 16 +-
.../plan/planner/LocalExecutionPlanContext.java | 5 +
.../db/mpp/plan/planner/LocalExecutionPlanner.java | 4 +-
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 20 +-
.../db/mpp/plan/planner/OperatorTreeGenerator.java | 169 +-
.../db/mpp/plan/planner/SubPlanTypeExtractor.java | 29 +
.../planner/distribution/ExchangeNodeAdder.java | 7 +
.../plan/planner/distribution/SourceRewriter.java | 248 ++-
.../plan/planner/plan/node/PlanGraphPrinter.java | 8 +
.../mpp/plan/planner/plan/node/PlanNodeType.java | 6 +-
.../db/mpp/plan/planner/plan/node/PlanVisitor.java | 5 +
.../{LimitNode.java => VerticallyConcatNode.java} | 56 +-
.../planner/plan/parameter/AggregationStep.java | 45 +-
.../db/mpp/plan/scheduler/StandaloneScheduler.java | 7 +-
.../scheduler/load/LoadTsFileDispatcherImpl.java | 9 +-
.../statement/metadata/GetRegionIdStatement.java | 30 +-
.../iotdb/db/mpp/statistics/QueryStatistics.java | 7 +
.../db/query/control/QueryResourceManager.java | 38 +-
.../groupby/GroupByWithValueFilterDataSet.java | 88 +-
.../groupby/GroupByWithoutValueFilterDataSet.java | 106 +-
.../db/query/executor/AggregationExecutor.java | 124 +-
.../iotdb/db/query/executor/FillQueryExecutor.java | 69 +-
.../iotdb/db/query/executor/LastQueryExecutor.java | 50 +-
.../db/query/executor/RawDataQueryExecutor.java | 138 +-
.../query/timegenerator/ServerTimeGenerator.java | 28 +-
.../org/apache/iotdb/db/rescon/SystemInfo.java | 10 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 21 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 5 +-
.../apache/iotdb/db/service/IoTDBShutdownHook.java | 4 +-
.../java/org/apache/iotdb/db/service/NewIoTDB.java | 8 +-
.../iotdb/db/service/RegionMigrateService.java | 4 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 8 +-
.../service/thrift/impl/DataNodeRegionManager.java | 4 +-
.../db/sync/sender/manager/LocalSyncManager.java | 2 +-
.../iotdb/db/sync/sender/pipe/TsFilePipe.java | 8 +-
.../sync/transport/client/SyncClientFactory.java | 7 +-
.../iotdb/db/tools/TsFileSplitByPartitionTool.java | 10 +-
.../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 6 +-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 259 +--
.../org/apache/iotdb/db/utils/ThreadUtils.java | 3 +-
.../java/org/apache/iotdb/db/wal/node/WALNode.java | 4 +-
...ageEngineV2Test.java => StorageEngineTest.java} | 14 +-
...CrossSpaceCompactionWithFastPerformerTest.java} | 19 +-
...CompactionWithFastPerformerValidationTest.java} | 201 +-
...SpaceCompactionWithReadPointPerformerTest.java} | 19 +-
...ctionWithReadPointPerformerValidationTest.java} | 253 +--
...CrossSpaceCompactionWithFastPerformerTest.java} | 2 +-
...SpaceCompactionWithReadPointPerformerTest.java} | 12 +-
.../utils/MultiTsFileDeviceIteratorTest.java | 75 +
.../db/engine/snapshot/IoTDBSnapshotTest.java | 2 +-
.../db/engine/storagegroup/DataRegionTest.java | 22 +-
...ocessorV2Test.java => TsFileProcessorTest.java} | 6 +-
.../mpp/execution/operator/OperatorMemoryTest.java | 5 +-
...Test.java => VerticallyConcatOperatorTest.java} | 139 +-
.../db/mpp/plan/StandaloneCoordinatorTest.java | 6 +-
.../distribution/AggregationDistributionTest.java | 70 +-
.../plan/scheduler/StandaloneSchedulerTest.java | 6 +-
.../iotdb/db/tools/TsFileAndModSettleToolTest.java | 373 ++--
.../apache/iotdb/db/utils/EnvironmentUtils.java | 4 +-
site/iotdb-doap.rdf | 8 +-
site/src/main/.vuepress/config.js | 29 +-
site/src/main/.vuepress/redirects | 3 +-
site/src/main/.vuepress/theme/components/Page.vue | 36 +-
site/src/main/package.json | 9 +-
.../apache/iotdb/spark/db/EnvironmentUtils.java | 7 +-
testcontainer/pom.xml | 2 +-
.../src/main/thrift/confignode.thrift | 6 +-
tsfile/pom.xml | 2 +-
215 files changed, 5612 insertions(+), 6965 deletions(-)
copy docs/UserGuide/{Maintenance-Tools/Maintenance-Command.md => Cluster/Cluster-Maintenance.md} (73%)
create mode 100644 docs/UserGuide/Cluster/Deployment-Recommendation.md
copy docs/{Blog/index.md => UserGuide/UDF-Library/M4.md} (85%)
copy docs/zh/UserGuide/{Maintenance-Tools/Maintenance-Command.md => Cluster/Cluster-Maintenance.md} (76%)
create mode 100644 docs/zh/UserGuide/Cluster/Deployment-Recommendation.md
create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationOptimizeIT.java
delete mode 100644 integration/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileIT.java
delete mode 100644 integration/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileWithVirtualSGIT.java
delete mode 100644 integration/src/test/java/org/apache/iotdb/db/integration/IoTDBSnapshotIT.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java
copy server/src/main/java/org/apache/iotdb/db/{qp/logical/sys/ShowStorageGroupOperator.java => engine/storagegroup/IDataRegionForQuery.java} (54%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/engine/storagegroup/VirtualDataRegion.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/join/VerticallyConcatOperator.java
copy server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/last/{UpdateLastCacheOperator.java => AbstractUpdateLastCacheOperator.java} (59%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/last/AlignedUpdateLastCacheOperator.java
copy server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/process/{LimitNode.java => VerticallyConcatNode.java} (64%)
rename server/src/test/java/org/apache/iotdb/db/engine/{StorageEngineV2Test.java => StorageEngineTest.java} (87%)
copy server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/{CrossSpaceCompactionTest.java => CrossSpaceCompactionWithFastPerformerTest.java} (98%)
copy server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/{CrossSpaceCompactionValidationTest.java => CrossSpaceCompactionWithFastPerformerValidationTest.java} (93%)
rename server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/{CrossSpaceCompactionTest.java => CrossSpaceCompactionWithReadPointPerformerTest.java} (98%)
rename server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/{CrossSpaceCompactionValidationTest.java => CrossSpaceCompactionWithReadPointPerformerValidationTest.java} (93%)
copy server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/{RewriteCrossSpaceCompactionTest.java => RewriteCrossSpaceCompactionWithFastPerformerTest.java} (99%)
rename server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/{RewriteCrossSpaceCompactionTest.java => RewriteCrossSpaceCompactionWithReadPointPerformerTest.java} (98%)
rename server/src/test/java/org/apache/iotdb/db/engine/storagegroup/{TsFileProcessorV2Test.java => TsFileProcessorTest.java} (99%)
copy server/src/test/java/org/apache/iotdb/db/mpp/execution/operator/{LimitOperatorTest.java => VerticallyConcatOperatorTest.java} (56%)
[iotdb] 02/02: add io metrics
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch QueryMetrics
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 72020ef96b74c12de0d9eebd03b216e734872d4f
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Tue Dec 6 23:13:16 2022 +0800
add io metrics
---
.../apache/iotdb/db/engine/cache/ChunkCache.java | 5 +
.../db/engine/cache/TimeSeriesMetadataCache.java | 6 +
.../iotdb/db/mpp/statistics/QueryStatistics.java | 7 +
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 259 +++++++++++----------
4 files changed, 159 insertions(+), 118 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
index 0fa63a5fdf..a47f9ecffe 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCache.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.commons.service.metric.MetricService;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.mpp.statistics.QueryStatistics;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
@@ -71,6 +72,7 @@ public class ChunkCache {
.recordStats()
.build(
chunkMetadata -> {
+ long startTime = System.nanoTime();
try {
TsFileSequenceReader reader =
FileReaderManager.getInstance()
@@ -79,6 +81,9 @@ public class ChunkCache {
} catch (IOException e) {
logger.error("Something wrong happened in reading {}", chunkMetadata, e);
throw e;
+ } finally {
+ QueryStatistics.getInstance()
+ .addCost(QueryStatistics.CHUNK_CACHE_MISS, System.nanoTime() - startTime);
}
});
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
index d70b2faf41..c9610d3354 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCache.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.commons.service.metric.MetricService;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.mpp.statistics.QueryStatistics;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
@@ -50,6 +51,8 @@ import java.util.Set;
import java.util.WeakHashMap;
import java.util.concurrent.atomic.AtomicLong;
+import static org.apache.iotdb.db.mpp.statistics.QueryStatistics.TIME_SERIES_METADATA_CACHE_MISS;
+
/**
* This class is used to cache <code>TimeSeriesMetadata</code> in IoTDB. The caching strategy is
* LRU.
@@ -137,6 +140,7 @@ public class TimeSeriesMetadataCache {
TimeseriesMetadata timeseriesMetadata = lruCache.getIfPresent(key);
if (timeseriesMetadata == null) {
+ long startTime = System.nanoTime();
if (debug) {
DEBUG_LOGGER.info(
"Cache miss: {}.{} in file: {}", key.device, key.measurement, key.filePath);
@@ -180,6 +184,8 @@ public class TimeSeriesMetadataCache {
}
}
}
+ QueryStatistics.getInstance()
+ .addCost(TIME_SERIES_METADATA_CACHE_MISS, System.nanoTime() - startTime);
}
if (timeseriesMetadata == null) {
if (debug) {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
index 9b0dd7e985..7f6074d709 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/statistics/QueryStatistics.java
@@ -84,6 +84,13 @@ public class QueryStatistics {
public static final String SERVER_RPC_RT = "ServerRpcRT";
+ public static final String LOAD_TIME_SERIES_METADATA_ALIGNED = "loadTimeSeriesMetadata-aligned";
+ public static final String LOAD_TIME_SERIES_METADATA = "loadTimeSeriesMetadata";
+ public static final String LOAD_CHUNK_METADATA_LIST = "loadChunkMetadataList";
+ public static final String LOAD_PAGE_READER_LIST = "loadPageReaderList";
+ public static final String TIME_SERIES_METADATA_CACHE_MISS = "TimeSeriesMetadataCacheMiss";
+ public static final String CHUNK_CACHE_MISS = "ChunkCacheMiss";
+
private QueryStatistics() {
ScheduledExecutorService scheduledExecutor =
IoTDBThreadPoolFactory.newScheduledThreadPool(1, "Query-Statistics-Print");
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
index bfd6a0bf80..bdaffb8360 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache.TimeSeriesMetada
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus;
+import org.apache.iotdb.db.mpp.statistics.QueryStatistics;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.reader.chunk.metadata.DiskAlignedChunkMetadataLoader;
import org.apache.iotdb.db.query.reader.chunk.metadata.DiskChunkMetadataLoader;
@@ -124,51 +125,56 @@ public class FileLoaderUtils {
Filter filter,
Set<String> allSensors)
throws IOException {
-
- // common path
- TimeseriesMetadata timeSeriesMetadata;
- // If the tsfile is closed, we need to load from tsfile
- if (resource.isClosed()) {
- // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
- // we should not ignore the non-exist of device in TsFileMetadata
- timeSeriesMetadata =
- TimeSeriesMetadataCache.getInstance()
- .get(
- new TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey(
- resource.getTsFilePath(),
- seriesPath.getDevice(),
- seriesPath.getMeasurement()),
- allSensors,
- resource.getTimeIndexType() != 1,
- context.isDebug());
- if (timeSeriesMetadata != null) {
- timeSeriesMetadata.setChunkMetadataLoader(
- new DiskChunkMetadataLoader(resource, seriesPath, context, filter));
- }
- } else { // if the tsfile is unclosed, we just get it directly from TsFileResource
- timeSeriesMetadata = (TimeseriesMetadata) resource.getTimeSeriesMetadata(seriesPath);
- if (timeSeriesMetadata != null) {
- timeSeriesMetadata.setChunkMetadataLoader(
- new MemChunkMetadataLoader(resource, seriesPath, context, filter));
+ long startTime = System.nanoTime();
+ try {
+ // common path
+ TimeseriesMetadata timeSeriesMetadata;
+ // If the tsfile is closed, we need to load from tsfile
+ if (resource.isClosed()) {
+ // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
+ // we should not ignore the non-exist of device in TsFileMetadata
+ timeSeriesMetadata =
+ TimeSeriesMetadataCache.getInstance()
+ .get(
+ new TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey(
+ resource.getTsFilePath(),
+ seriesPath.getDevice(),
+ seriesPath.getMeasurement()),
+ allSensors,
+ resource.getTimeIndexType() != 1,
+ context.isDebug());
+ if (timeSeriesMetadata != null) {
+ timeSeriesMetadata.setChunkMetadataLoader(
+ new DiskChunkMetadataLoader(resource, seriesPath, context, filter));
+ }
+ } else { // if the tsfile is unclosed, we just get it directly from TsFileResource
+ timeSeriesMetadata = (TimeseriesMetadata) resource.getTimeSeriesMetadata(seriesPath);
+ if (timeSeriesMetadata != null) {
+ timeSeriesMetadata.setChunkMetadataLoader(
+ new MemChunkMetadataLoader(resource, seriesPath, context, filter));
+ }
}
- }
- if (timeSeriesMetadata != null) {
- List<Modification> pathModifications =
- context.getPathModifications(resource.getModFile(), seriesPath);
- timeSeriesMetadata.setModified(!pathModifications.isEmpty());
- if (timeSeriesMetadata.getStatistics().getStartTime()
- > timeSeriesMetadata.getStatistics().getEndTime()) {
- return null;
- }
- if (filter != null
- && !filter.satisfyStartEndTime(
- timeSeriesMetadata.getStatistics().getStartTime(),
- timeSeriesMetadata.getStatistics().getEndTime())) {
- return null;
+ if (timeSeriesMetadata != null) {
+ List<Modification> pathModifications =
+ context.getPathModifications(resource.getModFile(), seriesPath);
+ timeSeriesMetadata.setModified(!pathModifications.isEmpty());
+ if (timeSeriesMetadata.getStatistics().getStartTime()
+ > timeSeriesMetadata.getStatistics().getEndTime()) {
+ return null;
+ }
+ if (filter != null
+ && !filter.satisfyStartEndTime(
+ timeSeriesMetadata.getStatistics().getStartTime(),
+ timeSeriesMetadata.getStatistics().getEndTime())) {
+ return null;
+ }
}
+ return timeSeriesMetadata;
+ } finally {
+ QueryStatistics.getInstance()
+ .addCost(QueryStatistics.LOAD_TIME_SERIES_METADATA, System.nanoTime() - startTime);
}
- return timeSeriesMetadata;
}
/**
@@ -181,87 +187,94 @@ public class FileLoaderUtils {
public static AlignedTimeSeriesMetadata loadTimeSeriesMetadata(
TsFileResource resource, AlignedPath vectorPath, QueryContext context, Filter filter)
throws IOException {
- AlignedTimeSeriesMetadata alignedTimeSeriesMetadata = null;
- // If the tsfile is closed, we need to load from tsfile
- if (resource.isClosed()) {
- // load all the TimeseriesMetadata of vector, the first one is for time column and the
- // remaining is for sub sensors
- // the order of timeSeriesMetadata list is same as subSensorList's order
- TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance();
- List<String> valueMeasurementList = vectorPath.getMeasurementList();
- Set<String> allSensors = new HashSet<>(valueMeasurementList);
- allSensors.add("");
- boolean isDebug = context.isDebug();
- String filePath = resource.getTsFilePath();
- String deviceId = vectorPath.getDevice();
+ long startTime = System.nanoTime();
+ try {
+ AlignedTimeSeriesMetadata alignedTimeSeriesMetadata = null;
+ // If the tsfile is closed, we need to load from tsfile
+ if (resource.isClosed()) {
+ // load all the TimeseriesMetadata of vector, the first one is for time column and the
+ // remaining is for sub sensors
+ // the order of timeSeriesMetadata list is same as subSensorList's order
+ TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance();
+ List<String> valueMeasurementList = vectorPath.getMeasurementList();
+ Set<String> allSensors = new HashSet<>(valueMeasurementList);
+ allSensors.add("");
+ boolean isDebug = context.isDebug();
+ String filePath = resource.getTsFilePath();
+ String deviceId = vectorPath.getDevice();
- // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
- // we should not ignore the non-exist of device in TsFileMetadata
- TimeseriesMetadata timeColumn =
- cache.get(
- new TimeSeriesMetadataCacheKey(filePath, deviceId, ""),
- allSensors,
- resource.getTimeIndexType() != 1,
- isDebug);
- if (timeColumn != null) {
- List<TimeseriesMetadata> valueTimeSeriesMetadataList =
- new ArrayList<>(valueMeasurementList.size());
- // if all the queried aligned sensors does not exist, we will return null
- boolean exist = false;
- for (String valueMeasurement : valueMeasurementList) {
- TimeseriesMetadata valueColumn =
- cache.get(
- new TimeSeriesMetadataCacheKey(filePath, deviceId, valueMeasurement),
- allSensors,
- resource.getTimeIndexType() != 1,
- isDebug);
- exist = (exist || (valueColumn != null));
- valueTimeSeriesMetadataList.add(valueColumn);
+ // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
+ // we should not ignore the non-exist of device in TsFileMetadata
+ TimeseriesMetadata timeColumn =
+ cache.get(
+ new TimeSeriesMetadataCacheKey(filePath, deviceId, ""),
+ allSensors,
+ resource.getTimeIndexType() != 1,
+ isDebug);
+ if (timeColumn != null) {
+ List<TimeseriesMetadata> valueTimeSeriesMetadataList =
+ new ArrayList<>(valueMeasurementList.size());
+ // if all the queried aligned sensors does not exist, we will return null
+ boolean exist = false;
+ for (String valueMeasurement : valueMeasurementList) {
+ TimeseriesMetadata valueColumn =
+ cache.get(
+ new TimeSeriesMetadataCacheKey(filePath, deviceId, valueMeasurement),
+ allSensors,
+ resource.getTimeIndexType() != 1,
+ isDebug);
+ exist = (exist || (valueColumn != null));
+ valueTimeSeriesMetadataList.add(valueColumn);
+ }
+ if (exist) {
+ alignedTimeSeriesMetadata =
+ new AlignedTimeSeriesMetadata(timeColumn, valueTimeSeriesMetadataList);
+ alignedTimeSeriesMetadata.setChunkMetadataLoader(
+ new DiskAlignedChunkMetadataLoader(resource, vectorPath, context, filter));
+ }
}
- if (exist) {
- alignedTimeSeriesMetadata =
- new AlignedTimeSeriesMetadata(timeColumn, valueTimeSeriesMetadataList);
+ } else { // if the tsfile is unclosed, we just get it directly from TsFileResource
+ alignedTimeSeriesMetadata =
+ (AlignedTimeSeriesMetadata) resource.getTimeSeriesMetadata(vectorPath);
+ if (alignedTimeSeriesMetadata != null) {
alignedTimeSeriesMetadata.setChunkMetadataLoader(
- new DiskAlignedChunkMetadataLoader(resource, vectorPath, context, filter));
+ new MemAlignedChunkMetadataLoader(resource, vectorPath, context, filter));
}
}
- } else { // if the tsfile is unclosed, we just get it directly from TsFileResource
- alignedTimeSeriesMetadata =
- (AlignedTimeSeriesMetadata) resource.getTimeSeriesMetadata(vectorPath);
- if (alignedTimeSeriesMetadata != null) {
- alignedTimeSeriesMetadata.setChunkMetadataLoader(
- new MemAlignedChunkMetadataLoader(resource, vectorPath, context, filter));
- }
- }
- if (alignedTimeSeriesMetadata != null) {
- if (alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime()
- > alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime()) {
- return null;
- }
- if (filter != null
- && !filter.satisfyStartEndTime(
- alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime(),
- alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime())) {
- return null;
- }
+ if (alignedTimeSeriesMetadata != null) {
+ if (alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime()
+ > alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime()) {
+ return null;
+ }
+ if (filter != null
+ && !filter.satisfyStartEndTime(
+ alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime(),
+ alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime())) {
+ return null;
+ }
- // set modifications to each aligned path
- List<TimeseriesMetadata> valueTimeSeriesMetadataList =
- alignedTimeSeriesMetadata.getValueTimeseriesMetadataList();
- boolean modified = false;
- for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) {
- if (valueTimeSeriesMetadataList.get(i) != null) {
- List<Modification> pathModifications =
- context.getPathModifications(
- resource.getModFile(), vectorPath.getPathWithMeasurement(i));
- valueTimeSeriesMetadataList.get(i).setModified(!pathModifications.isEmpty());
- modified = (modified || !pathModifications.isEmpty());
+ // set modifications to each aligned path
+ List<TimeseriesMetadata> valueTimeSeriesMetadataList =
+ alignedTimeSeriesMetadata.getValueTimeseriesMetadataList();
+ boolean modified = false;
+ for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) {
+ if (valueTimeSeriesMetadataList.get(i) != null) {
+ List<Modification> pathModifications =
+ context.getPathModifications(
+ resource.getModFile(), vectorPath.getPathWithMeasurement(i));
+ valueTimeSeriesMetadataList.get(i).setModified(!pathModifications.isEmpty());
+ modified = (modified || !pathModifications.isEmpty());
+ }
}
+ alignedTimeSeriesMetadata.getTimeseriesMetadata().setModified(modified);
}
- alignedTimeSeriesMetadata.getTimeseriesMetadata().setModified(modified);
+ return alignedTimeSeriesMetadata;
+ } finally {
+ QueryStatistics.getInstance()
+ .addCost(
+ QueryStatistics.LOAD_TIME_SERIES_METADATA_ALIGNED, System.nanoTime() - startTime);
}
- return alignedTimeSeriesMetadata;
}
/**
@@ -271,7 +284,11 @@ public class FileLoaderUtils {
*/
public static List<IChunkMetadata> loadChunkMetadataList(ITimeSeriesMetadata timeSeriesMetadata)
throws IOException {
- return timeSeriesMetadata.loadChunkMetadataList();
+ long startTime = System.nanoTime();
+ List<IChunkMetadata> chunkMetadataList = timeSeriesMetadata.loadChunkMetadataList();
+ QueryStatistics.getInstance()
+ .addCost(QueryStatistics.LOAD_CHUNK_METADATA_LIST, System.nanoTime() - startTime);
+ return chunkMetadataList;
}
/**
@@ -282,11 +299,17 @@ public class FileLoaderUtils {
*/
public static List<IPageReader> loadPageReaderList(
IChunkMetadata chunkMetaData, Filter timeFilter) throws IOException {
- if (chunkMetaData == null) {
- throw new IOException("Can't init null chunkMeta");
+ long startTime = System.nanoTime();
+ try {
+ if (chunkMetaData == null) {
+ throw new IOException("Can't init null chunkMeta");
+ }
+ IChunkLoader chunkLoader = chunkMetaData.getChunkLoader();
+ IChunkReader chunkReader = chunkLoader.getChunkReader(chunkMetaData, timeFilter);
+ return chunkReader.loadPageReaderList();
+ } finally {
+ QueryStatistics.getInstance()
+ .addCost(QueryStatistics.LOAD_PAGE_READER_LIST, System.nanoTime() - startTime);
}
- IChunkLoader chunkLoader = chunkMetaData.getChunkLoader();
- IChunkReader chunkReader = chunkLoader.getChunkReader(chunkMetaData, timeFilter);
- return chunkReader.loadPageReaderList();
}
}
[iotdb] 01/02: Merge remote-tracking branch 'origin/master' into QueryMetrics
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hui pushed a commit to branch QueryMetrics
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d209479edaef4df6d02a4cf60a5bc1bb2ec68121
Merge: 5d654bdbbd d6671b3662
Author: Minghui Liu <li...@foxmail.com>
AuthorDate: Tue Dec 6 22:24:56 2022 +0800
Merge remote-tracking branch 'origin/master' into QueryMetrics
# Conflicts:
# server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceManager.java
.github/workflows/main-unix.yml | 2 +-
README.md | 16 +-
README_ZH.md | 14 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 5 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 +
client-py/README.md | 7 +-
client-py/requirements_dev.txt | 2 +-
confignode/pom.xml | 2 +-
.../src/assembly/resources/conf/confignode-env.bat | 2 +
.../src/assembly/resources/conf/confignode-env.sh | 2 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 32 +-
.../confignode/conf/ConfigNodeDescriptor.java | 52 +-
.../request/read/region/GetRegionIdPlan.java | 2 +-
.../iotdb/confignode/manager/ConfigManager.java | 9 +-
.../iotdb/confignode/manager/ConsensusManager.java | 5 +-
.../apache/iotdb/confignode/manager/IManager.java | 4 +-
.../iotdb/confignode/manager/node/NodeManager.java | 6 +-
.../manager/partition/PartitionManager.java | 16 +-
.../persistence/executor/ConfigPlanExecutor.java | 14 +-
.../partition/StorageGroupPartitionTable.java | 2 +
.../iotdb/confignode/service/ConfigNode.java | 4 +
.../confignode/service/ConfigNodeCommandLine.java | 3 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 11 +-
consensus/pom.xml | 2 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 11 +-
.../iotdb/consensus/ratis/RatisConsensusTest.java | 5 +
cross-tests/pom.xml | 2 +-
docs/Download/README.md | 46 +-
docs/UserGuide/Cluster/Cluster-Concept.md | 5 +-
.../Cluster-Maintenance.md} | 274 +--
docs/UserGuide/Cluster/Cluster-Setup.md | 13 +-
.../UserGuide/Cluster/Deployment-Recommendation.md | 181 ++
docs/UserGuide/Data-Concept/Data-Type.md | 2 +-
.../Maintenance-Tools/Maintenance-Command.md | 383 ----
.../Operators-Functions/Data-Profiling.md | 4 +-
docs/UserGuide/Operators-Functions/String.md | 4 +-
.../Operators-Functions/User-Defined-Function.md | 2 +-
.../UserGuide/QuickStart/Command-Line-Interface.md | 7 +-
docs/UserGuide/QuickStart/QuickStart.md | 15 +-
docs/UserGuide/Reference/Common-Config-Manual.md | 23 +-
docs/UserGuide/UDF-Library/M4.md | 26 +
docs/zh/Download/README.md | 54 +-
docs/zh/UserGuide/Cluster/Cluster-Concept.md | 4 +-
.../Cluster-Maintenance.md} | 223 +--
docs/zh/UserGuide/Cluster/Cluster-Setup.md | 13 +-
.../UserGuide/Cluster/Deployment-Recommendation.md | 179 ++
docs/zh/UserGuide/Data-Concept/Data-Type.md | 2 +-
.../Maintenance-Tools/Maintenance-Command.md | 363 ----
.../Operators-Functions/Data-Profiling.md | 4 +-
docs/zh/UserGuide/Operators-Functions/Overview.md | 2 +-
docs/zh/UserGuide/Operators-Functions/String.md | 4 +-
.../Operators-Functions/User-Defined-Function.md | 2 +-
.../UserGuide/QuickStart/Command-Line-Interface.md | 4 +-
docs/zh/UserGuide/QuickStart/QuickStart.md | 14 +-
.../zh/UserGuide/Reference/Common-Config-Manual.md | 22 +-
.../Reference/ConfigNode-Config-Manual.md | 2 +-
integration-test/pom.xml | 212 +-
.../java/org/apache/iotdb/it/env/MppConfig.java | 9 +
.../org/apache/iotdb/itbase/env/BaseConfig.java | 8 +
.../IoTDBCustomRegionGroupExtensionIT.java | 39 +-
.../it/partition/IoTDBPartitionGetterIT.java | 4 +-
.../it/aggregation/IoTDBAggregationOptimizeIT.java | 181 ++
.../iotdb/db/it/env/StandaloneEnvConfig.java | 14 +
.../iotdb/db/it/selectinto/IoTDBSelectInto2IT.java | 3 +
.../iotdb/db/it/selectinto/IoTDBSelectInto3IT.java | 3 +
.../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 5 +
.../iotdb/session/it/IoTDBSessionSimpleIT.java | 31 +
integration/pom.xml | 20 +-
.../db/integration/IoTDBLoadExternalTsfileIT.java | 1032 ----------
.../IoTDBLoadExternalTsfileWithVirtualSGIT.java | 137 --
.../db/integration/IoTDBNewTsFileCompactionIT.java | 36 +-
.../db/integration/IoTDBRewriteTsFileToolIT.java | 59 +-
.../iotdb/db/integration/IoTDBSnapshotIT.java | 285 ---
.../aligned/IoTDBLoadExternalAlignedTsFileIT.java | 2027 ++++++++++----------
.../resources/conf/iotdb-common.properties | 13 +-
.../assembly/resources/sbin/start-standalone.bat | 3 +
.../assembly/resources/sbin/stop-standalone.bat | 5 +-
.../commons/auth/role/LocalFileRoleAccessor.java | 23 +-
.../apache/iotdb/commons/conf/IoTDBConstant.java | 8 +-
.../commons/partition/DataPartitionTable.java | 6 +-
.../commons/partition/SchemaPartitionTable.java | 6 +-
.../commons/partition/SeriesPartitionTable.java | 12 +-
.../org/apache/iotdb/commons/path/AlignedPath.java | 18 +
.../org/apache/iotdb/commons/path/PartialPath.java | 3 +
.../iotdb/commons/service/StartupChecks.java | 44 +-
pom.xml | 6 +-
server/pom.xml | 2 +-
.../src/assembly/resources/conf/datanode-env.bat | 2 +
server/src/assembly/resources/conf/datanode-env.sh | 2 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 43 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 22 +-
.../db/consensus/DataRegionConsensusImpl.java | 7 +-
.../db/consensus/SchemaRegionConsensusImpl.java | 3 +-
.../statemachine/DataRegionStateMachine.java | 16 +-
.../org/apache/iotdb/db/engine/StorageEngine.java | 777 +++++++-
.../apache/iotdb/db/engine/StorageEngineV2.java | 820 --------
.../db/engine/compaction/CompactionUtils.java | 40 +
.../compaction/cross/CrossSpaceCompactionTask.java | 12 +
.../rewrite/CrossSpaceCompactionResource.java | 13 +-
.../RewriteCrossSpaceCompactionSelector.java | 5 +-
.../compaction/inner/InnerSpaceCompactionTask.java | 11 +
.../writer/AbstractCompactionWriter.java | 6 +-
.../writer/AbstractCrossCompactionWriter.java | 2 +-
.../iotdb/db/engine/load/LoadTsFileManager.java | 2 +-
.../iotdb/db/engine/snapshot/SnapshotLoader.java | 4 +-
.../iotdb/db/engine/snapshot/SnapshotTaker.java | 8 +-
.../iotdb/db/engine/storagegroup/DataRegion.java | 173 +-
.../db/engine/storagegroup/DataRegionMetrics.java | 2 +-
.../engine/storagegroup/IDataRegionForQuery.java | 47 +
.../engine/storagegroup/TimePartitionManager.java | 4 +-
.../db/engine/storagegroup/TsFileProcessor.java | 4 +-
.../engine/storagegroup/TsFileProcessorInfo.java | 2 +-
.../db/engine/storagegroup/TsFileResourceList.java | 4 +-
.../db/engine/storagegroup/VirtualDataRegion.java | 78 +
.../storagegroup/timeindex/DeviceTimeIndex.java | 9 +-
.../storagegroup/timeindex/FileTimeIndex.java | 6 +-
.../iotdb/db/localconfignode/LocalConfigNode.java | 4 +-
.../db/mpp/aggregation/ExtremeAccumulator.java | 20 +-
.../db/mpp/aggregation/FirstValueAccumulator.java | 28 +-
.../db/mpp/aggregation/LastValueAccumulator.java | 27 +-
.../db/mpp/aggregation/MaxValueAccumulator.java | 21 +-
.../db/mpp/aggregation/MinTimeAccumulator.java | 1 +
.../db/mpp/aggregation/MinValueAccumulator.java | 21 +-
.../iotdb/db/mpp/aggregation/SumAccumulator.java | 1 +
.../db/mpp/common/header/ColumnHeaderConstant.java | 12 +-
.../iotdb/db/mpp/execution/driver/DataDriver.java | 4 +-
.../db/mpp/execution/driver/DataDriverContext.java | 8 +-
.../fragment/FragmentInstanceContext.java | 22 +-
.../fragment/FragmentInstanceManager.java | 21 +-
.../operator/process/AbstractIntoOperator.java | 215 ++-
.../operator/process/DeviceViewIntoOperator.java | 105 +-
.../execution/operator/process/IntoOperator.java | 48 +-
.../process/join/VerticallyConcatOperator.java | 207 ++
...r.java => AbstractUpdateLastCacheOperator.java} | 74 +-
.../last/AlignedUpdateLastCacheOperator.java | 89 +
.../operator/process/last/LastQueryOperator.java | 6 +-
.../process/last/LastQuerySortOperator.java | 11 +-
.../operator/process/last/LastQueryUtil.java | 18 +
.../process/last/UpdateLastCacheOperator.java | 84 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 1 -
.../plan/analyze/StandalonePartitionFetcher.java | 4 +-
.../config/executor/ClusterConfigTaskExecutor.java | 11 +-
.../config/metadata/GetTimeSlotListTask.java | 7 +-
.../memory/StatementMemorySourceVisitor.java | 10 +-
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 17 +-
.../db/mpp/plan/parser/StatementGenerator.java | 16 +-
.../plan/planner/LocalExecutionPlanContext.java | 5 +
.../db/mpp/plan/planner/LocalExecutionPlanner.java | 4 +-
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 20 +-
.../db/mpp/plan/planner/OperatorTreeGenerator.java | 169 +-
.../db/mpp/plan/planner/SubPlanTypeExtractor.java | 29 +
.../planner/distribution/ExchangeNodeAdder.java | 7 +
.../plan/planner/distribution/SourceRewriter.java | 248 ++-
.../plan/planner/plan/node/PlanGraphPrinter.java | 8 +
.../mpp/plan/planner/plan/node/PlanNodeType.java | 6 +-
.../db/mpp/plan/planner/plan/node/PlanVisitor.java | 5 +
.../plan/node/process/VerticallyConcatNode.java | 103 +
.../planner/plan/parameter/AggregationStep.java | 45 +-
.../db/mpp/plan/scheduler/StandaloneScheduler.java | 7 +-
.../scheduler/load/LoadTsFileDispatcherImpl.java | 9 +-
.../statement/metadata/GetRegionIdStatement.java | 30 +-
.../db/query/control/QueryResourceManager.java | 38 +-
.../groupby/GroupByWithValueFilterDataSet.java | 88 +-
.../groupby/GroupByWithoutValueFilterDataSet.java | 106 +-
.../db/query/executor/AggregationExecutor.java | 124 +-
.../iotdb/db/query/executor/FillQueryExecutor.java | 69 +-
.../iotdb/db/query/executor/LastQueryExecutor.java | 50 +-
.../db/query/executor/RawDataQueryExecutor.java | 138 +-
.../query/timegenerator/ServerTimeGenerator.java | 28 +-
.../org/apache/iotdb/db/rescon/SystemInfo.java | 10 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 21 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 5 +-
.../apache/iotdb/db/service/IoTDBShutdownHook.java | 4 +-
.../java/org/apache/iotdb/db/service/NewIoTDB.java | 8 +-
.../iotdb/db/service/RegionMigrateService.java | 4 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 8 +-
.../service/thrift/impl/DataNodeRegionManager.java | 4 +-
.../db/sync/sender/manager/LocalSyncManager.java | 2 +-
.../iotdb/db/sync/sender/pipe/TsFilePipe.java | 8 +-
.../sync/transport/client/SyncClientFactory.java | 7 +-
.../iotdb/db/tools/TsFileSplitByPartitionTool.java | 10 +-
.../db/tools/upgrade/TsFileOnlineUpgradeTool.java | 6 +-
.../org/apache/iotdb/db/utils/ThreadUtils.java | 3 +-
.../java/org/apache/iotdb/db/wal/node/WALNode.java | 4 +-
...ageEngineV2Test.java => StorageEngineTest.java} | 14 +-
...CrossSpaceCompactionWithFastPerformerTest.java} | 19 +-
...CompactionWithFastPerformerValidationTest.java} | 201 +-
...SpaceCompactionWithReadPointPerformerTest.java} | 19 +-
...ctionWithReadPointPerformerValidationTest.java} | 253 +--
...CrossSpaceCompactionWithFastPerformerTest.java} | 2 +-
...SpaceCompactionWithReadPointPerformerTest.java} | 12 +-
.../utils/MultiTsFileDeviceIteratorTest.java | 75 +
.../db/engine/snapshot/IoTDBSnapshotTest.java | 2 +-
.../db/engine/storagegroup/DataRegionTest.java | 22 +-
...ocessorV2Test.java => TsFileProcessorTest.java} | 6 +-
.../mpp/execution/operator/OperatorMemoryTest.java | 5 +-
.../operator/VerticallyConcatOperatorTest.java | 197 ++
.../db/mpp/plan/StandaloneCoordinatorTest.java | 6 +-
.../distribution/AggregationDistributionTest.java | 70 +-
.../plan/scheduler/StandaloneSchedulerTest.java | 6 +-
.../iotdb/db/tools/TsFileAndModSettleToolTest.java | 373 ++--
.../apache/iotdb/db/utils/EnvironmentUtils.java | 4 +-
site/iotdb-doap.rdf | 8 +-
site/src/main/.vuepress/config.js | 29 +-
site/src/main/.vuepress/redirects | 3 +-
site/src/main/.vuepress/theme/components/Page.vue | 36 +-
site/src/main/package.json | 9 +-
.../apache/iotdb/spark/db/EnvironmentUtils.java | 7 +-
testcontainer/pom.xml | 2 +-
.../src/main/thrift/confignode.thrift | 6 +-
tsfile/pom.xml | 2 +-
211 files changed, 5704 insertions(+), 6728 deletions(-)
diff --cc node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
index b5a31a5a95,2e58d01d60..45de4c9903
--- a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
@@@ -78,12 -81,10 +81,13 @@@ public class IoTDBConstant
public static final String AUDIT_LOGGER_NAME = "IoTDB_AUDIT_LOGGER";
public static final String SLOW_SQL_LOGGER_NAME = "SLOW_SQL";
+
+ public static final String QUERY_STATISTICS_LOGGER_NAME = "QUERY_STATISTICS";
+
public static final String COMPACTION_LOGGER_NAME = "COMPACTION";
- public static final String IOTDB_JMX_PORT = "iotdb.jmx.port";
+ public static final String IOTDB_JMX_LOCAL = "iotdb.jmx.local";
+ public static final String IOTDB_JMX_PORT = "com.sun.management.jmxremote.port";
public static final String IOTDB_PACKAGE = "org.apache.iotdb.service";
public static final String IOTDB_THREADPOOL_PACKAGE = "org.apache.iotdb.threadpool";
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceManager.java
index d9d0aa6b1f,5e37ac29aa..a502da89f0
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/fragment/FragmentInstanceManager.java
@@@ -70,8 -68,8 +70,10 @@@ public class FragmentInstanceManager
private static final long QUERY_TIMEOUT_MS =
IoTDBDescriptor.getInstance().getConfig().getQueryTimeoutThreshold();
+ private static final QueryStatistics QUERY_STATISTICS = QueryStatistics.getInstance();
+
+ private final ExecutorService intoOperationExecutor;
+
public static FragmentInstanceManager getInstance() {
return FragmentInstanceManager.InstanceHolder.INSTANCE;
}
@@@ -97,10 -100,9 +104,10 @@@
}
public FragmentInstanceInfo execDataQueryFragmentInstance(
- FragmentInstance instance, DataRegion dataRegion) {
+ FragmentInstance instance, IDataRegionForQuery dataRegion) {
FragmentInstanceId instanceId = instance.getId();
+ long startTime = System.nanoTime();
try (SetThreadName fragmentInstanceName = new SetThreadName(instanceId.getFullId())) {
FragmentInstanceExecution execution =
instanceExecution.computeIfAbsent(