You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2022/04/02 10:24:29 UTC
[iotdb] 01/02: Merge branch 'master' into xingtanzjr/query_execution
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch xingtanzjr/query_execution
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 02b017672861eecc18e17311ab108f1c677cea4c
Merge: 7922a2a d8a0ac4
Author: Jinrui.Zhang <xi...@gmail.com>
AuthorDate: Sat Apr 2 18:17:58 2022 +0800
Merge branch 'master' into xingtanzjr/query_execution
.github/workflows/client.yml | 8 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 47 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 3 +-
client-py/README.md | 277 +++-
client-py/SessionExample.py | 2 +-
client-py/SessionTest.py | 2 +-
.../org/apache/iotdb/cluster/ClusterIoTDB.java | 8 +-
.../iotdb/cluster/coordinator/Coordinator.java | 10 +-
.../cluster/log/applier/AsyncDataLogApplier.java | 8 +-
.../iotdb/cluster/log/applier/DataLogApplier.java | 6 +-
.../log/manage/MetaSingleSnapshotLogManager.java | 2 +-
.../log/manage/PartitionedSnapshotLogManager.java | 4 +-
.../cluster/log/snapshot/MetaSimpleSnapshot.java | 4 +-
.../{CSchemaEngine.java => CSchemaProcessor.java} | 30 +-
.../apache/iotdb/cluster/metadata/MetaPuller.java | 10 +-
.../iotdb/cluster/partition/PartitionTable.java | 4 +-
.../cluster/query/ClusterPhysicalGenerator.java | 8 +-
.../iotdb/cluster/query/ClusterPlanExecutor.java | 22 +-
.../iotdb/cluster/query/ClusterPlanRouter.java | 17 +-
.../iotdb/cluster/query/LocalQueryExecutor.java | 31 +-
.../iotdb/cluster/query/filter/SlotSgFilter.java | 2 +-
.../cluster/query/reader/ClusterTimeGenerator.java | 4 +-
.../cluster/server/member/DataGroupMember.java | 6 +-
.../cluster/server/member/MetaGroupMember.java | 2 +-
.../cluster/server/service/DataAsyncService.java | 15 +-
.../cluster/server/service/DataSyncService.java | 12 +-
.../iotdb/cluster/utils/ClusterQueryUtils.java | 2 +-
.../apache/iotdb/cluster/utils/ClusterUtils.java | 4 +-
.../log/applier/AsyncDataLogApplierTest.java | 6 +-
.../cluster/log/applier/DataLogApplierTest.java | 13 +-
.../cluster/log/applier/MetaLogApplierTest.java | 6 +-
.../iotdb/cluster/log/catchup/CatchUpTaskTest.java | 4 +-
.../cluster/log/snapshot/DataSnapshotTest.java | 2 +-
.../cluster/log/snapshot/FileSnapshotTest.java | 12 +-
.../log/snapshot/MetaSimpleSnapshotTest.java | 4 +-
.../log/snapshot/PartitionedSnapshotTest.java | 5 +-
.../cluster/log/snapshot/PullSnapshotTaskTest.java | 3 +-
...eWhiteBox.java => SchemaProcessorWhiteBox.java} | 20 +-
.../cluster/partition/SlotPartitionTableTest.java | 28 +-
.../cluster/query/ClusterPlanExecutorTest.java | 2 +-
.../clusterinfo/ClusterInfoServiceImplTest.java | 4 +-
.../iotdb/cluster/server/member/BaseMember.java | 10 +-
.../cluster/server/member/DataGroupMemberTest.java | 4 +-
.../cluster/server/member/MetaGroupMemberTest.java | 22 +-
.../resources/{logback.xml => logback-test.xml} | 2 +-
confignode/pom.xml | 7 +-
.../resources/conf/iotdb-confignode.properties | 44 +-
confignode/src/assembly/resources/conf/logback.xml | 4 +-
.../iotdb/confignode/conf/ConfigNodeConf.java | 38 +-
.../iotdb/confignode/conf/ConfigNodeConfCheck.java | 28 +-
.../iotdb/confignode/conf/ConfigNodeConstant.java | 1 +
.../confignode/conf/ConfigNodeDescriptor.java | 42 +-
.../consensus/response/DataNodesInfoDataSet.java | 12 +-
...chemaDataSet.java => DataPartitionDataSet.java} | 18 +-
.../consensus/response/SchemaPartitionDataSet.java | 82 ++
.../response/StorageGroupSchemaDataSet.java | 8 +-
.../statemachine/PartitionRegionStateMachine.java | 15 +-
.../iotdb/confignode/manager/ConfigManager.java | 171 ++-
.../iotdb/confignode/manager/ConsensusManager.java | 153 +++
.../iotdb/confignode/manager/DataNodeManager.java | 169 +++
.../apache/iotdb/confignode/manager/Manager.java | 119 ++
.../iotdb/confignode/manager/PartitionManager.java | 145 ++
.../iotdb/confignode/manager/RegionManager.java | 152 +++
...{DataPartitionInfo.java => DataRegionInfo.java} | 46 +-
.../iotdb/confignode/partition/PartitionTable.java | 186 ---
...emaPartitionInfo.java => SchemaRegionInfo.java} | 41 +-
.../persistence/DataNodeInfoPersistence.java | 186 +++
.../persistence/PartitionInfoPersistence.java | 157 +++
.../persistence/RegionInfoPersistence.java | 198 +++
.../iotdb/confignode/physical/PhysicalPlan.java | 23 +
.../confignode/physical/PhysicalPlanType.java | 6 +-
.../confignode/physical/sys/DataPartitionPlan.java | 78 ++
.../physical/sys/RegisterDataNodePlan.java | 12 +-
.../physical/sys/SchemaPartitionPlan.java | 97 ++
.../physical/sys/SetStorageGroupPlan.java | 30 +
.../iotdb/confignode/service/ConfigNode.java | 22 +-
.../confignode/service/ConfigNodeCommandLine.java | 6 +-
.../confignode/service/balancer/LoadBalancer.java | 12 -
.../confignode/service/executor/PlanExecutor.java | 32 +-
.../service/thrift/server/ConfigNodeRPCServer.java | 21 +-
.../server/ConfigNodeRPCServerProcessor.java | 108 +-
.../thrift/server/ConfigNodeRPCServiceHandler.java | 2 +-
.../confignode/util/SerializeDeserializeUtil.java | 242 ++++
.../confignode/consensus/RatisConsensusDemo.java | 136 ++
.../manager/ConfigManagerManualTest.java | 132 ++
...java => DeviceGroupHashExecutorManualTest.java} | 10 +-
.../server/ConfigNodeRPCServerProcessorTest.java | 163 ++-
.../utils/SerializeDeserializeUtilTest.java | 90 ++
consensus/README.md | 123 ++
consensus/pom.xml | 6 +
.../org/apache/iotdb/consensus/IConsensus.java | 4 +-
.../iotdb/consensus/common/ConsensusGroup.java | 2 +
.../common/{Endpoint.java => ConsensusType.java} | 52 +-
.../org/apache/iotdb/consensus/common/Peer.java | 3 +
.../ConsensusGroupAlreadyExistException.java | 2 +-
.../exception/ConsensusGroupNotExistException.java | 2 +-
.../PeerAlreadyInConsensusGroupException.java | 2 +-
.../PeerNotInConsensusGroupException.java | 2 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 68 +-
.../iotdb/consensus/ratis/RequestMessage.java | 29 +-
.../iotdb/consensus/ratis/ResponseMessage.java | 18 +-
.../org/apache/iotdb/consensus/ratis/Utils.java | 67 +-
.../consensus/standalone/StandAloneConsensus.java | 7 +-
.../consensus/statemachine/IStateMachine.java | 2 +-
consensus/src/main/resources/logback-test.xml | 2 +-
.../iotdb/consensus/ratis/RatisConsensusTest.java | 41 +-
.../apache/iotdb/consensus/ratis/UtilsTest.java | 6 +-
.../standalone/StandAloneConsensusTest.java | 6 +-
.../tests/tools/importCsv/ExportCsvTestIT.java | 31 +-
docs/UserGuide/API/Programming-MQTT.md | 2 +-
.../UserGuide/API/Programming-Python-Native-API.md | 19 +-
docs/UserGuide/Maintenance-Tools/Metric-Tool.md | 13 +-
docs/UserGuide/Process-Data/Alerting.md | 2 +-
docs/UserGuide/Query-Data/Select-Expression.md | 85 +-
docs/zh/UserGuide/API/Programming-MQTT.md | 2 +-
.../UserGuide/API/Programming-Python-Native-API.md | 17 +-
docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md | 15 +-
docs/zh/UserGuide/Process-Data/Alerting.md | 2 +-
docs/zh/UserGuide/Query-Data/Select-Expression.md | 80 +-
example/mqtt-customize/README.md | 4 +-
...apache.iotdb.db.protocol.mqtt.PayloadFormatter} | 0
example/rest-client-c-example/README.md | 111 ++
example/rest-client-c-example/base64.c | 100 ++
.../rest-client-c-example/base64.h | 14 +-
example/rest-client-c-example/main.c | 124 ++
.../iotdb/db/integration/IoTDBArithmeticIT.java | 18 +-
.../iotdb/db/integration/IoTDBCheckConfigIT.java | 4 +-
.../apache/iotdb/db/integration/IoTDBLastIT.java | 14 +-
.../iotdb/db/integration/IoTDBMetadataFetchIT.java | 35 -
.../iotdb/db/integration/IoTDBNestedQueryIT.java | 12 +-
.../IoTDBSelectCompareExpressionIT.java | 482 +++++++
.../iotdb/db/integration/IoTDBSelectIntoIT.java | 18 +-
.../iotdb/db/integration/IoTDBSimpleQueryIT.java | 8 +-
.../db/integration/IoTDBTriggerExecutionIT.java | 26 +-
.../db/integration/IoTDBTriggerManagementIT.java | 8 +-
.../iotdb/db/integration/IoTDBUDFManagementIT.java | 6 +-
.../versionadaption/IoTDBDDLVersionAdaptionIT.java | 4 +-
.../apache/iotdb/session/IoTDBSessionSimpleIT.java | 46 +-
.../src/test/resources/logback-test.xml | 2 +-
metrics/dropwizard-metrics/pom.xml | 1 -
.../iotdb/metrics/dropwizard/MetricName.java | 60 +-
.../reporter/DropwizardIoTDBReporter.java | 5 +-
.../reporter/DropwizardMetricsExporter.java | 169 +--
.../metrics/dropwizard/reporter/IoTDBReporter.java | 200 +--
.../org.apache.iotdb.metrics.reporter.Reporter | 3 +-
metrics/interface/pom.xml | 5 +
.../main/assembly/resources/conf/iotdb-metric.yml | 15 +-
.../org/apache/iotdb/metrics/MetricService.java | 22 +-
.../apache/iotdb/metrics/config/MetricConfig.java | 143 +-
.../iotdb/metrics/reporter/CompositeReporter.java | 14 +-
.../apache/iotdb/metrics/utils/MetricsUtils.java | 33 +-
.../interface/src/test/resources/iotdb-metric.yml | 15 +-
.../micrometer/reporter}/IoTDBMeterRegistry.java | 100 +-
.../micrometer/reporter}/IoTDBRegistryConfig.java | 7 +-
.../{IoTDBJmxConfig.java => JmxConfig.java} | 6 +-
.../reporter/MicrometerIoTDBReporter.java | 4 +-
.../micrometer/reporter/MicrometerJmxReporter.java | 2 +-
.../org.apache.iotdb.metrics.reporter.Reporter | 3 +-
node-commons/pom.xml | 2 +-
.../apache/iotdb/commons/cluster}/Endpoint.java | 26 +-
.../apache/iotdb/commons/conf/IoTDBConstant.java | 1 +
.../iotdb/commons/consensus}/ConsensusGroupId.java | 27 +-
.../apache/iotdb/commons/consensus}/GroupType.java | 2 +-
.../iotdb/commons/partition/DataNodeLocation.java | 38 +-
.../{DataPartitionInfo.java => DataPartition.java} | 20 +-
.../commons/partition/DataPartitionQueryParam.java | 20 +-
.../iotdb/commons/partition/DataRegionId.java | 39 -
.../commons/partition/DataRegionReplicaSet.java | 53 -
.../iotdb/commons/partition/PartitionInfo.java | 20 +-
.../iotdb/commons/partition/RegionReplicaSet.java | 89 ++
.../iotdb/commons/partition/SchemaPartition.java | 92 ++
.../commons/partition/SchemaPartitionInfo.java | 36 -
.../iotdb/commons/partition/SchemaRegionId.java | 31 -
.../commons/partition/SchemaRegionReplicaSet.java | 44 -
...DeviceGroupId.java => SeriesPartitionSlot.java} | 8 +-
...TimePartitionId.java => TimePartitionSlot.java} | 2 +-
.../iotdb/commons/service/RegisterManager.java | 2 +-
pom.xml | 12 +-
server/pom.xml | 4 +-
server/src/assembly/resources/sbin/stop-server.sh | 13 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 19 +-
.../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 3 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 57 +-
.../iotdb/db/consensus/ConsensusExample.java | 6 +-
.../org/apache/iotdb/db/engine/StorageEngine.java | 21 +-
.../engine/compaction/CompactionTaskManager.java | 14 +-
.../db/engine/compaction/CompactionUtils.java | 4 +-
.../cross/AbstractCrossSpaceCompactionTask.java | 13 +-
.../compaction/cross/CrossCompactionStrategy.java | 21 -
.../manage/CrossSpaceCompactionResource.java | 3 +-
.../selector/RewriteCompactionFileSelector.java | 8 +-
.../task/RewriteCrossCompactionRecoverTask.java | 467 -------
.../task/RewriteCrossSpaceCompactionTask.java | 13 +-
.../inner/AbstractInnerSpaceCompactionTask.java | 6 +-
.../compaction/inner/InnerCompactionStrategy.java | 25 -
.../InnerSpaceCompactionExceptionHandler.java | 277 ----
.../SizeTieredCompactionRecoverTask.java | 303 -----
.../sizetiered/SizeTieredCompactionSelector.java | 4 +-
.../inner/sizetiered/SizeTieredCompactionTask.java | 37 +-
.../inner/utils/InnerSpaceCompactionUtils.java | 13 +-
.../compaction/task/AbstractCompactionTask.java | 15 +-
.../CompactionExceptionHandler.java} | 123 +-
...overTask.java => CompactionRecoverManager.java} | 92 +-
.../compaction/task/CompactionRecoverTask.java | 519 +++++++-
.../compaction/utils/log/CompactionLogger.java | 7 +-
.../engine/cq/ContinuousQuerySchemaCheckTask.java | 2 +-
.../db/engine/storagegroup/TsFileResource.java | 4 +
.../storagegroup/VirtualStorageGroupProcessor.java | 106 +-
.../service/TriggerRegistrationService.java | 2 +-
.../trigger/sink/local/LocalIoTDBHandler.java | 6 +-
.../iotdb/db/metadata/LocalConfigManager.java | 784 +++++++++++
.../db/metadata/LocalSchemaPartitionTable.java | 123 ++
...SchemaEngine.java => LocalSchemaProcessor.java} | 686 +++-------
.../apache/iotdb/db/metadata/MetadataConstant.java | 1 +
.../db/metadata/idtable/IDTableHashmapImpl.java | 10 +-
.../iotdb/db/metadata/idtable/IDTableManager.java | 2 +-
.../idtable/entry/InsertMeasurementMNode.java | 2 +-
.../db/metadata/lastCache/LastCacheManager.java | 6 +-
.../db/metadata/mnode/IStorageGroupMNode.java | 6 -
.../apache/iotdb/db/metadata/mnode/MNodeUtils.java | 9 +-
.../db/metadata/mnode/StorageGroupEntityMNode.java | 17 +-
.../iotdb/db/metadata/mnode/StorageGroupMNode.java | 18 -
.../iotdb/db/metadata/mtree/MTreeAboveSG.java | 95 +-
.../iotdb/db/metadata/mtree/MTreeBelowSG.java | 79 +-
.../db/metadata/mtree/traverser/Traverser.java | 10 +-
.../traverser/collector/MNodeAboveSGCollector.java | 9 +-
.../mtree/traverser/collector/MNodeCollector.java | 2 +-
.../counter/MNodeAboveSGLevelCounter.java | 9 +-
.../mtree/traverser/counter/MNodeLevelCounter.java | 18 +-
.../iotdb/db/metadata/path/MeasurementPath.java | 5 +
.../db/metadata/schemaregion/SchemaEngine.java | 84 ++
.../metadata/{ => schemaregion}/SchemaRegion.java | 195 +--
.../IStorageGroupSchemaManager.java | 70 +-
.../storagegroup/StorageGroupLogReader.java} | 35 +-
.../storagegroup/StorageGroupLogWriter.java | 56 +
.../StorageGroupSchemaManager.java | 194 +--
.../iotdb/db/metadata/template/Template.java | 33 +-
.../db/metadata/template/TemplateManager.java | 56 +-
.../db/metadata/upgrade/MetadataUpgrader.java | 66 +-
.../iotdb/db/mpp/buffer/DataBlockServiceImpl.java | 4 +-
.../apache/iotdb/db/mpp/buffer/ISinkHandle.java | 2 +-
.../apache/iotdb/db/mpp/buffer/ISourceHandle.java | 2 +-
.../apache/iotdb/db/mpp/buffer/SourceHandle.java | 2 +-
.../org/apache/iotdb/db/mpp/common/TsBlock.java | 73 -
.../db/mpp/common/schematree/PathPatternNode.java | 68 +-
.../db/mpp/common/schematree/PathPatternTree.java | 104 +-
.../db/mpp/common/schematree/SchemaEntityNode.java | 43 +
.../mpp/common/schematree/SchemaInternalNode.java | 32 +-
.../common/schematree/SchemaMeasurementNode.java | 29 +-
.../iotdb/db/mpp/common/schematree/SchemaNode.java | 44 +-
.../iotdb/db/mpp/common/schematree/SchemaTree.java | 21 +-
.../mpp/common/schematree/SchemaTreeVisitor.java | 235 ++++
.../db/mpp/execution/FragmentInstanceContext.java | 11 +-
.../iotdb/db/mpp/execution/QueryExecution.java | 18 +-
.../execution/scheduler/StandaloneScheduler.java | 4 +-
.../org/apache/iotdb/db/mpp/operator/Operator.java | 8 +-
.../iotdb/db/mpp/operator/OperatorContext.java | 13 +-
.../db/mpp/operator/process/AggregateOperator.java | 2 +-
.../mpp/operator/process/DeviceMergeOperator.java | 2 +-
.../db/mpp/operator/process/FillOperator.java | 2 +-
.../mpp/operator/process/FilterNullOperator.java | 2 +-
.../mpp/operator/process/GroupByLevelOperator.java | 2 +-
.../db/mpp/operator/process/LimitOperator.java | 16 +-
.../db/mpp/operator/process/OffsetOperator.java | 2 +-
.../db/mpp/operator/process/SortOperator.java | 2 +-
.../db/mpp/operator/process/TimeJoinOperator.java | 144 +-
.../db/mpp/operator/sink/FragmentSinkOperator.java | 2 +-
.../iotdb/db/mpp/operator/sink/SinkOperator.java | 2 +-
.../mpp/operator/source/AlignedSeriesScanUtil.java | 88 ++
.../source/SeriesAggregateScanOperator.java | 2 +-
.../db/mpp/operator/source/SeriesScanOperator.java | 110 +-
.../db/mpp/operator/source/SeriesScanUtil.java | 1388 ++++++++++++++++++++
.../db/mpp/schedule/FragmentInstanceManager.java | 82 +-
.../db/mpp/schedule/IFragmentInstanceManager.java | 12 +-
.../iotdb/db/mpp/schedule/ITaskScheduler.java | 2 +-
.../mpp/schedule/queue/IndexedBlockingQueue.java | 13 +
.../db/mpp/schedule/queue/L1PriorityQueue.java | 5 +
.../db/mpp/schedule/queue/L2PriorityQueue.java | 6 +
.../db/mpp/schedule/task/FragmentInstanceTask.java | 6 +-
.../apache/iotdb/db/mpp/sql/analyze/Analysis.java | 58 +-
.../apache/iotdb/db/mpp/sql/analyze/Analyzer.java | 95 +-
.../mpp/sql/analyze/ClusterPartitionFetcher.java | 12 +-
.../db/mpp/sql/analyze/ClusterSchemaFetcher.java | 11 -
.../mpp/sql/analyze/FakePartitionFetcherImpl.java | 119 ++
.../analyze/FakeSchemaFetcherImpl.java} | 21 +-
.../db/mpp/sql/analyze/IPartitionFetcher.java | 12 +-
.../iotdb/db/mpp/sql/analyze/ISchemaFetcher.java | 8 -
.../sql/analyze/StandalonePartitionFetcher.java | 12 +-
.../mpp/sql/analyze/StandaloneSchemaFetcher.java | 11 -
.../iotdb/db/mpp/sql/constant/FilterConstant.java | 3 +-
.../apache/iotdb/db/mpp/sql/parser/ASTVisitor.java | 108 +-
.../db/mpp/sql/planner/DistributionPlanner.java | 25 +-
.../db/mpp/sql/planner/LocalExecutionPlanner.java | 37 +-
.../iotdb/db/mpp/sql/planner/LogicalPlanner.java | 213 ++-
.../db/mpp/sql/planner/plan/FragmentInstance.java | 17 +-
.../db/mpp/sql/planner/plan/PlanFragment.java | 10 +-
.../plan/SimpleFragmentParallelPlanner.java | 9 +-
.../db/mpp/sql/planner/plan/node/PlanNode.java | 30 +-
.../sql/planner/plan/node/PlanNodeIdAllocator.java | 7 +
.../db/mpp/sql/planner/plan/node/PlanNodeType.java | 18 +-
.../db/mpp/sql/planner/plan/node/PlanNodeUtil.java | 52 +-
.../db/mpp/sql/planner/plan/node/PlanVisitor.java | 16 +-
.../plan/node/metedata/read/ShowDevicesNode.java | 9 +-
.../node/metedata/write/AlterTimeSeriesNode.java | 140 ++
.../write/CreateAlignedTimeSeriesNode.java} | 132 +-
.../node/metedata/write/CreateTimeSeriesNode.java | 10 +-
.../planner/plan/node/process/AggregateNode.java | 11 +-
.../planner/plan/node/process/DeviceMergeNode.java | 37 +-
.../planner/plan/node/process/ExchangeNode.java | 26 +-
.../sql/planner/plan/node/process/FillNode.java | 35 +-
.../sql/planner/plan/node/process/FilterNode.java | 40 +-
.../planner/plan/node/process/FilterNullNode.java | 43 +-
.../plan/node/process/GroupByLevelNode.java | 32 +-
.../sql/planner/plan/node/process/LimitNode.java | 21 +-
.../sql/planner/plan/node/process/OffsetNode.java | 31 +-
.../sql/planner/plan/node/process/SortNode.java | 35 +-
.../planner/plan/node/process/TimeJoinNode.java | 33 +-
.../sql/planner/plan/node/sink/CsvSinkNode.java | 67 -
.../planner/plan/node/sink/FragmentSinkNode.java | 20 +-
.../sql/planner/plan/node/sink/ThriftSinkNode.java | 69 -
.../planner/plan/node/source/CsvSourceNode.java | 80 --
.../plan/node/source/SeriesAggregateScanNode.java | 42 +-
.../planner/plan/node/source/SeriesScanNode.java | 71 +-
.../sql/planner/plan/node/source/SourceNode.java | 28 +-
.../plan/node/write/InsertMultiTabletNode.java | 9 +-
.../sql/planner/plan/node/write/InsertRowNode.java | 9 +-
.../planner/plan/node/write/InsertRowsNode.java | 9 +-
.../plan/node/write/InsertRowsOfOneDeviceNode.java | 9 +-
.../planner/plan/node/write/InsertTabletNode.java | 9 +-
.../sql/rewriter/ColumnPaginationController.java | 101 ++
.../db/mpp/sql/rewriter/ConcatPathRewriter.java | 308 +----
.../db/mpp/sql/rewriter/IStatementRewriter.java | 33 -
.../db/mpp/sql/rewriter/WildcardsRemover.java | 339 ++++-
.../iotdb/db/mpp/sql/statement/StatementNode.java | 2 -
.../sql/{tree => statement}/StatementVisitor.java | 18 +-
.../statement/component}/FillPolicy.java | 2 +-
.../statement/component/FilterNullComponent.java | 8 +-
.../statement/component}/FilterNullPolicy.java | 2 +-
.../statement/component/GroupByLevelComponent.java | 6 +
.../component/GroupByLevelController.java | 10 +-
.../mpp/sql/statement/component/ResultColumn.java | 11 +-
.../sql/statement/component/SelectComponent.java | 21 +-
.../db/mpp/sql/statement/crud/InsertStatement.java | 2 +-
.../db/mpp/sql/statement/crud/QueryStatement.java | 2 +-
.../mpp/sql/statement/crud/UDAFQueryStatement.java | 2 +-
.../metadata/AlterTimeSeriesStatement.java | 119 ++
.../metadata/CreateAlignedTimeSeriesStatement.java | 17 +-
.../metadata/CreateTimeSeriesStatement.java | 2 +-
.../apache/iotdb/db/mpp/sql/tree/Expression.java | 21 -
.../db/protocol/mqtt/PayloadFormatManager.java | 89 +-
.../iotdb/db/qp/constant/FilterConstant.java | 3 +-
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 91 +-
.../iotdb/db/qp/logical/crud/QueryOperator.java | 4 +-
.../iotdb/db/qp/logical/crud/SelectComponent.java | 4 +-
.../db/qp/logical/crud/UDAFQueryOperator.java | 4 +-
.../iotdb/db/qp/physical/crud/InsertPlan.java | 2 +-
.../iotdb/db/qp/physical/crud/QueryPlan.java | 3 +-
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 38 +
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 3 +-
.../iotdb/db/qp/utils/GroupByLevelController.java | 4 +-
.../apache/iotdb/db/qp/utils/WildcardsRemover.java | 4 +-
.../iotdb/db/query/dataset/ShowDevicesDataSet.java | 2 +-
.../db/query/dataset/ShowTimeseriesDataSet.java | 2 +-
.../iotdb/db/query/executor/LastQueryExecutor.java | 16 +-
.../iotdb/db/query/expression/Expression.java | 12 +-
.../query/expression/binary/BinaryExpression.java | 53 +-
.../binary/EqualToExpression.java} | 21 +-
.../binary/GreaterEqualExpression.java} | 23 +-
.../binary/GreaterThanExpression.java} | 22 +-
.../binary/LessEqualExpression.java} | 22 +-
.../binary/LessThanExpression.java} | 22 +-
.../binary/LogicAndExpression.java} | 21 +-
.../binary/LogicOrExpression.java} | 21 +-
.../binary/NonEqualExpression.java} | 22 +-
.../db/query/expression/unary/ConstantOperand.java | 15 +
.../query/expression/unary/FunctionExpression.java | 56 +-
...tionExpression.java => LogicNotExpression.java} | 54 +-
.../query/expression/unary/NegationExpression.java | 21 +-
.../query/expression/unary/TimeSeriesOperand.java | 23 +-
.../query/reader/chunk/MemAlignedPageReader.java | 40 +
.../iotdb/db/query/reader/chunk/MemPageReader.java | 93 ++
.../transformer/ArithmeticAdditionTransformer.java | 2 +-
.../transformer/ArithmeticBinaryTransformer.java | 98 +-
.../transformer/ArithmeticDivisionTransformer.java | 2 +-
.../transformer/ArithmeticModuloTransformer.java | 2 +-
.../ArithmeticMultiplicationTransformer.java | 2 +-
.../ArithmeticSubtractionTransformer.java | 2 +-
...naryTransformer.java => BinaryTransformer.java} | 77 +-
...nsformer.java => CompareBinaryTransformer.java} | 14 +-
...sformer.java => CompareEqualToTransformer.java} | 13 +-
...er.java => CompareGreaterEqualTransformer.java} | 8 +-
...mer.java => CompareGreaterThanTransformer.java} | 8 +-
...ormer.java => CompareLessEqualTransformer.java} | 8 +-
...former.java => CompareLessThanTransformer.java} | 8 +-
...former.java => CompareNonEqualTransformer.java} | 13 +-
...loTransformer.java => LogicAndTransformer.java} | 9 +-
...ransformer.java => LogicBinaryTransformer.java} | 15 +-
.../udf/core/transformer/LogicNotTransformer.java | 66 +
...uloTransformer.java => LogicOrTransformer.java} | 9 +-
.../iotdb/db/service/InternalServiceImpl.java | 13 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 19 +-
.../iotdb/db/service/metrics/MetricsService.java | 4 +-
.../db/service/thrift/impl/TSServiceImpl.java | 20 +-
.../db/sync/receiver/transfer/SyncServiceImpl.java | 2 +-
.../db/sync/sender/manage/SyncFileManager.java | 2 +-
.../db/tools/virtualsg/DeviceMappingViewer.java | 11 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 6 +-
.../apache/iotdb/db/utils/ErrorHandlingUtils.java | 15 +-
.../org/apache/iotdb/db/utils/SchemaTestUtils.java | 2 +-
.../org/apache/iotdb/db/utils/SchemaUtils.java | 4 +-
.../iotdb/db/writelog/recover/LogReplayer.java | 4 +-
.../org.apache.iotdb.metrics.reporter.Reporter | 19 -
.../iotdb/db/engine/MetadataManagerHelper.java | 48 +-
.../iotdb/db/engine/cache/ChunkCacheTest.java | 8 +-
.../engine/compaction/AbstractCompactionTest.java | 10 +-
.../engine/compaction/CompactionSchedulerTest.java | 64 +-
.../compaction/CompactionTaskComparatorTest.java | 38 +-
.../compaction/TestUtilsForAlignedSeries.java | 6 +-
.../cross/CrossSpaceCompactionExceptionTest.java | 31 +-
.../compaction/cross/CrossSpaceCompactionTest.java | 8 +-
.../db/engine/compaction/cross/MergeTest.java | 8 +-
.../RewriteCrossSpaceCompactionRecoverTest.java | 74 +-
.../inner/AbstractInnerSpaceCompactionTest.java | 8 +-
.../inner/InnerCompactionMoreDataTest.java | 4 +-
.../inner/InnerCompactionSchedulerTest.java | 31 +
.../compaction/inner/InnerCompactionTest.java | 8 +-
.../compaction/inner/InnerSeqCompactionTest.java | 8 +-
.../inner/InnerSpaceCompactionExceptionTest.java | 64 +-
.../InnerSpaceCompactionUtilsAlignedTest.java | 4 +-
.../InnerSpaceCompactionUtilsNoAlignedTest.java | 6 +-
.../compaction/inner/InnerUnseqCompactionTest.java | 8 +-
.../SizeTieredCompactionRecoverTest.java | 121 +-
.../inner/sizetiered/SizeTieredCompactionTest.java | 8 +-
...eCrossSpaceCompactionRecoverCompatibleTest.java | 19 +-
.../SizeTieredCompactionRecoverCompatibleTest.java | 59 +-
.../recover/SizeTieredCompactionRecoverTest.java | 125 +-
.../engine/modification/DeletionFileNodeTest.java | 4 +-
.../db/engine/modification/DeletionQueryTest.java | 4 +-
.../storagegroup/FileNodeManagerBenchmark.java | 8 +-
.../iotdb/db/engine/storagegroup/TTLTest.java | 16 +-
...neAdvancedTest.java => SchemaAdvancedTest.java} | 72 +-
...maEngineBasicTest.java => SchemaBasicTest.java} | 974 +++++++-------
...gineImproveTest.java => SchemaImproveTest.java} | 51 +-
.../org/apache/iotdb/db/metadata/TemplateTest.java | 117 +-
.../iotdb/db/metadata/idtable/IDTableTest.java | 70 +-
.../db/metadata/idtable/InsertWithIDTableTest.java | 18 +-
.../iotdb/db/metadata/mtree/MTreeAboveSGTest.java | 18 +-
.../iotdb/db/metadata/mtree/MTreeBelowSGTest.java | 48 +-
.../db/metadata/upgrade/MetadataUpgradeTest.java | 24 +-
.../iotdb/db/mpp/common/PathPatternTreeTest.java | 131 ++
.../apache/iotdb/db/mpp/common/SchemaTreeTest.java | 214 +++
.../iotdb/db/mpp/operator/LimitOperatorTest.java | 170 +++
.../db/mpp/operator/SeriesScanOperatorTest.java | 125 ++
.../db/mpp/operator/TimeJoinOperatorTest.java | 156 +++
.../db/mpp/schedule/DefaultTaskSchedulerTest.java | 411 ++++++
.../mpp/schedule/FragmentInstanceManagerTest.java | 134 ++
.../FragmentInstanceTimeoutSentinelTest.java | 283 ++++
.../db/mpp/schedule/queue/L1PriorityQueueTest.java | 26 +
.../db/mpp/schedule/queue/L2PriorityQueueTest.java | 28 +-
.../iotdb/db/mpp/sql/analyze/AnalyzerTest.java | 10 +-
.../db/mpp/sql/parser/StatementGeneratorTest.java | 4 +-
.../db/mpp/sql/plan/DistributionPlannerTest.java | 91 +-
.../iotdb/db/mpp/sql/plan/LogicalPlanPrinter.java | 219 +++
.../iotdb/db/mpp/sql/plan/LogicalPlannerTest.java | 288 +++-
.../iotdb/db/mpp/sql/plan/QueryPlannerTest.java | 63 +
.../db/protocol/mqtt/PayloadFormatManagerTest.java | 7 +
.../java/org/apache/iotdb/db/qp/PlannerTest.java | 34 +-
.../iotdb/db/qp/logical/LogicalPlanSmallTest.java | 4 +-
.../iotdb/db/qp/physical/ConcatOptimizerTest.java | 18 +-
.../iotdb/db/qp/physical/InsertRowPlanTest.java | 12 +-
.../iotdb/db/qp/physical/InsertTabletPlanTest.java | 10 +-
.../iotdb/db/qp/physical/PhysicalPlanTest.java | 12 +-
.../iotdb/db/qp/physical/SerializationTest.java | 14 +-
.../dataset/EngineDataSetWithValueFilterTest.java | 2 +-
.../query/dataset/UDTFAlignByTimeDataSetTest.java | 14 +-
.../query/dataset/groupby/GroupByDataSetTest.java | 2 +-
.../dataset/groupby/GroupByFillDataSetTest.java | 2 +-
.../dataset/groupby/GroupByLevelDataSetTest.java | 2 +-
.../reader/series/SeriesAggregateReaderTest.java | 3 +-
.../reader/series/SeriesReaderByTimestampTest.java | 3 +-
.../db/query/reader/series/SeriesReaderTest.java | 3 +-
.../query/reader/series/SeriesReaderTestUtil.java | 33 +-
.../iotdb/db/rescon/ResourceManagerTest.java | 8 +-
.../db/sync/receiver/load/FileLoaderTest.java | 11 +-
.../recover/SyncReceiverLogAnalyzerTest.java | 11 +-
.../db/sync/sender/manage/SyncFileManagerTest.java | 2 +-
.../sender/recover/SyncSenderLogAnalyzerTest.java | 2 +-
.../org/apache/iotdb/db/tools/MLogParserTest.java | 123 +-
.../org/apache/iotdb/db/utils/SchemaUtilsTest.java | 8 +-
.../apache/iotdb/db/writelog/PerformanceTest.java | 10 +-
.../db/writelog/recover/DeviceStringTest.java | 12 +-
.../iotdb/db/writelog/recover/LogReplayerTest.java | 4 +-
.../recover/RecoverResourceFromReaderTest.java | 8 +-
.../db/writelog/recover/SeqTsFileRecoverTest.java | 8 +-
.../writelog/recover/UnseqTsFileRecoverTest.java | 8 +-
.../src/test/resources/logback-test.xml | 4 +-
.../org/apache/iotdb/rpc/RedirectException.java | 6 +-
.../apache/iotdb/spark/db/EnvironmentUtils.java | 4 +-
.../src/main/thrift/confignode.thrift | 63 +-
thrift/src/main/thrift/mpp.thrift | 16 +-
tsfile/pom.xml | 8 +
.../iotdb/tsfile/read/common/block/TsBlock.java | 317 +++++
.../tsfile/read/common/block/TsBlockBuilder.java | 295 +++++
.../read/common/block/TsBlockBuilderStatus.java | 75 ++
.../tsfile/read/common/block}/TsBlockMetadata.java | 8 +-
.../read/common/block/column/BinaryColumn.java | 110 ++
.../common/block/column/BinaryColumnBuilder.java | 148 +++
.../read/common/block/column/BooleanColumn.java | 109 ++
.../common/block/column/BooleanColumnBuilder.java | 150 +++
.../tsfile/read/common/block/column/Column.java | 87 ++
.../read/common/block/column/ColumnBuilder.java | 81 ++
.../common/block/column/ColumnBuilderStatus.java | 91 ++
.../read/common/block/column/ColumnUtil.java | 97 ++
.../read/common/block/column/DoubleColumn.java | 109 ++
.../common/block/column/DoubleColumnBuilder.java | 150 +++
.../read/common/block/column/FloatColumn.java | 108 ++
.../common/block/column/FloatColumnBuilder.java | 150 +++
.../tsfile/read/common/block/column/IntColumn.java | 108 ++
.../read/common/block/column/IntColumnBuilder.java | 150 +++
.../read/common/block/column/LongColumn.java | 108 ++
.../common/block/column/LongColumnBuilder.java | 150 +++
.../block/column/RunLengthEncodedColumn.java | 133 ++
.../read/common/block/column/TimeColumn.java | 95 ++
.../common/block/column/TimeColumnBuilder.java | 126 ++
.../iotdb/tsfile/read/reader/IPageReader.java | 3 +
.../tsfile/read/reader/page/AlignedPageReader.java | 44 +
.../iotdb/tsfile/read/reader/page/PageReader.java | 86 ++
.../apache/iotdb/tsfile/write/record/Tablet.java | 4 +-
.../iotdb/tsfile/common/block/TsBlockTest.java | 317 +++++
529 files changed, 20348 insertions(+), 6923 deletions(-)
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/execution/QueryExecution.java
index b35c88a,4cbbb44..f35693d
--- a/server/src/main/java/org/apache/iotdb/db/mpp/execution/QueryExecution.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/execution/QueryExecution.java
@@@ -35,8 -31,10 +35,9 @@@ import org.apache.iotdb.rpc.RpcUtils
import org.apache.iotdb.rpc.TSStatusCode;
import java.nio.ByteBuffer;
+ import java.util.ArrayList;
import java.util.List;
-
-import static org.apache.iotdb.rpc.RpcUtils.getStatus;
+import java.util.concurrent.ExecutionException;
/**
* QueryExecution stores all the status of a query which is being prepared or running inside the MPP
@@@ -61,17 -56,8 +62,18 @@@ public class QueryExecution
public QueryExecution(Statement statement, MPPQueryContext context) {
this.context = context;
+ this.planOptimizers = new ArrayList<>();
this.analysis = analyze(statement, context);
+ this.stateMachine = new QueryStateMachine(context.getQueryId());
+
+ // We add the abort logic inside the QueryExecution.
+ // So that the other components can only focus on the state change.
+ stateMachine.addStateChangeListener(state -> {
+ if (!state.isDone()) {
+ return;
+ }
+ this.cleanup();
+ });
}
public void start() {
@@@ -134,23 -100,20 +136,35 @@@
return null;
}
- public ExecutionResult getResult() {
-
- return new ExecutionResult(context.getQueryId(), getStatus(TSStatusCode.SUCCESS_STATUS));
+ /**
+ * This method is a synchronized method.
+ * For READ, it will block until all the FragmentInstances have been submitted.
+ * For WRITE, it will block until all the FragmentInstances have finished.
+ * @return ExecutionStatus. Contains the QueryId and the TSStatus.
+ */
+ public ExecutionStatus getStatus() {
+ // Although we monitor the state to transition to RUNNING, the future will return if any Terminated state is triggered
+ ListenableFuture<QueryState> future = stateMachine.getStateChange(QueryState.RUNNING);
+ try {
+ QueryState state = future.get();
+ // TODO: (xingtanzjr) use more TSStatusCode if the QueryState isn't FINISHED
+ TSStatusCode statusCode = state == QueryState.FINISHED ? TSStatusCode.SUCCESS_STATUS : TSStatusCode.QUERY_PROCESS_ERROR;
+ return new ExecutionStatus(context.getQueryId(), RpcUtils.getStatus(statusCode));
+ } catch (InterruptedException | ExecutionException e) {
+ // TODO: (xingtanzjr) use more accurate error handling
+ return new ExecutionStatus(context.getQueryId(), RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR));
+ }
}
+
+ public DistributedQueryPlan getDistributedPlan() {
+ return distributedPlan;
+ }
+
+ public LogicalQueryPlan getLogicalPlan() {
+ return logicalPlan;
+ }
+
+ public List<FragmentInstance> getFragmentInstances() {
+ return fragmentInstances;
+ }
}
diff --cc server/src/main/java/org/apache/iotdb/db/mpp/sql/planner/plan/node/sink/FragmentSinkNode.java
index b9940c8,94f2d1d..2656205
--- a/server/src/main/java/org/apache/iotdb/db/mpp/sql/planner/plan/node/sink/FragmentSinkNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/sql/planner/plan/node/sink/FragmentSinkNode.java
@@@ -18,7 -18,7 +18,8 @@@
*/
package org.apache.iotdb.db.mpp.sql.planner.plan.node.sink;
+import org.apache.commons.lang.Validate;
+ import org.apache.iotdb.commons.cluster.Endpoint;
import org.apache.iotdb.db.mpp.common.FragmentInstanceId;
import org.apache.iotdb.db.mpp.sql.planner.plan.node.PlanNode;
import org.apache.iotdb.db.mpp.sql.planner.plan.node.PlanNodeId;
@@@ -48,9 -47,11 +48,6 @@@ public class FragmentSinkNode extends S
}
@Override
- public void addChildren(PlanNode child) {}
- public void addChild(PlanNode child) {
- this.child = child;
- }
--
-- @Override
public PlanNode clone() {
FragmentSinkNode sinkNode = new FragmentSinkNode(getId());
sinkNode.setDownStream(downStreamEndpoint, downStreamInstanceId, downStreamPlanNodeId);
@@@ -59,15 -60,10 +56,24 @@@
}
@Override
+ public PlanNode cloneWithChildren(List<PlanNode> children) {
+ Validate.isTrue(children == null || children.size() == 1, "Children size of FragmentSinkNode should be 0 or 1");
+ FragmentSinkNode sinkNode = (FragmentSinkNode) clone();
+ if (children != null) {
+ sinkNode.setChild(children.get(0));
+ }
+ return sinkNode;
+ }
+
++ public void addChild(PlanNode child) {
++ this.child = child;
++ }
++
++ @Override
+ public int allowedChildCount() {
+ return ONE_CHILD;
+ }
+
@Override
public List<String> getOutputColumnNames() {
return null;