You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2023/12/18 07:28:18 UTC
(iotdb) branch 2G_metadata_read updated (df77d64e335 -> e86c93b9c62)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch 2G_metadata_read
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from df77d64e335 support 2G metadata read
add ec118416f36 Added getRegionId api for partitionManager (#11663)
add 24f0a606cbc Ignore MismatchedType caused by delete
add 788b0bc9523 Refactor Device Node Implementation in SchemaRegion (#11661)
add dcaedd336e3 Extend `Filter` interface to support aligned timeseries
add 235a3f3c14e Pipe Schema: Added ConcurrentIterableLinkedQueue to record the ConsensusRequest and SnapShot event (#11650)
add 5e89a90ece5 TopKNode will use IdentityNode but not ShuffleSinkNode
add 9b19cf72ac4 [IOTDB-6174] Support ColumnInjectOperator for '__endTime' output
add 6c92fbc8c77 Optimize the efficiency of map-related code in IRegionGroupAllocator (#11673)
add 78ce7e74bce [IOTDB-6259][RatisConsensus] Fix missing prefix in gauge metrics (#11677)
add c185c6c9c92 [IOTDB-6263] Enhance IoTDBPartitionCreationIT (#11676)
add 0fa7f403130 Fix reconnect bug for JDBC
add 3b4788614c2 Pipe: avoid useless pipe meta sync (stopped status) to DN to achieve high availability (#11641)
add 90ce01eb7e8 [RatisConsensus] Retry Read Requests on StatusRuntimeException (#11679)
add 11adc7c920a [IOTDB-6252] Greedy CopySet allocation algorithm (#11572)
add ee5e23dbb94 Rest service code optimization and fixed insertRecords error prompt returning null issue (#11678)
add 9ded0bdf7dd Bump ch.qos.logback:logback-core from 1.3.13 to 1.3.14 (#11683)
add 60d66ae873f Enhance the note of parameter seed_config_node (#11685)
add a93cf09d1bd Enhance the exapmle of parameter dn_seed_config_node (#11686)
add c450e768980 Enhance parameter dn_seed_config_node commets order (#11687)
add 4922dad4d17 Add last time check for inner space compaction task (#11553)
add 9af1e765c5d Cache the internal compaction estimation result if compaction task is failed to run (#11586)
add acde4457a3d [IOTDB-6264] Load: Optimized Validation Messages with 'write_data' Permissions When Loading TsFiles Without schema Creation (#11684)
add 09b7e86fd63 add config for inner compaction selector (#11693)
add b1d3dc63c9e [IOTDB-6265] Construct SessionPool using empty nodeUrls should throw exception
add 5caeeb3e3b7 Pipe: automatically start pipe upon creation & allow to start RUNNING pipe & allow to stop STOPPED pipe (#11690)
add a061c0d4514 Pipe: Make error message reported by pipe parameter check clearer (#11695)
add b57f320153b Make slimit and soffset available in align by device with template
add 1dfdae910bf [IOTDB-6266] Add the ability to flush syncIndex and update reader periodically for IoTConsensus (#11691)
add e75bc1c4675 Enable Jenkins Log when CI failed (#11700)
add f5a05fba466 Enable jenkins log when ci failed (#11701)
add bcc3b3dd4a1 Pipe schema: RPC request & response datastructures (#11680)
add 63475a16592 Pipe: Add pipeSinkHandshakeTimeout parameter to distinguish it from pipeSinkTransferTimeout & Call handshake() method of temporary connector's to validate createPipeStatement (#11655)
add baae7ce0332 Fix snapshot UT (#11697)
add 4e72d07de9b Featrue add insertRecords and insertTablets to the rest v2 service for data type verification (#11708)
add fb23bd43639 fix(cli): add jna for windows (#11707)
add f61f310c277 fix Precondition failure (#11710)
add f12b5eb5f51 add IT testing to rest service (#11437)
add 795d420db86 Pipe: rename iotdb-thrift-ssl-sink & allow not specifying connector in kv attrs & support kv attrs that don't have sink. / processor. / sink. prefixes (#11715)
add 3e4c6a4df91 Pipe Schema: Receiver Agent: Added pipe enriched planNode to enable pipe request detection to configure "forwarding-pipe-request" (#11672)
add ac326a59e40 Improved stability of Ratis addMemberToGroup and testThresholdSnapshot UT (#11717)
add 88549541f6b Add a param to switch whether to enable data separation or not (#11692)
add 8c8c8d58523 [IOTDB-5706] Data inconsistency between IoT protocol replications (#11727)
add 13c047fe7bc IT: enable REST service when needed to fix IT (#11726)
add 54a3733263f Take snapshot for mods using copy instead of hard link (#11728)
add 1c886d255b6 change default config value of inner_compaction_task_selection_mods_file_threshold (#11703)
add fd9cb6ed8b7 Implement Cluster ID (#11702)
add 301033fe015 Show devices with template (#11681)
add 40c32518da0 Pipe Schema: Receiver Agent: config plan node to statement transformation logic (#11689)
add ef6dc0a8e14 Pipe: enhance the idempotence of start / stop pipe (#11713)
add 720d4729bd1 SeriesScanUtil support predicate pushdown
new a0f9fc0f842 Merge branch 'master' of github.com:apache/iotdb into 2G_metadata_read
new a25aa657843 finish dev sketch tool
new ec191a2a4b2 fix query
new e86c93b9c62 fix load and query error
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
Jenkinsfile | 4 +
.../java/org/apache/iotdb/tsfile/TsFileRead.java | 16 +-
.../apache/iotdb/tsfile/TsFileSequenceRead.java | 4 +-
.../it/env/cluster/config/MppDataNodeConfig.java | 6 +
.../it/env/cluster/node/AbstractNodeWrapper.java | 26 +-
.../iotdb/it/env/cluster/node/DataNodeWrapper.java | 11 +
.../it/env/remote/config/RemoteDataNodeConfig.java | 5 +
.../apache/iotdb/itbase/constant/TestConstant.java | 1 +
.../apache/iotdb/itbase/env/DataNodeConfig.java | 2 +
.../it/cluster/IoTDBClusterRestartIT.java | 17 +-
.../confignode/it/cluster/IoTDBClusterStartIT.java | 93 +
.../it/partition/IoTDBPartitionCreationIT.java | 4 +-
.../apache/iotdb/db/it/GrafanaApiServiceIT.java | 451 +++++
.../org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 2 +-
.../org/apache/iotdb/db/it/IoTDBRestServiceIT.java | 2119 ++++++++++++++++++++
.../IoTDBAlignByDeviceWithTemplateIT.java | 82 +
.../iotdb/db/it/auth/IoTDBClusterAuthorityIT.java | 5 +-
.../iotdb/db/it/auth/IoTDBSeriesPermissionIT.java | 4 +-
.../iotdb/db/it/auth/IoTDBSystemPermissionIT.java | 75 +-
.../apache/iotdb/db/it/groupby/IOTDBGroupByIT.java | 196 ++
.../apache/iotdb/db/it/query/IoTDBResultSetIT.java | 7 +-
.../iotdb/db/it/schema/IoTDBMetadataFetchIT.java | 85 +-
.../org/apache/iotdb/db/it/utils/TestUtils.java | 108 +-
.../apache/iotdb/pipe/it/IoTDBPipeExtractorIT.java | 22 +-
.../apache/iotdb/pipe/it/IoTDBPipeLifeCycleIT.java | 97 +
.../iotdb/pipe/it/IoTDBPipeSwitchStatusIT.java | 22 +-
.../apache/iotdb/pipe/it/IoTDBPipeSyntaxIT.java | 2 +-
.../iotdb/zeppelin/it/IoTDBInterpreterIT.java | 10 +-
.../parameter/PipeParameterValidator.java | 6 +-
.../api/customizer/parameter/PipeParameters.java | 104 +-
iotdb-client/cli/pom.xml | 17 +
.../java/org/apache/iotdb/cli/AbstractCli.java | 2 +-
.../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 15 +-
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../java/org/apache/iotdb/session/Session.java | 8 +-
.../org/apache/iotdb/session/pool/SessionPool.java | 8 +-
.../java/org/apache/iotdb/session/SessionTest.java | 12 +
.../session/pool/SessionPoolExceptionTest.java | 12 +
.../iotdb/spark/tsfile/NarrowConverter.scala | 56 +-
.../apache/iotdb/spark/tsfile/WideConverter.scala | 56 +-
.../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 1 +
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 13 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 +
.../resources/conf/iotdb-confignode.properties | 2 +
.../iotdb/confignode/conf/ConfigNodeConfig.java | 2 +-
.../consensus/request/ConfigPhysicalPlan.java | 8 +
.../consensus/request/ConfigPhysicalPlanType.java | 5 +-
.../write/confignode/UpdateClusterIdPlan.java} | 46 +-
.../request/write/pipe/PipeEnrichedPlan.java | 79 +
.../statemachine/ConfigRegionStateMachine.java | 6 +-
.../iotdb/confignode/manager/ClusterManager.java | 67 +
.../iotdb/confignode/manager/ConfigManager.java | 21 +-
.../apache/iotdb/confignode/manager/IManager.java | 10 +
.../iotdb/confignode/manager/ProcedureManager.java | 17 +-
.../manager/load/balancer/RegionBalancer.java | 21 +-
.../region/CopySetRegionGroupAllocator.java | 160 --
.../region/GreedyCopySetRegionGroupAllocator.java | 191 ++
.../region/GreedyRegionGroupAllocator.java | 31 +-
.../manager/partition/PartitionManager.java | 15 +-
.../payload/request/PipeTransferConfigPlanReq.java | 78 +
.../manager/pipe/runtime/PipeHeartbeatParser.java | 4 +-
.../manager/pipe/runtime/PipeMetaSyncer.java | 15 +-
.../manager/pipe/task/PipeTaskCoordinator.java | 29 +-
.../iotdb/confignode/persistence/ClusterInfo.java | 109 +
.../persistence/executor/ConfigPlanExecutor.java | 13 +
.../partition/DatabasePartitionTable.java | 8 +
.../persistence/partition/PartitionInfo.java | 8 +
.../persistence/pipe/PipePluginInfo.java | 13 +-
.../confignode/persistence/pipe/PipeTaskInfo.java | 32 +-
.../confignode/persistence/schema/ConfigMTree.java | 2 +-
.../persistence/schema/mnode/IConfigMNode.java | 28 +
.../schema/mnode/basic/ConfigBasicMNode.java | 50 +-
.../mnode/impl/ConfigBasicInternalMNode.java | 37 +-
.../impl/pipe/AbstractOperatePipeProcedureV2.java | 18 +-
.../runtime/PipeHandleLeaderChangeProcedure.java | 3 +-
.../runtime/PipeHandleMetaChangeProcedure.java | 3 +-
.../impl/pipe/runtime/PipeMetaSyncProcedure.java | 3 +-
.../impl/pipe/task/CreatePipeProcedureV2.java | 6 +-
.../impl/pipe/task/DropPipeProcedureV2.java | 4 +-
.../impl/pipe/task/StartPipeProcedureV2.java | 5 +-
.../impl/pipe/task/StopPipeProcedureV2.java | 5 +-
.../confignode/service/ConfigNodeShutdownHook.java | 2 +
.../thrift/ConfigNodeRPCServiceProcessor.java | 45 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 12 +
.../GreedyCopySetRegionGroupAllocatorTest.java | 203 ++
.../confignode/persistence/ClusterInfoTest.java | 67 +
.../confignode/persistence/PartitionInfoTest.java | 70 +-
.../apache/iotdb/consensus/iot/IoTConsensus.java | 31 +-
.../consensus/iot/IoTConsensusServerImpl.java | 14 +-
.../consensus/iot/client/DispatchLogHandler.java | 5 +-
.../iot/logdispatcher/IndexController.java | 3 +-
.../consensus/iot/logdispatcher/LogDispatcher.java | 11 +
.../consensus/iot/logdispatcher/SyncStatus.java | 4 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 26 +-
.../ratis/metrics/IoTDBMetricRegistry.java | 8 +-
.../iot/logdispatcher/IndexControllerTest.java | 14 +-
.../iotdb/consensus/ratis/DiskGuardianTest.java | 3 +-
.../iotdb/consensus/ratis/RatisConsensusTest.java | 25 +-
.../iotdb/consensus/ratis/RecoverReadTest.java | 4 +-
.../apache/iotdb/consensus/ratis/TestUtils.java | 25 +-
.../resources/conf/iotdb-datanode.properties | 10 +-
.../org/apache/iotdb/db/audit/AuditLogger.java | 2 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 56 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 29 +-
.../iotdb/db/conf/rest/IoTDBRestServiceConfig.java | 1 +
.../db/conf/rest/IoTDBRestServiceDescriptor.java | 119 +-
.../dataregion/DataExecutionVisitor.java | 29 +-
.../schemaregion/SchemaExecutionVisitor.java | 14 +
.../agent/plugin/PipeConnectorConstructor.java | 13 +-
.../db/pipe/agent/plugin/PipePluginAgent.java | 7 +
.../iotdb/db/pipe/agent/task/PipeTaskAgent.java | 6 +-
.../env/PipeTaskTemporaryRuntimeEnvironment.java} | 15 +-
.../request/PipeTransferFilePieceReq.java | 4 +-
.../evolvable/request/PipeTransferFileSealReq.java | 4 +-
.../request/PipeTransferHandshakeReq.java | 4 +-
.../request/PipeTransferSchemaPlanReq.java | 105 +
.../request/PipeTransferTabletBatchReq.java | 4 +-
.../request/PipeTransferTabletBinaryReq.java | 4 +-
.../request/PipeTransferTabletInsertNodeReq.java | 4 +-
.../request/PipeTransferTabletRawReq.java | 4 +-
.../protocol/airgap/IoTDBAirGapConnector.java | 2 +-
.../thrift/sync/IoTDBThriftSyncConnector.java | 48 +-
.../sync/IoTDBThriftSyncConnectorClient.java | 12 +-
.../protocol/writeback/WriteBackConnector.java | 4 +-
.../tsfile/TsFileInsertionDataContainer.java | 6 +-
.../pipe/receiver/PipePlanToStatementVisitor.java | 186 ++
.../pipe/receiver/airgap/IoTDBAirGapReceiver.java | 2 +-
.../pipe/receiver/thrift/IoTDBThriftReceiver.java | 2 +-
.../receiver/thrift/IoTDBThriftReceiverAgent.java | 2 +-
.../receiver/thrift/IoTDBThriftReceiverV1.java | 46 +-
.../iotdb/db/protocol/client/ConfigNodeClient.java | 37 +-
.../rest/v1/handler/RequestValidationHandler.java | 12 -
.../v1/handler/StatementConstructionHandler.java | 59 -
.../protocol/rest/v1/impl/RestApiServiceImpl.java | 47 -
.../rest/v2/handler/RequestValidationHandler.java | 43 +
.../protocol/rest/v2/impl/RestApiServiceImpl.java | 36 +-
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 4 +-
.../common/header/ColumnHeaderConstant.java | 16 +-
.../common/header/DatasetHeaderFactory.java | 4 +
.../execution/executor/RegionWriteExecutor.java | 276 ++-
.../execution/load/LoadTsFileManager.java | 4 +-
.../queryengine/execution/load/TsFileSplitter.java | 2 +-
.../execution/operator/AggregationUtil.java | 16 +-
.../operator/process/AggregationOperator.java | 19 +-
.../operator/process/ColumnInjectOperator.java | 115 ++
.../process/SlidingWindowAggregationOperator.java | 18 +-
.../execution/operator/process/TopKOperator.java | 8 +-
.../operator/schema/source/DeviceSchemaSource.java | 15 +
.../AbstractSeriesAggregationScanOperator.java | 52 +-
.../AlignedSeriesAggregationScanOperator.java | 25 +
.../operator/source/AlignedSeriesScanUtil.java | 163 +-
.../operator/source}/FileLoaderUtils.java | 109 +-
.../source/SeriesAggregationScanOperator.java | 25 +
.../execution/operator/source/SeriesScanUtil.java | 565 +++---
.../queryengine/plan/analyze/AnalyzeVisitor.java | 44 +-
.../plan/analyze/LoadTsfileAnalyzer.java | 43 +-
.../queryengine/plan/analyze/TemplatedAnalyze.java | 42 +-
.../queryengine/plan/execution/QueryExecution.java | 10 +-
.../plan/execution/config/ConfigTaskVisitor.java | 8 +
.../config/executor/ClusterConfigTaskExecutor.java | 102 +-
.../config/executor/IConfigTaskExecutor.java | 10 +
.../config/metadata/ShowClusterIdTask.java | 65 +
.../ConvertPredicateToTimeFilterVisitor.java | 30 +-
.../plan/optimization/LimitOffsetPushDown.java | 11 +-
.../plan/optimization/PlanNodePushDown.java | 76 +
.../base/ColumnInjectionPushDown.java} | 9 +-
.../db/queryengine/plan/parser/ASTVisitor.java | 20 +-
.../plan/planner/LogicalPlanBuilder.java | 16 +-
.../plan/planner/LogicalPlanVisitor.java | 72 +-
.../plan/planner/OperatorTreeGenerator.java | 61 +-
.../planner/distribution/DistributionPlanner.java | 36 +-
.../plan/planner/plan/node/PlanGraphPrinter.java | 13 +
.../plan/planner/plan/node/PlanNode.java | 10 +
.../plan/planner/plan/node/PlanNodeType.java | 22 +-
.../plan/planner/plan/node/PlanVisitor.java | 28 +-
.../planner/plan/node/load/LoadTsFileNode.java | 7 +-
.../node/metedata/write/CreateTimeSeriesNode.java | 2 +-
.../metedata/write/view/AlterLogicalViewNode.java | 43 +-
.../node/pipe/PipeEnrichedConfigSchemaNode.java | 161 ++
.../plan/node/pipe/PipeEnrichedDeleteDataNode.java | 169 ++
.../{write => pipe}/PipeEnrichedInsertNode.java | 19 +-
.../node/pipe/PipeEnrichedWriteSchemaNode.java | 192 ++
.../planner/plan/node/process/AggregationNode.java | 8 +-
.../plan/node/process/ColumnInjectNode.java | 123 ++
.../node/process/SlidingWindowAggregationNode.java | 56 +-
.../source/AlignedSeriesAggregationScanNode.java | 34 +-
.../node/source/SeriesAggregationScanNode.java | 34 +-
.../node/source/SeriesAggregationSourceNode.java | 37 +-
.../planner/plan/node/write/DeleteDataNode.java | 4 +-
.../plan/planner/plan/node/write/InsertNode.java | 10 -
.../planner/plan/parameter/SeriesScanOptions.java | 10 +-
.../queryengine/plan/statement/StatementType.java | 3 +-
.../plan/statement/StatementVisitor.java | 18 +-
.../crud/PipeEnrichedInsertBaseStatement.java | 220 --
.../crud/PipeEnrichedLoadTsFileStatement.java | 137 --
.../metadata/ShowClusterIdStatement.java} | 24 +-
.../metadata/view/CreateLogicalViewStatement.java | 10 +-
.../plan/statement/pipe/PipeEnrichedStatement.java | 72 +
.../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 31 +-
.../schemaregion/mtree/impl/mem/MemMTreeStore.java | 23 +-
.../mtree/impl/mem/mnode/basic/BasicMNode.java | 26 +-
.../impl/mem/mnode/factory/MemMNodeFactory.java | 11 +-
.../impl/mem/mnode/impl/BasicInternalMNode.java | 83 +-
.../impl/mem/mnode/impl/DatabaseDeviceMNode.java | 37 -
.../mtree/impl/mem/mnode/impl/DatabaseMNode.java | 17 +-
.../mtree/impl/mem/mnode/impl/DeviceMNode.java | 36 -
.../mtree/impl/mem/mnode/info/LogicalViewInfo.java | 2 +-
.../impl/mem/snapshot/MemMTreeSnapshotUtil.java | 75 +-
.../mtree/impl/pbtree/CachedMTreeStore.java | 43 +-
.../mtree/impl/pbtree/MTreeBelowSGCachedImpl.java | 31 +-
.../impl/pbtree/mnode/basic/CachedBasicMNode.java | 25 +-
.../pbtree/mnode/factory/CacheMNodeFactory.java | 11 +-
.../mnode/impl/CachedBasicInternalMNode.java | 91 +-
.../mnode/impl/CachedDatabaseDeviceMNode.java | 49 -
.../pbtree/mnode/impl/CachedDatabaseMNode.java | 18 +-
.../impl/pbtree/mnode/impl/CachedDeviceMNode.java | 48 -
.../read/resp/info/IDeviceSchemaInfo.java | 2 +
.../read/resp/info/impl/ShowDevicesResult.java | 11 +
.../schemaregion/utils/MNodeUtils.java | 54 +-
.../utils/filter/DeviceFilterVisitor.java | 26 +
.../apache/iotdb/db/service/IoTDBShutdownHook.java | 3 +-
.../db/storageengine/dataregion/DataRegion.java | 193 +-
.../dataregion/HashLastFlushTimeMap.java | 115 +-
.../dataregion/IDataRegionForQuery.java | 5 +-
.../dataregion/ILastFlushTimeMap.java | 31 +-
.../dataregion/VirtualDataRegion.java | 5 +-
.../CompactionLastTimeCheckFailedException.java | 18 +-
.../execute/task/AbstractCompactionTask.java | 2 +-
.../execute/task/InnerSpaceCompactionTask.java | 8 +-
.../fast/AlignedSeriesCompactionExecutor.java | 19 +-
.../fast/NonAlignedSeriesCompactionExecutor.java | 15 +-
.../executor/fast/SeriesCompactionExecutor.java | 46 +-
.../executor/fast/element/AlignedPageElement.java | 94 +
.../fast/element/NonAlignedPageElement.java | 74 +
.../utils/executor/fast/element/PageElement.java | 96 +-
.../utils/executor/fast/element/PointElement.java | 11 +-
.../fast/reader/CompactionAlignedChunkReader.java | 122 ++
.../fast/reader/CompactionChunkReader.java | 94 +
.../readchunk/AlignedSeriesCompactionExecutor.java | 12 +
.../readchunk/SingleSeriesCompactionExecutor.java | 15 +-
.../execute/utils/reader/PointPriorityReader.java | 2 +-
.../utils/writer/AbstractCompactionWriter.java | 11 +
.../writer/AbstractCrossCompactionWriter.java | 11 +-
.../writer/AbstractInnerCompactionWriter.java | 2 +
.../utils/writer/FastInnerCompactionWriter.java | 4 +
.../compaction/schedule/CompactionWorker.java | 9 +-
.../estimator/AbstractCompactionEstimator.java | 27 +-
.../impl/RewriteCrossSpaceCompactionSelector.java | 6 +-
.../impl/SizeTieredCompactionSelector.java | 8 +-
.../memtable/AlignedReadOnlyMemChunk.java | 12 +-
.../read/reader/chunk/DiskAlignedChunkLoader.java | 13 +-
.../read/reader/chunk/DiskChunkLoader.java | 4 +-
.../read/reader/chunk/MemAlignedChunkLoader.java | 13 +-
.../read/reader/chunk/MemAlignedChunkReader.java | 6 +-
.../read/reader/chunk/MemAlignedPageReader.java | 127 +-
.../read/reader/chunk/MemChunkLoader.java | 4 +-
.../read/reader/chunk/MemChunkReader.java | 13 +-
.../read/reader/chunk/MemPageReader.java | 281 +--
.../metadata/DiskAlignedChunkMetadataLoader.java | 57 +-
.../chunk/metadata/DiskChunkMetadataLoader.java | 48 +-
.../metadata/MemAlignedChunkMetadataLoader.java | 40 +-
.../chunk/metadata/MemChunkMetadataLoader.java | 31 +-
.../dataregion/snapshot/SnapshotTaker.java | 14 +-
.../dataregion/tsfile/TsFileResource.java | 8 +-
.../dataregion/utils/TsFileResourceUtils.java | 52 +-
.../file/AbstractTsFileRecoverPerformer.java | 4 +-
.../quotas/DataNodeThrottleQuotaManager.java | 2 +-
.../rescon/quotas/DefaultOperationQuota.java | 16 +-
.../apache/iotdb/db/tools/TsFileSketchTool.java | 41 +-
.../iotdb/db/tools/TsFileSplitByPartitionTool.java | 2 +-
.../org/apache/iotdb/db/tools/TsFileSplitTool.java | 2 +-
.../db/tools/validate/TsFileValidationTool.java | 2 +-
.../db/trigger/executor/TriggerFireVisitor.java | 17 +-
.../db/utils/columngenerator/ColumnGenerator.java} | 11 +-
.../columngenerator/ColumnGeneratorType.java} | 15 +-
.../SlidingTimeColumnGenerator.java | 52 +
.../parameter/ColumnGeneratorParameter.java | 82 +
.../SlidingTimeColumnGeneratorParameter.java | 80 +
.../apache/iotdb/db/metadata/mnode/MNodeTest.java | 31 +-
.../metadata/mtree/schemafile/SchemaFileTest.java | 2 +-
.../schemaRegion/SchemaRegionBasicTest.java | 42 +-
...est.java => PipeDataNodeThriftRequestTest.java} | 80 +-
.../db/protocol/rest/IoTDBRestServiceTest.java | 15 +
.../operator/AggregationOperatorTest.java | 1 +
.../AlignedSeriesAggregationScanOperatorTest.java | 12 +-
.../execution/operator/OperatorMemoryTest.java | 2 +
.../operator/RawDataAggregationOperatorTest.java | 4 +-
.../SeriesAggregationScanOperatorTest.java | 12 +-
.../SlidingWindowAggregationOperatorTest.java | 1 +
.../operator/UpdateLastCacheOperatorTest.java | 6 +-
.../schema/SchemaQueryScanOperatorTest.java | 6 +-
.../ConvertPredicateToTimeFilterTest.java | 46 +-
.../plan/analyze/QueryTimePartitionTest.java | 108 +-
.../plan/optimization/LimitOffsetPushDownTest.java | 10 +-
.../plan/plan/QueryLogicalPlanUtil.java | 8 +-
.../distribution/AggregationDistributionTest.java | 3 +-
.../storageengine/dataregion/DataRegionTest.java | 48 +-
.../compaction/CompactionOverlapCheckTest.java | 269 +++
...InconsistentCompressionTypeAndEncodingTest.java | 4 +-
.../ReadPointCompactionPerformerTest.java | 2 +-
.../compaction/utils/CompactionCheckerUtils.java | 7 +-
.../utils/CompactionUpdateFileCountTest.java | 2 +-
.../reader/chunk/AlignedMemPageReaderTest.java | 250 +++
.../reader/chunk/MemAlignedChunkLoaderTest.java | 13 +-
.../read/reader/chunk/MemChunkLoaderTest.java | 12 +-
.../read/reader/chunk/MemPageReaderTest.java | 134 ++
...est.java => AbstractAlignedSeriesScanTest.java} | 307 +--
.../read/reader/series/AbstractSeriesScanTest.java | 236 +++
.../AlignedSeriesScanLimitOffsetPushDownTest.java | 404 +---
.../AlignedSeriesScanPredicatePushDownTest.java | 225 +++
...gleColumnSeriesScanLimitOffsetPushDownTest.java | 201 ++
.../series/SeriesScanLimitOffsetPushDownTest.java | 210 +-
.../series/SeriesScanPredicatePushDownTest.java | 318 +++
.../dataregion/snapshot/IoTDBSnapshotTest.java | 4 +
.../datanode/src/test/resources/logback-test.xml | 1 +
iotdb-core/node-commons/pom.xml | 5 +
.../resources/conf/iotdb-common.properties | 14 +-
.../iotdb/commons/client/ClientPoolFactory.java | 2 +-
.../iotdb/commons/concurrent/ThreadName.java | 4 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 19 +-
.../iotdb/commons/conf/CommonDescriptor.java | 13 +-
.../apache/iotdb/commons/conf/IoTDBConstant.java | 1 +
.../iotdb/commons/pipe/config/PipeConfig.java | 11 +-
.../request}/IoTDBConnectorRequestVersion.java | 2 +-
.../payload/request}/PipeRequestType.java | 8 +-
.../request/PipeTransferSnapshotPieceReq.java} | 78 +-
.../request/PipeTransferSnapshotSealReq.java} | 74 +-
.../payload/request/TransferConfigPlanReq.java} | 7 +-
.../response/PipeTransferSnapshotPieceResp.java | 81 +
.../ConcurrentIterableLinkedQueue.java | 345 ++++
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 8 +-
.../connector/IoTDBThriftSslConnector.java} | 16 +-
.../iotdb/commons/schema/filter/SchemaFilter.java | 3 +
.../commons/schema/filter/SchemaFilterFactory.java | 5 +
.../commons/schema/filter/SchemaFilterType.java | 3 +
.../commons/schema/filter/SchemaFilterVisitor.java | 5 +
.../commons/schema/filter/impl/TemplateFilter.java | 73 +
.../apache/iotdb/commons/schema/node/IMNode.java | 12 +-
.../node/common/AbstractAboveDatabaseMNode.java | 18 +-
.../node/common/AbstractDatabaseDeviceMNode.java | 299 ---
.../schema/node/common/AbstractDatabaseMNode.java | 26 +-
.../node/common/AbstractMeasurementMNode.java | 19 +-
...actDeviceMNode.java => DeviceMNodeWrapper.java} | 130 +-
.../commons/schema/node/role/IDeviceMNode.java | 2 +-
.../commons/schema/node/role/IInternalMNode.java} | 43 +-
.../commons/schema/node/visitor/MNodeVisitor.java | 7 -
.../commons/utils/ThriftConfigNodeSerDeUtils.java | 3 +
.../ConcurrentIterableLinkedQueueTest.java | 439 ++++
.../pipe/{ => task/meta}/PipeMetaDeSerTest.java | 7 +-
.../ByteBufferAllocateException.java} | 12 +-
.../iotdb/tsfile/file/header/ChunkGroupHeader.java | 2 +-
.../iotdb/tsfile/file/header/PageHeader.java | 23 +-
.../tsfile/file/metadata/AlignedChunkMetadata.java | 44 +-
.../file/metadata/AlignedTimeSeriesMetadata.java | 59 +-
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 22 +
.../iotdb/tsfile/file/metadata/IChunkMetadata.java | 6 +-
.../metadata/IMetadata.java} | 21 +-
.../tsfile/file/metadata/ITimeSeriesMetadata.java | 9 +-
.../tsfile/file/metadata/TimeseriesMetadata.java | 31 +
.../apache/iotdb/tsfile/read/TimeValuePair.java | 20 +
.../read/TsFileAlignedSeriesReaderIterator.java | 3 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 122 +-
...leSequenceReaderTimeseriesMetadataIterator.java | 30 +-
.../apache/iotdb/tsfile/read/common/TimeRange.java | 10 +-
.../iotdb/tsfile/read/common/block/TsBlock.java | 32 +-
.../tsfile/read/common/block/TsBlockBuilder.java | 7 +-
.../read/controller/CachedChunkLoaderImpl.java | 4 +-
.../iotdb/tsfile/read/controller/IChunkLoader.java | 3 +-
.../base => basic}/BinaryLogicalFilter.java | 10 +-
...ilter.java => DisableStatisticsTimeFilter.java} | 8 +-
...lter.java => DisableStatisticsValueFilter.java} | 22 +-
.../iotdb/tsfile/read/filter/basic/Filter.java | 80 +-
.../tsfile/read/filter/basic/OperatorType.java | 68 +-
.../iotdb/tsfile/read/filter/basic/TimeFilter.java | 68 +
.../tsfile/read/filter/basic/ValueFilter.java | 137 ++
.../tsfile/read/filter/factory/FilterFactory.java | 21 +-
.../{TimeFilter.java => TimeFilterApi.java} | 4 +-
.../tsfile/read/filter/factory/ValueFilter.java | 168 --
.../tsfile/read/filter/factory/ValueFilterApi.java | 186 ++
.../iotdb/tsfile/read/filter/operator/And.java | 30 +-
.../tsfile/read/filter/operator/GroupByFilter.java | 8 +-
.../read/filter/operator/GroupByMonthFilter.java | 7 +-
.../iotdb/tsfile/read/filter/operator/Not.java | 40 +-
.../iotdb/tsfile/read/filter/operator/Or.java | 30 +-
.../read/filter/operator/TimeFilterOperators.java | 196 +-
.../read/filter/operator/ValueFilterOperators.java | 649 +++---
.../filter/operator/base/ColumnCompareFilter.java | 51 -
.../filter/operator/base/ColumnRangeFilter.java | 51 -
.../read/filter/operator/base/ColumnSetFilter.java | 50 -
.../iotdb/tsfile/read/reader/IPageReader.java | 9 +-
.../read/reader/chunk/AbstractChunkReader.java | 83 +
.../read/reader/chunk/AlignedChunkReader.java | 474 ++---
.../tsfile/read/reader/chunk/ChunkReader.java | 239 +--
.../tsfile/read/reader/page/AlignedPageReader.java | 505 ++---
.../iotdb/tsfile/read/reader/page/PageReader.java | 414 ++--
.../tsfile/read/reader/page/ValuePageReader.java | 3 +-
.../reader/series/AbstractFileSeriesReader.java | 16 +-
.../read/reader/series/EmptyFileSeriesReader.java | 4 +-
.../read/reader/series/FileSeriesReader.java | 6 +-
.../reader/series/FileSeriesReaderByTimestamp.java | 4 +-
.../read/reader/series/PaginationController.java | 1 -
.../ITimeFilter.java => utils/Preconditions.java} | 20 +-
.../iotdb/tsfile/common/block/TsBlockTest.java | 77 +
.../apache/iotdb/tsfile/read/ExpressionTest.java | 36 +-
.../iotdb/tsfile/read/ReadInPartitionTest.java | 8 +-
.../org/apache/iotdb/tsfile/read/ReadTest.java | 60 +-
.../iotdb/tsfile/read/TimePlainEncodeReadTest.java | 64 +-
.../apache/iotdb/tsfile/read/TsFileReaderTest.java | 66 +-
.../tsfile/read/filter/FilterSerializeTest.java | 70 +-
.../iotdb/tsfile/read/filter/FilterTestUtil.java | 105 +
.../tsfile/read/filter/GroupByFilterTest.java | 33 +-
.../tsfile/read/filter/GroupByMonthFilterTest.java | 35 +-
.../read/filter/IExpressionOptimizerTest.java | 83 +-
.../read/filter/MinTimeMaxTimeFilterTest.java | 70 +-
.../iotdb/tsfile/read/filter/OperatorTest.java | 95 +-
.../filter/PredicateRemoveNotRewriterTest.java | 62 +-
.../tsfile/read/filter/StatisticsFilterTest.java | 300 +--
.../read/query/executor/QueryExecutorTest.java | 11 +-
.../read/query/timegenerator/ReadWriteTest.java | 10 +-
.../timegenerator/TimeGeneratorReadEmptyTest.java | 10 +-
.../timegenerator/TimeGeneratorReadWriteTest.java | 10 +-
.../query/timegenerator/TimeGeneratorTest.java | 10 +-
.../read/reader/AlignedPageReaderPushDownTest.java | 322 +++
.../iotdb/tsfile/read/reader/ChunkReaderTest.java | 4 +-
.../tsfile/read/reader/PageReaderPushDownTest.java | 145 ++
.../iotdb/tsfile/read/reader/PageReaderTest.java | 6 +-
.../iotdb/tsfile/read/reader/ReaderTest.java | 8 +-
.../tsfile/write/TsFileIntegrityCheckingTool.java | 7 +-
.../openapi/src/main/openapi3/iotdb_rest_v1.yaml | 52 -
.../src/main/thrift/confignode.thrift | 44 +-
pom.xml | 8 +-
431 files changed, 15984 insertions(+), 7676 deletions(-)
create mode 100644 integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterStartIT.java
create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/GrafanaApiServiceIT.java
create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestServiceIT.java
copy iotdb-core/{tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/base/BinaryLogicalFilter.java => confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateClusterIdPlan.java} (51%)
create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/PipeEnrichedPlan.java
create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ClusterManager.java
delete mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/region/CopySetRegionGroupAllocator.java
create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/region/GreedyCopySetRegionGroupAllocator.java
create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/payload/request/PipeTransferConfigPlanReq.java
create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/ClusterInfo.java
create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/region/GreedyCopySetRegionGroupAllocatorTest.java
create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/ClusterInfoTest.java
copy iotdb-core/{tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/ITimeFilter.java => datanode/src/main/java/org/apache/iotdb/db/pipe/config/plugin/env/PipeTaskTemporaryRuntimeEnvironment.java} (63%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferSchemaPlanReq.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/PipePlanToStatementVisitor.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/ColumnInjectOperator.java
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{utils => queryengine/execution/operator/source}/FileLoaderUtils.java (75%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterIdTask.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/optimization/PlanNodePushDown.java
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/{schemaengine/schemaregion/read/resp/info/IDeviceSchemaInfo.java => queryengine/plan/optimization/base/ColumnInjectionPushDown.java} (78%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedConfigSchemaNode.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedDeleteDataNode.java
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/{write => pipe}/PipeEnrichedInsertNode.java (89%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedWriteSchemaNode.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/process/ColumnInjectNode.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/PipeEnrichedInsertBaseStatement.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/PipeEnrichedLoadTsFileStatement.java
copy iotdb-core/{tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/IValueFilter.java => datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowClusterIdStatement.java} (60%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/pipe/PipeEnrichedStatement.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/DatabaseDeviceMNode.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/DeviceMNode.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedDatabaseDeviceMNode.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedDeviceMNode.java
copy integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteDataNodeConfig.java => iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionLastTimeCheckFailedException.java (62%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/element/AlignedPageElement.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/element/NonAlignedPageElement.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionAlignedChunkReader.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionChunkReader.java
copy iotdb-core/{tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/IAlignedPageReader.java => datanode/src/main/java/org/apache/iotdb/db/utils/columngenerator/ColumnGenerator.java} (77%)
copy iotdb-core/{tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/IAlignedPageReader.java => datanode/src/main/java/org/apache/iotdb/db/utils/columngenerator/ColumnGeneratorType.java} (76%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/columngenerator/SlidingTimeColumnGenerator.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/columngenerator/parameter/ColumnGeneratorParameter.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/columngenerator/parameter/SlidingTimeColumnGeneratorParameter.java
rename iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/{PipeThriftRequestTest.java => PipeDataNodeThriftRequestTest.java} (67%)
create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionOverlapCheckTest.java
create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/AlignedMemPageReaderTest.java
create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReaderTest.java
copy iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/{AlignedSeriesScanLimitOffsetPushDownTest.java => AbstractAlignedSeriesScanTest.java} (60%)
create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AbstractSeriesScanTest.java
create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanPredicatePushDownTest.java
create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSingleColumnSeriesScanLimitOffsetPushDownTest.java
create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/SeriesScanPredicatePushDownTest.java
rename iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol => node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/payload/request}/IoTDBConnectorRequestVersion.java (94%)
rename iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable => node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/payload/request}/PipeRequestType.java (89%)
copy iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferFilePieceReq.java => node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/payload/request/PipeTransferSnapshotPieceReq.java} (53%)
copy iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/pipe/connector/payload/evolvable/request/PipeTransferFileSealReq.java => node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/payload/request/PipeTransferSnapshotSealReq.java} (52%)
copy iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/read/resp/info/IDeviceSchemaInfo.java => node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/payload/request/TransferConfigPlanReq.java} (80%)
create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/payload/response/PipeTransferSnapshotPieceResp.java
create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/ConcurrentIterableLinkedQueue.java
rename iotdb-core/{tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/IStatefulFilter.java => node-commons/src/main/java/org/apache/iotdb/commons/pipe/plugin/builtin/connector/IoTDBThriftSslConnector.java} (64%)
create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/filter/impl/TemplateFilter.java
delete mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/common/AbstractDatabaseDeviceMNode.java
rename iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/common/{AbstractDeviceMNode.java => DeviceMNodeWrapper.java} (65%)
rename iotdb-core/{tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/base/ColumnPatternMatchFilter.java => node-commons/src/main/java/org/apache/iotdb/commons/schema/node/role/IInternalMNode.java} (51%)
create mode 100644 iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/datastructure/ConcurrentIterableLinkedQueueTest.java
rename iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/pipe/{ => task/meta}/PipeMetaDeSerTest.java (94%)
rename iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/{read/reader/IAlignedPageReader.java => exception/ByteBufferAllocateException.java} (78%)
rename iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/{read/filter/basic/IDisableStatisticsValueFilter.java => file/metadata/IMetadata.java} (65%)
rename iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/{operator/base => basic}/BinaryLogicalFilter.java (85%)
rename iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/{IDisableStatisticsTimeFilter.java => DisableStatisticsTimeFilter.java} (79%)
rename iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/{IValueFilter.java => DisableStatisticsValueFilter.java} (61%)
create mode 100644 iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/TimeFilter.java
create mode 100644 iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/basic/ValueFilter.java
rename iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/{TimeFilter.java => TimeFilterApi.java} (98%)
delete mode 100644 iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/ValueFilter.java
create mode 100644 iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/factory/ValueFilterApi.java
delete mode 100644 iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/base/ColumnCompareFilter.java
delete mode 100644 iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/base/ColumnRangeFilter.java
delete mode 100644 iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/operator/base/ColumnSetFilter.java
create mode 100644 iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/AbstractChunkReader.java
rename iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/{read/filter/basic/ITimeFilter.java => utils/Preconditions.java} (65%)
create mode 100644 iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/FilterTestUtil.java
create mode 100644 iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/AlignedPageReaderPushDownTest.java
create mode 100644 iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/read/reader/PageReaderPushDownTest.java
(iotdb) 02/04: finish dev sketch tool
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch 2G_metadata_read
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a25aa657843876785a6e6e3eecda9cecbfbfab00
Author: HTHou <hh...@outlook.com>
AuthorDate: Mon Dec 18 12:40:31 2023 +0800
finish dev sketch tool
---
.../apache/iotdb/tsfile/TsFileSequenceRead.java | 2 +-
.../apache/iotdb/db/tools/TsFileSketchTool.java | 41 ++++++++----
.../exception/ByteBufferAllocateException.java | 27 ++++++++
.../iotdb/tsfile/read/TsFileSequenceReader.java | 74 ++++++++++++++++------
4 files changed, 110 insertions(+), 34 deletions(-)
diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java
index ab6d5d67e87..4ea4b42c6e8 100644
--- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java
+++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileSequenceRead.java
@@ -55,7 +55,7 @@ public class TsFileSequenceRead {
"squid:S106"
}) // Suppress high Cognitive Complexity and Standard outputs warning
public static void main(String[] args) throws IOException {
- String filename = "/Volumes/ExtHD/ExtDownloads/2811/1700281998534-405-4-0.tsfile";
+ String filename = "test.tsfile";
if (args.length >= 1) {
filename = args[0];
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
index a13afc9e61a..0f621cf4d5e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.tools;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.exception.ByteBufferAllocateException;
import org.apache.iotdb.tsfile.file.MetaMarker;
import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader;
import org.apache.iotdb.tsfile.file.header.PageHeader;
@@ -62,7 +63,7 @@ public class TsFileSketchTool {
public static void main(String[] args) throws IOException {
Pair<String, String> fileNames = checkArgs(args);
- String filename = "/Volumes/ExtHD/ExtDownloads/2811/1700281998534-405-4-0.tsfile";
+ String filename = fileNames.left;
String outFile = fileNames.right;
System.out.println("TsFile path:" + filename);
System.out.println("Sketch save path:" + outFile);
@@ -109,7 +110,7 @@ public class TsFileSketchTool {
printFileInfo();
// print chunk
- // printChunk(allChunkGroupMetadata);
+ printChunk(allChunkGroupMetadata);
// metadata begins
if (tsFileMetaData.getMetadataIndex().getChildren().isEmpty()) {
@@ -539,14 +540,29 @@ public class TsFileSketchTool {
if (i != metadataIndexListSize - 1) {
endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset();
}
- generateMetadataIndexWithOffset(
- metadataIndexNode.getChildren().get(i).getOffset(),
- endOffset,
- metadataIndexNode.getChildren().get(i),
- deviceId,
- metadataIndexNode.getNodeType(),
- timeseriesMetadataMap,
- needChunkMetadata);
+ try {
+ ByteBuffer nextBuffer =
+ readData(metadataIndexNode.getChildren().get(i).getOffset(), endOffset);
+ generateMetadataIndexWithOffset(
+ metadataIndexNode.getChildren().get(i).getOffset(),
+ metadataIndexNode.getChildren().get(i),
+ nextBuffer,
+ deviceId,
+ metadataIndexNode.getNodeType(),
+ timeseriesMetadataMap,
+ needChunkMetadata);
+ } catch (ByteBufferAllocateException e) {
+ // when the buffer length is over than Integer.MAX_VALUE,
+ // using tsFileInput to get timeseriesMetadataList
+ generateMetadataIndexWithOffsetUsingTsFileInput(
+ metadataIndexNode.getChildren().get(i).getOffset(),
+ endOffset,
+ metadataIndexNode.getChildren().get(i),
+ deviceId,
+ metadataIndexNode.getNodeType(),
+ timeseriesMetadataMap,
+ needChunkMetadata);
+ }
}
}
} catch (BufferOverflowException e) {
@@ -558,12 +574,11 @@ public class TsFileSketchTool {
* Traverse the metadata index from MetadataIndexEntry to get TimeseriesMetadatas
*
* @param metadataIndex MetadataIndexEntry
- * @param buffer byte buffer
* @param deviceId String
* @param timeseriesMetadataMap map: deviceId -> timeseriesMetadata list
* @param needChunkMetadata deserialize chunk metadata list or not
*/
- private void generateMetadataIndexWithOffset(
+ private void generateMetadataIndexWithOffsetUsingTsFileInput(
long start,
long end,
MetadataIndexEntry metadataIndex,
@@ -598,7 +613,7 @@ public class TsFileSketchTool {
if (i != metadataIndexListSize - 1) {
endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset();
}
- generateMetadataIndexWithOffset(
+ generateMetadataIndexWithOffsetUsingTsFileInput(
metadataIndexNode.getChildren().get(i).getOffset(),
endOffset,
metadataIndexNode.getChildren().get(i),
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/exception/ByteBufferAllocateException.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/exception/ByteBufferAllocateException.java
new file mode 100644
index 00000000000..1c8c6193316
--- /dev/null
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/exception/ByteBufferAllocateException.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.exception;
+
+public class ByteBufferAllocateException extends TsFileRuntimeException {
+
+ public ByteBufferAllocateException(String message) {
+ super(message);
+ }
+}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 45a95f5a61d..45c63122246 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.compress.IUnCompressor;
import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
+import org.apache.iotdb.tsfile.exception.ByteBufferAllocateException;
import org.apache.iotdb.tsfile.exception.TsFileRuntimeException;
import org.apache.iotdb.tsfile.exception.TsFileStatisticsMistakesException;
import org.apache.iotdb.tsfile.file.MetaMarker;
@@ -401,17 +402,32 @@ public class TsFileSequenceReader implements AutoCloseable {
return null;
}
List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
- // buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
- tsFileInput.position(metadataIndexPair.left.getOffset());
- while (tsFileInput.position() < metadataIndexPair.right) {
- try {
- timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(tsFileInput, true));
- } catch (Exception e) {
- logger.error(
- "Something error happened while deserializing TimeseriesMetadata of file {}", file);
- throw e;
+ try {
+ buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
+ while (buffer.hasRemaining()) {
+ try {
+ timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer, true));
+ } catch (Exception e) {
+ logger.error(
+ "Something error happened while deserializing TimeseriesMetadata of file {}", file);
+ throw e;
+ }
+ }
+ } catch (ByteBufferAllocateException e) {
+ // when the buffer length is over than Integer.MAX_VALUE,
+ // using tsFileInput to get timeseriesMetadataList
+ tsFileInput.position(metadataIndexPair.left.getOffset());
+ while (tsFileInput.position() < metadataIndexPair.right) {
+ try {
+ timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(tsFileInput, true));
+ } catch (Exception e1) {
+ logger.error(
+ "Something error happened while deserializing TimeseriesMetadata of file {}", file);
+ throw e1;
+ }
}
}
+
// return null if path does not exist in the TsFile
int searchResult = binarySearchInTimeseriesMetadataList(timeseriesMetadataList, measurement);
return searchResult >= 0 ? timeseriesMetadataList.get(searchResult) : null;
@@ -1084,14 +1100,28 @@ public class TsFileSequenceReader implements AutoCloseable {
if (i != metadataIndexListSize - 1) {
endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset();
}
- generateMetadataIndex(
- metadataIndexNode.getChildren().get(i),
- metadataIndexNode.getChildren().get(i).getOffset(),
- endOffset,
- deviceId,
- metadataIndexNode.getNodeType(),
- timeseriesMetadataMap,
- needChunkMetadata);
+ try {
+ ByteBuffer nextBuffer =
+ readData(metadataIndexNode.getChildren().get(i).getOffset(), endOffset);
+ generateMetadataIndex(
+ metadataIndexNode.getChildren().get(i),
+ nextBuffer,
+ deviceId,
+ metadataIndexNode.getNodeType(),
+ timeseriesMetadataMap,
+ needChunkMetadata);
+ } catch (ByteBufferAllocateException e) {
+ // when the buffer length is over than Integer.MAX_VALUE,
+ // using tsFileInput to get timeseriesMetadataList
+ generateMetadataIndexUsingTsFileInput(
+ metadataIndexNode.getChildren().get(i),
+ metadataIndexNode.getChildren().get(i).getOffset(),
+ endOffset,
+ deviceId,
+ metadataIndexNode.getNodeType(),
+ timeseriesMetadataMap,
+ needChunkMetadata);
+ }
}
}
} catch (Exception e) {
@@ -1100,7 +1130,7 @@ public class TsFileSequenceReader implements AutoCloseable {
}
}
- private void generateMetadataIndex(
+ private void generateMetadataIndexUsingTsFileInput(
MetadataIndexEntry metadataIndex,
long start,
long end,
@@ -1133,7 +1163,7 @@ public class TsFileSequenceReader implements AutoCloseable {
if (i != metadataIndexListSize - 1) {
endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset();
}
- generateMetadataIndex(
+ generateMetadataIndexUsingTsFileInput(
metadataIndexNode.getChildren().get(i),
metadataIndexNode.getChildren().get(i).getOffset(),
endOffset,
@@ -1555,8 +1585,12 @@ public class TsFileSequenceReader implements AutoCloseable {
* @return data that been read.
*/
protected ByteBuffer readData(long start, long end) throws IOException {
+ int totalSize = (int) (end - start);
+ if (totalSize < 0) {
+ throw new ByteBufferAllocateException("ByteBuffer capacity < 0: (" + totalSize + " < 0)");
+ }
try {
- return readData(start, (int) (end - start));
+ return readData(start, totalSize);
} catch (Throwable t) {
logger.warn("Exception {} happened while reading data of {}", t.getMessage(), file);
throw t;
(iotdb) 04/04: fix load and query error
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch 2G_metadata_read
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e86c93b9c62a164eea83192be61bba82f7d48d76
Author: HTHou <hh...@outlook.com>
AuthorDate: Mon Dec 18 15:27:56 2023 +0800
fix load and query error
---
.../iotdb/tsfile/read/TsFileSequenceReader.java | 26 ++++++++++---------
...leSequenceReaderTimeseriesMetadataIterator.java | 30 +++++++++++++++++-----
2 files changed, 38 insertions(+), 18 deletions(-)
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index ee0d9c953d2..f7a032d3be0 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -518,18 +518,20 @@ public class TsFileSequenceReader implements AutoCloseable {
} catch (ByteBufferAllocateException e) {
// when the buffer length is over than Integer.MAX_VALUE,
// using tsFileInput to get timeseriesMetadataList
- tsFileInput.position(metadataIndexPair.left.getOffset());
- while (tsFileInput.position() < metadataIndexPair.right) {
- TimeseriesMetadata timeseriesMetadata;
- try {
- timeseriesMetadata = TimeseriesMetadata.deserializeFrom(tsFileInput, true);
- } catch (Exception e1) {
- logger.error(
- "Something error happened while deserializing TimeseriesMetadata of file {}", file);
- throw e1;
- }
- if (allSensors.contains(timeseriesMetadata.getMeasurementId())) {
- timeseriesMetadataList.add(timeseriesMetadata);
+ synchronized (this) {
+ tsFileInput.position(metadataIndexPair.left.getOffset());
+ while (tsFileInput.position() < metadataIndexPair.right) {
+ TimeseriesMetadata timeseriesMetadata;
+ try {
+ timeseriesMetadata = TimeseriesMetadata.deserializeFrom(tsFileInput, true);
+ } catch (Exception e1) {
+ logger.error(
+ "Something error happened while deserializing TimeseriesMetadata of file {}", file);
+ throw e1;
+ }
+ if (allSensors.contains(timeseriesMetadata.getMeasurementId())) {
+ timeseriesMetadataList.add(timeseriesMetadata);
+ }
}
}
}
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
index 7648e402e97..0b53757171c 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReaderTimeseriesMetadataIterator.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.tsfile.read;
+import org.apache.iotdb.tsfile.exception.ByteBufferAllocateException;
import org.apache.iotdb.tsfile.exception.TsFileSequenceReaderTimeseriesMetadataIteratorException;
import org.apache.iotdb.tsfile.file.metadata.MetadataIndexEntry;
import org.apache.iotdb.tsfile.file.metadata.MetadataIndexNode;
@@ -151,12 +152,17 @@ public class TsFileSequenceReaderTimeseriesMetadataIterator
throw new TsFileSequenceReaderTimeseriesMetadataIteratorException(
"currentBuffer still has some data left before deserializeLeafMeasurement");
}
-
- currentBuffer = reader.readData(metadataIndexEntry.getOffset(), endOffset);
-
- timeseriesMetadataMap
- .computeIfAbsent(currentDeviceId, k -> new ArrayList<>())
- .addAll(deserializeTimeseriesMetadata());
+ try {
+ currentBuffer = reader.readData(metadataIndexEntry.getOffset(), endOffset);
+ timeseriesMetadataMap
+ .computeIfAbsent(currentDeviceId, k -> new ArrayList<>())
+ .addAll(deserializeTimeseriesMetadata());
+ } catch (ByteBufferAllocateException e) {
+ reader.position(metadataIndexEntry.getOffset());
+ timeseriesMetadataMap
+ .computeIfAbsent(currentDeviceId, k -> new ArrayList<>())
+ .addAll(deserializeTimeseriesMetadataUsingTsFileInput(endOffset));
+ }
}
private List<TimeseriesMetadata> deserializeTimeseriesMetadata() {
@@ -170,6 +176,18 @@ public class TsFileSequenceReaderTimeseriesMetadataIterator
return timeseriesMetadataList;
}
+ private List<TimeseriesMetadata> deserializeTimeseriesMetadataUsingTsFileInput(long endOffset)
+ throws IOException {
+ final List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
+ while (reader.position() < endOffset
+ && currentTimeseriesMetadataCount < MAX_TIMESERIES_METADATA_COUNT) {
+ timeseriesMetadataList.add(
+ TimeseriesMetadata.deserializeFrom(reader.tsFileInput, needChunkMetadata));
+ currentTimeseriesMetadataCount++;
+ }
+ return timeseriesMetadataList;
+ }
+
private void deserializeInternalNode(
MetadataIndexEntry metadataIndexEntry, long endOffset, boolean isLeafDevice)
throws IOException {
(iotdb) 01/04: Merge branch 'master' of github.com:apache/iotdb into 2G_metadata_read
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch 2G_metadata_read
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a0f9fc0f842e739522a39a0ac0a42dac1a6a08a2
Merge: df77d64e335 720d4729bd1
Author: HTHou <hh...@outlook.com>
AuthorDate: Mon Dec 18 11:35:54 2023 +0800
Merge branch 'master' of github.com:apache/iotdb into 2G_metadata_read
Jenkinsfile | 4 +
.../java/org/apache/iotdb/tsfile/TsFileRead.java | 16 +-
.../apache/iotdb/tsfile/TsFileSequenceRead.java | 2 +-
.../it/env/cluster/config/MppDataNodeConfig.java | 6 +
.../it/env/cluster/node/AbstractNodeWrapper.java | 26 +-
.../iotdb/it/env/cluster/node/DataNodeWrapper.java | 11 +
.../it/env/remote/config/RemoteDataNodeConfig.java | 5 +
.../apache/iotdb/itbase/constant/TestConstant.java | 1 +
.../apache/iotdb/itbase/env/DataNodeConfig.java | 2 +
.../it/cluster/IoTDBClusterRestartIT.java | 17 +-
.../confignode/it/cluster/IoTDBClusterStartIT.java | 93 +
.../it/partition/IoTDBPartitionCreationIT.java | 4 +-
.../apache/iotdb/db/it/GrafanaApiServiceIT.java | 451 +++++
.../org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 2 +-
.../org/apache/iotdb/db/it/IoTDBRestServiceIT.java | 2119 ++++++++++++++++++++
.../IoTDBAlignByDeviceWithTemplateIT.java | 82 +
.../iotdb/db/it/auth/IoTDBClusterAuthorityIT.java | 5 +-
.../iotdb/db/it/auth/IoTDBSeriesPermissionIT.java | 4 +-
.../iotdb/db/it/auth/IoTDBSystemPermissionIT.java | 75 +-
.../apache/iotdb/db/it/groupby/IOTDBGroupByIT.java | 196 ++
.../apache/iotdb/db/it/query/IoTDBResultSetIT.java | 7 +-
.../iotdb/db/it/schema/IoTDBMetadataFetchIT.java | 85 +-
.../org/apache/iotdb/db/it/utils/TestUtils.java | 108 +-
.../apache/iotdb/pipe/it/IoTDBPipeExtractorIT.java | 22 +-
.../apache/iotdb/pipe/it/IoTDBPipeLifeCycleIT.java | 97 +
.../iotdb/pipe/it/IoTDBPipeSwitchStatusIT.java | 22 +-
.../apache/iotdb/pipe/it/IoTDBPipeSyntaxIT.java | 2 +-
.../iotdb/zeppelin/it/IoTDBInterpreterIT.java | 10 +-
.../parameter/PipeParameterValidator.java | 6 +-
.../api/customizer/parameter/PipeParameters.java | 104 +-
iotdb-client/cli/pom.xml | 17 +
.../java/org/apache/iotdb/cli/AbstractCli.java | 2 +-
.../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 15 +-
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../java/org/apache/iotdb/session/Session.java | 8 +-
.../org/apache/iotdb/session/pool/SessionPool.java | 8 +-
.../java/org/apache/iotdb/session/SessionTest.java | 12 +
.../session/pool/SessionPoolExceptionTest.java | 12 +
.../iotdb/spark/tsfile/NarrowConverter.scala | 56 +-
.../apache/iotdb/spark/tsfile/WideConverter.scala | 56 +-
.../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 1 +
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 13 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 4 +
.../resources/conf/iotdb-confignode.properties | 2 +
.../iotdb/confignode/conf/ConfigNodeConfig.java | 2 +-
.../consensus/request/ConfigPhysicalPlan.java | 8 +
.../consensus/request/ConfigPhysicalPlanType.java | 5 +-
.../write/confignode/UpdateClusterIdPlan.java} | 46 +-
.../request/write/pipe/PipeEnrichedPlan.java | 79 +
.../statemachine/ConfigRegionStateMachine.java | 6 +-
.../iotdb/confignode/manager/ClusterManager.java | 67 +
.../iotdb/confignode/manager/ConfigManager.java | 21 +-
.../apache/iotdb/confignode/manager/IManager.java | 10 +
.../iotdb/confignode/manager/ProcedureManager.java | 17 +-
.../manager/load/balancer/RegionBalancer.java | 21 +-
.../region/CopySetRegionGroupAllocator.java | 160 --
.../region/GreedyCopySetRegionGroupAllocator.java | 191 ++
.../region/GreedyRegionGroupAllocator.java | 31 +-
.../manager/partition/PartitionManager.java | 15 +-
.../payload/request/PipeTransferConfigPlanReq.java | 78 +
.../manager/pipe/runtime/PipeHeartbeatParser.java | 4 +-
.../manager/pipe/runtime/PipeMetaSyncer.java | 15 +-
.../manager/pipe/task/PipeTaskCoordinator.java | 29 +-
.../iotdb/confignode/persistence/ClusterInfo.java | 109 +
.../persistence/executor/ConfigPlanExecutor.java | 13 +
.../partition/DatabasePartitionTable.java | 8 +
.../persistence/partition/PartitionInfo.java | 8 +
.../persistence/pipe/PipePluginInfo.java | 13 +-
.../confignode/persistence/pipe/PipeTaskInfo.java | 32 +-
.../confignode/persistence/schema/ConfigMTree.java | 2 +-
.../persistence/schema/mnode/IConfigMNode.java | 28 +
.../schema/mnode/basic/ConfigBasicMNode.java | 50 +-
.../mnode/impl/ConfigBasicInternalMNode.java | 37 +-
.../impl/pipe/AbstractOperatePipeProcedureV2.java | 18 +-
.../runtime/PipeHandleLeaderChangeProcedure.java | 3 +-
.../runtime/PipeHandleMetaChangeProcedure.java | 3 +-
.../impl/pipe/runtime/PipeMetaSyncProcedure.java | 3 +-
.../impl/pipe/task/CreatePipeProcedureV2.java | 6 +-
.../impl/pipe/task/DropPipeProcedureV2.java | 4 +-
.../impl/pipe/task/StartPipeProcedureV2.java | 5 +-
.../impl/pipe/task/StopPipeProcedureV2.java | 5 +-
.../confignode/service/ConfigNodeShutdownHook.java | 2 +
.../thrift/ConfigNodeRPCServiceProcessor.java | 45 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 12 +
.../GreedyCopySetRegionGroupAllocatorTest.java | 203 ++
.../confignode/persistence/ClusterInfoTest.java | 67 +
.../confignode/persistence/PartitionInfoTest.java | 70 +-
.../apache/iotdb/consensus/iot/IoTConsensus.java | 31 +-
.../consensus/iot/IoTConsensusServerImpl.java | 14 +-
.../consensus/iot/client/DispatchLogHandler.java | 5 +-
.../iot/logdispatcher/IndexController.java | 3 +-
.../consensus/iot/logdispatcher/LogDispatcher.java | 11 +
.../consensus/iot/logdispatcher/SyncStatus.java | 4 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 26 +-
.../ratis/metrics/IoTDBMetricRegistry.java | 8 +-
.../iot/logdispatcher/IndexControllerTest.java | 14 +-
.../iotdb/consensus/ratis/DiskGuardianTest.java | 3 +-
.../iotdb/consensus/ratis/RatisConsensusTest.java | 25 +-
.../iotdb/consensus/ratis/RecoverReadTest.java | 4 +-
.../apache/iotdb/consensus/ratis/TestUtils.java | 25 +-
.../resources/conf/iotdb-datanode.properties | 10 +-
.../org/apache/iotdb/db/audit/AuditLogger.java | 2 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 56 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 29 +-
.../iotdb/db/conf/rest/IoTDBRestServiceConfig.java | 1 +
.../db/conf/rest/IoTDBRestServiceDescriptor.java | 119 +-
.../dataregion/DataExecutionVisitor.java | 29 +-
.../schemaregion/SchemaExecutionVisitor.java | 14 +
.../agent/plugin/PipeConnectorConstructor.java | 13 +-
.../db/pipe/agent/plugin/PipePluginAgent.java | 7 +
.../iotdb/db/pipe/agent/task/PipeTaskAgent.java | 6 +-
.../env/PipeTaskTemporaryRuntimeEnvironment.java} | 15 +-
.../request/PipeTransferFilePieceReq.java | 4 +-
.../evolvable/request/PipeTransferFileSealReq.java | 4 +-
.../request/PipeTransferHandshakeReq.java | 4 +-
.../request/PipeTransferSchemaPlanReq.java | 105 +
.../request/PipeTransferTabletBatchReq.java | 4 +-
.../request/PipeTransferTabletBinaryReq.java | 4 +-
.../request/PipeTransferTabletInsertNodeReq.java | 4 +-
.../request/PipeTransferTabletRawReq.java | 4 +-
.../protocol/airgap/IoTDBAirGapConnector.java | 2 +-
.../thrift/sync/IoTDBThriftSyncConnector.java | 48 +-
.../sync/IoTDBThriftSyncConnectorClient.java | 12 +-
.../protocol/writeback/WriteBackConnector.java | 4 +-
.../tsfile/TsFileInsertionDataContainer.java | 6 +-
.../pipe/receiver/PipePlanToStatementVisitor.java | 186 ++
.../pipe/receiver/airgap/IoTDBAirGapReceiver.java | 2 +-
.../pipe/receiver/thrift/IoTDBThriftReceiver.java | 2 +-
.../receiver/thrift/IoTDBThriftReceiverAgent.java | 2 +-
.../receiver/thrift/IoTDBThriftReceiverV1.java | 46 +-
.../iotdb/db/protocol/client/ConfigNodeClient.java | 37 +-
.../rest/v1/handler/RequestValidationHandler.java | 12 -
.../v1/handler/StatementConstructionHandler.java | 59 -
.../protocol/rest/v1/impl/RestApiServiceImpl.java | 47 -
.../rest/v2/handler/RequestValidationHandler.java | 43 +
.../protocol/rest/v2/impl/RestApiServiceImpl.java | 36 +-
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 4 +-
.../common/header/ColumnHeaderConstant.java | 16 +-
.../common/header/DatasetHeaderFactory.java | 4 +
.../execution/executor/RegionWriteExecutor.java | 276 ++-
.../execution/load/LoadTsFileManager.java | 4 +-
.../queryengine/execution/load/TsFileSplitter.java | 2 +-
.../execution/operator/AggregationUtil.java | 16 +-
.../operator/process/AggregationOperator.java | 19 +-
.../operator/process/ColumnInjectOperator.java | 115 ++
.../process/SlidingWindowAggregationOperator.java | 18 +-
.../execution/operator/process/TopKOperator.java | 8 +-
.../operator/schema/source/DeviceSchemaSource.java | 15 +
.../AbstractSeriesAggregationScanOperator.java | 52 +-
.../AlignedSeriesAggregationScanOperator.java | 25 +
.../operator/source/AlignedSeriesScanUtil.java | 163 +-
.../operator/source}/FileLoaderUtils.java | 109 +-
.../source/SeriesAggregationScanOperator.java | 25 +
.../execution/operator/source/SeriesScanUtil.java | 565 +++---
.../queryengine/plan/analyze/AnalyzeVisitor.java | 44 +-
.../plan/analyze/LoadTsfileAnalyzer.java | 43 +-
.../queryengine/plan/analyze/TemplatedAnalyze.java | 42 +-
.../queryengine/plan/execution/QueryExecution.java | 10 +-
.../plan/execution/config/ConfigTaskVisitor.java | 8 +
.../config/executor/ClusterConfigTaskExecutor.java | 102 +-
.../config/executor/IConfigTaskExecutor.java | 10 +
.../config/metadata/ShowClusterIdTask.java | 65 +
.../ConvertPredicateToTimeFilterVisitor.java | 30 +-
.../plan/optimization/LimitOffsetPushDown.java | 11 +-
.../plan/optimization/PlanNodePushDown.java | 76 +
.../base/ColumnInjectionPushDown.java} | 9 +-
.../db/queryengine/plan/parser/ASTVisitor.java | 20 +-
.../plan/planner/LogicalPlanBuilder.java | 16 +-
.../plan/planner/LogicalPlanVisitor.java | 72 +-
.../plan/planner/OperatorTreeGenerator.java | 61 +-
.../planner/distribution/DistributionPlanner.java | 36 +-
.../plan/planner/plan/node/PlanGraphPrinter.java | 13 +
.../plan/planner/plan/node/PlanNode.java | 10 +
.../plan/planner/plan/node/PlanNodeType.java | 22 +-
.../plan/planner/plan/node/PlanVisitor.java | 28 +-
.../planner/plan/node/load/LoadTsFileNode.java | 7 +-
.../node/metedata/write/CreateTimeSeriesNode.java | 2 +-
.../metedata/write/view/AlterLogicalViewNode.java | 43 +-
.../node/pipe/PipeEnrichedConfigSchemaNode.java | 161 ++
.../plan/node/pipe/PipeEnrichedDeleteDataNode.java | 169 ++
.../{write => pipe}/PipeEnrichedInsertNode.java | 19 +-
.../node/pipe/PipeEnrichedWriteSchemaNode.java | 192 ++
.../planner/plan/node/process/AggregationNode.java | 8 +-
.../plan/node/process/ColumnInjectNode.java | 123 ++
.../node/process/SlidingWindowAggregationNode.java | 56 +-
.../source/AlignedSeriesAggregationScanNode.java | 34 +-
.../node/source/SeriesAggregationScanNode.java | 34 +-
.../node/source/SeriesAggregationSourceNode.java | 37 +-
.../planner/plan/node/write/DeleteDataNode.java | 4 +-
.../plan/planner/plan/node/write/InsertNode.java | 10 -
.../planner/plan/parameter/SeriesScanOptions.java | 10 +-
.../queryengine/plan/statement/StatementType.java | 3 +-
.../plan/statement/StatementVisitor.java | 18 +-
.../crud/PipeEnrichedInsertBaseStatement.java | 220 --
.../crud/PipeEnrichedLoadTsFileStatement.java | 137 --
.../metadata/ShowClusterIdStatement.java} | 24 +-
.../metadata/view/CreateLogicalViewStatement.java | 10 +-
.../plan/statement/pipe/PipeEnrichedStatement.java | 72 +
.../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 31 +-
.../schemaregion/mtree/impl/mem/MemMTreeStore.java | 23 +-
.../mtree/impl/mem/mnode/basic/BasicMNode.java | 26 +-
.../impl/mem/mnode/factory/MemMNodeFactory.java | 11 +-
.../impl/mem/mnode/impl/BasicInternalMNode.java | 83 +-
.../impl/mem/mnode/impl/DatabaseDeviceMNode.java | 37 -
.../mtree/impl/mem/mnode/impl/DatabaseMNode.java | 17 +-
.../mtree/impl/mem/mnode/impl/DeviceMNode.java | 36 -
.../mtree/impl/mem/mnode/info/LogicalViewInfo.java | 2 +-
.../impl/mem/snapshot/MemMTreeSnapshotUtil.java | 75 +-
.../mtree/impl/pbtree/CachedMTreeStore.java | 43 +-
.../mtree/impl/pbtree/MTreeBelowSGCachedImpl.java | 31 +-
.../impl/pbtree/mnode/basic/CachedBasicMNode.java | 25 +-
.../pbtree/mnode/factory/CacheMNodeFactory.java | 11 +-
.../mnode/impl/CachedBasicInternalMNode.java | 91 +-
.../mnode/impl/CachedDatabaseDeviceMNode.java | 49 -
.../pbtree/mnode/impl/CachedDatabaseMNode.java | 18 +-
.../impl/pbtree/mnode/impl/CachedDeviceMNode.java | 48 -
.../read/resp/info/IDeviceSchemaInfo.java | 2 +
.../read/resp/info/impl/ShowDevicesResult.java | 11 +
.../schemaregion/utils/MNodeUtils.java | 54 +-
.../utils/filter/DeviceFilterVisitor.java | 26 +
.../apache/iotdb/db/service/IoTDBShutdownHook.java | 3 +-
.../db/storageengine/dataregion/DataRegion.java | 193 +-
.../dataregion/HashLastFlushTimeMap.java | 115 +-
.../dataregion/IDataRegionForQuery.java | 5 +-
.../dataregion/ILastFlushTimeMap.java | 31 +-
.../dataregion/VirtualDataRegion.java | 5 +-
.../CompactionLastTimeCheckFailedException.java | 18 +-
.../execute/task/AbstractCompactionTask.java | 2 +-
.../execute/task/InnerSpaceCompactionTask.java | 8 +-
.../fast/AlignedSeriesCompactionExecutor.java | 19 +-
.../fast/NonAlignedSeriesCompactionExecutor.java | 15 +-
.../executor/fast/SeriesCompactionExecutor.java | 46 +-
.../executor/fast/element/AlignedPageElement.java | 94 +
.../fast/element/NonAlignedPageElement.java | 74 +
.../utils/executor/fast/element/PageElement.java | 96 +-
.../utils/executor/fast/element/PointElement.java | 11 +-
.../fast/reader/CompactionAlignedChunkReader.java | 122 ++
.../fast/reader/CompactionChunkReader.java | 94 +
.../readchunk/AlignedSeriesCompactionExecutor.java | 12 +
.../readchunk/SingleSeriesCompactionExecutor.java | 15 +-
.../execute/utils/reader/PointPriorityReader.java | 2 +-
.../utils/writer/AbstractCompactionWriter.java | 11 +
.../writer/AbstractCrossCompactionWriter.java | 11 +-
.../writer/AbstractInnerCompactionWriter.java | 2 +
.../utils/writer/FastInnerCompactionWriter.java | 4 +
.../compaction/schedule/CompactionWorker.java | 9 +-
.../estimator/AbstractCompactionEstimator.java | 27 +-
.../impl/RewriteCrossSpaceCompactionSelector.java | 6 +-
.../impl/SizeTieredCompactionSelector.java | 8 +-
.../memtable/AlignedReadOnlyMemChunk.java | 12 +-
.../read/reader/chunk/DiskAlignedChunkLoader.java | 13 +-
.../read/reader/chunk/DiskChunkLoader.java | 4 +-
.../read/reader/chunk/MemAlignedChunkLoader.java | 13 +-
.../read/reader/chunk/MemAlignedChunkReader.java | 6 +-
.../read/reader/chunk/MemAlignedPageReader.java | 127 +-
.../read/reader/chunk/MemChunkLoader.java | 4 +-
.../read/reader/chunk/MemChunkReader.java | 13 +-
.../read/reader/chunk/MemPageReader.java | 281 +--
.../metadata/DiskAlignedChunkMetadataLoader.java | 57 +-
.../chunk/metadata/DiskChunkMetadataLoader.java | 48 +-
.../metadata/MemAlignedChunkMetadataLoader.java | 40 +-
.../chunk/metadata/MemChunkMetadataLoader.java | 31 +-
.../dataregion/snapshot/SnapshotTaker.java | 14 +-
.../dataregion/tsfile/TsFileResource.java | 8 +-
.../dataregion/utils/TsFileResourceUtils.java | 52 +-
.../file/AbstractTsFileRecoverPerformer.java | 4 +-
.../quotas/DataNodeThrottleQuotaManager.java | 2 +-
.../rescon/quotas/DefaultOperationQuota.java | 16 +-
.../iotdb/db/tools/TsFileSplitByPartitionTool.java | 2 +-
.../org/apache/iotdb/db/tools/TsFileSplitTool.java | 2 +-
.../db/tools/validate/TsFileValidationTool.java | 2 +-
.../db/trigger/executor/TriggerFireVisitor.java | 17 +-
.../db/utils/columngenerator/ColumnGenerator.java} | 11 +-
.../columngenerator/ColumnGeneratorType.java} | 15 +-
.../SlidingTimeColumnGenerator.java | 52 +
.../parameter/ColumnGeneratorParameter.java | 82 +
.../SlidingTimeColumnGeneratorParameter.java | 80 +
.../apache/iotdb/db/metadata/mnode/MNodeTest.java | 31 +-
.../metadata/mtree/schemafile/SchemaFileTest.java | 2 +-
.../schemaRegion/SchemaRegionBasicTest.java | 42 +-
...est.java => PipeDataNodeThriftRequestTest.java} | 80 +-
.../db/protocol/rest/IoTDBRestServiceTest.java | 15 +
.../operator/AggregationOperatorTest.java | 1 +
.../AlignedSeriesAggregationScanOperatorTest.java | 12 +-
.../execution/operator/OperatorMemoryTest.java | 2 +
.../operator/RawDataAggregationOperatorTest.java | 4 +-
.../SeriesAggregationScanOperatorTest.java | 12 +-
.../SlidingWindowAggregationOperatorTest.java | 1 +
.../operator/UpdateLastCacheOperatorTest.java | 6 +-
.../schema/SchemaQueryScanOperatorTest.java | 6 +-
.../ConvertPredicateToTimeFilterTest.java | 46 +-
.../plan/analyze/QueryTimePartitionTest.java | 108 +-
.../plan/optimization/LimitOffsetPushDownTest.java | 10 +-
.../plan/plan/QueryLogicalPlanUtil.java | 8 +-
.../distribution/AggregationDistributionTest.java | 3 +-
.../storageengine/dataregion/DataRegionTest.java | 48 +-
.../compaction/CompactionOverlapCheckTest.java | 269 +++
...InconsistentCompressionTypeAndEncodingTest.java | 4 +-
.../ReadPointCompactionPerformerTest.java | 2 +-
.../compaction/utils/CompactionCheckerUtils.java | 7 +-
.../utils/CompactionUpdateFileCountTest.java | 2 +-
.../reader/chunk/AlignedMemPageReaderTest.java | 250 +++
.../reader/chunk/MemAlignedChunkLoaderTest.java | 13 +-
.../read/reader/chunk/MemChunkLoaderTest.java | 12 +-
.../read/reader/chunk/MemPageReaderTest.java | 134 ++
...est.java => AbstractAlignedSeriesScanTest.java} | 307 +--
.../read/reader/series/AbstractSeriesScanTest.java | 236 +++
.../AlignedSeriesScanLimitOffsetPushDownTest.java | 404 +---
.../AlignedSeriesScanPredicatePushDownTest.java | 225 +++
...gleColumnSeriesScanLimitOffsetPushDownTest.java | 201 ++
.../series/SeriesScanLimitOffsetPushDownTest.java | 210 +-
.../series/SeriesScanPredicatePushDownTest.java | 318 +++
.../dataregion/snapshot/IoTDBSnapshotTest.java | 4 +
.../datanode/src/test/resources/logback-test.xml | 1 +
iotdb-core/node-commons/pom.xml | 5 +
.../resources/conf/iotdb-common.properties | 14 +-
.../iotdb/commons/client/ClientPoolFactory.java | 2 +-
.../iotdb/commons/concurrent/ThreadName.java | 4 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 19 +-
.../iotdb/commons/conf/CommonDescriptor.java | 13 +-
.../apache/iotdb/commons/conf/IoTDBConstant.java | 1 +
.../iotdb/commons/pipe/config/PipeConfig.java | 11 +-
.../request}/IoTDBConnectorRequestVersion.java | 2 +-
.../payload/request}/PipeRequestType.java | 8 +-
.../request/PipeTransferSnapshotPieceReq.java} | 78 +-
.../request/PipeTransferSnapshotSealReq.java} | 74 +-
.../payload/request/TransferConfigPlanReq.java} | 11 +-
.../response/PipeTransferSnapshotPieceResp.java | 81 +
.../ConcurrentIterableLinkedQueue.java | 345 ++++
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 8 +-
.../connector/IoTDBThriftSslConnector.java} | 16 +-
.../iotdb/commons/schema/filter/SchemaFilter.java | 3 +
.../commons/schema/filter/SchemaFilterFactory.java | 5 +
.../commons/schema/filter/SchemaFilterType.java | 3 +
.../commons/schema/filter/SchemaFilterVisitor.java | 5 +
.../commons/schema/filter/impl/TemplateFilter.java | 73 +
.../apache/iotdb/commons/schema/node/IMNode.java | 12 +-
.../node/common/AbstractAboveDatabaseMNode.java | 18 +-
.../node/common/AbstractDatabaseDeviceMNode.java | 299 ---
.../schema/node/common/AbstractDatabaseMNode.java | 26 +-
.../node/common/AbstractMeasurementMNode.java | 19 +-
...actDeviceMNode.java => DeviceMNodeWrapper.java} | 130 +-
.../commons/schema/node/role/IDeviceMNode.java | 2 +-
.../commons/schema/node/role/IInternalMNode.java} | 43 +-
.../commons/schema/node/visitor/MNodeVisitor.java | 7 -
.../commons/utils/ThriftConfigNodeSerDeUtils.java | 3 +
.../ConcurrentIterableLinkedQueueTest.java | 439 ++++
.../pipe/{ => task/meta}/PipeMetaDeSerTest.java | 7 +-
.../iotdb/tsfile/file/header/ChunkGroupHeader.java | 2 +-
.../iotdb/tsfile/file/header/PageHeader.java | 23 +-
.../tsfile/file/metadata/AlignedChunkMetadata.java | 44 +-
.../file/metadata/AlignedTimeSeriesMetadata.java | 59 +-
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 22 +
.../iotdb/tsfile/file/metadata/IChunkMetadata.java | 6 +-
.../metadata/IMetadata.java} | 21 +-
.../tsfile/file/metadata/ITimeSeriesMetadata.java | 9 +-
.../tsfile/file/metadata/TimeseriesMetadata.java | 31 +
.../apache/iotdb/tsfile/read/TimeValuePair.java | 20 +
.../read/TsFileAlignedSeriesReaderIterator.java | 3 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 5 +-
.../apache/iotdb/tsfile/read/common/TimeRange.java | 10 +-
.../iotdb/tsfile/read/common/block/TsBlock.java | 32 +-
.../tsfile/read/common/block/TsBlockBuilder.java | 7 +-
.../read/controller/CachedChunkLoaderImpl.java | 4 +-
.../iotdb/tsfile/read/controller/IChunkLoader.java | 3 +-
.../base => basic}/BinaryLogicalFilter.java | 10 +-
...ilter.java => DisableStatisticsTimeFilter.java} | 8 +-
...lter.java => DisableStatisticsValueFilter.java} | 22 +-
.../iotdb/tsfile/read/filter/basic/Filter.java | 80 +-
.../tsfile/read/filter/basic/OperatorType.java | 68 +-
.../iotdb/tsfile/read/filter/basic/TimeFilter.java | 68 +
.../tsfile/read/filter/basic/ValueFilter.java | 137 ++
.../tsfile/read/filter/factory/FilterFactory.java | 21 +-
.../{TimeFilter.java => TimeFilterApi.java} | 4 +-
.../tsfile/read/filter/factory/ValueFilter.java | 168 --
.../tsfile/read/filter/factory/ValueFilterApi.java | 186 ++
.../iotdb/tsfile/read/filter/operator/And.java | 30 +-
.../tsfile/read/filter/operator/GroupByFilter.java | 8 +-
.../read/filter/operator/GroupByMonthFilter.java | 7 +-
.../iotdb/tsfile/read/filter/operator/Not.java | 40 +-
.../iotdb/tsfile/read/filter/operator/Or.java | 30 +-
.../read/filter/operator/TimeFilterOperators.java | 196 +-
.../read/filter/operator/ValueFilterOperators.java | 649 +++---
.../filter/operator/base/ColumnCompareFilter.java | 51 -
.../filter/operator/base/ColumnRangeFilter.java | 51 -
.../read/filter/operator/base/ColumnSetFilter.java | 50 -
.../iotdb/tsfile/read/reader/IPageReader.java | 9 +-
.../read/reader/chunk/AbstractChunkReader.java | 83 +
.../read/reader/chunk/AlignedChunkReader.java | 474 ++---
.../tsfile/read/reader/chunk/ChunkReader.java | 239 +--
.../tsfile/read/reader/page/AlignedPageReader.java | 505 ++---
.../iotdb/tsfile/read/reader/page/PageReader.java | 414 ++--
.../tsfile/read/reader/page/ValuePageReader.java | 3 +-
.../reader/series/AbstractFileSeriesReader.java | 16 +-
.../read/reader/series/EmptyFileSeriesReader.java | 4 +-
.../read/reader/series/FileSeriesReader.java | 6 +-
.../reader/series/FileSeriesReaderByTimestamp.java | 4 +-
.../read/reader/series/PaginationController.java | 1 -
.../ITimeFilter.java => utils/Preconditions.java} | 20 +-
.../iotdb/tsfile/common/block/TsBlockTest.java | 77 +
.../apache/iotdb/tsfile/read/ExpressionTest.java | 36 +-
.../iotdb/tsfile/read/ReadInPartitionTest.java | 8 +-
.../org/apache/iotdb/tsfile/read/ReadTest.java | 60 +-
.../iotdb/tsfile/read/TimePlainEncodeReadTest.java | 64 +-
.../apache/iotdb/tsfile/read/TsFileReaderTest.java | 66 +-
.../tsfile/read/filter/FilterSerializeTest.java | 70 +-
.../iotdb/tsfile/read/filter/FilterTestUtil.java | 105 +
.../tsfile/read/filter/GroupByFilterTest.java | 33 +-
.../tsfile/read/filter/GroupByMonthFilterTest.java | 35 +-
.../read/filter/IExpressionOptimizerTest.java | 83 +-
.../read/filter/MinTimeMaxTimeFilterTest.java | 70 +-
.../iotdb/tsfile/read/filter/OperatorTest.java | 95 +-
.../filter/PredicateRemoveNotRewriterTest.java | 62 +-
.../tsfile/read/filter/StatisticsFilterTest.java | 300 +--
.../read/query/executor/QueryExecutorTest.java | 11 +-
.../read/query/timegenerator/ReadWriteTest.java | 10 +-
.../timegenerator/TimeGeneratorReadEmptyTest.java | 10 +-
.../timegenerator/TimeGeneratorReadWriteTest.java | 10 +-
.../query/timegenerator/TimeGeneratorTest.java | 10 +-
.../read/reader/AlignedPageReaderPushDownTest.java | 322 +++
.../iotdb/tsfile/read/reader/ChunkReaderTest.java | 4 +-
.../tsfile/read/reader/PageReaderPushDownTest.java | 145 ++
.../iotdb/tsfile/read/reader/PageReaderTest.java | 6 +-
.../iotdb/tsfile/read/reader/ReaderTest.java | 8 +-
.../tsfile/write/TsFileIntegrityCheckingTool.java | 7 +-
.../openapi/src/main/openapi3/iotdb_rest_v1.yaml | 52 -
.../src/main/thrift/confignode.thrift | 44 +-
pom.xml | 8 +-
428 files changed, 15840 insertions(+), 7622 deletions(-)
(iotdb) 03/04: fix query
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch 2G_metadata_read
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ec191a2a4b2f68e1a99ff2784ea73fe5dbd5a159
Author: HTHou <hh...@outlook.com>
AuthorDate: Mon Dec 18 12:47:14 2023 +0800
fix query
---
.../iotdb/tsfile/read/TsFileSequenceReader.java | 41 ++++++++++++++++------
1 file changed, 30 insertions(+), 11 deletions(-)
diff --git a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 45c63122246..ee0d9c953d2 100644
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -500,18 +500,37 @@ public class TsFileSequenceReader implements AutoCloseable {
}
List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
- ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
- while (buffer.hasRemaining()) {
- TimeseriesMetadata timeseriesMetadata;
- try {
- timeseriesMetadata = TimeseriesMetadata.deserializeFrom(buffer, true);
- } catch (Exception e) {
- logger.error(
- "Something error happened while deserializing TimeseriesMetadata of file {}", file);
- throw e;
+ try {
+ ByteBuffer buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
+ while (buffer.hasRemaining()) {
+ TimeseriesMetadata timeseriesMetadata;
+ try {
+ timeseriesMetadata = TimeseriesMetadata.deserializeFrom(buffer, true);
+ } catch (Exception e) {
+ logger.error(
+ "Something error happened while deserializing TimeseriesMetadata of file {}", file);
+ throw e;
+ }
+ if (allSensors.contains(timeseriesMetadata.getMeasurementId())) {
+ timeseriesMetadataList.add(timeseriesMetadata);
+ }
}
- if (allSensors.contains(timeseriesMetadata.getMeasurementId())) {
- timeseriesMetadataList.add(timeseriesMetadata);
+ } catch (ByteBufferAllocateException e) {
+ // when the buffer length is over than Integer.MAX_VALUE,
+ // using tsFileInput to get timeseriesMetadataList
+ tsFileInput.position(metadataIndexPair.left.getOffset());
+ while (tsFileInput.position() < metadataIndexPair.right) {
+ TimeseriesMetadata timeseriesMetadata;
+ try {
+ timeseriesMetadata = TimeseriesMetadata.deserializeFrom(tsFileInput, true);
+ } catch (Exception e1) {
+ logger.error(
+ "Something error happened while deserializing TimeseriesMetadata of file {}", file);
+ throw e1;
+ }
+ if (allSensors.contains(timeseriesMetadata.getMeasurementId())) {
+ timeseriesMetadataList.add(timeseriesMetadata);
+ }
}
}
return timeseriesMetadataList;