You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2023/11/16 02:20:45 UTC
(iotdb) branch load_v2 updated (f43bb655f29 -> b4b31e4ac66)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a change to branch load_v2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from f43bb655f29 fix tests
add cc5ed977507 Pipe: refined key set of tags for pipe metrics & fixed hit rate in PipeWALInsertNodeCacheMetrics always `1.0`(#11484)
add fd3911f4ca6 Pipe: convert the value of source, processor, sink key to lowercase & convert `SHOW_PIPE_PLUGINS_BLACKLIST` keys to uppercase (#11487)
add 0f328ce5eb9 Fix empty tsfile genarated when all datatypes in one insert mismatching (#11456)
add f7b26846a0a Add Metric Prometheus Reporter IT (#11443)
add 0b8faac5f75 PBTree: Logging Disabled with Ratis and Pre-Allocation Disabled for devices using template (#11467)
add f594958f0da Feat/configurable home directory (#11492)
add ed3817eaac3 fix: Increase the timeout even more to finally make the build succeed.
add 9077f03e1a4 Fix NPE in SchemaStatisticsTest (#11490)
add 4b0fc1d7ec6 [IOTDB-6239] Show regions display error create time (#11481)
add b478d96ad21 Fix datanode stuck when meets disk error (#11486)
add 16eee23443b Simplify library-udf packaging result
add 058c5f6eab6 [IOTDB-6241] Load: Check System READ_ONLY mode in Analysis, Dispatch and Writing Stage (#11488)
add 3e1feac4d9e [IOTDB-1476]IoTDB support IPv6 (#11457)
add 8f3729fb825 Update ReadMe in IT module (#11499)
add 31d0c5bc1e8 Pipe: Added check in PipeSubtask submit logic to ensure one PipeConnectorSubtask instance is running in only one thread (#11474)
add a4510266fdb [IOTDB-6206] Metric Core: Shrink ringBuffer length to optimize histogram and timer's record performance (#11500)
add 1f518743f9f Fix aggregation error when inserting negative timestamp
add 05a608f8ea1 Pipe: Make each connector subtask inject cron event at fix rate to avoid random time of batch transmission (#11501)
add 1ea88fa2030 [IOTDB-6238] Fix delete timeSeries / deactivate template procedure rollback logic (#11479)
add 45c2fdf5535 Pipe: introduce pipe_max_allowed_pinned_memtable_count param for realtime hybrid mode (#11505)
add e094b3649f5 add time cost for datanode start (#11498)
add 33a3d73ed34 Pipe IT: adjust -DintegrationTest.forkCount & -DDataNodeMaxHeapSize in pipe-it-2cluster.yml to provide more stable env (#11504)
add bbc57ec98ff Check empty target file after sequence InnerSpaceCompactionTask (#11454)
add bb79cf16592 Add compaction logs and metrics (#11448)
add 98c4aa324ec Pipe: skip pipe CN heartbeat and sync when `PipeTaskCoordinatorLock` is held by another thread (#11509)
add d3548ed2acd [IOTDB-6209] Pipe: Solving the topological order of the progress index in the historical data collection phase (#11478)
add 8901076ec17 [IOTDB-6227] Add retry for RatisConsensus read (#11458)
add 58d3a129113 Fix exception add database name message (#11507)
add 47bc342f407 [IOTDB-6245] Load: Adjust forceCloseWriterManager execution delay in LoadTsFileManager to 15mins to avoid OOM & clear all data structures after use to help GC (#11506)
add 4eb2015166b [IOTDB-6186] Grafana does not support group by time and level (#11338)
add 872482ce8b9 fix(conf): add e.g. for ssl (#11512)
add 93295a1cf8e Check and update searchIndex when restarting dataNode if necessary (#11510)
add 369041f3672 Optimize batch fetch schema for data insertion
add aa9f7f1f424 Pipe: collect total memory size with pipe module instead of memory usage (#11513)
add c86cb0e76c6 [IOTDB-6246] Pipe: Validator validates the range of values for optional parameters improperly (#11520)
add 27d219d07b4 remove duplicate description of metric (#11519)
add bbc053fbc4b Pipe: Added concurrency check to avoid WALEntryHandler#getInsertNode returning null to fix potential NPE (#11523)
add acbf2057ad8 Add logs in compaction recovery (#11433)
add 918ee105303 [IOTDB-6247] Remove redundant error log when restarting simple consensus ConfigNode (#11529)
add 726c4733910 [IOTDB-6239] Revert updation of CreateRegionGroupsPlan (#11528)
add 3a4c085afee Add OFF level (#11530)
add f7b899b1e05 fix insertion compaction replace keyword bug (#11514)
add ebca20e5d83 Validate flushed tsfile without holding closeQueryLock (#11508)
add 29c9186525d Pipe: Enable dynamic adjusting of allocated pipe memory (#11518)
add 070a514f141 Pipe IT: Enable Pipe IT to tolerate failures caused by resource shortage (#11527)
add 3eb76822ca0 Support high available while meeting timeout exception
add 6bb3d93be13 Pipe IT: Fix WALFlushListener.testConcurrentGetValue() & Fix smells reported by sonar (#11525)
add 723b5718f3b Bump go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc (#11517)
add 180e5f1fe73 Implement more fine-grained time slice control for the AggregationScanOperator (#11521)
add 31ad3ffc825 Pipe IT: Enable IoTDBPipeExtractorIT & IoTDBPipeSwitchStatusIT to tolerate failures caused by resource shortage (#11539)
add 0d88353b824 fix compaction logs and metrics (#11533)
add f11c252809e DataNode & ConfigNode exit directly while meeting OutOfMemoryError
add 71043c1fed6 add nonnull to ProgressIndex's topological sort (#11542)
add f1fc5b1de10 return WRITE_PROCESS_ERROR when catching BatchProcessException in writing process (#11543)
add 6730cb106d9 add file validate when rename file, avoid rename error because file is not exist (#11536)
add 2bfffe6f73a Fix insertRecordsOfOneDevice causes data out of order (#11535)
add 36bb554e756 Pipe: Make the pipe to detect illegal parameters when there is no data (#11541)
add de96005b9bd Fix aligned series modification not matched in MultiTsFileDeviceIterator (#11544)
add fbbb0200d2b Increase retry timeout for ratis retry mechanism (#11548)
add 93cc6aa56ec Refactor CachedMTreeStore.updateMNode interface usage and implementation in PBTree (#11546)
add b4a1d020ca5 Fix update and recovery of LastFlushTimeMap (#11538)
add f3b3d3979c6 use fast constructor of PartialPath in Compaction (#11547)
new b4b31e4ac66 Merge branch 'master' into load_v2
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/pipe-it-2cluster.yml | 6 +-
Jenkinsfile | 2 +-
distribution/src/assembly/library-udf.xml | 23 +-
.../java/org/apache/iotdb/CountPointProcessor.java | 15 +-
integration-test/README.md | 24 +-
.../org/apache/iotdb/it/env/cluster/EnvUtils.java | 5 +-
.../it/env/cluster/config/MppCommonConfig.java | 16 ++
.../it/env/cluster/config/MppConfigNodeConfig.java | 7 +
.../it/env/cluster/config/MppDataNodeConfig.java | 7 +
.../env/cluster/config/MppSharedCommonConfig.java | 15 +
.../iotdb/it/env/cluster/env/AbstractEnv.java | 53 ++--
.../it/env/cluster/node/AbstractNodeWrapper.java | 7 +
.../it/env/cluster/node/ConfigNodeWrapper.java | 7 +-
.../iotdb/it/env/cluster/node/DataNodeWrapper.java | 5 +-
.../it/env/remote/config/RemoteCommonConfig.java | 11 +
.../env/remote/config/RemoteConfigNodeConfig.java | 9 +-
.../it/env/remote/config/RemoteDataNodeConfig.java | 9 +-
.../iotdb/it/env/remote/env/RemoteServerEnv.java | 17 ++
.../java/org/apache/iotdb/itbase/env/BaseEnv.java | 32 +++
.../apache/iotdb/itbase/env/BaseNodeWrapper.java | 2 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 4 +
.../apache/iotdb/itbase/env/ConfigNodeConfig.java | 6 +-
.../apache/iotdb/itbase/env/DataNodeConfig.java | 6 +-
.../apache/iotdb/db/it/IoTDBInsertMultiRowIT.java | 21 ++
.../apache/iotdb/db/it/IoTDBInsertWithQueryIT.java | 23 ++
...ery2IT.java => IoTDBAlignedSeriesQuery5IT.java} | 8 +-
.../IoTDBMetricIT.java} | 32 ++-
.../org/apache/iotdb/db/it/utils/TestUtils.java | 57 +++-
.../apache/iotdb/pipe/it/IoTDBPipeClusterIT.java | 130 ++++++---
.../pipe/it/IoTDBPipeConnectorParallelIT.java | 11 +-
.../apache/iotdb/pipe/it/IoTDBPipeDataSyncIT.java | 14 +-
.../apache/iotdb/pipe/it/IoTDBPipeLifeCycleIT.java | 192 +++++++++----
.../apache/iotdb/pipe/it/IoTDBPipeProtocolIT.java | 48 ++--
.../iotdb/pipe/it/IoTDBPipeSwitchStatusIT.java | 10 +-
.../apache/iotdb/pipe/it/IoTDBPipeSyntaxIT.java | 43 ++-
.../pipe/it/extractor/IoTDBPipeExtractorIT.java | 231 ++++++++++-----
.../parameter/PipeParameterValidator.java | 4 -
.../main/java/org/apache/iotdb/jdbc/Config.java | 2 +
.../src/main/java/org/apache/iotdb/jdbc/Utils.java | 27 +-
.../test/java/org/apache/iotdb/jdbc/UtilsTest.java | 25 ++
.../main/java/org/apache/iotdb/rpc/UrlUtils.java | 52 ++++
.../java/org/apache/iotdb/rpc/UrlUtilsTest.java | 61 ++++
.../apache/iotdb/session/util/SessionUtils.java | 3 +-
iotdb-connector/grafana-plugin/go.mod | 22 +-
.../src/assembly/resources/conf/confignode-env.bat | 4 +
.../src/assembly/resources/conf/confignode-env.sh | 3 +
.../resources/conf/iotdb-confignode.properties | 7 +-
.../write/region/CreateRegionGroupsPlan.java | 35 +--
.../manager/consensus/ConsensusManager.java | 18 +-
.../manager/load/balancer/RegionBalancer.java | 2 +-
.../pipe/runtime/PipeHeartbeatScheduler.java | 6 +
.../manager/pipe/runtime/PipeMetaSyncer.java | 8 +-
.../manager/pipe/task/PipeTaskCoordinator.java | 7 +-
.../manager/pipe/task/PipeTaskCoordinatorLock.java | 4 +
.../partition/DatabasePartitionTable.java | 4 +-
.../persistence/partition/PartitionInfo.java | 4 +-
.../persistence/schema/ConfigMTreeStore.java | 3 +-
.../impl/schema/DeactivateTemplateProcedure.java | 40 +--
.../impl/schema/DeleteDatabaseProcedure.java | 6 +-
.../impl/schema/DeleteTimeSeriesProcedure.java | 44 +--
.../impl/schema/UnsetTemplateProcedure.java | 9 +-
.../statemachine/CreateRegionGroupsProcedure.java | 10 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 1 -
.../impl/CreateRegionGroupsProcedureTest.java | 1 +
.../confignode1conf/iotdb-confignode.properties | 1 -
.../confignode2conf/iotdb-confignode.properties | 1 -
.../confignode3conf/iotdb-confignode.properties | 1 -
.../consensus/iot/IoTConsensusServerImpl.java | 21 ++
.../ratis/ApplicationStateMachineProxy.java | 16 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 91 +++---
.../iotdb/consensus/ratis/utils/Retriable.java | 107 +++++++
.../iotdb/consensus/ratis/utils/RetryPolicy.java | 79 ++++++
.../apache/iotdb/consensus/ratis/utils/Utils.java | 10 +
.../iotdb/consensus/ratis/RecoverReadTest.java | 3 +-
.../src/assembly/resources/conf/datanode-env.bat | 3 +-
.../src/assembly/resources/conf/datanode-env.sh | 3 +-
.../resources/conf/iotdb-datanode.properties | 19 +-
.../org/apache/iotdb/db/audit/AuditLogger.java | 4 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 +-
.../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 5 +
.../db/consensus/DataRegionConsensusImpl.java | 2 +
.../db/consensus/SchemaRegionConsensusImpl.java | 2 +
.../dataregion/DataExecutionVisitor.java | 4 +-
...geException.java => LoadReadOnlyException.java} | 9 +-
.../agent/plugin/PipeConnectorConstructor.java | 112 ++++++++
.../agent/plugin/PipeExtractorConstructor.java | 58 ++++
.../db/pipe/agent/plugin/PipePluginAgent.java | 111 ++++----
.../pipe/agent/plugin/PipePluginConstructor.java | 85 ++++++
.../agent/plugin/PipeProcessorConstructor.java | 52 ++++
.../pipe/agent/runtime/PipeCronEventInjector.java | 70 -----
.../agent/runtime/PipePeriodicalJobExecutor.java | 100 +++++++
.../db/pipe/agent/runtime/PipeRuntimeAgent.java | 7 +-
...oTDBThriftAsyncPipeTransferBatchReqBuilder.java | 2 +-
...IoTDBThriftSyncPipeTransferBatchReqBuilder.java | 2 +-
.../builder/PipeTransferBatchReqBuilder.java | 30 +-
.../thrift/async/IoTDBThriftAsyncConnector.java | 2 +-
.../pipe/extractor/IoTDBDataRegionExtractor.java | 21 +-
.../PipeHistoricalDataRegionTsFileExtractor.java | 21 +-
.../PipeRealtimeDataRegionHybridExtractor.java | 43 ++-
.../PipeRealtimeDataRegionLogExtractor.java | 2 +-
.../realtime/assigner/DisruptorQueue.java | 2 +-
.../iotdb/db/pipe/metric/PipeConnectorMetrics.java | 86 +++++-
.../iotdb/db/pipe/metric/PipeExtractorMetrics.java | 114 ++++++--
.../iotdb/db/pipe/metric/PipeProcessorMetrics.java | 86 +++++-
.../iotdb/db/pipe/metric/PipeResourceMetrics.java | 23 +-
.../db/pipe/resource/memory/PipeMemoryBlock.java | 137 ++++++++-
.../db/pipe/resource/memory/PipeMemoryManager.java | 119 ++++++--
.../db/pipe/task/stage/PipeTaskExtractorStage.java | 15 +-
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 16 +-
.../iotdb/db/pipe/task/subtask/PipeSubtask.java | 14 +-
.../subtask/connector/PipeConnectorSubtask.java | 121 ++++++--
.../connector/PipeConnectorSubtaskManager.java | 78 +-----
.../subtask/processor/PipeProcessorSubtask.java | 21 +-
.../iotdb/db/protocol/client/ConfigNodeInfo.java | 14 +-
.../rest/v1/impl/GrafanaApiServiceImpl.java | 3 +-
.../rest/v2/impl/GrafanaApiServiceImpl.java | 3 +-
.../SingleTimeWindowIterator.java | 3 +-
.../execution/operator/AggregationUtil.java | 2 +-
.../operator/schema/SchemaFetchScanOperator.java | 8 +-
.../AbstractSeriesAggregationScanOperator.java | 142 ++++++----
.../queryengine/plan/analyze/AnalyzeVisitor.java | 22 ++
.../plan/analyze/LoadTsfileAnalyzer.java | 23 ++
.../config/executor/ClusterConfigTaskExecutor.java | 11 +
.../db/queryengine/plan/parser/ASTVisitor.java | 19 +-
.../plan/planner/plan/node/write/InsertNode.java | 11 +
.../plan/node/write/InsertRowsOfOneDeviceNode.java | 46 +--
.../planner/plan/parameter/SeriesScanOptions.java | 7 +-
.../plan/scheduler/AsyncSendPlanNodeHandler.java | 7 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 16 +-
.../schemaengine/schemaregion/ISchemaRegion.java | 4 +
.../schemaregion/impl/SchemaRegionMemoryImpl.java | 15 +
.../schemaregion/impl/SchemaRegionPBTreeImpl.java | 41 ++-
.../schemaregion/mtree/IMTreeStore.java | 9 +-
.../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 25 ++
.../schemaregion/mtree/impl/mem/MemMTreeStore.java | 2 +-
.../mtree/impl/pbtree/CachedMTreeStore.java | 36 ++-
.../mtree/impl/pbtree/MTreeBelowSGCachedImpl.java | 100 ++++---
.../pbtree/ReentrantReadOnlyCachedMTreeStore.java | 5 +-
.../mtree/impl/pbtree/cache/CacheManager.java | 35 ++-
.../mtree/impl/pbtree/schemafile/SchemaFile.java | 35 ++-
.../schemafile/pagemgr/BTreePageManager.java | 6 +-
.../pbtree/schemafile/pagemgr/PageManager.java | 71 +++--
.../schemaregion/mtree/traverser/Traverser.java | 16 ++
.../traverser/basic/MeasurementTraverser.java | 13 +
.../traverser/collector/MeasurementCollector.java | 5 +
.../java/org/apache/iotdb/db/service/DataNode.java | 63 +++--
.../apache/iotdb/db/service/IoTDBShutdownHook.java | 3 +
.../db/service/metrics/CompactionMetrics.java | 311 +++++++++++++++++++++
.../iotdb/db/storageengine/StorageEngine.java | 7 +
.../db/storageengine/dataregion/DataRegion.java | 102 +++++--
.../dataregion/HashLastFlushTimeMap.java | 2 +-
.../impl/ReadChunkCompactionPerformer.java | 10 +-
.../execute/recover/CompactionRecoverManager.java | 18 +-
.../execute/task/AbstractCompactionTask.java | 63 ++++-
.../execute/task/CrossSpaceCompactionTask.java | 7 +-
.../execute/task/InnerSpaceCompactionTask.java | 35 +--
.../task/InsertionCrossSpaceCompactionTask.java | 24 +-
.../execute/utils/MultiTsFileDeviceIterator.java | 20 +-
.../fast/AlignedSeriesCompactionExecutor.java | 18 +-
.../fast/NonAlignedSeriesCompactionExecutor.java | 9 +-
.../compaction/io/CompactionTsFileWriter.java | 11 +
.../schedule/CompactionScheduleSummary.java | 72 +++++
.../compaction/schedule/CompactionScheduler.java | 36 ++-
.../compaction/schedule/CompactionWorker.java | 13 +-
.../impl/RewriteCrossSpaceCompactionSelector.java | 11 +-
.../dataregion/flush/MemTableFlushTask.java | 4 +-
.../dataregion/tsfile/TsFileManager.java | 6 +-
.../dataregion/tsfile/TsFileResource.java | 5 +-
.../storageengine/dataregion/wal/node/WALNode.java | 7 +-
.../dataregion/wal/utils/WALEntryHandler.java | 2 +-
.../dataregion/wal/utils/WALEntryPosition.java | 8 +-
.../dataregion/wal/utils/WALInsertNodeCache.java | 63 ++++-
.../wal/utils/listener/AbstractResultListener.java | 2 +
.../wal/utils/listener/WALFlushListener.java | 2 +-
.../db/storageengine/rescon/disk/TierManager.java | 3 +
.../db/storageengine/rescon/memory/SystemInfo.java | 52 +++-
.../rescon/memory/TimePartitionManager.java | 6 +
.../org/apache/iotdb/db/utils/DateTimeUtils.java | 14 -
.../mtree/schemafile/SchemaFileLogTest.java | 13 +
.../schemaRegion/SchemaRegionTemplateTest.java | 42 +--
.../db/pipe/agent/plugin/PipePluginAgentTest.java | 6 +-
.../PipeConnectorSubtaskExecutorTest.java | 3 +
.../PipeProcessorSubtaskExecutorTest.java | 3 +
.../db/protocol/rest/GrafanaApiServiceTest.java | 92 +++++-
.../schema/SchemaFetchScanOperatorTest.java | 7 +
.../plan/node/write/WritePlanNodeSplitTest.java | 80 ++++++
.../storageengine/dataregion/DataRegionTest.java | 120 ++++++++
.../TsFileResourceProgressIndexTest.java | 162 ++++++++++-
.../FastInnerCompactionPerformerTest.java | 2 +-
.../compaction/ReadChunkInnerCompactionTest.java | 190 +++++++++++++
...sSpaceCompactionWithReadPointPerformerTest.java | 4 +-
.../inner/InnerCompactionEmptyTsFileTest.java | 2 +-
.../inner/InnerCompactionSchedulerTest.java | 10 +-
.../dataregion/wal/node/WALEntryHandlerTest.java | 9 +-
.../datanode1conf/iotdb-datanode.properties | 1 -
.../datanode2conf/iotdb-datanode.properties | 1 -
.../datanode3conf/iotdb-datanode.properties | 1 -
iotdb-core/metrics/ReadMe.md | 18 +-
.../iotdb/metrics/core/IoTDBMetricManager.java | 3 +-
.../iotdb/metrics/AbstractMetricManager.java | 10 +-
.../iotdb/metrics/impl/DoNothingMetricManager.java | 2 +-
...nager.java => DoNothingDiskMetricsManager.java} | 3 +-
.../metricsets/disk/IDiskMetricsManager.java | 24 +-
...Manager.java => DoNothingNetMetricManager.java} | 2 +-
.../metrics/metricsets/net/INetMetricManager.java | 33 ++-
.../metrics/metricsets/system/SystemMetrics.java | 40 +--
.../apache/iotdb/metrics/utils/MetricLevel.java | 2 +-
.../iotdb/metrics/config/MetricConfigTest.java | 2 -
.../iotdb/commons/concurrent/ThreadName.java | 4 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 36 +++
.../iotdb/commons/conf/CommonDescriptor.java | 15 +
.../apache/iotdb/commons/conf/IoTDBConstant.java | 5 +
.../commons/consensus/index/ProgressIndex.java | 144 ++++++++--
.../consensus/index/impl/HybridProgressIndex.java | 36 ++-
.../consensus/index/impl/IoTProgressIndex.java | 19 +-
.../consensus/index/impl/MinimumProgressIndex.java | 12 +-
.../consensus/index/impl/RecoverProgressIndex.java | 22 +-
.../consensus/index/impl/SimpleProgressIndex.java | 11 +-
.../apache/iotdb/commons/path/PathPatternNode.java | 6 +-
.../apache/iotdb/commons/path/PathPatternTree.java | 17 +-
.../apache/iotdb/commons/path/fa/dfa/DFAState.java | 6 +-
.../iotdb/commons/path/fa/dfa/PatternDFA.java | 63 +++--
.../iotdb/commons/path/fa/dfa/graph/DFAGraph.java | 80 ++++--
.../iotdb/commons/pipe/config/PipeConfig.java | 16 ++
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 31 +-
.../commons/schema/tree/AbstractTreeVisitor.java | 15 +-
.../iotdb/commons/service/RegisterManager.java | 6 +
.../iotdb/commons/service/metric/enums/Metric.java | 8 +-
.../iotdb/commons/service/metric/enums/Tag.java | 4 +-
.../iotdb/commons/utils/CommonDateTimeUtils.java | 15 +
.../apache/iotdb/commons/utils/NodeUrlUtils.java | 23 +-
.../apache/iotdb/commons/path/PatternDFATest.java | 35 ++-
.../iotdb/commons/pipe/PipeMetaDeSerTest.java | 4 +-
.../iotdb/commons/utils/NodeUrlUtilsTest.java | 33 +++
.../tsfile/write/chunk/AlignedChunkWriterImpl.java | 5 +
.../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 5 +
.../iotdb/tsfile/write/chunk/IChunkWriter.java | 3 +
237 files changed, 5315 insertions(+), 1477 deletions(-)
copy integration-test/src/test/java/org/apache/iotdb/db/it/aligned/{IoTDBAlignedSeriesQuery2IT.java => IoTDBAlignedSeriesQuery5IT.java} (90%)
copy integration-test/src/test/java/org/apache/iotdb/db/it/{selectinto/IoTDBSelectInto2IT.java => metric/IoTDBMetricIT.java} (65%)
create mode 100644 iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/UrlUtils.java
create mode 100644 iotdb-client/service-rpc/src/test/java/org/apache/iotdb/rpc/UrlUtilsTest.java
create mode 100644 iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/utils/Retriable.java
create mode 100644 iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/utils/RetryPolicy.java
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/{MergeException.java => LoadReadOnlyException.java} (77%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/PipeConnectorConstructor.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/PipeExtractorConstructor.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/PipePluginConstructor.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/plugin/PipeProcessorConstructor.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipeCronEventInjector.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/runtime/PipePeriodicalJobExecutor.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleSummary.java
copy iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/disk/{MacDiskMetricsManager.java => DoNothingDiskMetricsManager.java} (86%)
copy iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/net/{MacNetMetricManager.java => DoNothingNetMetricManager.java} (92%)
(iotdb) 01/01: Merge branch 'master' into load_v2
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch load_v2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b4b31e4ac660a0cd62f46927ec4ec0743dbf04ad
Merge: f43bb655f29 f3b3d3979c6
Author: Tian Jiang <jt...@163.com>
AuthorDate: Thu Nov 16 10:25:33 2023 +0800
Merge branch 'master' into load_v2
# Conflicts:
# iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java
# iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
.github/workflows/pipe-it-2cluster.yml | 6 +-
Jenkinsfile | 2 +-
distribution/src/assembly/library-udf.xml | 23 +-
.../java/org/apache/iotdb/CountPointProcessor.java | 15 +-
integration-test/README.md | 24 +-
.../org/apache/iotdb/it/env/cluster/EnvUtils.java | 5 +-
.../it/env/cluster/config/MppCommonConfig.java | 16 ++
.../it/env/cluster/config/MppConfigNodeConfig.java | 7 +
.../it/env/cluster/config/MppDataNodeConfig.java | 7 +
.../env/cluster/config/MppSharedCommonConfig.java | 15 +
.../iotdb/it/env/cluster/env/AbstractEnv.java | 53 ++--
.../it/env/cluster/node/AbstractNodeWrapper.java | 7 +
.../it/env/cluster/node/ConfigNodeWrapper.java | 7 +-
.../iotdb/it/env/cluster/node/DataNodeWrapper.java | 5 +-
.../it/env/remote/config/RemoteCommonConfig.java | 11 +
.../env/remote/config/RemoteConfigNodeConfig.java | 9 +-
.../it/env/remote/config/RemoteDataNodeConfig.java | 9 +-
.../iotdb/it/env/remote/env/RemoteServerEnv.java | 17 ++
.../java/org/apache/iotdb/itbase/env/BaseEnv.java | 32 +++
.../apache/iotdb/itbase/env/BaseNodeWrapper.java | 2 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 4 +
.../apache/iotdb/itbase/env/ConfigNodeConfig.java | 6 +-
.../apache/iotdb/itbase/env/DataNodeConfig.java | 6 +-
.../apache/iotdb/db/it/IoTDBInsertMultiRowIT.java | 21 ++
.../apache/iotdb/db/it/IoTDBInsertWithQueryIT.java | 23 ++
.../db/it/aligned/IoTDBAlignedSeriesQuery5IT.java | 56 ++++
.../apache/iotdb/db/it/metric/IoTDBMetricIT.java | 68 +++++
.../org/apache/iotdb/db/it/utils/TestUtils.java | 57 +++-
.../apache/iotdb/pipe/it/IoTDBPipeClusterIT.java | 130 ++++++---
.../pipe/it/IoTDBPipeConnectorParallelIT.java | 11 +-
.../apache/iotdb/pipe/it/IoTDBPipeDataSyncIT.java | 14 +-
.../apache/iotdb/pipe/it/IoTDBPipeLifeCycleIT.java | 192 +++++++++----
.../apache/iotdb/pipe/it/IoTDBPipeProtocolIT.java | 48 ++--
.../iotdb/pipe/it/IoTDBPipeSwitchStatusIT.java | 10 +-
.../apache/iotdb/pipe/it/IoTDBPipeSyntaxIT.java | 43 ++-
.../pipe/it/extractor/IoTDBPipeExtractorIT.java | 231 ++++++++++-----
.../parameter/PipeParameterValidator.java | 4 -
.../main/java/org/apache/iotdb/jdbc/Config.java | 2 +
.../src/main/java/org/apache/iotdb/jdbc/Utils.java | 27 +-
.../test/java/org/apache/iotdb/jdbc/UtilsTest.java | 25 ++
.../main/java/org/apache/iotdb/rpc/UrlUtils.java | 52 ++++
.../java/org/apache/iotdb/rpc/UrlUtilsTest.java | 61 ++++
.../apache/iotdb/session/util/SessionUtils.java | 3 +-
iotdb-connector/grafana-plugin/go.mod | 22 +-
.../src/assembly/resources/conf/confignode-env.bat | 4 +
.../src/assembly/resources/conf/confignode-env.sh | 3 +
.../resources/conf/iotdb-confignode.properties | 7 +-
.../write/region/CreateRegionGroupsPlan.java | 35 +--
.../manager/consensus/ConsensusManager.java | 18 +-
.../manager/load/balancer/RegionBalancer.java | 2 +-
.../pipe/runtime/PipeHeartbeatScheduler.java | 6 +
.../manager/pipe/runtime/PipeMetaSyncer.java | 8 +-
.../manager/pipe/task/PipeTaskCoordinator.java | 7 +-
.../manager/pipe/task/PipeTaskCoordinatorLock.java | 4 +
.../partition/DatabasePartitionTable.java | 4 +-
.../persistence/partition/PartitionInfo.java | 4 +-
.../persistence/schema/ConfigMTreeStore.java | 3 +-
.../impl/schema/DeactivateTemplateProcedure.java | 40 +--
.../impl/schema/DeleteDatabaseProcedure.java | 6 +-
.../impl/schema/DeleteTimeSeriesProcedure.java | 44 +--
.../impl/schema/UnsetTemplateProcedure.java | 9 +-
.../statemachine/CreateRegionGroupsProcedure.java | 10 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 1 -
.../impl/CreateRegionGroupsProcedureTest.java | 1 +
.../confignode1conf/iotdb-confignode.properties | 1 -
.../confignode2conf/iotdb-confignode.properties | 1 -
.../confignode3conf/iotdb-confignode.properties | 1 -
.../consensus/iot/IoTConsensusServerImpl.java | 21 ++
.../ratis/ApplicationStateMachineProxy.java | 16 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 91 +++---
.../iotdb/consensus/ratis/utils/Retriable.java | 107 +++++++
.../iotdb/consensus/ratis/utils/RetryPolicy.java | 79 ++++++
.../apache/iotdb/consensus/ratis/utils/Utils.java | 10 +
.../iotdb/consensus/ratis/RecoverReadTest.java | 3 +-
.../src/assembly/resources/conf/datanode-env.bat | 3 +-
.../src/assembly/resources/conf/datanode-env.sh | 3 +-
.../resources/conf/iotdb-datanode.properties | 19 +-
.../org/apache/iotdb/db/audit/AuditLogger.java | 4 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 +-
.../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 5 +
.../db/consensus/DataRegionConsensusImpl.java | 2 +
.../db/consensus/SchemaRegionConsensusImpl.java | 2 +
.../dataregion/DataExecutionVisitor.java | 4 +-
.../iotdb/db/exception/LoadReadOnlyException.java} | 26 +-
.../agent/plugin/PipeConnectorConstructor.java | 112 ++++++++
.../agent/plugin/PipeExtractorConstructor.java | 58 ++++
.../db/pipe/agent/plugin/PipePluginAgent.java | 111 ++++----
.../pipe/agent/plugin/PipePluginConstructor.java | 85 ++++++
.../agent/plugin/PipeProcessorConstructor.java | 52 ++++
.../pipe/agent/runtime/PipeCronEventInjector.java | 70 -----
.../agent/runtime/PipePeriodicalJobExecutor.java | 100 +++++++
.../db/pipe/agent/runtime/PipeRuntimeAgent.java | 7 +-
...oTDBThriftAsyncPipeTransferBatchReqBuilder.java | 2 +-
...IoTDBThriftSyncPipeTransferBatchReqBuilder.java | 2 +-
.../builder/PipeTransferBatchReqBuilder.java | 30 +-
.../thrift/async/IoTDBThriftAsyncConnector.java | 2 +-
.../pipe/extractor/IoTDBDataRegionExtractor.java | 21 +-
.../PipeHistoricalDataRegionTsFileExtractor.java | 21 +-
.../PipeRealtimeDataRegionHybridExtractor.java | 43 ++-
.../PipeRealtimeDataRegionLogExtractor.java | 2 +-
.../realtime/assigner/DisruptorQueue.java | 2 +-
.../iotdb/db/pipe/metric/PipeConnectorMetrics.java | 86 +++++-
.../iotdb/db/pipe/metric/PipeExtractorMetrics.java | 114 ++++++--
.../iotdb/db/pipe/metric/PipeProcessorMetrics.java | 86 +++++-
.../iotdb/db/pipe/metric/PipeResourceMetrics.java | 23 +-
.../db/pipe/resource/memory/PipeMemoryBlock.java | 137 ++++++++-
.../db/pipe/resource/memory/PipeMemoryManager.java | 119 ++++++--
.../db/pipe/task/stage/PipeTaskExtractorStage.java | 15 +-
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 16 +-
.../iotdb/db/pipe/task/subtask/PipeSubtask.java | 14 +-
.../subtask/connector/PipeConnectorSubtask.java | 121 ++++++--
.../connector/PipeConnectorSubtaskManager.java | 78 +-----
.../subtask/processor/PipeProcessorSubtask.java | 21 +-
.../iotdb/db/protocol/client/ConfigNodeInfo.java | 14 +-
.../rest/v1/impl/GrafanaApiServiceImpl.java | 3 +-
.../rest/v2/impl/GrafanaApiServiceImpl.java | 3 +-
.../SingleTimeWindowIterator.java | 3 +-
.../execution/operator/AggregationUtil.java | 2 +-
.../operator/schema/SchemaFetchScanOperator.java | 8 +-
.../AbstractSeriesAggregationScanOperator.java | 142 ++++++----
.../queryengine/plan/analyze/AnalyzeVisitor.java | 22 ++
.../plan/analyze/LoadTsfileAnalyzer.java | 23 ++
.../config/executor/ClusterConfigTaskExecutor.java | 11 +
.../db/queryengine/plan/parser/ASTVisitor.java | 19 +-
.../plan/planner/plan/node/write/InsertNode.java | 11 +
.../plan/node/write/InsertRowsOfOneDeviceNode.java | 46 +--
.../planner/plan/parameter/SeriesScanOptions.java | 7 +-
.../plan/scheduler/AsyncSendPlanNodeHandler.java | 7 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 16 +-
.../schemaengine/schemaregion/ISchemaRegion.java | 4 +
.../schemaregion/impl/SchemaRegionMemoryImpl.java | 15 +
.../schemaregion/impl/SchemaRegionPBTreeImpl.java | 41 ++-
.../schemaregion/mtree/IMTreeStore.java | 9 +-
.../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 25 ++
.../schemaregion/mtree/impl/mem/MemMTreeStore.java | 2 +-
.../mtree/impl/pbtree/CachedMTreeStore.java | 36 ++-
.../mtree/impl/pbtree/MTreeBelowSGCachedImpl.java | 100 ++++---
.../pbtree/ReentrantReadOnlyCachedMTreeStore.java | 5 +-
.../mtree/impl/pbtree/cache/CacheManager.java | 35 ++-
.../mtree/impl/pbtree/schemafile/SchemaFile.java | 35 ++-
.../schemafile/pagemgr/BTreePageManager.java | 6 +-
.../pbtree/schemafile/pagemgr/PageManager.java | 71 +++--
.../schemaregion/mtree/traverser/Traverser.java | 16 ++
.../traverser/basic/MeasurementTraverser.java | 13 +
.../traverser/collector/MeasurementCollector.java | 5 +
.../java/org/apache/iotdb/db/service/DataNode.java | 63 +++--
.../apache/iotdb/db/service/IoTDBShutdownHook.java | 3 +
.../db/service/metrics/CompactionMetrics.java | 311 +++++++++++++++++++++
.../iotdb/db/storageengine/StorageEngine.java | 7 +
.../db/storageengine/dataregion/DataRegion.java | 102 +++++--
.../dataregion/HashLastFlushTimeMap.java | 2 +-
.../impl/ReadChunkCompactionPerformer.java | 10 +-
.../execute/recover/CompactionRecoverManager.java | 18 +-
.../execute/task/AbstractCompactionTask.java | 63 ++++-
.../execute/task/CrossSpaceCompactionTask.java | 7 +-
.../execute/task/InnerSpaceCompactionTask.java | 35 +--
.../task/InsertionCrossSpaceCompactionTask.java | 24 +-
.../execute/utils/MultiTsFileDeviceIterator.java | 20 +-
.../fast/AlignedSeriesCompactionExecutor.java | 18 +-
.../fast/NonAlignedSeriesCompactionExecutor.java | 9 +-
.../compaction/io/CompactionTsFileWriter.java | 11 +
.../schedule/CompactionScheduleSummary.java | 72 +++++
.../compaction/schedule/CompactionScheduler.java | 36 ++-
.../compaction/schedule/CompactionWorker.java | 13 +-
.../impl/RewriteCrossSpaceCompactionSelector.java | 11 +-
.../dataregion/flush/MemTableFlushTask.java | 4 +-
.../dataregion/tsfile/TsFileManager.java | 6 +-
.../dataregion/tsfile/TsFileResource.java | 5 +-
.../storageengine/dataregion/wal/node/WALNode.java | 7 +-
.../dataregion/wal/utils/WALEntryHandler.java | 2 +-
.../dataregion/wal/utils/WALEntryPosition.java | 8 +-
.../dataregion/wal/utils/WALInsertNodeCache.java | 63 ++++-
.../wal/utils/listener/AbstractResultListener.java | 2 +
.../wal/utils/listener/WALFlushListener.java | 2 +-
.../db/storageengine/rescon/disk/TierManager.java | 3 +
.../db/storageengine/rescon/memory/SystemInfo.java | 52 +++-
.../rescon/memory/TimePartitionManager.java | 6 +
.../org/apache/iotdb/db/utils/DateTimeUtils.java | 14 -
.../mtree/schemafile/SchemaFileLogTest.java | 13 +
.../schemaRegion/SchemaRegionTemplateTest.java | 42 +--
.../db/pipe/agent/plugin/PipePluginAgentTest.java | 6 +-
.../PipeConnectorSubtaskExecutorTest.java | 3 +
.../PipeProcessorSubtaskExecutorTest.java | 3 +
.../db/protocol/rest/GrafanaApiServiceTest.java | 92 +++++-
.../schema/SchemaFetchScanOperatorTest.java | 7 +
.../plan/node/write/WritePlanNodeSplitTest.java | 80 ++++++
.../storageengine/dataregion/DataRegionTest.java | 120 ++++++++
.../TsFileResourceProgressIndexTest.java | 162 ++++++++++-
.../FastInnerCompactionPerformerTest.java | 2 +-
.../compaction/ReadChunkInnerCompactionTest.java | 190 +++++++++++++
...sSpaceCompactionWithReadPointPerformerTest.java | 4 +-
.../inner/InnerCompactionEmptyTsFileTest.java | 2 +-
.../inner/InnerCompactionSchedulerTest.java | 10 +-
.../dataregion/wal/node/WALEntryHandlerTest.java | 9 +-
.../datanode1conf/iotdb-datanode.properties | 1 -
.../datanode2conf/iotdb-datanode.properties | 1 -
.../datanode3conf/iotdb-datanode.properties | 1 -
iotdb-core/metrics/ReadMe.md | 18 +-
.../iotdb/metrics/core/IoTDBMetricManager.java | 3 +-
.../iotdb/metrics/AbstractMetricManager.java | 10 +-
.../iotdb/metrics/impl/DoNothingMetricManager.java | 2 +-
.../disk/DoNothingDiskMetricsManager.java | 5 +-
.../metricsets/disk/IDiskMetricsManager.java | 24 +-
.../metricsets/net/DoNothingNetMetricManager.java | 5 +-
.../metrics/metricsets/net/INetMetricManager.java | 33 ++-
.../metrics/metricsets/system/SystemMetrics.java | 40 +--
.../apache/iotdb/metrics/utils/MetricLevel.java | 2 +-
.../iotdb/metrics/config/MetricConfigTest.java | 2 -
.../iotdb/commons/concurrent/ThreadName.java | 4 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 36 +++
.../iotdb/commons/conf/CommonDescriptor.java | 15 +
.../apache/iotdb/commons/conf/IoTDBConstant.java | 5 +
.../commons/consensus/index/ProgressIndex.java | 144 ++++++++--
.../consensus/index/impl/HybridProgressIndex.java | 36 ++-
.../consensus/index/impl/IoTProgressIndex.java | 19 +-
.../consensus/index/impl/MinimumProgressIndex.java | 12 +-
.../consensus/index/impl/RecoverProgressIndex.java | 22 +-
.../consensus/index/impl/SimpleProgressIndex.java | 11 +-
.../apache/iotdb/commons/path/PathPatternNode.java | 6 +-
.../apache/iotdb/commons/path/PathPatternTree.java | 17 +-
.../apache/iotdb/commons/path/fa/dfa/DFAState.java | 6 +-
.../iotdb/commons/path/fa/dfa/PatternDFA.java | 63 +++--
.../iotdb/commons/path/fa/dfa/graph/DFAGraph.java | 80 ++++--
.../iotdb/commons/pipe/config/PipeConfig.java | 16 ++
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 31 +-
.../commons/schema/tree/AbstractTreeVisitor.java | 15 +-
.../iotdb/commons/service/RegisterManager.java | 6 +
.../iotdb/commons/service/metric/enums/Metric.java | 8 +-
.../iotdb/commons/service/metric/enums/Tag.java | 4 +-
.../iotdb/commons/utils/CommonDateTimeUtils.java | 15 +
.../apache/iotdb/commons/utils/NodeUrlUtils.java | 23 +-
.../apache/iotdb/commons/path/PatternDFATest.java | 35 ++-
.../iotdb/commons/pipe/PipeMetaDeSerTest.java | 4 +-
.../iotdb/commons/utils/NodeUrlUtilsTest.java | 33 +++
.../tsfile/write/chunk/AlignedChunkWriterImpl.java | 5 +
.../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 5 +
.../iotdb/tsfile/write/chunk/IChunkWriter.java | 3 +
237 files changed, 5416 insertions(+), 1482 deletions(-)
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java
index 9782d1b63bf,17059ff5fc6..89bde8e31e2
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java
@@@ -199,12 -197,7 +199,12 @@@ public class DataExecutionVisitor exten
return StatusUtils.OK;
} catch (IOException e) {
LOGGER.error("Error in executing plan node: {}", node, e);
- return StatusUtils.EXECUTE_STATEMENT_ERROR;
+ return new TSStatus(TSStatusCode.WRITE_PROCESS_ERROR.getStatusCode());
}
}
+
+ @Override
+ public TSStatus visitLoadCommand(LoadCommandNode node, DataRegion context) {
+ return StorageEngine.getInstance().executeLoadCommand(node);
+ }
}
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/connector/PipeConnectorSubtask.java
index 3b2bcb4cbcf,c9b8c20a3db..0cbab51b2a0
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/connector/PipeConnectorSubtask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/connector/PipeConnectorSubtask.java
@@@ -123,20 -138,8 +139,11 @@@ public class PipeConnectorSubtask exten
} else if (event instanceof TsFileInsertionEvent) {
outputPipeConnector.transfer((TsFileInsertionEvent) event);
PipeConnectorMetrics.getInstance().markTsFileEvent(taskID);
+ } else if (event instanceof TsFileBatchInsertionEvent) {
+ outputPipeConnector.transfer((TsFileBatchInsertionEvent) event);
+ PipeConnectorMetrics.getInstance().markTsFileEvent(taskID);
} else if (event instanceof PipeHeartbeatEvent) {
- try {
- outputPipeConnector.heartbeat();
- outputPipeConnector.transfer(event);
- } catch (Exception e) {
- throw new PipeConnectionException(
- "PipeConnector: " + outputPipeConnector.getClass().getName() + " heartbeat failed",
- e);
- }
- ((PipeHeartbeatEvent) event).onTransferred();
- PipeConnectorMetrics.getInstance().markPipeHeartbeatEvent(taskID);
+ transferHeartbeatEvent((PipeHeartbeatEvent) event);
} else {
outputPipeConnector.transfer(event);
}
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
index df818ee63c3,92a3ecd1047..6d94ab8a38b
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
@@@ -79,11 -91,10 +83,11 @@@ import java.util.concurrent.TimeoutExce
* href="https://apache-iotdb.feishu.cn/docx/doxcnyBYWzek8ksSEU6obZMpYLe">...</a>;
*/
public class LoadTsFileScheduler implements IScheduler {
+
private static final Logger logger = LoggerFactory.getLogger(LoadTsFileScheduler.class);
- public static final long LOAD_TASK_MAX_TIME_IN_SECOND = 5184000L; // one day
+ public static final long LOAD_TASK_MAX_TIME_IN_SECOND = 900L; // 15min
- private static final long MAX_MEMORY_SIZE;
- private static final int TRANSMIT_LIMIT;
+ public static final long MAX_MEMORY_SIZE;
+ public static final int TRANSMIT_LIMIT;
static {
IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
@@@ -196,17 -203,10 +200,18 @@@
}
private boolean firstPhase(LoadSingleTsFileNode node) {
+ final TsFileDataManager tsFileDataManager = new TsFileDataManager(this, node);
try {
+ TsFileDataManager tsFileDataManager =
+ new TsFileDataManager(
+ this::dispatchOnePieceNode,
+ node.getPlanNodeId(),
+ node.getTsFileResource().getTsFile(),
+ partitionFetcher,
+ MAX_MEMORY_SIZE,
+ queryContext.getSession().getUserName());
new TsFileSplitter(
- node.getTsFileResource().getTsFile(), tsFileDataManager::addOrSendTsFileData)
+ node.getTsFileResource().getTsFile(), tsFileDataManager::addOrSendTsFileData, 0)
.splitTsFileByDataPartition();
if (!tsFileDataManager.sendAllTsFileData()) {
stateMachine.transitionToFailed(new TSStatus(TSStatusCode.LOAD_FILE_ERROR.getStatusCode()));