You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2023/11/08 02:58:35 UTC
(iotdb) branch load_v2 updated (270f6f6efce -> c1201f5b035)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a change to branch load_v2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 270f6f6efce Merge branch 'load_v2' of github.com:apache/iotdb into load_v2
add b13edf3d099 [IOTDB-6207] Load: Add Write Point Rate Metric (#11379)
add 811f019a6e4 PBTree mode support logical view (#11401)
add 7b55646f51d Add fsync for QuotaInfo
add 41a516618fd [IOTDB-6219] Fix the display problem of explain that the print result is not aligned
add bd9c3fa7e37 fix datanode can't stop because wal thread. (#11314)
add e3d2a2ae613 [IOTDB-6151] Move datanode system.properties safer (#11415)
add 7c5feb13613 datanode's dn_rpc_port supports ssl (#11218)
add f3e0f35a09d Fixed Ratis server RPC timeout parameter setting invalidation bug (#11418)
add 5fac123f292 [IOTDB-6190] Increase the threshold for Ratis to shut itself down if it detects that a process is stuck
add 100c7b9afc2 Remove IoTDB-ML Module
add 374b851389d degrade log level of pbtree memory status monitor (#11419)
add 538fd5c43c8 Fix empty tsfile resource didn't remove (#11413)
add 3e01d33e517 Add java doc for SessionDataSet.next() method to let users prefer to use SessionDataSet.DataIterator
add 32e3d066813 Pipe: rename PipeExtractor -> PipeSource, PipeConnector -> PipeSink in pipe-api, antlr (#11427)
add 3adb2e079cc Pipe: Disable batch mode for retry connector in case some events are never retried & Pipe won't collect TsFile without tail magic & Make async connector stop properly when retry count exceeds limit (#11425)
add 4040bfccce4 [IOTDB-6218] Rename storage_query_schema_consensus_free_memory_proportion to datanode_memory_proportion (#11408)
add 1dd5a200281 Speed up recover stage when DataNode restarting (#11421)
add ce82c2b76a6 [IOTDB-6206] Metric: merge metric algorithm library and optimize some core components (#11410)
add 457848c39fb [IOTDB-6220] Pipe: Added connector endpoint check logic to avoid self-transmission (#11417)
add f47675302df Fix PBTree view device isAligned should be null
add c2c70755e54 Extend expected test time of SharedTsBlockQueueTest
add 91310539d9a Add PrivilegeType USE_MODEL
add 6b513b5e933 [IOTDB-6225] Support batch processing in UDF interface
add a4879954d5c Add InsertionCrossSpaceCompaction to move non overlapped unsequence file to sequence space directly (#11414)
add ea3537bf7ed [IOTDB-6223] Pipe: Pipe Memory Management integrates into the Pipe Module (#11424)
add 726403f11ab fix compile error caused by concurrent PR merge in github (#11438)
add ad980dbf973 Fix Compaction Schedule didn't start (#11439)
add f94565d1919 Pipe: rename plugins *-extractor -> *-source, *-connector -> *-sink (#11440)
add 2feb202ca9d add tsfile correctness validation for write compaction and load (#11392)
add 9c05a54f238 feat(env): set MAX_CACHED_BUFFER_SIZE equals OFF_HEAP_MEMORY / IO_THREADS_NUMBER
add a457e8f71fc fixed the issue of missing enableQueryRedirection parameter in SSL (#11436)
add e614c5e8f6c Pipe: remove unnecessary waiting operation for memory alloc calculation (#11445)
add e7592c09d91 fix the issue that compaction scheduler is not started when DataRegion created (#11446)
add 3d9cd3c3052 Refactor Series/Device Quota Limitation (#11422)
add 7915e0a19c4 [IOTDB-6227] Remove unused retry policy in IStateMachine (#11451)
add 2c64ea41227 Fix concurrent read lock bug in StampedWriterPreferredLock (#11453)
add 54b7e6d4cd9 [IOTDB-6220] Pipe: Changed the sink loopback detection logic to support hostName and IPv6 specification. (#11455)
add 13f6e7e61d3 [IOTDB-6234] Pipe: Added client example for opc-ua-connector (#11460)
add a7b745842d4 Pipe: add stream & batch options for source.realtime.mode & remove some pipe plugins from show pipe plugins' dataset & make pipe-api compatible with the v1.2.x releases (#11461)
add c712ce8980e Bump master branch version from 1.3.0-SNAPSHOT to 1.3.1-SNAPSHOT (#11450)
add bce85d58c1f Fix opc-ua-sink version error
add 3b179eeb3ab To adapt activation (#11463)
add da8b9c0bf59 Disable spotless for JDK 21 (#11469)
add ff19689fa6a [IOTDB-6224] Fix group by month in non-ms precision and mixed unit
add ac7629e78f4 [IOTDB-6236] Pipe: CountPointProcessor example (#11470)
add d8580c95122 Implement iterative volatile node process during pbtree flush (#11432)
add 83447eb30b1 [IOTDB-6235] Pipe: Implement write-back-sink to transfer data back to the sender (#11466)
add 1f34ad7c9f8 Fix the path of TsFileResource in unit test
add 93de59e4db0 Pipe: add connector's pending tsfile event count to help isTsFileEventCountInQueueExceededLimit judgement when hybrid mode is enabled (#11476)
add 1df8ec25eea Make all built-in scalar UDF into batch processing
add aa99a220bc3 Pipe: preliminary SSL support for pipe connector (sink) sync client (#11465)
add 3244255e940 fix empty resource recovered from wal (#11452)
add 4e686b9a824 Pipe: Enable github machines instead of self-hosted machines to run pipe IT (#11435)
add 256194a9e28 [IOTDB-6240] Pipe: Fix bug with PipeMemoryManager allocating memory for the DisruptorQueue (#11485)
new c1201f5b035 Merge branch 'master' into load_v2
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/iotdb-ml.yml | 61 -
.github/workflows/pipe-it-2cluster.yml | 7 +-
.github/workflows/sonar-codecov.yml | 2 +-
.gitignore | 3 -
code-coverage/pom.xml | 2 +-
distribution/pom.xml | 14 +-
example/client-cpp-example/pom.xml | 2 +-
example/flink-sql/pom.xml | 2 +-
example/flink/pom.xml | 2 +-
.../org/apache/iotdb/flink/FlinkIoTDBSink.java | 2 +-
.../org/apache/iotdb/flink/FlinkIoTDBSource.java | 2 +-
.../apache/iotdb/flink/FlinkTsFileBatchSink.java | 2 +-
.../apache/iotdb/flink/FlinkTsFileStreamSink.java | 2 +-
.../java/org/apache/iotdb/flink/TsFileUtils.java | 2 +-
example/hadoop/pom.xml | 2 +-
.../iotdb/hadoop/tsfile/TSMRWriteExample.java | 2 +-
.../apache/iotdb/hadoop/tsfile/TsFileHelper.java | 2 +-
.../iotdb/hadoop/tsfile/TsFileWriteToHDFS.java | 2 +-
example/jdbc/pom.xml | 2 +-
example/kafka/pom.xml | 2 +-
.../main/java/org/apache/iotdb/kafka/Consumer.java | 2 +-
.../org/apache/iotdb/kafka/ConsumerThread.java | 2 +-
example/mqtt-customize/pom.xml | 2 +-
example/mqtt/pom.xml | 2 +-
.../{flink => pipe-count-point-processor}/pom.xml | 43 +-
.../java/org/apache/iotdb/CountPointProcessor.java | 83 +
example/{session => pipe-opc-ua-sink}/pom.xml | 19 +-
.../java/org/apache/iotdb/opcua/ClientExample.java | 50 +
.../apache/iotdb/opcua/ClientExampleRunner.java | 144 ++
.../java/org/apache/iotdb/opcua/ClientTest.java | 129 ++
.../iotdb/opcua/IoTDBKeyStoreLoaderClient.java | 128 ++
example/pom.xml | 4 +-
example/pulsar/pom.xml | 2 +-
.../org/apache/iotdb/pulsar/PulsarConsumer.java | 2 +-
.../apache/iotdb/pulsar/PulsarConsumerThread.java | 2 +-
example/rabbitmq/pom.xml | 2 +-
.../apache/iotdb/rabbitmq/RabbitMQConsumer.java | 2 +-
example/rest-java-example/pom.xml | 2 +-
example/rocketmq/pom.xml | 2 +-
.../apache/iotdb/rocketmq/RocketMQConsumer.java | 2 +-
example/schema/pom.xml | 4 +-
example/session/pom.xml | 2 +-
.../iotdb/AlignedTimeseriesSessionExample.java | 2 +-
.../org/apache/iotdb/DataMigrationExample.java | 2 +-
.../iotdb/HybridTimeseriesSessionExample.java | 2 +-
.../org/apache/iotdb/SessionConcurrentExample.java | 2 +-
.../main/java/org/apache/iotdb/SessionExample.java | 5 +-
.../java/org/apache/iotdb/SessionPoolExample.java | 2 +-
.../iotdb/SyntaxConventionRelatedExample.java | 2 +-
.../main/java/org/apache/iotdb/TabletExample.java | 6 +-
example/trigger/pom.xml | 2 +-
.../org/apache/iotdb/trigger/LoggerTrigger.java | 2 +-
.../iotdb/trigger/StatisticsUpdaterTrigger.java | 2 +-
example/tsfile/pom.xml | 2 +-
.../iotdb/tsfile/TsFileForceAppendWrite.java | 2 +-
.../apache/iotdb/tsfile/TsFileSequenceRead.java | 2 +-
.../tsfile/TsFileWriteAlignedWithTSRecord.java | 2 +-
.../iotdb/tsfile/TsFileWriteAlignedWithTablet.java | 5 +-
.../iotdb/tsfile/TsFileWriteWithTSRecord.java | 2 +-
.../apache/iotdb/tsfile/TsFileWriteWithTablet.java | 5 +-
example/udf/pom.xml | 2 +-
integration-test/pom.xml | 40 +-
.../apache/iotdb/db/query/udf/example/Adder.java | 2 +-
.../apache/iotdb/db/query/udf/example/TwoSum.java | 43 +-
.../iotdb/db/query/udf/example/TwoSumBlock.java | 107 +
.../it/env/cluster/config/MppCommonConfig.java | 24 +-
.../env/cluster/config/MppSharedCommonConfig.java | 29 +-
.../iotdb/it/env/cluster/env/AbstractEnv.java | 18 +
.../it/env/remote/config/RemoteCommonConfig.java | 13 +-
.../org/apache/iotdb/it/utils/TsFileGenerator.java | 4 +-
.../java/org/apache/iotdb/itbase/env/BaseEnv.java | 8 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 8 +-
.../org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 2 +-
.../iotdb/db/it/IoTDBPartialInsertionIT.java | 2 +-
.../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 5 +
.../db/it/groupby/IoTDBGroupByNaturalMonthIT.java | 108 +-
.../IoTDBGroupByNaturalMonthNsPrecisionIT.java | 107 +
.../IoTDBGroupByNaturalMonthUsPrecisionIT.java | 92 +
.../{ => quota}/IoTDBClusterDeviceQuotaIT.java | 7 +-
.../IoTDBClusterMixQuotaIT.java} | 77 +-
.../IoTDBClusterQuotaIT.java} | 39 +-
.../quota/IoTDBClusterTimeSeriesQuotaIT.java | 37 +-
.../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 2 +-
.../iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java | 155 ++
.../org/apache/iotdb/db/it/utils/TestUtils.java | 47 +-
.../test/java/org/apache/iotdb/flink/it/Utils.java | 2 +-
.../session/it/IoTDBSessionAlignedInsertIT.java | 2 +-
.../iotdb/session/it/IoTDBSessionComplexIT.java | 2 +-
.../it/IoTDBSessionDisableMemControlIT.java | 7 +-
.../iotdb/session/it/IoTDBSessionInsertNullIT.java | 2 +-
.../IoTDBSessionInsertWithTriggerExecutionIT.java | 2 +-
.../session/it/IoTDBSessionSchemaTemplateIT.java | 2 +-
.../iotdb/session/it/IoTDBSessionSimpleIT.java | 31 +-
.../session/it/IoTDBSessionSyntaxConventionIT.java | 5 +-
.../org/apache/iotdb/session/it/SessionIT.java | 5 +-
.../iotdb/session/it/pool/SessionPoolIT.java | 2 +-
iotdb-api/{external-api => common-api}/pom.xml | 14 +-
.../org/apache/iotdb/tsfile/access}/Column.java | 5 +-
.../apache/iotdb/tsfile/access}/ColumnBuilder.java | 4 +-
.../iotdb/tsfile/access}/ColumnBuilderStatus.java | 4 +-
.../iotdb/tsfile/access}/TsBlockBuilderStatus.java | 12 +-
.../apache/iotdb/tsfile/enums}/ColumnEncoding.java | 2 +-
.../org/apache/iotdb/tsfile}/enums/TSDataType.java | 4 +-
.../exception}/UnSupportedDataTypeException.java | 4 +-
.../java/org/apache/iotdb/tsfile/utils/Binary.java | 25 +-
.../apache/iotdb/tsfile/utils/TsPrimitiveType.java | 4 +-
iotdb-api/external-api/pom.xml | 2 +-
iotdb-api/pipe-api/pom.xml | 4 +-
.../org/apache/iotdb/pipe/api/PipeConnector.java | 3 +-
.../org/apache/iotdb/pipe/api/PipeExtractor.java | 3 +-
.../pipe/api/{PipeConnector.java => PipeSink.java} | 77 +-
.../api/{PipeExtractor.java => PipeSource.java} | 48 +-
.../PipeSinkRuntimeConfiguration.java | 4 +-
.../PipeSourceRuntimeConfiguration.java | 7 +-
.../parameter/PipeParameterValidator.java | 2 +-
.../api/customizer/parameter/PipeParameters.java | 41 +-
.../pipe/api/exception/PipeSinkException.java | 10 +-
iotdb-api/pom.xml | 3 +-
iotdb-api/trigger-api/pom.xml | 4 +-
iotdb-api/udf-api/pom.xml | 9 +-
.../main/java/org/apache/iotdb/udf/api/UDTF.java | 46 +
.../org/apache/iotdb/udf/api/utils/RowImpl.java | 47 +-
iotdb-client/cli/pom.xml | 25 +-
.../java/org/apache/iotdb/cli/AbstractCli.java | 42 +
.../src/main/java/org/apache/iotdb/cli/Cli.java | 23 +-
.../main/java/org/apache/iotdb/tool/ExportCsv.java | 2 +-
.../java/org/apache/iotdb/tool/ExportTsFile.java | 2 +-
.../main/java/org/apache/iotdb/tool/ImportCsv.java | 14 +-
iotdb-client/client-cpp/pom.xml | 4 +-
iotdb-client/client-py/pom.xml | 4 +-
iotdb-client/isession/pom.xml | 15 +-
.../java/org/apache/iotdb/isession/ISession.java | 2 +-
.../org/apache/iotdb/isession/SessionDataSet.java | 13 +-
.../apache/iotdb/isession/pool/ISessionPool.java | 2 +-
iotdb-client/jdbc/pom.xml | 15 +-
.../main/java/org/apache/iotdb/jdbc/Config.java | 6 +
.../iotdb/jdbc/GroupedLSBWatermarkEncoder.java | 2 +-
.../org/apache/iotdb/jdbc/IoTDBConnection.java | 17 +-
.../apache/iotdb/jdbc/IoTDBConnectionParams.java | 28 +
.../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 6 +-
.../iotdb/jdbc/IoTDBNonAlignJDBCResultSet.java | 4 +-
.../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 3 +-
.../src/main/java/org/apache/iotdb/jdbc/Utils.java | 15 +
.../apache/iotdb/jdbc/IoTDBJDBCResultSetTest.java | 2 +-
iotdb-client/pom.xml | 2 +-
iotdb-client/service-rpc/pom.xml | 13 +-
.../org/apache/iotdb/rpc/IoTDBJDBCDataSet.java | 4 +-
.../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 8 +-
.../org/apache/iotdb/rpc/RpcTransportFactory.java | 11 +
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 7 +-
iotdb-client/session/pom.xml | 17 +-
.../java/org/apache/iotdb/session/Session.java | 77 +-
.../apache/iotdb/session/SessionConnection.java | 27 +-
.../org/apache/iotdb/session/pool/SessionPool.java | 140 +-
.../iotdb/session/template/MeasurementNode.java | 2 +-
.../apache/iotdb/session/util/SessionUtils.java | 4 +-
.../iotdb/session/SessionCacheLeaderTest.java | 2 +-
.../java/org/apache/iotdb/session/SessionTest.java | 12 +-
.../java/org/apache/iotdb/session/TabletTest.java | 2 +-
.../session/pool/SessionPoolExceptionTest.java | 2 +-
.../apache/iotdb/session/pool/SessionPoolTest.java | 2 +-
.../iotdb/session/util/SessionUtilsTest.java | 2 +-
iotdb-connector/flink-iotdb-connector/pom.xml | 15 +-
.../iotdb/flink/DefaultIoTSerializationSchema.java | 2 +-
.../main/java/org/apache/iotdb/flink/Event.java | 2 +-
.../java/org/apache/iotdb/flink/IoTDBSink.java | 2 +-
.../iotdb/flink/options/IoTDBSinkOptions.java | 2 +-
iotdb-connector/flink-sql-iotdb-connector/pom.xml | 17 +-
.../org/apache/iotdb/flink/sql/common/Utils.java | 7 +-
.../sql/function/IoTDBBoundedScanFunction.java | 2 +-
.../flink/sql/function/IoTDBCDCSourceFunction.java | 2 +-
.../flink/sql/function/IoTDBLookupFunction.java | 2 +-
.../flink/sql/function/IoTDBSinkFunction.java | 2 +-
iotdb-connector/flink-tsfile-connector/pom.xml | 11 +-
.../iotdb/flink/tsfile/RowTSRecordConverter.java | 8 +-
.../tsfile/RowTsFileOutputFormatTestBase.java | 2 +-
.../apache/iotdb/flink/util/TsFileWriteUtil.java | 2 +-
iotdb-connector/grafana-connector/pom.xml | 4 +-
iotdb-connector/grafana-plugin/pom.xml | 2 +-
iotdb-connector/hadoop/pom.xml | 9 +-
.../iotdb/hadoop/tsfile/TSFRecordReader.java | 4 +-
.../iotdb/hadoop/tsfile/record/HDFSTSRecord.java | 4 +-
.../iotdb/hadoop/tsfile/TsFileTestHelper.java | 2 +-
iotdb-connector/hive-connector/pom.xml | 12 +-
.../org/apache/iotdb/hive/TsFileTestHelper.java | 2 +-
iotdb-connector/pom.xml | 2 +-
iotdb-connector/spark-iotdb-connector/pom.xml | 10 +-
.../spark-iotdb-connector/scala_2.11/pom.xml | 2 +-
.../spark-iotdb-connector/scala_2.12/pom.xml | 2 +-
.../org/apache/iotdb/spark/db/DataFrameTools.scala | 4 +-
iotdb-connector/spark-tsfile/pom.xml | 11 +-
.../tsfile/qp/optimizer/PhysicalOptimizer.java | 2 +-
.../org/apache/iotdb/spark/tsfile/Converter.scala | 2 +-
.../iotdb/spark/tsfile/NarrowConverter.scala | 3 +-
.../apache/iotdb/spark/tsfile/WideConverter.scala | 6 +-
.../apache/iotdb/spark/tsfile/ConverterTest.scala | 5 +-
iotdb-connector/zeppelin-interpreter/pom.xml | 6 +-
iotdb-core/antlr/pom.xml | 2 +-
.../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 3 -
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 46 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 17 +-
iotdb-core/confignode/pom.xml | 31 +-
.../src/assembly/resources/conf/confignode-env.bat | 21 +-
.../src/assembly/resources/conf/confignode-env.sh | 24 +-
.../confignode/client/DataNodeRequestType.java | 3 -
.../async/AsyncConfigNodeHeartbeatClientPool.java | 7 +-
.../heartbeat/ConfigNodeHeartbeatHandler.java | 14 +-
.../heartbeat/DataNodeHeartbeatHandler.java | 31 +-
.../client/sync/SyncDataNodeClientPool.java | 3 -
.../confignode/conf/ConfigNodeDescriptor.java | 17 +-
.../consensus/request/ConfigPhysicalPlan.java | 28 -
.../consensus/request/ConfigPhysicalPlanType.java | 9 +-
.../request/read/model/GetModelInfoPlan.java | 79 -
.../request/read/model/ShowModelPlan.java | 89 -
.../request/read/model/ShowTrialPlan.java | 97 -
.../request/write/model/CreateModelPlan.java | 78 -
.../request/write/model/DropModelPlan.java | 81 -
.../request/write/model/UpdateModelInfoPlan.java | 98 -
.../request/write/model/UpdateModelStatePlan.java | 105 -
.../consensus/response/model/GetModelInfoResp.java | 46 -
.../consensus/response/model/ModelTableResp.java | 55 -
.../consensus/response/model/TrialTableResp.java | 55 -
.../statemachine/ConfigRegionStateMachine.java | 3 +-
.../iotdb/confignode/manager/ConfigManager.java | 106 +-
.../apache/iotdb/confignode/manager/IManager.java | 38 -
.../iotdb/confignode/manager/ModelManager.java | 204 --
.../iotdb/confignode/manager/ProcedureManager.java | 31 -
.../confignode/manager/load/cache/LoadCache.java | 8 +-
.../load/cache/node/NodeHeartbeatSample.java | 8 +
.../manager/load/service/HeartbeatService.java | 33 +-
.../iotdb/confignode/manager/node/NodeManager.java | 4 +-
.../manager/schema/ClusterSchemaManager.java | 31 +-
.../schema/ClusterSchemaQuotaStatistics.java | 73 +-
.../iotdb/confignode/persistence/ModelInfo.java | 278 ---
.../persistence/executor/ConfigPlanExecutor.java | 28 -
.../persistence/pipe/PipePluginInfo.java | 2 +-
.../confignode/persistence/quota/QuotaInfo.java | 1 +
.../persistence/schema/TemplateTable.java | 2 +-
.../schema/mnode/factory/ConfigMNodeFactory.java | 3 +-
.../procedure/impl/model/CreateModelProcedure.java | 223 --
.../procedure/impl/model/DropModelProcedure.java | 209 --
.../procedure/state/model/DropModelState.java | 28 -
.../procedure/store/ProcedureFactory.java | 12 -
.../confignode/procedure/store/ProcedureType.java | 4 +-
.../iotdb/confignode/service/ConfigNode.java | 10 +-
.../confignode/service/ConfigNodeCommandLine.java | 8 +-
.../confignode/service/ConfigNodeShutdownHook.java | 6 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 53 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 5 +-
.../iotdb/confignode/persistence/PipeInfoTest.java | 5 +-
.../persistence/schema/ClusterSchemaInfoTest.java | 2 +-
.../persistence/schema/TemplateTableTest.java | 2 +-
.../schema/DeactivateTemplateProcedureTest.java | 2 +-
.../impl/schema/UnsetTemplateProcedureTest.java | 2 +-
iotdb-core/consensus/pom.xml | 14 +-
.../org/apache/iotdb/consensus/IStateMachine.java | 32 -
.../apache/iotdb/consensus/config/RatisConfig.java | 8 +-
.../ratis/ApplicationStateMachineProxy.java | 44 +-
.../consensus/ratis/metrics/CounterProxy.java | 2 +-
.../apache/iotdb/consensus/ratis/utils/Utils.java | 1 +
iotdb-core/datanode/pom.xml | 60 +-
.../src/assembly/resources/conf/datanode-env.bat | 24 +-
.../src/assembly/resources/conf/datanode-env.sh | 28 +-
.../resources/conf/iotdb-datanode.properties | 55 +-
.../src/main/codegen/templates/ModeAccumulator.ftl | 6 +-
.../codegen/templates/abstractVariationWindow.ftl | 2 +-
.../src/main/codegen/templates/constantFill.ftl | 2 +-
.../src/main/codegen/templates/linearFill.ftl | 2 +-
.../src/main/codegen/templates/previousFill.ftl | 2 +-
.../src/main/codegen/templates/variationWindow.ftl | 2 +-
.../codegen/templates/variationWindowManager.ftl | 2 +-
.../org/apache/iotdb/db/audit/AuditLogger.java | 9 +-
.../org/apache/iotdb/db/auth/AuthorityChecker.java | 14 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 100 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 105 +-
.../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 9 +-
.../iotdb/db/conf/rest/IoTDBRestServiceConfig.java | 2 +-
.../consensus/statemachine/BaseStateMachine.java | 3 +-
.../dataregion/DataRegionStateMachine.java | 18 -
.../schemaregion/SchemaExecutionVisitor.java | 2 +-
.../schemaregion/SchemaRegionStateMachine.java | 18 -
.../metadata/DataTypeMismatchException.java | 2 +-
.../metadata/SchemaQuotaExceededException.java | 8 +-
.../db/pipe/agent/plugin/PipePluginAgent.java | 2 +-
.../config/constant/PipeConnectorConstant.java | 4 +
.../config/constant/PipeExtractorConstant.java | 2 +
.../builder/PipeTransferBatchReqBuilder.java | 26 +-
.../request/PipeTransferTabletInsertNodeReq.java | 14 +-
.../request/PipeTransferTabletRawReq.java | 15 +-
.../db/pipe/connector/protocol/IoTDBConnector.java | 33 +-
.../protocol/airgap/IoTDBAirGapConnector.java | 51 +-
.../protocol/legacy/IoTDBLegacyPipeConnector.java | 125 +-
.../connector/protocol/opcua/OpcUaConnector.java | 4 +-
.../thrift/async/IoTDBThriftAsyncConnector.java | 42 +-
.../thrift/sync/IoTDBThriftSyncConnector.java | 82 +-
.../sync/IoTDBThriftSyncConnectorClient.java | 29 +-
.../protocol/websocket/WebSocketConnector.java | 4 +-
.../protocol/writeback/WriteBackConnector.java | 176 ++
.../apache/iotdb/db/pipe/event/EnrichedEvent.java | 2 +
.../event/common/heartbeat/PipeHeartbeatEvent.java | 18 +-
.../event/common/row/PipeDataTypeTransformer.java | 7 +-
.../iotdb/db/pipe/event/common/row/PipeRow.java | 5 +-
.../common/tablet/PipeRawTabletInsertionEvent.java | 6 +
.../tablet/TabletInsertionDataContainer.java | 4 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 9 +-
.../tsfile/TsFileInsertionDataContainer.java | 14 +-
.../tsfile/TsFileInsertionDataTabletIterator.java | 2 +-
.../tsfile/TsFileListInsertionDataContainer.java | 2 +-
.../pipe/extractor/IoTDBDataRegionExtractor.java | 12 +-
.../PipeHistoricalDataRegionTsFileExtractor.java | 5 +-
.../PipeRealtimeDataRegionHybridExtractor.java | 15 +-
.../realtime/assigner/DisruptorQueue.java | 19 +-
.../db/pipe/resource/memory/PipeMemoryBlock.java | 4 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 142 +-
.../resource/tsfile/PipeTsFileResourceManager.java | 8 +-
.../db/pipe/task/stage/PipeTaskExtractorStage.java | 12 +-
.../connector/PipeConnectorSubtaskManager.java | 28 +
.../iotdb/db/protocol/client/ConfigNodeClient.java | 65 +-
.../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 2 +-
.../org/apache/iotdb/db/protocol/mqtt/Message.java | 2 +-
.../db/protocol/rest/utils/InsertRowDataUtils.java | 2 +-
.../rest/v1/handler/QueryDataSetHandler.java | 9 +-
.../v1/handler/StatementConstructionHandler.java | 2 +-
.../rest/v2/handler/QueryDataSetHandler.java | 9 +-
.../v2/handler/StatementConstructionHandler.java | 2 +-
.../handler/MLNodeRPCServiceThriftHandler.java | 60 -
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 18 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 61 +-
.../protocol/thrift/impl/MLNodeRPCServiceImpl.java | 217 --
.../db/queryengine/common/header/ColumnHeader.java | 2 +-
.../common/header/ColumnHeaderConstant.java | 30 +-
.../queryengine/common/header/DatasetHeader.java | 2 +-
.../common/header/DatasetHeaderFactory.java | 8 -
.../execution/aggregation/Accumulator.java | 6 +-
.../execution/aggregation/AccumulatorFactory.java | 2 +-
.../execution/aggregation/Aggregator.java | 6 +-
.../execution/aggregation/AvgAccumulator.java | 8 +-
.../execution/aggregation/CountAccumulator.java | 6 +-
.../execution/aggregation/CountIfAccumulator.java | 6 +-
.../aggregation/CountTimeAccumulator.java | 6 +-
.../execution/aggregation/ExtremeAccumulator.java | 8 +-
.../aggregation/FirstValueAccumulator.java | 8 +-
.../aggregation/FirstValueDescAccumulator.java | 4 +-
.../aggregation/LastValueAccumulator.java | 8 +-
.../aggregation/LastValueDescAccumulator.java | 4 +-
.../execution/aggregation/MaxTimeAccumulator.java | 6 +-
.../aggregation/MaxTimeDescAccumulator.java | 2 +-
.../execution/aggregation/MaxValueAccumulator.java | 8 +-
.../execution/aggregation/MinTimeAccumulator.java | 6 +-
.../aggregation/MinTimeDescAccumulator.java | 2 +-
.../execution/aggregation/MinValueAccumulator.java | 8 +-
.../execution/aggregation/SumAccumulator.java | 8 +-
.../aggregation/TimeDurationAccumulator.java | 6 +-
.../MonotonicQueueSlidingWindowAggregator.java | 2 +-
.../slidingwindow/SlidingWindowAggregator.java | 8 +-
.../SlidingWindowAggregatorFactory.java | 4 +-
.../timerangeiterator/AggrWindowIterator.java | 91 +-
.../PreAggrWindowWithNaturalMonthIterator.java | 17 +-
.../TimeRangeIteratorFactory.java | 41 +-
.../execution/exchange/sink/SinkChannel.java | 5 +-
.../fragment/FragmentInstanceManager.java | 11 -
.../execution/load/AlignedChunkData.java | 4 +-
.../execution/load/LoadTsFileManager.java | 29 +-
.../execution/load/MergedTsFileSplitter.java | 2 +-
.../execution/load/NonAlignedChunkData.java | 2 +-
.../execution/load/TsFileSplitSender.java | 8 +-
.../queryengine/execution/load/TsFileSplitter.java | 2 +-
.../nodesplit/ClusteringMeasurementSplitter.java | 2 +-
.../execution/operator/AggregationUtil.java | 9 +-
.../operator/process/AbstractIntoOperator.java | 11 +-
.../operator/process/AggregationOperator.java | 2 +-
.../operator/process/DeviceViewIntoOperator.java | 13 +-
.../operator/process/DeviceViewOperator.java | 9 +-
.../execution/operator/process/FillOperator.java | 2 +-
.../operator/process/FilterAndProjectOperator.java | 6 +-
.../execution/operator/process/IntoOperator.java | 11 +-
.../operator/process/LinearFillOperator.java | 2 +-
.../operator/process/MergeSortOperator.java | 4 +-
.../process/RawDataAggregationOperator.java | 2 +-
.../operator/process/SingleDeviceViewOperator.java | 9 +-
.../process/SlidingWindowAggregationOperator.java | 2 +-
.../execution/operator/process/SortOperator.java | 11 +-
.../operator/process/TagAggregationOperator.java | 9 +-
.../execution/operator/process/TopKOperator.java | 8 +-
.../operator/process/TransformOperator.java | 6 +-
.../execution/operator/process/fill/IFill.java | 2 +-
.../operator/process/fill/ILinearFill.java | 2 +-
.../process/fill/identity/IdentityFill.java | 2 +-
.../process/fill/identity/IdentityLinearFill.java | 2 +-
.../operator/process/fill/linear/LinearFill.java | 2 +-
.../process/join/HorizontallyConcatOperator.java | 6 +-
.../process/join/RowBasedTimeJoinOperator.java | 2 +-
.../operator/process/join/merge/ColumnMerger.java | 2 +-
.../process/join/merge/MergeSortComparator.java | 2 +-
.../process/join/merge/MultiColumnMerger.java | 4 +-
.../join/merge/NonOverlappedMultiColumnMerger.java | 2 +-
.../process/join/merge/SingleColumnMerger.java | 4 +-
.../last/AbstractUpdateLastCacheOperator.java | 2 +-
.../operator/process/last/LastQueryOperator.java | 4 +-
.../process/last/LastQuerySortOperator.java | 4 +-
.../operator/process/last/LastQueryUtil.java | 13 +-
.../process/last/UpdateLastCacheOperator.java | 2 +-
.../last/UpdateViewPathLastCacheOperator.java | 2 +-
.../operator/process/ml/ForecastOperator.java | 241 --
.../schema/CountGroupByLevelMergeOperator.java | 10 +-
.../schema/CountGroupByLevelScanOperator.java | 12 +-
.../operator/schema/CountMergeOperator.java | 2 +-
.../schema/NodeManageMemoryMergeOperator.java | 16 +-
.../operator/schema/NodePathsConvertOperator.java | 7 +-
.../operator/schema/NodePathsCountOperator.java | 7 +-
.../operator/schema/SchemaCountOperator.java | 2 +-
.../operator/schema/SchemaFetchScanOperator.java | 6 +-
.../schema/SchemaQueryOrderByHeatOperator.java | 7 +-
.../operator/schema/SchemaQueryScanOperator.java | 7 +-
.../operator/schema/SchemaTsBlockUtil.java | 9 +-
.../operator/schema/source/DeviceSchemaSource.java | 15 +-
.../operator/schema/source/NodeSchemaSource.java | 10 +-
.../schema/source/PathsUsingTemplateSource.java | 5 +-
.../AbstractSeriesAggregationScanOperator.java | 2 +-
.../operator/source/AlignedSeriesScanOperator.java | 4 +-
.../operator/source/AlignedSeriesScanUtil.java | 2 +-
.../operator/source/ExchangeOperator.java | 6 +-
.../operator/source/SeriesScanOperator.java | 4 +-
.../execution/operator/source/SeriesScanUtil.java | 4 +-
.../operator/source/ShowQueriesOperator.java | 16 +-
.../operator/window/AbstractVariationWindow.java | 2 +-
.../execution/operator/window/ConditionWindow.java | 2 +-
.../operator/window/ConditionWindowManager.java | 4 +-
.../execution/operator/window/CountWindow.java | 2 +-
.../operator/window/CountWindowManager.java | 4 +-
.../execution/operator/window/IWindow.java | 2 +-
.../execution/operator/window/IWindowManager.java | 4 +-
.../execution/operator/window/SessionWindow.java | 2 +-
.../operator/window/SessionWindowManager.java | 2 +-
.../execution/operator/window/TimeWindow.java | 2 +-
.../operator/window/TimeWindowManager.java | 2 +-
.../operator/window/VariationWindowManager.java | 2 +-
.../operator/window/VariationWindowParameter.java | 2 +-
.../operator/window/WindowManagerFactory.java | 2 +-
.../db/queryengine/plan/analyze/Analysis.java | 13 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 125 +-
.../plan/analyze/ExpressionAnalyzer.java | 4 +-
.../plan/analyze/ExpressionTypeAnalyzer.java | 4 +-
.../queryengine/plan/analyze/ExpressionUtils.java | 2 +-
.../db/queryengine/plan/analyze/IModelFetcher.java | 27 -
.../plan/analyze/LoadTsfileAnalyzer.java | 24 +-
.../db/queryengine/plan/analyze/ModelFetcher.java | 75 -
.../queryengine/plan/analyze/SelectIntoUtils.java | 2 +-
.../db/queryengine/plan/analyze/TypeProvider.java | 2 +-
.../analyze/cache/schema/SchemaCacheEntry.java | 2 +-
.../analyze/schema/AutoCreateSchemaExecutor.java | 2 +-
.../analyze/schema/ClusterSchemaFetchExecutor.java | 2 +-
.../plan/analyze/schema/ClusterSchemaFetcher.java | 2 +-
.../plan/analyze/schema/ISchemaAutoCreation.java | 2 +-
.../plan/analyze/schema/ISchemaFetcher.java | 2 +-
.../plan/analyze/schema/NormalSchemaFetcher.java | 2 +-
.../plan/analyze/schema/SchemaValidator.java | 2 +-
.../plan/analyze/schema/TemplateSchemaFetcher.java | 2 +-
.../plan/execution/config/ConfigTaskVisitor.java | 33 -
.../config/executor/ClusterConfigTaskExecutor.java | 184 +-
.../config/executor/IConfigTaskExecutor.java | 10 -
.../config/metadata/CountDatabaseTask.java | 2 +-
.../config/metadata/CountTimeSlotListTask.java | 2 +-
.../execution/config/metadata/GetRegionIdTask.java | 2 +-
.../config/metadata/GetSeriesSlotListTask.java | 2 +-
.../config/metadata/GetTimeSlotListTask.java | 8 +-
.../config/metadata/ShowClusterDetailsTask.java | 75 +-
.../execution/config/metadata/ShowClusterTask.java | 18 +-
.../config/metadata/ShowConfigNodesTask.java | 10 +-
.../config/metadata/ShowContinuousQueriesTask.java | 12 +-
.../config/metadata/ShowDataNodesTask.java | 9 +-
.../config/metadata/ShowFunctionsTask.java | 21 +-
.../config/metadata/ShowPipePluginsTask.java | 21 +-
.../execution/config/metadata/ShowRegionTask.java | 21 +-
.../execution/config/metadata/ShowTTLTask.java | 7 +-
.../config/metadata/ShowTriggersTask.java | 20 +-
.../config/metadata/ShowVariablesTask.java | 90 +-
.../config/metadata/model/CreateModelTask.java | 45 -
.../config/metadata/model/DropModelTask.java | 41 -
.../config/metadata/model/ShowModelsTask.java | 100 -
.../config/metadata/model/ShowTrialsTask.java | 90 -
.../template/ShowNodesInSchemaTemplateTask.java | 18 +-
.../metadata/template/ShowPathSetTemplateTask.java | 7 +-
.../metadata/template/ShowSchemaTemplateTask.java | 7 +-
.../execution/config/sys/pipe/ShowPipeTask.java | 31 +-
.../config/sys/quota/ShowSpaceQuotaTask.java | 32 +-
.../config/sys/quota/ShowThrottleQuotaTask.java | 36 +-
.../memory/StatementMemorySourceVisitor.java | 25 +-
.../db/queryengine/plan/expression/Expression.java | 2 +-
.../plan/expression/ExpressionFactory.java | 2 +-
.../plan/expression/binary/BinaryExpression.java | 2 +-
.../plan/expression/leaf/ConstantOperand.java | 2 +-
.../plan/expression/leaf/LeafOperand.java | 2 +-
.../plan/expression/leaf/TimeSeriesOperand.java | 2 +-
.../plan/expression/multi/FunctionExpression.java | 12 +-
.../plan/expression/multi/FunctionType.java | 3 +-
.../multi/builtin/BuiltInScalarFunctionHelper.java | 2 +-
.../multi/builtin/helper/CastFunctionHelper.java | 2 +-
.../multi/builtin/helper/DiffFunctionHelper.java | 2 +-
.../builtin/helper/ReplaceFunctionHelper.java | 2 +-
.../multi/builtin/helper/RoundFunctionHelper.java | 2 +-
.../builtin/helper/SubStringFunctionHelper.java | 2 +-
.../expression/other/CaseWhenThenExpression.java | 2 +-
.../plan/expression/ternary/TernaryExpression.java | 2 +-
.../plan/expression/unary/UnaryExpression.java | 2 +-
.../visitor/ColumnTransformerVisitor.java | 6 +-
.../visitor/IntermediateLayerVisitor.java | 2 +-
.../plan/optimization/LimitOffsetPushDown.java | 15 +-
.../db/queryengine/plan/parser/ASTVisitor.java | 122 +-
.../plan/parser/StatementGenerator.java | 52 +-
.../plan/planner/LocalExecutionPlanContext.java | 6 +-
.../plan/planner/LogicalPlanBuilder.java | 12 +-
.../plan/planner/LogicalPlanVisitor.java | 20 -
.../plan/planner/OperatorTreeGenerator.java | 51 +-
.../plan/planner/plan/node/PlanGraphPrinter.java | 92 +-
.../plan/planner/plan/node/PlanNodeType.java | 7 +-
.../plan/planner/plan/node/PlanVisitor.java | 5 -
.../plan/node/load/LoadSingleTsFileNode.java | 19 +-
.../planner/plan/node/load/LoadTsFileNode.java | 10 +-
.../write/CreateAlignedTimeSeriesNode.java | 2 +-
.../metedata/write/CreateMultiTimeSeriesNode.java | 2 +-
.../node/metedata/write/CreateTimeSeriesNode.java | 2 +-
.../plan/node/metedata/write/MeasurementGroup.java | 2 +-
.../planner/plan/node/process/ml/ForecastNode.java | 122 -
.../plan/planner/plan/node/write/InsertNode.java | 2 +-
.../planner/plan/node/write/InsertRowNode.java | 4 +-
.../plan/node/write/InsertRowsOfOneDeviceNode.java | 2 +-
.../planner/plan/node/write/InsertTabletNode.java | 4 +-
.../plan/node/write/PipeEnrichedInsertNode.java | 2 +-
.../parameter/DeviceViewIntoPathDescriptor.java | 2 +-
.../plan/parameter/GroupByTimeParameter.java | 85 +-
.../planner/plan/parameter/IntoPathDescriptor.java | 2 +-
.../model/ForecastModelInferenceDescriptor.java | 176 --
.../parameter/model/ModelInferenceDescriptor.java | 111 -
.../plan/scheduler/load/LoadTsFileScheduler.java | 27 +
.../plan/statement/StatementVisitor.java | 21 -
.../statement/component/GroupByTimeComponent.java | 52 +-
.../plan/statement/component/ResultColumn.java | 3 +-
.../plan/statement/component/SelectComponent.java | 8 -
.../plan/statement/crud/InsertBaseStatement.java | 2 +-
.../crud/InsertMultiTabletsStatement.java | 2 +-
.../plan/statement/crud/InsertRowStatement.java | 2 +-
.../crud/InsertRowsOfOneDeviceStatement.java | 2 +-
.../plan/statement/crud/InsertRowsStatement.java | 2 +-
.../plan/statement/crud/InsertTabletStatement.java | 4 +-
.../plan/statement/crud/LoadTsFileStatement.java | 11 +
.../crud/PipeEnrichedInsertBaseStatement.java | 2 +-
.../crud/PipeEnrichedLoadTsFileStatement.java | 10 +
.../plan/statement/crud/QueryStatement.java | 50 -
.../InternalCreateTimeSeriesStatement.java | 2 +-
.../plan/statement/literal/BooleanLiteral.java | 5 +-
.../plan/statement/literal/DoubleLiteral.java | 5 +-
.../plan/statement/literal/Literal.java | 2 +-
.../plan/statement/literal/LongLiteral.java | 5 +-
.../plan/statement/literal/NullLiteral.java | 5 +-
.../plan/statement/literal/StringLiteral.java | 5 +-
.../metadata/CreateAlignedTimeSeriesStatement.java | 2 +-
.../metadata/CreateMultiTimeSeriesStatement.java | 2 +-
.../metadata/CreateTimeSeriesStatement.java | 2 +-
.../statement/metadata/ShowDatabaseStatement.java | 7 +-
.../metadata/model/CreateModelStatement.java | 128 --
.../metadata/model/DropModelStatement.java | 57 -
.../metadata/model/ShowModelsStatement.java | 51 -
.../metadata/model/ShowTrialsStatement.java | 57 -
.../template/AlterSchemaTemplateStatement.java | 2 +-
.../template/CreateSchemaTemplateStatement.java | 2 +-
.../transformation/api/LayerPointReader.java | 2 +-
.../transformation/api/LayerRowReader.java | 2 +-
.../transformation/api/LayerRowWindowReader.java | 2 +-
...ializableRowRecordListBackedMultiColumnRow.java | 6 +-
...izableRowRecordListBackedMultiColumnWindow.java | 2 +-
.../LayerPointReaderBackedSingleColumnRow.java | 3 +-
.../dag/builder/EvaluationDAGBuilder.java | 2 +-
.../dag/column/CaseWhenThenColumnTransformer.java | 4 +-
.../transformation/dag/column/ColumnCache.java | 2 +-
.../dag/column/ColumnTransformer.java | 2 +-
.../binary/ArithmeticBinaryColumnTransformer.java | 4 +-
.../dag/column/binary/BinaryColumnTransformer.java | 4 +-
.../binary/CompareBinaryColumnTransformer.java | 15 +-
.../column/binary/LogicAndColumnTransformer.java | 4 +-
.../column/binary/LogicOrColumnTransformer.java | 4 +-
.../dag/column/leaf/ConstantColumnTransformer.java | 2 +-
.../column/multi/MappableUDFColumnTransformer.java | 48 +-
.../column/ternary/BetweenColumnTransformer.java | 13 +-
.../ternary/CompareTernaryColumnTransformer.java | 4 +-
.../unary/ArithmeticNegationColumnTransformer.java | 4 +-
.../dag/column/unary/InColumnTransformer.java | 8 +-
.../dag/column/unary/IsNullColumnTransformer.java | 4 +-
.../column/unary/LogicNotColumnTransformer.java | 4 +-
.../dag/column/unary/RegularColumnTransformer.java | 11 +-
.../dag/column/unary/UnaryColumnTransformer.java | 4 +-
.../scalar/CastFunctionColumnTransformer.java | 18 +-
.../scalar/DiffFunctionColumnTransformer.java | 4 +-
.../scalar/ReplaceFunctionColumnTransformer.java | 11 +-
.../scalar/RoundFunctionColumnTransformer.java | 4 +-
.../scalar/SubStringFunctionColumnTransformer.java | 11 +-
.../dag/input/ConstantInputReader.java | 2 +-
.../transformation/dag/input/IUDFInputDataSet.java | 2 +-
.../dag/input/QueryDataSetInputLayer.java | 2 +-
.../dag/input/TsBlockInputDataSet.java | 2 +-
.../MultiInputColumnIntermediateLayer.java | 4 +-
...InputColumnMultiReferenceIntermediateLayer.java | 2 +-
...nputColumnSingleReferenceIntermediateLayer.java | 2 +-
.../binary/ArithmeticBinaryTransformer.java | 4 +-
.../dag/transformer/binary/BinaryTransformer.java | 2 +-
.../binary/CompareBinaryTransformer.java | 4 +-
.../binary/CompareEqualToTransformer.java | 5 +-
.../binary/CompareGreaterEqualTransformer.java | 5 +-
.../binary/CompareGreaterThanTransformer.java | 5 +-
.../binary/CompareLessEqualTransformer.java | 5 +-
.../binary/CompareLessThanTransformer.java | 5 +-
.../binary/CompareNonEqualTransformer.java | 7 +-
.../transformer/binary/LogicBinaryTransformer.java | 4 +-
.../multi/MappableUDFQueryRowTransformer.java | 2 +-
.../dag/transformer/multi/UDFQueryTransformer.java | 2 +-
.../multi/UniversalUDFQueryTransformer.java | 2 +-
.../transformer/ternary/BetweenTransformer.java | 17 +-
.../ternary/CompareTernaryTransformer.java | 4 +-
.../transformer/ternary/TernaryTransformer.java | 2 +-
.../unary/ArithmeticNegationTransformer.java | 2 +-
.../dag/transformer/unary/InTransformer.java | 5 +-
.../dag/transformer/unary/IsNullTransformer.java | 2 +-
.../dag/transformer/unary/LogicNotTransformer.java | 4 +-
.../dag/transformer/unary/RegularTransformer.java | 7 +-
.../transformer/unary/TransparentTransformer.java | 2 +-
.../dag/transformer/unary/UnaryTransformer.java | 2 +-
.../unary/scalar/CastFunctionTransformer.java | 18 +-
.../unary/scalar/DiffFunctionTransformer.java | 2 +-
.../unary/scalar/ReplaceFunctionTransformer.java | 11 +-
.../unary/scalar/RoundFunctionTransformer.java | 2 +-
.../unary/scalar/SubStringFunctionTransformer.java | 10 +-
.../transformation/dag/udf/UDTFExecutor.java | 12 +-
.../dag/udf/UDTFInformationInferrer.java | 2 +-
.../transformation/dag/util/LayerCacheUtils.java | 2 +-
.../transformation/dag/util/TransformUtils.java | 6 +-
.../row/ElasticSerializableRowRecordList.java | 2 +-
.../row/SerializableRowRecordList.java | 4 +-
.../tv/ElasticSerializableBinaryTVList.java | 8 +-
.../tv/ElasticSerializableTVList.java | 12 +-
.../datastructure/tv/SerializableBinaryTVList.java | 2 +-
.../tv/SerializableBooleanTVList.java | 2 +-
.../datastructure/tv/SerializableDoubleTVList.java | 2 +-
.../datastructure/tv/SerializableFloatTVList.java | 2 +-
.../datastructure/tv/SerializableIntTVList.java | 2 +-
.../datastructure/tv/SerializableLongTVList.java | 2 +-
.../datastructure/tv/SerializableTVList.java | 4 +-
.../apache/iotdb/db/schemaengine/SchemaEngine.java | 66 +-
.../rescon/CachedSchemaEngineStatistics.java | 4 +
.../rescon/DataNodeSchemaQuotaManager.java | 85 +-
.../rescon/ISchemaEngineStatistics.java | 2 +
.../rescon/MemSchemaEngineStatistics.java | 39 +-
.../schemaregion/impl/SchemaRegionMemoryImpl.java | 13 +-
.../schemaregion/impl/SchemaRegionPBTreeImpl.java | 87 +-
.../visitor/SchemaRegionPlanDeserializer.java | 2 +-
.../visitor/SchemaRegionPlanSerializer.java | 2 +-
.../visitor/SchemaRegionPlanTxtSerializer.java | 2 +-
.../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 7 +-
.../impl/mem/mnode/factory/MemMNodeFactory.java | 3 +-
.../mtree/impl/mem/mnode/info/LogicalViewInfo.java | 2 +-
.../mtree/impl/mem/mnode/info/MeasurementInfo.java | 2 +-
.../impl/mem/snapshot/MemMTreeSnapshotUtil.java | 3 +-
.../mtree/impl/pbtree/CachedMTreeStore.java | 78 +-
.../mtree/impl/pbtree/MTreeBelowSGCachedImpl.java | 170 +-
.../impl/pbtree/StampedWriterPreferredLock.java | 10 +-
.../mtree/impl/pbtree/cache/CacheManager.java | 285 ++-
.../mtree/impl/pbtree/cache/ICacheManager.java | 10 +-
.../impl/pbtree/flush/PBTreeFlushExecutor.java | 104 +
.../mnode/container/CachedMNodeContainer.java | 5 +
.../mnode/container/ICachedMNodeContainer.java | 4 +-
.../pbtree/mnode/factory/CacheMNodeFactory.java | 3 +-
.../mtree/impl/pbtree/schemafile/RecordUtils.java | 99 +-
.../impl/pbtree/schemafile/WrappedSegment.java | 2 +-
.../read/resp/info/impl/ShowTimeSeriesResult.java | 2 +-
.../reader/impl/TimeseriesReaderWithViewFetch.java | 2 +-
.../schemaregion/utils/ResourceByPathUtils.java | 2 +-
.../write/req/ICreateAlignedTimeSeriesPlan.java | 2 +-
.../write/req/ICreateTimeSeriesPlan.java | 2 +-
.../write/req/SchemaRegionWritePlanFactory.java | 2 +-
.../req/impl/CreateAlignedTimeSeriesPlanImpl.java | 2 +-
.../write/req/impl/CreateTimeSeriesPlanImpl.java | 2 +-
.../template/ClusterTemplateManager.java | 6 +-
.../iotdb/db/schemaengine/template/Template.java | 2 +-
.../template/alter/TemplateExtendInfo.java | 2 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 8 +-
.../apache/iotdb/db/service/IoTDBShutdownHook.java | 7 +-
.../apache/iotdb/db/service/MLNodeRPCService.java | 92 -
.../org/apache/iotdb/db/service/RPCService.java | 40 +-
.../db/service/metrics/CompactionMetrics.java | 101 +-
.../metrics/IoTDBInternalLocalReporter.java | 2 +-
.../iotdb/db/storageengine/StorageEngine.java | 6 +-
.../db/storageengine/dataregion/DataRegion.java | 171 +-
.../dataregion/HashLastFlushTimeMap.java | 10 +-
.../compaction/constant/CompactionTaskType.java | 3 +-
.../execute/recover/CompactionRecoverManager.java | 16 +-
.../execute/task/AbstractCompactionTask.java | 53 +-
...skType.java => CompactionTaskPriorityType.java} | 2 +-
.../execute/task/CrossSpaceCompactionTask.java | 18 +-
.../execute/task/InnerSpaceCompactionTask.java | 20 +-
.../task/InsertionCrossSpaceCompactionTask.java | 363 +++
.../compaction/execute/utils/CompactionUtils.java | 167 +-
.../execute/utils/log/CompactionLogger.java | 13 +-
.../execute/utils/log/SimpleCompactionLogger.java | 4 +
.../execute/utils/log/TsFileIdentifier.java | 32 +-
.../utils/validator/CompactionValidator.java | 51 -
.../ResourceAndTsfileCompactionValidator.java | 57 -
.../utils/writer/AbstractCompactionWriter.java | 2 +-
.../writer/AbstractCrossCompactionWriter.java | 2 +-
.../writer/AbstractInnerCompactionWriter.java | 2 +-
.../utils/writer/FastCrossCompactionWriter.java | 2 +-
.../utils/writer/FastInnerCompactionWriter.java | 2 +-
.../writer/ReadPointCrossCompactionWriter.java | 2 +-
.../writer/ReadPointInnerCompactionWriter.java | 2 +-
.../compaction/io/CompactionTsFileWriter.java | 2 +-
.../compaction/schedule/CompactionScheduler.java | 56 +
.../compaction/schedule/CompactionTaskManager.java | 20 +-
.../compaction/schedule/CompactionWorker.java | 27 +-
.../DefaultCompactionTaskComparatorImpl.java | 15 +-
.../impl/RewriteCrossSpaceCompactionSelector.java | 206 +-
.../impl/SizeTieredCompactionSelector.java | 10 +-
.../utils/CrossSpaceCompactionCandidate.java | 116 -
.../utils/DeviceInfo.java} | 16 +-
.../InsertionCrossCompactionTaskResource.java} | 33 +-
.../selector/utils/TsFileResourceCandidate.java | 130 ++
.../dataregion/flush/FlushManager.java | 7 +-
.../dataregion/memtable/AbstractMemTable.java | 4 +-
.../memtable/AlignedReadOnlyMemChunk.java | 2 +-
.../memtable/AlignedWritableMemChunk.java | 4 +-
.../dataregion/memtable/IWritableMemChunk.java | 2 +-
.../dataregion/memtable/ReadOnlyMemChunk.java | 2 +-
.../dataregion/memtable/TsFileProcessor.java | 28 +-
.../dataregion/memtable/WritableMemChunk.java | 4 +-
.../read/reader/chunk/MemAlignedPageReader.java | 6 +-
.../read/reader/chunk/MemPageReader.java | 6 +-
.../reader/common/CachedPriorityMergeReader.java | 2 +-
.../dataregion/snapshot/SnapshotLoader.java | 20 +-
.../dataregion/tsfile/TsFileManager.java | 20 +-
.../dataregion/tsfile/TsFileResource.java | 22 +-
.../dataregion/utils/TsFileResourceUtils.java | 378 ++++
.../validate/TsFileResourceAndDataValidator.java | 58 +
.../validate/TsFileResourceValidator.java} | 43 +-
.../validate/TsFileValidator.java} | 36 +-
.../storageengine/dataregion/wal/WALManager.java | 40 +-
.../dataregion/wal/utils/WALInsertNodeCache.java | 21 +-
.../dataregion/wal/utils/WALWriteUtils.java | 2 +-
.../db/storageengine/rescon/disk/TierManager.java | 4 +-
.../rescon/memory/PrimitiveArrayManager.java | 4 +-
.../rescon/quotas/DefaultOperationQuota.java | 2 +-
.../iotdb/db/tools/TsFileSplitByPartitionTool.java | 4 +-
.../org/apache/iotdb/db/tools/TsFileSplitTool.java | 4 +-
.../db/tools/validate/TsFileValidationTool.java | 2 +-
.../org/apache/iotdb/db/utils/CommonUtils.java | 12 +-
.../org/apache/iotdb/db/utils/DateTimeUtils.java | 99 +-
.../iotdb/db/utils/EncodingInferenceUtils.java | 4 +-
.../java/org/apache/iotdb/db/utils/MemUtils.java | 5 +-
.../apache/iotdb/db/utils/QueryDataSetUtils.java | 6 +-
.../org/apache/iotdb/db/utils/SchemaUtils.java | 2 +-
.../apache/iotdb/db/utils/TimeValuePairUtils.java | 7 +-
.../iotdb/db/utils/TimestampPrecisionUtils.java | 19 +-
.../apache/iotdb/db/utils/TypeInferenceUtils.java | 2 +-
.../iotdb/db/utils/constant/SqlConstant.java | 3 -
.../db/utils/datastructure/AlignedTVList.java | 4 +-
.../db/utils/datastructure/BackAlignedTVList.java | 2 +-
.../db/utils/datastructure/BackBinaryTVList.java | 2 +-
.../db/utils/datastructure/BackBooleanTVList.java | 2 +-
.../db/utils/datastructure/BackDoubleTVList.java | 2 +-
.../db/utils/datastructure/BackFloatTVList.java | 2 +-
.../db/utils/datastructure/BackIntTVList.java | 2 +-
.../db/utils/datastructure/BackLongTVList.java | 2 +-
.../iotdb/db/utils/datastructure/BinaryTVList.java | 2 +-
.../db/utils/datastructure/BooleanTVList.java | 2 +-
.../iotdb/db/utils/datastructure/DoubleTVList.java | 2 +-
.../iotdb/db/utils/datastructure/FloatTVList.java | 2 +-
.../iotdb/db/utils/datastructure/IntTVList.java | 2 +-
.../iotdb/db/utils/datastructure/LongTVList.java | 2 +-
.../db/utils/datastructure/QuickAlignedTVList.java | 2 +-
.../iotdb/db/utils/datastructure/TVList.java | 2 +-
.../db/utils/datastructure/TimAlignedTVList.java | 2 +-
.../db/utils/datastructure/TimBinaryTVList.java | 2 +-
.../db/utils/datastructure/TimBooleanTVList.java | 2 +-
.../db/utils/datastructure/TimDoubleTVList.java | 2 +-
.../db/utils/datastructure/TimFloatTVList.java | 2 +-
.../iotdb/db/utils/datastructure/TimIntTVList.java | 2 +-
.../db/utils/datastructure/TimLongTVList.java | 2 +-
.../apache/iotdb/db/utils/sort/DiskSpiller.java | 4 +-
.../iotdb/db/utils/sort/FileSpillerReader.java | 6 +-
.../iotdb/db/utils/sort/SortBufferManager.java | 5 +-
.../iotdb/db/utils/windowing/api/Window.java | 2 +-
.../windowing/configuration/Configuration.java | 2 +-
.../SlidingSizeWindowConfiguration.java | 2 +-
.../SlidingTimeWindowConfiguration.java | 2 +-
.../handler/SlidingWindowEvaluationHandler.java | 3 +-
.../utils/windowing/window/EvictableBatchList.java | 2 +-
.../db/utils/windowing/window/WindowImpl.java | 4 +-
.../apache/iotdb/db/metadata/MetaUtilsTest.java | 2 +-
.../db/metadata/cache/DataNodeSchemaCacheTest.java | 2 +-
.../cache/dualkeycache/DualKeyCacheTest.java | 2 +-
.../mtree/lock/StampedWriterPreferredLockTest.java | 43 +
.../metadata/mtree/schemafile/RecordUtilTests.java | 2 +-
.../metadata/mtree/schemafile/SchemaFileTest.java | 2 +-
.../metadata/mtree/schemafile/SchemaPageTest.java | 2 +-
.../mtree/schemafile/WrappedSegmentTest.java | 2 +-
.../db/metadata/path/MeasurementPathTest.java | 2 +-
.../schemaRegion/SchemaRegionAliasAndTagTest.java | 2 +-
.../schemaRegion/SchemaRegionBasicTest.java | 2 +-
.../schemaRegion/SchemaRegionManagementTest.java | 2 +-
.../schemaRegion/SchemaRegionTemplateTest.java | 2 +-
.../schemaRegion/SchemaRegionTestUtil.java | 2 +-
.../schemaRegion/SchemaStatisticsTest.java | 2 +-
.../metadata/view/ViewExpressionToStringTest.java | 2 +-
.../apache/iotdb/db/metric/MetricServiceTest.java | 48 +-
.../iotdb/db/pipe/connector/PipeConnectorTest.java | 76 +-
.../iotdb/db/pipe/connector/PipeReceiverTest.java | 2 +-
.../db/pipe/connector/PipeThriftRequestTest.java | 2 +-
.../db/pipe/event/PipeDataTypeTransformerTest.java | 2 +-
.../pipe/event/PipeTabletInsertionEventTest.java | 11 +-
.../db/pipe/extractor/PipeRealtimeExtractTest.java | 2 +-
.../resource/PipeTsFileResourceManagerTest.java | 4 +-
.../apache/iotdb/db/query/udf/example/Adder.java | 2 +-
.../common/schematree/ClusterSchemaTreeTest.java | 2 +-
.../queryengine/common/schematree/NodeRefTest.java | 2 +-
.../queryengine/execution/ConfigExecutionTest.java | 2 +-
.../db/queryengine/execution/DataDriverTest.java | 2 +-
.../execution/aggregation/AccumulatorTest.java | 6 +-
.../aggregation/TimeRangeIteratorTest.java | 183 +-
.../execution/exchange/SharedTsBlockQueueTest.java | 2 +-
.../execution/exchange/SinkChannelTest.java | 5 +-
.../executor/RegionWriteExecutorTest.java | 2 +-
.../execution/load/LoadTsFileSchedulerTest.java | 2 +-
.../db/queryengine/execution/load/TestBase.java | 2 +-
.../operator/AggregationOperatorTest.java | 14 +-
.../AlignedSeriesAggregationScanOperatorTest.java | 29 +-
.../operator/AlignedSeriesScanOperatorTest.java | 2 +-
.../execution/operator/AlignedSeriesTestUtil.java | 2 +-
.../execution/operator/DeviceViewOperatorTest.java | 5 +-
.../execution/operator/FillOperatorTest.java | 2 +-
.../operator/HorizontallyConcatOperatorTest.java | 11 +-
.../execution/operator/LastQueryOperatorTest.java | 7 +-
.../operator/LastQuerySortOperatorTest.java | 7 +-
.../execution/operator/LimitOperatorTest.java | 2 +-
.../execution/operator/LinearFillOperatorTest.java | 2 +-
.../execution/operator/MergeSortOperatorTest.java | 4 +-
.../execution/operator/MultiColumnMergerTest.java | 6 +-
.../NonOverlappedMultiColumnMergerTest.java | 6 +-
.../execution/operator/OffsetOperatorTest.java | 2 +-
.../execution/operator/OperatorMemoryTest.java | 27 +-
.../operator/RawDataAggregationOperatorTest.java | 23 +-
.../SeriesAggregationScanOperatorTest.java | 29 +-
.../execution/operator/SeriesScanOperatorTest.java | 2 +-
.../execution/operator/SingleColumnMergerTest.java | 6 +-
.../operator/SingleDeviceViewOperatorTest.java | 2 +-
.../SlidingWindowAggregationOperatorTest.java | 10 +-
.../execution/operator/SortOperatorTest.java | 2 +-
.../execution/operator/TopKOperatorTest.java | 4 +-
.../operator/UpdateLastCacheOperatorTest.java | 7 +-
.../schema/CountGroupByLevelMergeOperatorTest.java | 4 +-
.../operator/schema/SchemaCountOperatorTest.java | 3 +-
.../schema/SchemaFetchScanOperatorTest.java | 2 +-
.../schema/SchemaQueryScanOperatorTest.java | 2 +-
.../operator/sink/IdentitySinkOperatorTest.java | 2 +-
.../plan/analyze/AggregationDescriptorTest.java | 2 +-
.../db/queryengine/plan/analyze/AnalyzeTest.java | 2 +-
.../plan/analyze/FakeSchemaFetcherImpl.java | 2 +-
.../plan/optimization/LimitOffsetPushDownTest.java | 2 +-
.../plan/parser/StatementGeneratorTest.java | 34 +-
.../db/queryengine/plan/plan/ExpressionTest.java | 2 +-
.../queryengine/plan/plan/LogicalPlannerTest.java | 2 +-
.../queryengine/plan/plan/PipelineBuilderTest.java | 2 +-
.../plan/plan/QueryLogicalPlanUtil.java | 2 +-
.../distribution/AggregationDistributionTest.java | 2 +-
.../distribution/DistributionPlannerBasicTest.java | 2 +-
.../queryengine/plan/plan/distribution/Util.java | 2 +-
.../queryengine/plan/plan/distribution/Util2.java | 2 +-
.../plan/plan/node/PlanGraphPrinterTest.java | 70 +
.../plan/plan/node/load/LoadTsFileNodeTest.java | 2 +-
.../metadata/write/MetadataWriteNodeSerDeTest.java | 2 +-
.../node/process/AggregationNodeSerdeTest.java | 5 +-
.../plan/node/process/FilterNodeSerdeTest.java | 2 +-
.../node/process/GroupByLevelNodeSerdeTest.java | 5 +-
.../plan/node/process/GroupByTagNodeSerdeTest.java | 5 +-
.../plan/plan/node/process/LimitNodeSerdeTest.java | 2 +-
.../plan/node/process/OffsetNodeSerdeTest.java | 2 +-
.../plan/plan/node/process/SortNodeSerdeTest.java | 2 +-
.../plan/node/process/TimeJoinNodeSerdeTest.java | 2 +-
.../source/SeriesAggregationScanNodeSerdeTest.java | 5 +-
.../plan/node/source/SeriesScanNodeSerdeTest.java | 2 +-
.../write/InsertMultiTabletsNodeSerdeTest.java | 5 +-
.../plan/node/write/InsertRowNodeSerdeTest.java | 2 +-
.../plan/node/write/InsertRowsNodeSerdeTest.java | 2 +-
.../write/InsertRowsOfOneDeviceNodeSerdeTest.java | 2 +-
.../plan/node/write/InsertTabletNodeSerdeTest.java | 2 +-
.../plan/node/write/WritePlanNodeSplitTest.java | 2 +-
...meticAndCompareBinaryColumnTransformerTest.java | 6 +-
.../binary/LogicBinaryColumnTransformerTest.java | 6 +-
.../column/unary/UnaryColumnTransformerTest.java | 6 +-
.../ElasticSerializableRowRecordListTest.java | 10 +-
.../ElasticSerializableTVListTest.java | 7 +-
.../SerializableBinaryTVListTest.java | 5 +-
.../SerializableBooleanTVListTest.java | 2 +-
.../SerializableDoubleTVListTest.java | 2 +-
.../datastructure/SerializableFloatTVListTest.java | 2 +-
.../datastructure/SerializableIntTVListTest.java | 2 +-
.../datastructure/SerializableLongTVListTest.java | 2 +-
.../SerializableRowRecordListTest.java | 6 +-
.../DataNodeInternalRPCServiceImplTest.java | 2 +-
.../storageengine/buffer/BloomFilterCacheTest.java | 2 +-
.../db/storageengine/buffer/ChunkCacheTest.java | 2 +-
.../storageengine/dataregion/DataRegionTest.java | 2 +-
.../dataregion/LastFlushTimeMapTest.java | 86 +-
.../iotdb/db/storageengine/dataregion/TTLTest.java | 2 +-
.../compaction/AbstractCompactionTest.java | 83 +-
.../compaction/CompactionTaskComparatorTest.java | 21 +-
.../compaction/CompactionValidationTest.java | 47 +-
.../CrossSpaceCompactionWithUnusualCasesTest.java | 122 +-
.../compaction/FastAlignedCrossCompactionTest.java | 2 +-
.../FastCrossCompactionPerformerTest.java | 6 +-
.../FastInnerCompactionPerformerTest.java | 4 +-
.../FastNonAlignedCrossCompactionTest.java | 2 +-
.../compaction/ReadChunkInnerCompactionTest.java | 2 +-
.../ReadPointAlignedCrossCompactionTest.java | 2 +-
.../ReadPointCompactionPerformerTest.java | 2 +-
.../ReadPointNonAlignedCrossCompactionTest.java | 2 +-
.../compaction/TestUtilsForAlignedSeries.java | 2 +-
.../TsFileValidationCorrectnessTests.java | 296 +++
...eCompactionWithFastPerformerValidationTest.java | 12 +-
...actionWithReadPointPerformerValidationTest.java | 2 +-
.../InsertionCrossSpaceCompactionRecoverTest.java | 372 +++
.../InsertionCrossSpaceCompactionSelectorTest.java | 2372 ++++++++++++++++++++
.../cross/InsertionCrossSpaceCompactionTest.java | 510 +++++
.../dataregion/compaction/cross/MergeTest.java | 2 +-
.../compaction/cross/MergeUpgradeTest.java | 2 +-
...eCrossSpaceCompactionWithFastPerformerTest.java | 14 +-
...sSpaceCompactionWithReadPointPerformerTest.java | 6 +-
.../inner/AbstractInnerSpaceCompactionTest.java | 2 +-
.../inner/FastCompactionPerformerAlignedTest.java | 2 +-
.../FastCompactionPerformerNoAlignedTest.java | 2 +-
.../inner/InnerCompactionEmptyTsFileTest.java | 2 +-
.../inner/InnerCompactionMoreDataTest.java | 2 +-
.../compaction/inner/InnerCompactionTest.java | 2 +-
.../ReadChunkCompactionPerformerAlignedTest.java | 5 +-
.../ReadChunkCompactionPerformerNoAlignedTest.java | 2 +-
.../inner/sizetiered/SizeTieredCompactionTest.java | 2 +-
.../recover/SizeTieredCompactionRecoverTest.java | 2 +-
.../settle/SettleRequestHandlerTest.java | 2 +-
.../compaction/utils/CompactionCheckerUtils.java | 2 +-
.../utils/CompactionFileGeneratorUtils.java | 2 +-
.../compaction/utils/CompactionTestFileWriter.java | 6 +-
.../utils/MultiTsFileDeviceIteratorTest.java | 2 +-
.../compaction/utils/TsFileGeneratorUtils.java | 134 +-
.../dataregion/memtable/MemTableFlushTaskTest.java | 2 +-
.../dataregion/memtable/MemTableTestUtils.java | 2 +-
.../dataregion/memtable/MemtableBenchmark.java | 2 +-
.../dataregion/memtable/PrimitiveMemTableTest.java | 8 +-
.../dataregion/memtable/TsFileProcessorTest.java | 2 +-
.../reader/chunk/MemAlignedChunkLoaderTest.java | 5 +-
.../read/reader/chunk/MemChunkLoaderTest.java | 7 +-
.../read/reader/common/FakedSeriesReader.java | 2 +-
.../AlignedSeriesScanLimitOffsetPushDownTest.java | 2 +-
.../read/reader/series/SeriesReaderTestUtil.java | 2 +-
.../series/SeriesScanLimitOffsetPushDownTest.java | 2 +-
.../dataregion/wal/WALManagerTest.java | 7 +-
.../wal/allocation/ElasticStrategyTest.java | 5 +-
.../wal/allocation/FirstCreateStrategyTest.java | 5 +-
.../wal/allocation/RoundRobinStrategyTest.java | 5 +-
.../dataregion/wal/buffer/WALBufferCommonTest.java | 5 +-
.../dataregion/wal/io/WALFileTest.java | 7 +-
.../wal/node/ConsensusReqReaderTest.java | 7 +-
.../dataregion/wal/node/WALEntryHandlerTest.java | 5 +-
.../dataregion/wal/node/WALNodeTest.java | 5 +-
.../wal/recover/WALRecoverManagerTest.java | 5 +-
.../wal/recover/WALRecoverWriterTest.java | 5 +-
.../file/SealedTsFileRecoverPerformerTest.java | 2 +-
.../wal/recover/file/TsFilePlanRedoerTest.java | 20 +-
.../file/UnsealedTsFileRecoverPerformerTest.java | 2 +-
.../wal/utils/WALInsertNodeCacheTest.java | 5 +-
.../rescon/memory/ResourceManagerTest.java | 2 +-
.../org/apache/iotdb/db/tools/MLogParserTest.java | 2 +-
.../iotdb/db/tools/PBTreeFileSketchTest.java | 2 +-
.../iotdb/db/tools/TsFileAndModSettleToolTest.java | 2 +-
.../iotdb/db/tools/TsFileSelfCheckToolTest.java | 2 +-
.../iotdb/db/tools/TsFileSketchToolTest.java | 2 +-
.../db/trigger/executor/TriggerExecuteTest.java | 2 +-
.../DateTimeUtilsTest.java} | 33 +-
.../iotdb/db/utils/EncodingInferenceUtilsTest.java | 2 +-
.../org/apache/iotdb/db/utils/MemUtilsTest.java | 8 +-
.../iotdb/db/utils/QueryDataSetUtilsTest.java | 8 +-
.../org/apache/iotdb/db/utils/SchemaTestUtils.java | 2 +-
.../org/apache/iotdb/db/utils/SchemaUtilsTest.java | 2 +-
.../apache/iotdb/db/utils/SerializeUtilsTest.java | 70 +-
.../iotdb/db/utils/TypeInferenceUtilsTest.java | 2 +-
.../db/utils/datastructure/AlignedTVListTest.java | 9 +-
.../db/utils/datastructure/BinaryTVListTest.java | 15 +-
.../db/utils/datastructure/MergeSortHeapTest.java | 19 +-
.../datastructure/PrimitiveArrayManagerTest.java | 2 +-
.../apache/iotdb/db/utils/sort/SortUtilTest.java | 4 +-
.../SlidingSizeWindowEvaluationHandlerTest.java | 2 +-
.../SlidingTimeWindowEvaluationHandlerTest.java | 2 +-
.../metrics/{dropwizard-metrics => core}/pom.xml | 16 +-
.../iotdb/metrics/core/IoTDBMetricManager.java | 156 ++
.../metrics/core/reporter/IoTDBJmxReporter.java | 183 ++
.../iotdb/metrics/core/type/IoTDBAutoGauge.java} | 18 +-
.../metrics/core/type/IoTDBAutoGaugeMBean.java} | 14 +-
.../iotdb/metrics/core/type/IoTDBCounter.java} | 21 +-
.../metrics/core/type/IoTDBCounterMBean.java} | 14 +-
.../iotdb/metrics/core/type/IoTDBGauge.java} | 16 +-
.../iotdb/metrics/core/type/IoTDBGaugeMBean.java} | 14 +-
.../iotdb/metrics/core/type/IoTDBHistogram.java} | 37 +-
.../metrics/core/type/IoTDBHistogramMBean.java} | 23 +-
.../metrics/core/type/IoTDBHistogramSnapshot.java} | 13 +-
.../apache/iotdb/metrics/core/type/IoTDBRate.java} | 27 +-
.../iotdb/metrics/core/type/IoTDBRateMBean.java} | 17 +-
.../iotdb/metrics/core/type/IoTDBTimer.java} | 39 +-
.../core/type/IoTDBTimerHistogramSnapshot.java} | 13 +-
.../iotdb/metrics/core/type/IoTDBTimerMBean.java} | 26 +-
.../iotdb/metrics/core/uitls/IoTDBCachedGauge.java | 111 +
.../metrics/core/uitls/IoTDBCachedGaugeMBean.java} | 10 +-
.../core/uitls/IoTDBMetricObjNameFactory.java | 104 +
.../metrics/core/uitls/IoTDBMovingAverage.java | 86 +
.../metrics/core/uitls/ObjectNameFactory.java} | 20 +-
.../org.apache.iotdb.metrics.AbstractMetricManager | 2 +-
.../org.apache.iotdb.metrics.reporter.JmxReporter} | 2 +-
.../dropwizard/DropwizardMetricManager.java | 129 --
.../dropwizard/DropwizardMetricNameTool.java | 82 -
.../dropwizard/reporter/DropwizardJmxReporter.java | 83 -
.../metrics/dropwizard/type/DropwizardGauge.java | 72 -
.../dropwizard/type/DropwizardHistogram.java | 47 -
.../type/DropwizardHistogramSnapshot.java | 58 -
.../metrics/dropwizard/type/DropwizardRate.java | 94 -
.../metrics/dropwizard/type/DropwizardTimer.java | 48 -
.../org.apache.iotdb.metrics.reporter.JmxReporter | 18 -
iotdb-core/metrics/interface/pom.xml | 15 +-
.../iotdb/metrics/AbstractMetricManager.java | 69 +-
.../iotdb/metrics/AbstractMetricService.java | 71 +-
.../iotdb/metrics/DoNothingMetricService.java | 10 +
.../apache/iotdb/metrics/config/MetricConfig.java | 17 +-
.../metrics/config/MetricConfigDescriptor.java | 9 +-
.../iotdb/metrics/impl/DoNothingAutoGauge.java | 5 +-
.../iotdb/metrics/impl/DoNothingCounter.java | 5 +-
.../apache/iotdb/metrics/impl/DoNothingGauge.java | 5 +-
.../iotdb/metrics/impl/DoNothingHistogram.java | 5 +-
.../metrics/impl/DoNothingHistogramSnapshot.java | 4 +-
.../iotdb/metrics/impl/DoNothingMetricManager.java | 8 +-
.../apache/iotdb/metrics/impl/DoNothingRate.java | 13 +-
.../apache/iotdb/metrics/impl/DoNothingTimer.java | 3 +-
.../metrics/metricsets/cpu/CpuUsageMetrics.java | 2 +-
.../apache/iotdb/metrics/reporter/JmxReporter.java | 7 +-
.../metrics/reporter/iotdb/IoTDBReporter.java | 2 +-
.../reporter/iotdb/IoTDBSessionReporter.java | 2 +-
.../reporter/prometheus/PrometheusReporter.java | 12 +-
.../org/apache/iotdb/metrics/type/AutoGauge.java | 4 +-
.../org/apache/iotdb/metrics/type/Counter.java | 4 +-
.../java/org/apache/iotdb/metrics/type/Gauge.java | 4 +-
.../org/apache/iotdb/metrics/type/Histogram.java | 4 +-
.../org/apache/iotdb/metrics/type/IMetric.java | 4 +
.../java/org/apache/iotdb/metrics/type/Rate.java | 8 -
...tricFrameType.java => AbstractMetricMBean.java} | 17 +-
.../iotdb/metrics/config/MetricConfigTest.java | 3 -
iotdb-core/metrics/micrometer-metrics/pom.xml | 54 -
.../micrometer/MicrometerMetricManager.java | 153 --
.../micrometer/reporter/MicrometerJmxReporter.java | 90 -
.../micrometer/type/MicrometerAutoGauge.java | 52 -
.../metrics/micrometer/type/MicrometerCounter.java | 45 -
.../org.apache.iotdb.metrics.reporter.JmxReporter | 18 -
iotdb-core/metrics/pom.xml | 5 +-
iotdb-core/mlnode/.flake8 | 33 -
iotdb-core/mlnode/.gitignore | 9 -
iotdb-core/mlnode/README.md | 47 -
iotdb-core/mlnode/iotdb/__init__.py | 17 -
iotdb-core/mlnode/iotdb/mlnode/__init__.py | 17 -
.../mlnode/iotdb/mlnode/algorithm/factory.py | 66 -
.../iotdb/mlnode/algorithm/hyperparameter.py | 407 ----
iotdb-core/mlnode/iotdb/mlnode/algorithm/metric.py | 80 -
.../mlnode/algorithm/models/forecast/__init__.py | 17 -
.../mlnode/algorithm/models/forecast/dlinear.py | 133 --
.../mlnode/algorithm/models/forecast/nbeats.py | 131 --
.../mlnode/iotdb/mlnode/algorithm/validator.py | 45 -
iotdb-core/mlnode/iotdb/mlnode/client.py | 274 ---
iotdb-core/mlnode/iotdb/mlnode/config.py | 168 --
iotdb-core/mlnode/iotdb/mlnode/constant.py | 99 -
iotdb-core/mlnode/iotdb/mlnode/dataset/dataset.py | 74 -
iotdb-core/mlnode/iotdb/mlnode/dataset/factory.py | 50 -
iotdb-core/mlnode/iotdb/mlnode/dataset/source.py | 80 -
.../mlnode/iotdb/mlnode/dataset/utils/__init__.py | 17 -
.../iotdb/mlnode/dataset/utils/time_features.py | 149 --
iotdb-core/mlnode/iotdb/mlnode/exception.py | 66 -
iotdb-core/mlnode/iotdb/mlnode/handler.py | 95 -
iotdb-core/mlnode/iotdb/mlnode/log.py | 34 -
iotdb-core/mlnode/iotdb/mlnode/parser.py | 99 -
iotdb-core/mlnode/iotdb/mlnode/process/__init__.py | 17 -
iotdb-core/mlnode/iotdb/mlnode/process/manager.py | 124 -
iotdb-core/mlnode/iotdb/mlnode/process/task.py | 265 ---
iotdb-core/mlnode/iotdb/mlnode/process/trial.py | 224 --
iotdb-core/mlnode/iotdb/mlnode/script.py | 36 -
iotdb-core/mlnode/iotdb/mlnode/serde.py | 508 -----
iotdb-core/mlnode/iotdb/mlnode/service.py | 58 -
iotdb-core/mlnode/iotdb/mlnode/storage.py | 110 -
iotdb-core/mlnode/iotdb/mlnode/util.py | 80 -
iotdb-core/mlnode/pom.xml | 175 --
iotdb-core/mlnode/pyproject.toml | 55 -
iotdb-core/mlnode/requirements.txt | 30 -
iotdb-core/mlnode/requirements_dev.txt | 28 -
iotdb-core/mlnode/resources/conf/iotdb-mlnode.toml | 70 -
.../mlnode/resources/conf/logging_config.ini | 40 -
.../mlnode/test/test_create_forecast_model.py | 100 -
iotdb-core/mlnode/test/test_model_storage.py | 100 -
iotdb-core/mlnode/test/test_serde.py | 171 --
iotdb-core/node-commons/pom.xml | 33 +-
.../resources/conf/iotdb-common.properties | 101 +-
.../commons/auth/entity/PriPrivilegeType.java | 1 +
.../iotdb/commons/auth/entity/PrivilegeType.java | 2 +
.../org/apache/iotdb/commons/auth/entity/Role.java | 4 +
.../iotdb/commons/client/ClientPoolFactory.java | 26 -
.../iotdb/commons/client/mlnode/MLNodeClient.java | 212 --
.../commons/client/mlnode/MLNodeClientManager.java | 40 -
.../iotdb/commons/client/mlnode/MLNodeInfo.java | 29 -
.../iotdb/commons/concurrent/ThreadName.java | 2 -
.../apache/iotdb/commons/conf/CommonConfig.java | 83 +-
.../iotdb/commons/conf/CommonDescriptor.java | 43 +-
.../apache/iotdb/commons/conf/IoTDBConstant.java | 4 +-
.../exception/pipe/PipeRuntimeExceptionType.java | 5 +
.../PipeRuntimeOutOfMemoryCriticalException.java | 100 +
.../commons/model/ForecastModeInformation.java | 153 --
.../iotdb/commons/model/ModelHyperparameter.java | 81 -
.../iotdb/commons/model/ModelInformation.java | 261 ---
.../org/apache/iotdb/commons/model/ModelTable.java | 91 -
.../iotdb/commons/model/TrialInformation.java | 108 -
.../model/exception/ModelManagementException.java | 26 -
.../org/apache/iotdb/commons/path/AlignedPath.java | 10 +-
.../apache/iotdb/commons/path/MeasurementPath.java | 2 +-
.../org/apache/iotdb/commons/path/PartialPath.java | 2 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 24 +
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 37 +
.../builtin/connector/WebSocketConnector.java | 4 +-
.../builtin/connector/WriteBackConnector.java} | 14 +-
.../commons/schema/filter/SchemaFilterFactory.java | 2 +-
.../commons/schema/filter/impl/DataTypeFilter.java | 2 +-
.../node/common/AbstractMeasurementMNode.java | 2 +-
.../commons/schema/node/info/IMeasurementInfo.java | 2 +-
.../schema/node/role/IMeasurementMNode.java | 2 +-
.../commons/schema/node/utils/IMNodeFactory.java | 3 +-
.../commons/schema/view/LogicalViewSchema.java | 2 +-
.../viewExpression/leaf/ConstantViewOperand.java | 2 +-
.../service/AbstractThriftServiceThread.java | 36 +
.../apache/iotdb/commons/service/ServiceType.java | 1 -
.../iotdb/commons/service/ThriftServiceThread.java | 28 +
.../commons/service/metric/MetricService.java | 49 +
.../iotdb/commons/service/metric/enums/Metric.java | 1 +
.../commons/udf/builtin/String/UDTFConcat.java | 63 +-
.../commons/udf/builtin/String/UDTFEndsWith.java | 20 +
.../commons/udf/builtin/String/UDTFLower.java | 23 +-
.../commons/udf/builtin/String/UDTFStartsWith.java | 36 +-
.../commons/udf/builtin/String/UDTFStrCompare.java | 40 +-
.../commons/udf/builtin/String/UDTFStrLength.java | 36 +-
.../commons/udf/builtin/String/UDTFStrLocate.java | 37 +-
.../iotdb/commons/udf/builtin/String/UDTFTrim.java | 36 +-
.../commons/udf/builtin/String/UDTFUpper.java | 23 +-
.../apache/iotdb/commons/udf/builtin/UDTFAbs.java | 85 +
.../iotdb/commons/udf/builtin/UDTFConst.java | 132 +-
.../iotdb/commons/udf/builtin/UDTFConstE.java | 34 +-
.../iotdb/commons/udf/builtin/UDTFConstPi.java | 34 +-
.../iotdb/commons/udf/builtin/UDTFContains.java | 36 +-
.../udf/builtin/UDTFContinuouslySatisfy.java | 2 +-
.../udf/builtin/UDTFEqualSizeBucketAggSample.java | 2 +-
.../udf/builtin/UDTFEqualSizeBucketSample.java | 2 +-
.../iotdb/commons/udf/builtin/UDTFInRange.java | 123 +-
.../apache/iotdb/commons/udf/builtin/UDTFJexl.java | 2 +-
.../apache/iotdb/commons/udf/builtin/UDTFM4.java | 2 +-
.../iotdb/commons/udf/builtin/UDTFMatches.java | 35 +-
.../apache/iotdb/commons/udf/builtin/UDTFMath.java | 87 +-
.../iotdb/commons/udf/builtin/UDTFOnOff.java | 87 +-
.../iotdb/commons/udf/builtin/UDTFSelectK.java | 2 +-
.../iotdb/commons/udf/builtin/UDTFValueTrend.java | 2 +-
.../commons/udf/service/UDFManagementService.java | 19 -
.../commons/udf/utils/UDFDataTypeTransformer.java | 7 +-
.../apache/iotdb/commons/utils/NodeUrlUtils.java | 65 +-
.../apache/iotdb/commons/utils/SerializeUtils.java | 2 +-
iotdb-core/pom.xml | 3 +-
iotdb-core/tsfile/pom.xml | 9 +-
.../iotdb/tsfile/common/conf/TSFileConfig.java | 2 +-
.../iotdb/tsfile/encoding/decoder/Decoder.java | 2 +-
.../tsfile/encoding/decoder/FloatDecoder.java | 2 +-
.../tsfile/encoding/encoder/FloatEncoder.java | 2 +-
.../tsfile/encoding/encoder/PlainEncoder.java | 2 +-
.../tsfile/encoding/encoder/TSEncodingBuilder.java | 4 +-
.../write/UnSupportedDataTypeException.java | 31 -
.../iotdb/tsfile/file/header/ChunkHeader.java | 2 +-
.../iotdb/tsfile/file/header/PageHeader.java | 2 +-
.../tsfile/file/metadata/AlignedChunkMetadata.java | 2 +-
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 2 +-
.../iotdb/tsfile/file/metadata/IChunkMetadata.java | 2 +-
.../tsfile/file/metadata/TimeseriesMetadata.java | 2 +-
.../file/metadata/statistics/BinaryStatistics.java | 7 +-
.../metadata/statistics/BooleanStatistics.java | 2 +-
.../file/metadata/statistics/DoubleStatistics.java | 2 +-
.../file/metadata/statistics/FloatStatistics.java | 2 +-
.../metadata/statistics/IntegerStatistics.java | 2 +-
.../file/metadata/statistics/LongStatistics.java | 2 +-
.../file/metadata/statistics/Statistics.java | 2 +-
.../file/metadata/statistics/TimeStatistics.java | 2 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 2 +-
.../apache/iotdb/tsfile/read/common/BatchData.java | 4 +-
.../iotdb/tsfile/read/common/BatchDataFactory.java | 2 +-
.../tsfile/read/common/DescReadBatchData.java | 2 +-
.../tsfile/read/common/DescReadWriteBatchData.java | 4 +-
.../org/apache/iotdb/tsfile/read/common/Field.java | 4 +-
.../apache/iotdb/tsfile/read/common/RowRecord.java | 2 +-
.../iotdb/tsfile/read/common/block/TsBlock.java | 4 +-
.../tsfile/read/common/block/TsBlockBuilder.java | 14 +-
.../block/column/BinaryArrayColumnEncoder.java | 3 +-
.../read/common/block/column/BinaryColumn.java | 4 +-
.../common/block/column/BinaryColumnBuilder.java | 7 +-
.../read/common/block/column/BooleanColumn.java | 4 +-
.../common/block/column/BooleanColumnBuilder.java | 7 +-
.../block/column/ByteArrayColumnEncoder.java | 3 +-
.../read/common/block/column/ColumnEncoder.java | 3 +-
.../common/block/column/ColumnEncoderFactory.java | 2 +
.../read/common/block/column/DoubleColumn.java | 4 +-
.../common/block/column/DoubleColumnBuilder.java | 7 +-
.../read/common/block/column/FloatColumn.java | 4 +-
.../common/block/column/FloatColumnBuilder.java | 7 +-
.../block/column/Int32ArrayColumnEncoder.java | 3 +-
.../block/column/Int64ArrayColumnEncoder.java | 3 +-
.../tsfile/read/common/block/column/IntColumn.java | 4 +-
.../read/common/block/column/IntColumnBuilder.java | 7 +-
.../read/common/block/column/LongColumn.java | 4 +-
.../common/block/column/LongColumnBuilder.java | 7 +-
.../read/common/block/column/NullColumn.java | 4 +-
.../block/column/RunLengthColumnEncoder.java | 4 +-
.../block/column/RunLengthEncodedColumn.java | 4 +-
.../read/common/block/column/TimeColumn.java | 4 +-
.../common/block/column/TimeColumnBuilder.java | 7 +-
.../read/common/block/column/TsBlockSerde.java | 4 +-
.../iotdb/tsfile/read/common/type/BinaryType.java | 4 +-
.../iotdb/tsfile/read/common/type/BooleanType.java | 4 +-
.../iotdb/tsfile/read/common/type/DoubleType.java | 4 +-
.../iotdb/tsfile/read/common/type/FloatType.java | 4 +-
.../iotdb/tsfile/read/common/type/IntType.java | 4 +-
.../iotdb/tsfile/read/common/type/LongType.java | 4 +-
.../apache/iotdb/tsfile/read/common/type/Type.java | 4 +-
.../iotdb/tsfile/read/common/type/TypeFactory.java | 2 +-
.../tsfile/read/controller/IMetadataQuerier.java | 2 +-
.../read/controller/MetadataQuerierByFileImpl.java | 2 +-
.../tsfile/read/expression/QueryExpression.java | 2 +-
.../iotdb/tsfile/read/filter/GroupByFilter.java | 5 +
.../tsfile/read/filter/GroupByMonthFilter.java | 183 +-
.../iotdb/tsfile/read/filter/operator/Between.java | 2 +-
.../iotdb/tsfile/read/filter/operator/Eq.java | 2 +-
.../iotdb/tsfile/read/filter/operator/Gt.java | 2 +-
.../iotdb/tsfile/read/filter/operator/GtEq.java | 2 +-
.../iotdb/tsfile/read/filter/operator/Lt.java | 2 +-
.../iotdb/tsfile/read/filter/operator/LtEq.java | 2 +-
.../iotdb/tsfile/read/filter/operator/NotEq.java | 2 +-
.../query/dataset/DataSetWithTimeGenerator.java | 2 +-
.../query/dataset/DataSetWithoutTimeGenerator.java | 4 +-
.../tsfile/read/query/dataset/QueryDataSet.java | 2 +-
.../query/executor/ExecutorWithTimeGenerator.java | 2 +-
.../tsfile/read/query/executor/TsFileExecutor.java | 2 +-
.../read/query/timegenerator/TimeGenerator.java | 2 +-
.../iotdb/tsfile/read/reader/IPageReader.java | 2 +-
.../read/reader/chunk/AlignedChunkReader.java | 2 +-
.../tsfile/read/reader/chunk/ChunkReader.java | 2 +-
.../tsfile/read/reader/page/AlignedPageReader.java | 2 +-
.../iotdb/tsfile/read/reader/page/PageReader.java | 6 +-
.../tsfile/read/reader/page/ValuePageReader.java | 6 +-
.../reader/series/FileSeriesReaderByTimestamp.java | 2 +-
.../org/apache/iotdb/tsfile/utils/BytesUtils.java | 4 +
.../iotdb/tsfile/utils/ReadWriteIOUtils.java | 2 +-
.../apache/iotdb/tsfile/utils/TimeDuration.java | 194 ++
.../iotdb/tsfile/utils/TsFileGeneratorUtils.java | 11 +-
.../write/chunk/AlignedChunkGroupWriterImpl.java | 4 +-
.../tsfile/write/chunk/AlignedChunkWriterImpl.java | 4 +-
.../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 2 +-
.../chunk/NonAlignedChunkGroupWriterImpl.java | 4 +-
.../iotdb/tsfile/write/chunk/TimeChunkWriter.java | 2 +-
.../iotdb/tsfile/write/chunk/ValueChunkWriter.java | 2 +-
.../apache/iotdb/tsfile/write/page/PageWriter.java | 2 +-
.../iotdb/tsfile/write/page/ValuePageWriter.java | 2 +-
.../apache/iotdb/tsfile/write/record/Tablet.java | 10 +-
.../write/record/datapoint/BooleanDataPoint.java | 2 +-
.../tsfile/write/record/datapoint/DataPoint.java | 8 +-
.../write/record/datapoint/DoubleDataPoint.java | 2 +-
.../write/record/datapoint/FloatDataPoint.java | 2 +-
.../write/record/datapoint/IntDataPoint.java | 2 +-
.../write/record/datapoint/LongDataPoint.java | 2 +-
.../write/record/datapoint/StringDataPoint.java | 2 +-
.../tsfile/write/schema/IMeasurementSchema.java | 2 +-
.../tsfile/write/schema/MeasurementSchema.java | 2 +-
.../tsfile/write/schema/TimeseriesSchema.java | 2 +-
.../write/schema/VectorMeasurementSchema.java | 2 +-
.../write/writer/RestorableTsFileIOWriter.java | 2 +-
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 2 +-
.../write/writer/TsFileIOWriterEndFileTest.java | 2 +-
.../write/writer/tsmiterator/DiskTSMIterator.java | 2 +-
.../write/writer/tsmiterator/TSMIterator.java | 2 +-
.../common/block/BinaryArrayColumnEncoderTest.java | 4 +-
.../common/block/ByteArrayColumnEncoderTest.java | 4 +-
.../tsfile/common/block/ColumnEncoderTest.java | 2 +-
.../common/block/Int32ArrayColumnEncoderTest.java | 4 +-
.../common/block/Int64ArrayColumnEncoderTest.java | 4 +-
.../tsfile/common/block/NullColumnUnitTest.java | 4 +-
.../common/block/RunLengthColumnEncoderTest.java | 9 +-
.../tsfile/common/block/TsBlockSerdeTest.java | 9 +-
.../iotdb/tsfile/common/block/TsBlockTest.java | 19 +-
.../encoding/decoder/DictionaryDecoderTest.java | 5 +-
.../tsfile/encoding/decoder/FloatDecoderTest.java | 2 +-
.../tsfile/exception/TsFileExceptionTest.java | 1 -
.../iotdb/tsfile/file/header/PageHeaderTest.java | 2 +-
.../metadata/statistics/StringStatisticsTest.java | 37 +-
.../tsfile/file/metadata/utils/TestHelper.java | 2 +-
.../org/apache/iotdb/tsfile/read/ReadTest.java | 3 +-
.../iotdb/tsfile/read/TimePlainEncodeReadTest.java | 9 +-
.../apache/iotdb/tsfile/read/TsFileReaderTest.java | 4 +-
.../tsfile/read/TsFileSequenceReaderTest.java | 2 +-
.../iotdb/tsfile/read/common/BatchDataTest.java | 2 +-
.../iotdb/tsfile/read/common/ColumnTest.java | 6 +-
.../tsfile/read/filter/FilterSerializeTest.java | 18 +-
.../tsfile/read/filter/GroupByMonthFilterTest.java | 60 +-
.../iotdb/tsfile/read/filter/OperatorTest.java | 9 +-
.../tsfile/read/filter/StatisticsFilterTest.java | 2 +-
.../read/query/executor/QueryExecutorTest.java | 3 +-
.../read/query/timegenerator/ReadWriteTest.java | 2 +-
.../timegenerator/TimeGeneratorReadEmptyTest.java | 2 +-
.../timegenerator/TimeGeneratorReadWriteTest.java | 2 +-
.../query/timegenerator/TimeGeneratorTest.java | 3 +-
.../TsFileGeneratorForSeriesReaderByTimestamp.java | 2 +-
.../iotdb/tsfile/read/reader/FakedBatchReader.java | 2 +-
.../tsfile/read/reader/FakedMultiBatchReader.java | 2 +-
.../iotdb/tsfile/read/reader/PageReaderTest.java | 5 +-
.../apache/iotdb/tsfile/utils/FileGenerator.java | 2 +-
.../org/apache/iotdb/tsfile/utils/RecordUtils.java | 4 +-
.../apache/iotdb/tsfile/utils/RecordUtilsTest.java | 2 +-
.../iotdb/tsfile/utils/TimeDurationTest.java | 67 +
.../iotdb/tsfile/utils/TsFileGeneratorForTest.java | 2 +-
.../iotdb/tsfile/utils/TsPrimitiveTypeTest.java | 10 +-
.../tsfile/write/DefaultSchemaTemplateTest.java | 2 +-
.../tsfile/write/MetadataIndexConstructorTest.java | 2 +-
.../org/apache/iotdb/tsfile/write/PerfTest.java | 2 +-
.../iotdb/tsfile/write/ReadPageInMemTest.java | 2 +-
...SameMeasurementsWithDifferentDataTypesTest.java | 2 +-
.../iotdb/tsfile/write/TsFileIOWriterTest.java | 2 +-
.../tsfile/write/TsFileIntegrityCheckingTool.java | 2 +-
.../iotdb/tsfile/write/TsFileReadWriteTest.java | 2 +-
.../iotdb/tsfile/write/TsFileWriteApiTest.java | 7 +-
.../iotdb/tsfile/write/TsFileWriterTest.java | 2 +-
.../org/apache/iotdb/tsfile/write/WriteTest.java | 2 +-
.../iotdb/tsfile/write/record/TabletTest.java | 2 +-
.../write/schema/converter/SchemaBuilderTest.java | 2 +-
.../write/writer/AlignedChunkWriterImplTest.java | 2 +-
.../write/writer/ForceAppendTsFileWriterTest.java | 2 +-
.../writer/MeasurementSchemaSerializeTest.java | 2 +-
.../iotdb/tsfile/write/writer/PageWriterTest.java | 7 +-
.../write/writer/RestorableTsFileIOWriterTest.java | 2 +-
.../tsfile/write/writer/TimeChunkWriterTest.java | 2 +-
.../tsfile/write/writer/TimePageWriterTest.java | 2 +-
.../writer/TsFileIOWriterMemoryControlTest.java | 8 +-
.../tsfile/write/writer/ValueChunkWriterTest.java | 2 +-
.../tsfile/write/writer/ValuePageWriterTest.java | 2 +-
.../write/writer/VectorMeasurementSchemaStub.java | 2 +-
iotdb-protocol/openapi/pom.xml | 2 +-
iotdb-protocol/pom.xml | 3 +-
iotdb-protocol/thrift-commons/pom.xml | 2 +-
.../thrift-commons/src/main/thrift/common.thrift | 25 +-
iotdb-protocol/thrift-confignode/pom.xml | 4 +-
.../src/main/thrift/confignode.thrift | 126 +-
iotdb-protocol/thrift-consensus/pom.xml | 4 +-
iotdb-protocol/thrift-datanode/pom.xml | 4 +-
.../src/main/thrift/datanode.thrift | 115 +-
iotdb-protocol/thrift-mlnode/pom.xml | 69 -
.../thrift-mlnode/src/main/thrift/mlnode.thrift | 60 -
library-udf/pom.xml | 9 +-
.../apache/iotdb/library/anomaly/UDTFRange.java | 9 +-
.../apache/iotdb/library/dprofile/UDTFSample.java | 8 +-
.../iotdb/library/string/UDTFRegexMatch.java | 44 +-
.../iotdb/library/string/UDTFRegexReplace.java | 46 +-
.../iotdb/library/string/UDTFRegexSplit.java | 64 +-
.../iotdb/library/string/UDTFStrReplace.java | 45 +-
pom.xml | 19 +-
1376 files changed, 15945 insertions(+), 16278 deletions(-)
delete mode 100644 .github/workflows/iotdb-ml.yml
copy example/{flink => pipe-count-point-processor}/pom.xml (64%)
create mode 100644 example/pipe-count-point-processor/src/main/java/org/apache/iotdb/CountPointProcessor.java
copy example/{session => pipe-opc-ua-sink}/pom.xml (75%)
create mode 100644 example/pipe-opc-ua-sink/src/main/java/org/apache/iotdb/opcua/ClientExample.java
create mode 100644 example/pipe-opc-ua-sink/src/main/java/org/apache/iotdb/opcua/ClientExampleRunner.java
create mode 100644 example/pipe-opc-ua-sink/src/main/java/org/apache/iotdb/opcua/ClientTest.java
create mode 100644 example/pipe-opc-ua-sink/src/main/java/org/apache/iotdb/opcua/IoTDBKeyStoreLoaderClient.java
copy iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/String/UDTFUpper.java => integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/TwoSum.java (64%)
create mode 100644 integration-test/src/main/java/org/apache/iotdb/db/query/udf/example/TwoSumBlock.java
create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthNsPrecisionIT.java
create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/groupby/IoTDBGroupByNaturalMonthUsPrecisionIT.java
rename integration-test/src/test/java/org/apache/iotdb/db/it/schema/{ => quota}/IoTDBClusterDeviceQuotaIT.java (85%)
copy integration-test/src/test/java/org/apache/iotdb/db/it/schema/{IoTDBClusterMeasurementQuotaIT.java => quota/IoTDBClusterMixQuotaIT.java} (57%)
rename integration-test/src/test/java/org/apache/iotdb/db/it/schema/{IoTDBClusterMeasurementQuotaIT.java => quota/IoTDBClusterQuotaIT.java} (76%)
rename iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/ModelInferenceFunction.java => integration-test/src/test/java/org/apache/iotdb/db/it/schema/quota/IoTDBClusterTimeSeriesQuotaIT.java (52%)
create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java
copy iotdb-api/{external-api => common-api}/pom.xml (89%)
rename {iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column => iotdb-api/common-api/src/main/java/org/apache/iotdb/tsfile/access}/Column.java (97%)
rename {iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column => iotdb-api/common-api/src/main/java/org/apache/iotdb/tsfile/access}/ColumnBuilder.java (96%)
rename {iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column => iotdb-api/common-api/src/main/java/org/apache/iotdb/tsfile/access}/ColumnBuilderStatus.java (95%)
rename {iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block => iotdb-api/common-api/src/main/java/org/apache/iotdb/tsfile/access}/TsBlockBuilderStatus.java (81%)
rename {iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/block/column => iotdb-api/common-api/src/main/java/org/apache/iotdb/tsfile/enums}/ColumnEncoding.java (96%)
rename {iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata => iotdb-api/common-api/src/main/java/org/apache/iotdb/tsfile}/enums/TSDataType.java (97%)
rename {iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/runtime => iotdb-api/common-api/src/main/java/org/apache/iotdb/tsfile/exception}/UnSupportedDataTypeException.java (92%)
rename {iotdb-core/tsfile => iotdb-api/common-api}/src/main/java/org/apache/iotdb/tsfile/utils/Binary.java (82%)
rename {iotdb-core/tsfile => iotdb-api/common-api}/src/main/java/org/apache/iotdb/tsfile/utils/TsPrimitiveType.java (99%)
copy iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/{PipeConnector.java => PipeSink.java} (62%)
copy iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/{PipeExtractor.java => PipeSource.java} (61%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/MLNodeRPCServiceMBean.java => iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/configuration/PipeSinkRuntimeConfiguration.java (84%)
rename iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/model/CreateModelState.java => iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/configuration/PipeSourceRuntimeConfiguration.java (84%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/runtime/ModelInferenceProcessException.java => iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/exception/PipeSinkException.java (77%)
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/adapter/ElasticSerializableRowRecordListBackedMultiColumnRow.java => iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/utils/RowImpl.java (65%)
delete mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/GetModelInfoPlan.java
delete mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/ShowModelPlan.java
delete mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/ShowTrialPlan.java
delete mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/CreateModelPlan.java
delete mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/DropModelPlan.java
delete mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/UpdateModelInfoPlan.java
delete mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/UpdateModelStatePlan.java
delete mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/model/GetModelInfoResp.java
delete mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/model/ModelTableResp.java
delete mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/model/TrialTableResp.java
delete mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ModelManager.java
delete mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/ModelInfo.java
delete mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/model/CreateModelProcedure.java
delete mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/model/DropModelProcedure.java
delete mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/model/DropModelState.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/writeback/WriteBackConnector.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/handler/MLNodeRPCServiceThriftHandler.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/MLNodeRPCServiceImpl.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/ml/ForecastOperator.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IModelFetcher.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ModelFetcher.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/model/CreateModelTask.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/model/DropModelTask.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/model/ShowModelsTask.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/model/ShowTrialsTask.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/process/ml/ForecastNode.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/model/ForecastModelInferenceDescriptor.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/model/ModelInferenceDescriptor.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/model/CreateModelStatement.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/model/DropModelStatement.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/model/ShowModelsStatement.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/model/ShowTrialsStatement.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/flush/PBTreeFlushExecutor.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/MLNodeRPCService.java
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/{CompactionTaskType.java => CompactionTaskPriorityType.java} (96%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/validator/CompactionValidator.java
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/validator/ResourceAndTsfileCompactionValidator.java
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/{constant/CompactionValidationLevel.java => selector/utils/DeviceInfo.java} (75%)
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/{execute/utils/validator/NoneCompactionValidator.java => selector/utils/InsertionCrossCompactionTaskResource.java} (56%)
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/utils/TsFileResourceCandidate.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java
create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/validate/TsFileResourceAndDataValidator.java
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/{compaction/execute/utils/validator/ResourceOnlyCompactionValidator.java => utils/validate/TsFileResourceValidator.java} (53%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/{compaction/execute/utils/validator/NoneCompactionValidator.java => utils/validate/TsFileValidator.java} (53%)
create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/plan/node/PlanGraphPrinterTest.java
create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/TsFileValidationCorrectnessTests.java
create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java
create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionSelectorTest.java
create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionTest.java
rename iotdb-core/datanode/src/test/java/org/apache/iotdb/db/{qp/utils/DatetimeQueryDataSetUtilsTest.java => utils/DateTimeUtilsTest.java} (89%)
rename iotdb-core/metrics/{dropwizard-metrics => core}/pom.xml (85%)
create mode 100644 iotdb-core/metrics/core/src/main/java/org/apache/iotdb/metrics/core/IoTDBMetricManager.java
create mode 100644 iotdb-core/metrics/core/src/main/java/org/apache/iotdb/metrics/core/reporter/IoTDBJmxReporter.java
rename iotdb-core/metrics/{dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardAutoGauge.java => core/src/main/java/org/apache/iotdb/metrics/core/type/IoTDBAutoGauge.java} (77%)
rename iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/constant/CompactionValidationLevel.java => metrics/core/src/main/java/org/apache/iotdb/metrics/core/type/IoTDBAutoGaugeMBean.java} (77%)
rename iotdb-core/metrics/{dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardCounter.java => core/src/main/java/org/apache/iotdb/metrics/core/type/IoTDBCounter.java} (68%)
copy iotdb-core/metrics/{interface/src/main/java/org/apache/iotdb/metrics/utils/MetricFrameType.java => core/src/main/java/org/apache/iotdb/metrics/core/type/IoTDBCounterMBean.java} (83%)
rename iotdb-core/metrics/{micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerGauge.java => core/src/main/java/org/apache/iotdb/metrics/core/type/IoTDBGauge.java} (76%)
copy iotdb-core/metrics/{interface/src/main/java/org/apache/iotdb/metrics/utils/MetricFrameType.java => core/src/main/java/org/apache/iotdb/metrics/core/type/IoTDBGaugeMBean.java} (83%)
rename iotdb-core/metrics/{micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerHistogram.java => core/src/main/java/org/apache/iotdb/metrics/core/type/IoTDBHistogram.java} (59%)
copy iotdb-core/metrics/{micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/reporter/MicrometerJmxConfig.java => core/src/main/java/org/apache/iotdb/metrics/core/type/IoTDBHistogramMBean.java} (73%)
rename iotdb-core/metrics/{micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerHistogramSnapshot.java => core/src/main/java/org/apache/iotdb/metrics/core/type/IoTDBHistogramSnapshot.java} (88%)
rename iotdb-core/metrics/{micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerRate.java => core/src/main/java/org/apache/iotdb/metrics/core/type/IoTDBRate.java} (76%)
rename iotdb-core/metrics/{micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/reporter/MicrometerJmxConfig.java => core/src/main/java/org/apache/iotdb/metrics/core/type/IoTDBRateMBean.java} (76%)
rename iotdb-core/metrics/{micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerTimer.java => core/src/main/java/org/apache/iotdb/metrics/core/type/IoTDBTimer.java} (59%)
rename iotdb-core/metrics/{micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerTimerHistogramSnapshot.java => core/src/main/java/org/apache/iotdb/metrics/core/type/IoTDBTimerHistogramSnapshot.java} (87%)
copy iotdb-core/metrics/{interface/src/main/java/org/apache/iotdb/metrics/utils/MetricFrameType.java => core/src/main/java/org/apache/iotdb/metrics/core/type/IoTDBTimerMBean.java} (72%)
create mode 100644 iotdb-core/metrics/core/src/main/java/org/apache/iotdb/metrics/core/uitls/IoTDBCachedGauge.java
copy iotdb-core/metrics/{interface/src/main/java/org/apache/iotdb/metrics/type/IMetric.java => core/src/main/java/org/apache/iotdb/metrics/core/uitls/IoTDBCachedGaugeMBean.java} (82%)
create mode 100644 iotdb-core/metrics/core/src/main/java/org/apache/iotdb/metrics/core/uitls/IoTDBMetricObjNameFactory.java
create mode 100644 iotdb-core/metrics/core/src/main/java/org/apache/iotdb/metrics/core/uitls/IoTDBMovingAverage.java
copy iotdb-core/metrics/{interface/src/main/java/org/apache/iotdb/metrics/utils/MetricFrameType.java => core/src/main/java/org/apache/iotdb/metrics/core/uitls/ObjectNameFactory.java} (68%)
rename iotdb-core/metrics/{dropwizard-metrics => core}/src/main/resources/META-INF/services/org.apache.iotdb.metrics.AbstractMetricManager (93%)
rename iotdb-core/metrics/{micrometer-metrics/src/main/resources/META-INF/services/org.apache.iotdb.metrics.AbstractMetricManager => core/src/main/resources/META-INF/services/org.apache.iotdb.metrics.reporter.JmxReporter} (92%)
delete mode 100644 iotdb-core/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/DropwizardMetricManager.java
delete mode 100644 iotdb-core/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/DropwizardMetricNameTool.java
delete mode 100644 iotdb-core/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/reporter/DropwizardJmxReporter.java
delete mode 100644 iotdb-core/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardGauge.java
delete mode 100644 iotdb-core/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardHistogram.java
delete mode 100644 iotdb-core/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardHistogramSnapshot.java
delete mode 100644 iotdb-core/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardRate.java
delete mode 100644 iotdb-core/metrics/dropwizard-metrics/src/main/java/org/apache/iotdb/metrics/dropwizard/type/DropwizardTimer.java
delete mode 100644 iotdb-core/metrics/dropwizard-metrics/src/main/resources/META-INF/services/org.apache.iotdb.metrics.reporter.JmxReporter
rename iotdb-core/metrics/interface/src/main/java/org/apache/iotdb/metrics/utils/{MetricFrameType.java => AbstractMetricMBean.java} (70%)
delete mode 100644 iotdb-core/metrics/micrometer-metrics/pom.xml
delete mode 100644 iotdb-core/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/MicrometerMetricManager.java
delete mode 100644 iotdb-core/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/reporter/MicrometerJmxReporter.java
delete mode 100644 iotdb-core/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerAutoGauge.java
delete mode 100644 iotdb-core/metrics/micrometer-metrics/src/main/java/org/apache/iotdb/metrics/micrometer/type/MicrometerCounter.java
delete mode 100644 iotdb-core/metrics/micrometer-metrics/src/main/resources/META-INF/services/org.apache.iotdb.metrics.reporter.JmxReporter
delete mode 100644 iotdb-core/mlnode/.flake8
delete mode 100644 iotdb-core/mlnode/.gitignore
delete mode 100644 iotdb-core/mlnode/README.md
delete mode 100644 iotdb-core/mlnode/iotdb/__init__.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/__init__.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/algorithm/factory.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/algorithm/hyperparameter.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/algorithm/metric.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/algorithm/models/forecast/__init__.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/algorithm/models/forecast/dlinear.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/algorithm/models/forecast/nbeats.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/algorithm/validator.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/client.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/config.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/constant.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/dataset/dataset.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/dataset/factory.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/dataset/source.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/dataset/utils/__init__.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/dataset/utils/time_features.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/exception.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/handler.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/log.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/parser.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/process/__init__.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/process/manager.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/process/task.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/process/trial.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/script.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/serde.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/service.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/storage.py
delete mode 100644 iotdb-core/mlnode/iotdb/mlnode/util.py
delete mode 100644 iotdb-core/mlnode/pom.xml
delete mode 100644 iotdb-core/mlnode/pyproject.toml
delete mode 100644 iotdb-core/mlnode/requirements.txt
delete mode 100644 iotdb-core/mlnode/requirements_dev.txt
delete mode 100644 iotdb-core/mlnode/resources/conf/iotdb-mlnode.toml
delete mode 100644 iotdb-core/mlnode/resources/conf/logging_config.ini
delete mode 100644 iotdb-core/mlnode/test/test_create_forecast_model.py
delete mode 100644 iotdb-core/mlnode/test/test_model_storage.py
delete mode 100644 iotdb-core/mlnode/test/test_serde.py
delete mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/mlnode/MLNodeClient.java
delete mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/mlnode/MLNodeClientManager.java
delete mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/mlnode/MLNodeInfo.java
create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/pipe/PipeRuntimeOutOfMemoryCriticalException.java
delete mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/model/ForecastModeInformation.java
delete mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/model/ModelHyperparameter.java
delete mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/model/ModelInformation.java
delete mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/model/ModelTable.java
delete mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/model/TrialInformation.java
delete mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/model/exception/ModelManagementException.java
rename iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/IMLNodeRPCServiceWithHandler.java => node-commons/src/main/java/org/apache/iotdb/commons/pipe/plugin/builtin/connector/WriteBackConnector.java} (63%)
delete mode 100644 iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/exception/write/UnSupportedDataTypeException.java
create mode 100644 iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/TimeDuration.java
create mode 100644 iotdb-core/tsfile/src/test/java/org/apache/iotdb/tsfile/utils/TimeDurationTest.java
delete mode 100644 iotdb-protocol/thrift-mlnode/pom.xml
delete mode 100644 iotdb-protocol/thrift-mlnode/src/main/thrift/mlnode.thrift
(iotdb) 01/01: Merge branch 'master' into load_v2
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch load_v2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c1201f5b035f9dfad7feeb51558082ca9645286a
Merge: 270f6f6efce 256194a9e28
Author: Tian Jiang <jt...@163.com>
AuthorDate: Wed Nov 8 11:02:52 2023 +0800
Merge branch 'master' into load_v2
# Conflicts:
# iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
# iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
# iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/config/constant/PipeExtractorConstant.java
# iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBThriftAsyncConnector.java
# iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java
# iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java
# iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
# iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/MLNodeRPCServiceImpl.java
# iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/LoadTsFileManager.java
# iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
# iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
# iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
.github/workflows/iotdb-ml.yml | 61 -
.github/workflows/pipe-it-2cluster.yml | 7 +-
.github/workflows/sonar-codecov.yml | 2 +-
.gitignore | 3 -
code-coverage/pom.xml | 2 +-
distribution/pom.xml | 14 +-
example/client-cpp-example/pom.xml | 2 +-
example/flink-sql/pom.xml | 2 +-
example/flink/pom.xml | 2 +-
.../org/apache/iotdb/flink/FlinkIoTDBSink.java | 2 +-
.../org/apache/iotdb/flink/FlinkIoTDBSource.java | 2 +-
.../apache/iotdb/flink/FlinkTsFileBatchSink.java | 2 +-
.../apache/iotdb/flink/FlinkTsFileStreamSink.java | 2 +-
.../java/org/apache/iotdb/flink/TsFileUtils.java | 2 +-
example/hadoop/pom.xml | 2 +-
.../iotdb/hadoop/tsfile/TSMRWriteExample.java | 2 +-
.../apache/iotdb/hadoop/tsfile/TsFileHelper.java | 2 +-
.../iotdb/hadoop/tsfile/TsFileWriteToHDFS.java | 2 +-
example/jdbc/pom.xml | 2 +-
example/kafka/pom.xml | 2 +-
.../main/java/org/apache/iotdb/kafka/Consumer.java | 2 +-
.../org/apache/iotdb/kafka/ConsumerThread.java | 2 +-
example/mqtt-customize/pom.xml | 2 +-
example/mqtt/pom.xml | 2 +-
.../{flink => pipe-count-point-processor}/pom.xml | 43 +-
.../java/org/apache/iotdb/CountPointProcessor.java | 83 +
example/{session => pipe-opc-ua-sink}/pom.xml | 19 +-
.../java/org/apache/iotdb/opcua/ClientExample.java | 50 +
.../apache/iotdb/opcua/ClientExampleRunner.java | 144 ++
.../java/org/apache/iotdb/opcua/ClientTest.java | 129 ++
.../iotdb/opcua/IoTDBKeyStoreLoaderClient.java | 128 ++
example/pom.xml | 4 +-
example/pulsar/pom.xml | 2 +-
.../org/apache/iotdb/pulsar/PulsarConsumer.java | 2 +-
.../apache/iotdb/pulsar/PulsarConsumerThread.java | 2 +-
example/rabbitmq/pom.xml | 2 +-
.../apache/iotdb/rabbitmq/RabbitMQConsumer.java | 2 +-
example/rest-java-example/pom.xml | 2 +-
example/rocketmq/pom.xml | 2 +-
.../apache/iotdb/rocketmq/RocketMQConsumer.java | 2 +-
example/schema/pom.xml | 4 +-
example/session/pom.xml | 2 +-
.../iotdb/AlignedTimeseriesSessionExample.java | 2 +-
.../org/apache/iotdb/DataMigrationExample.java | 2 +-
.../iotdb/HybridTimeseriesSessionExample.java | 2 +-
.../org/apache/iotdb/SessionConcurrentExample.java | 2 +-
.../main/java/org/apache/iotdb/SessionExample.java | 5 +-
.../java/org/apache/iotdb/SessionPoolExample.java | 2 +-
.../iotdb/SyntaxConventionRelatedExample.java | 2 +-
.../main/java/org/apache/iotdb/TabletExample.java | 6 +-
example/trigger/pom.xml | 2 +-
.../org/apache/iotdb/trigger/LoggerTrigger.java | 2 +-
.../iotdb/trigger/StatisticsUpdaterTrigger.java | 2 +-
example/tsfile/pom.xml | 2 +-
.../iotdb/tsfile/TsFileForceAppendWrite.java | 2 +-
.../apache/iotdb/tsfile/TsFileSequenceRead.java | 2 +-
.../tsfile/TsFileWriteAlignedWithTSRecord.java | 2 +-
.../iotdb/tsfile/TsFileWriteAlignedWithTablet.java | 5 +-
.../iotdb/tsfile/TsFileWriteWithTSRecord.java | 2 +-
.../apache/iotdb/tsfile/TsFileWriteWithTablet.java | 5 +-
example/udf/pom.xml | 2 +-
integration-test/pom.xml | 40 +-
.../apache/iotdb/db/query/udf/example/Adder.java | 2 +-
.../apache/iotdb/db/query/udf/example/TwoSum.java | 43 +-
.../iotdb/db/query/udf/example/TwoSumBlock.java | 107 +
.../it/env/cluster/config/MppCommonConfig.java | 24 +-
.../env/cluster/config/MppSharedCommonConfig.java | 29 +-
.../iotdb/it/env/cluster/env/AbstractEnv.java | 18 +
.../it/env/remote/config/RemoteCommonConfig.java | 13 +-
.../org/apache/iotdb/it/utils/TsFileGenerator.java | 4 +-
.../java/org/apache/iotdb/itbase/env/BaseEnv.java | 8 +
.../org/apache/iotdb/itbase/env/CommonConfig.java | 8 +-
.../org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 2 +-
.../iotdb/db/it/IoTDBPartialInsertionIT.java | 2 +-
.../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 5 +
.../db/it/groupby/IoTDBGroupByNaturalMonthIT.java | 108 +-
.../IoTDBGroupByNaturalMonthNsPrecisionIT.java | 107 +
.../IoTDBGroupByNaturalMonthUsPrecisionIT.java | 92 +
.../{ => quota}/IoTDBClusterDeviceQuotaIT.java | 7 +-
.../IoTDBClusterMixQuotaIT.java} | 77 +-
.../IoTDBClusterQuotaIT.java} | 39 +-
.../quota/IoTDBClusterTimeSeriesQuotaIT.java | 37 +-
.../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 2 +-
.../iotdb/db/it/udf/IoTDBUDFBlockQueryIT.java | 155 ++
.../org/apache/iotdb/db/it/utils/TestUtils.java | 47 +-
.../test/java/org/apache/iotdb/flink/it/Utils.java | 2 +-
.../session/it/IoTDBSessionAlignedInsertIT.java | 2 +-
.../iotdb/session/it/IoTDBSessionComplexIT.java | 2 +-
.../it/IoTDBSessionDisableMemControlIT.java | 7 +-
.../iotdb/session/it/IoTDBSessionInsertNullIT.java | 2 +-
.../IoTDBSessionInsertWithTriggerExecutionIT.java | 2 +-
.../session/it/IoTDBSessionSchemaTemplateIT.java | 2 +-
.../iotdb/session/it/IoTDBSessionSimpleIT.java | 31 +-
.../session/it/IoTDBSessionSyntaxConventionIT.java | 5 +-
.../org/apache/iotdb/session/it/SessionIT.java | 5 +-
.../iotdb/session/it/pool/SessionPoolIT.java | 2 +-
iotdb-api/{external-api => common-api}/pom.xml | 14 +-
.../org/apache/iotdb/tsfile/access}/Column.java | 5 +-
.../apache/iotdb/tsfile/access}/ColumnBuilder.java | 4 +-
.../iotdb/tsfile/access}/ColumnBuilderStatus.java | 4 +-
.../iotdb/tsfile/access}/TsBlockBuilderStatus.java | 12 +-
.../apache/iotdb/tsfile/enums}/ColumnEncoding.java | 2 +-
.../org/apache/iotdb/tsfile}/enums/TSDataType.java | 4 +-
.../exception}/UnSupportedDataTypeException.java | 4 +-
.../java/org/apache/iotdb/tsfile/utils/Binary.java | 25 +-
.../apache/iotdb/tsfile/utils/TsPrimitiveType.java | 4 +-
iotdb-api/external-api/pom.xml | 2 +-
iotdb-api/pipe-api/pom.xml | 4 +-
.../org/apache/iotdb/pipe/api/PipeConnector.java | 3 +-
.../org/apache/iotdb/pipe/api/PipeExtractor.java | 3 +-
.../pipe/api/{PipeConnector.java => PipeSink.java} | 77 +-
.../api/{PipeExtractor.java => PipeSource.java} | 48 +-
.../PipeSinkRuntimeConfiguration.java | 4 +-
.../PipeSourceRuntimeConfiguration.java | 7 +-
.../parameter/PipeParameterValidator.java | 2 +-
.../api/customizer/parameter/PipeParameters.java | 41 +-
.../pipe/api/exception/PipeSinkException.java | 10 +-
iotdb-api/pom.xml | 3 +-
iotdb-api/trigger-api/pom.xml | 4 +-
iotdb-api/udf-api/pom.xml | 9 +-
.../main/java/org/apache/iotdb/udf/api/UDTF.java | 46 +
.../org/apache/iotdb/udf/api/utils/RowImpl.java | 47 +-
iotdb-client/cli/pom.xml | 25 +-
.../java/org/apache/iotdb/cli/AbstractCli.java | 42 +
.../src/main/java/org/apache/iotdb/cli/Cli.java | 23 +-
.../main/java/org/apache/iotdb/tool/ExportCsv.java | 2 +-
.../java/org/apache/iotdb/tool/ExportTsFile.java | 2 +-
.../main/java/org/apache/iotdb/tool/ImportCsv.java | 14 +-
iotdb-client/client-cpp/pom.xml | 4 +-
iotdb-client/client-py/pom.xml | 4 +-
iotdb-client/isession/pom.xml | 15 +-
.../java/org/apache/iotdb/isession/ISession.java | 2 +-
.../org/apache/iotdb/isession/SessionDataSet.java | 13 +-
.../apache/iotdb/isession/pool/ISessionPool.java | 2 +-
iotdb-client/jdbc/pom.xml | 15 +-
.../main/java/org/apache/iotdb/jdbc/Config.java | 6 +
.../iotdb/jdbc/GroupedLSBWatermarkEncoder.java | 2 +-
.../org/apache/iotdb/jdbc/IoTDBConnection.java | 17 +-
.../apache/iotdb/jdbc/IoTDBConnectionParams.java | 28 +
.../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 6 +-
.../iotdb/jdbc/IoTDBNonAlignJDBCResultSet.java | 4 +-
.../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 3 +-
.../src/main/java/org/apache/iotdb/jdbc/Utils.java | 15 +
.../apache/iotdb/jdbc/IoTDBJDBCResultSetTest.java | 2 +-
iotdb-client/pom.xml | 2 +-
iotdb-client/service-rpc/pom.xml | 13 +-
.../org/apache/iotdb/rpc/IoTDBJDBCDataSet.java | 4 +-
.../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 8 +-
.../org/apache/iotdb/rpc/RpcTransportFactory.java | 11 +
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 7 +-
iotdb-client/session/pom.xml | 17 +-
.../java/org/apache/iotdb/session/Session.java | 77 +-
.../apache/iotdb/session/SessionConnection.java | 27 +-
.../org/apache/iotdb/session/pool/SessionPool.java | 140 +-
.../iotdb/session/template/MeasurementNode.java | 2 +-
.../apache/iotdb/session/util/SessionUtils.java | 4 +-
.../iotdb/session/SessionCacheLeaderTest.java | 2 +-
.../java/org/apache/iotdb/session/SessionTest.java | 12 +-
.../java/org/apache/iotdb/session/TabletTest.java | 2 +-
.../session/pool/SessionPoolExceptionTest.java | 2 +-
.../apache/iotdb/session/pool/SessionPoolTest.java | 2 +-
.../iotdb/session/util/SessionUtilsTest.java | 2 +-
iotdb-connector/flink-iotdb-connector/pom.xml | 15 +-
.../iotdb/flink/DefaultIoTSerializationSchema.java | 2 +-
.../main/java/org/apache/iotdb/flink/Event.java | 2 +-
.../java/org/apache/iotdb/flink/IoTDBSink.java | 2 +-
.../iotdb/flink/options/IoTDBSinkOptions.java | 2 +-
iotdb-connector/flink-sql-iotdb-connector/pom.xml | 17 +-
.../org/apache/iotdb/flink/sql/common/Utils.java | 7 +-
.../sql/function/IoTDBBoundedScanFunction.java | 2 +-
.../flink/sql/function/IoTDBCDCSourceFunction.java | 2 +-
.../flink/sql/function/IoTDBLookupFunction.java | 2 +-
.../flink/sql/function/IoTDBSinkFunction.java | 2 +-
iotdb-connector/flink-tsfile-connector/pom.xml | 11 +-
.../iotdb/flink/tsfile/RowTSRecordConverter.java | 8 +-
.../tsfile/RowTsFileOutputFormatTestBase.java | 2 +-
.../apache/iotdb/flink/util/TsFileWriteUtil.java | 2 +-
iotdb-connector/grafana-connector/pom.xml | 4 +-
iotdb-connector/grafana-plugin/pom.xml | 2 +-
iotdb-connector/hadoop/pom.xml | 9 +-
.../iotdb/hadoop/tsfile/TSFRecordReader.java | 4 +-
.../iotdb/hadoop/tsfile/record/HDFSTSRecord.java | 4 +-
.../iotdb/hadoop/tsfile/TsFileTestHelper.java | 2 +-
iotdb-connector/hive-connector/pom.xml | 12 +-
.../org/apache/iotdb/hive/TsFileTestHelper.java | 2 +-
iotdb-connector/pom.xml | 2 +-
iotdb-connector/spark-iotdb-connector/pom.xml | 10 +-
.../spark-iotdb-connector/scala_2.11/pom.xml | 2 +-
.../spark-iotdb-connector/scala_2.12/pom.xml | 2 +-
.../org/apache/iotdb/spark/db/DataFrameTools.scala | 4 +-
iotdb-connector/spark-tsfile/pom.xml | 11 +-
.../tsfile/qp/optimizer/PhysicalOptimizer.java | 2 +-
.../org/apache/iotdb/spark/tsfile/Converter.scala | 2 +-
.../iotdb/spark/tsfile/NarrowConverter.scala | 3 +-
.../apache/iotdb/spark/tsfile/WideConverter.scala | 6 +-
.../apache/iotdb/spark/tsfile/ConverterTest.scala | 5 +-
iotdb-connector/zeppelin-interpreter/pom.xml | 6 +-
iotdb-core/antlr/pom.xml | 2 +-
.../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 3 -
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 46 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 17 +-
iotdb-core/confignode/pom.xml | 31 +-
.../src/assembly/resources/conf/confignode-env.bat | 21 +-
.../src/assembly/resources/conf/confignode-env.sh | 24 +-
.../confignode/client/DataNodeRequestType.java | 3 -
.../async/AsyncConfigNodeHeartbeatClientPool.java | 7 +-
.../heartbeat/ConfigNodeHeartbeatHandler.java | 14 +-
.../heartbeat/DataNodeHeartbeatHandler.java | 31 +-
.../client/sync/SyncDataNodeClientPool.java | 3 -
.../confignode/conf/ConfigNodeDescriptor.java | 17 +-
.../consensus/request/ConfigPhysicalPlan.java | 28 -
.../consensus/request/ConfigPhysicalPlanType.java | 9 +-
.../request/read/model/GetModelInfoPlan.java | 79 -
.../request/read/model/ShowModelPlan.java | 89 -
.../request/read/model/ShowTrialPlan.java | 97 -
.../request/write/model/CreateModelPlan.java | 78 -
.../request/write/model/DropModelPlan.java | 81 -
.../request/write/model/UpdateModelInfoPlan.java | 98 -
.../request/write/model/UpdateModelStatePlan.java | 105 -
.../consensus/response/model/GetModelInfoResp.java | 46 -
.../consensus/response/model/ModelTableResp.java | 55 -
.../consensus/response/model/TrialTableResp.java | 55 -
.../statemachine/ConfigRegionStateMachine.java | 3 +-
.../iotdb/confignode/manager/ConfigManager.java | 106 +-
.../apache/iotdb/confignode/manager/IManager.java | 38 -
.../iotdb/confignode/manager/ModelManager.java | 204 --
.../iotdb/confignode/manager/ProcedureManager.java | 31 -
.../confignode/manager/load/cache/LoadCache.java | 8 +-
.../load/cache/node/NodeHeartbeatSample.java | 8 +
.../manager/load/service/HeartbeatService.java | 33 +-
.../iotdb/confignode/manager/node/NodeManager.java | 4 +-
.../manager/schema/ClusterSchemaManager.java | 31 +-
.../schema/ClusterSchemaQuotaStatistics.java | 73 +-
.../iotdb/confignode/persistence/ModelInfo.java | 278 ---
.../persistence/executor/ConfigPlanExecutor.java | 28 -
.../persistence/pipe/PipePluginInfo.java | 2 +-
.../confignode/persistence/quota/QuotaInfo.java | 1 +
.../persistence/schema/TemplateTable.java | 2 +-
.../schema/mnode/factory/ConfigMNodeFactory.java | 3 +-
.../procedure/impl/model/CreateModelProcedure.java | 223 --
.../procedure/impl/model/DropModelProcedure.java | 209 --
.../procedure/state/model/DropModelState.java | 28 -
.../procedure/store/ProcedureFactory.java | 12 -
.../confignode/procedure/store/ProcedureType.java | 4 +-
.../iotdb/confignode/service/ConfigNode.java | 10 +-
.../confignode/service/ConfigNodeCommandLine.java | 8 +-
.../confignode/service/ConfigNodeShutdownHook.java | 6 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 53 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 5 +-
.../iotdb/confignode/persistence/PipeInfoTest.java | 5 +-
.../persistence/schema/ClusterSchemaInfoTest.java | 2 +-
.../persistence/schema/TemplateTableTest.java | 2 +-
.../schema/DeactivateTemplateProcedureTest.java | 2 +-
.../impl/schema/UnsetTemplateProcedureTest.java | 2 +-
iotdb-core/consensus/pom.xml | 14 +-
.../org/apache/iotdb/consensus/IStateMachine.java | 32 -
.../apache/iotdb/consensus/config/RatisConfig.java | 8 +-
.../ratis/ApplicationStateMachineProxy.java | 44 +-
.../consensus/ratis/metrics/CounterProxy.java | 2 +-
.../apache/iotdb/consensus/ratis/utils/Utils.java | 1 +
iotdb-core/datanode/pom.xml | 60 +-
.../src/assembly/resources/conf/datanode-env.bat | 24 +-
.../src/assembly/resources/conf/datanode-env.sh | 28 +-
.../resources/conf/iotdb-datanode.properties | 55 +-
.../src/main/codegen/templates/ModeAccumulator.ftl | 6 +-
.../codegen/templates/abstractVariationWindow.ftl | 2 +-
.../src/main/codegen/templates/constantFill.ftl | 2 +-
.../src/main/codegen/templates/linearFill.ftl | 2 +-
.../src/main/codegen/templates/previousFill.ftl | 2 +-
.../src/main/codegen/templates/variationWindow.ftl | 2 +-
.../codegen/templates/variationWindowManager.ftl | 2 +-
.../org/apache/iotdb/db/audit/AuditLogger.java | 9 +-
.../org/apache/iotdb/db/auth/AuthorityChecker.java | 14 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 100 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 105 +-
.../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 9 +-
.../iotdb/db/conf/rest/IoTDBRestServiceConfig.java | 2 +-
.../consensus/statemachine/BaseStateMachine.java | 3 +-
.../dataregion/DataRegionStateMachine.java | 18 -
.../schemaregion/SchemaExecutionVisitor.java | 2 +-
.../schemaregion/SchemaRegionStateMachine.java | 18 -
.../metadata/DataTypeMismatchException.java | 2 +-
.../metadata/SchemaQuotaExceededException.java | 8 +-
.../db/pipe/agent/plugin/PipePluginAgent.java | 2 +-
.../config/constant/PipeConnectorConstant.java | 4 +
.../config/constant/PipeExtractorConstant.java | 2 +
.../builder/PipeTransferBatchReqBuilder.java | 26 +-
.../request/PipeTransferTabletInsertNodeReq.java | 14 +-
.../request/PipeTransferTabletRawReq.java | 15 +-
.../db/pipe/connector/protocol/IoTDBConnector.java | 33 +-
.../protocol/airgap/IoTDBAirGapConnector.java | 51 +-
.../protocol/legacy/IoTDBLegacyPipeConnector.java | 125 +-
.../connector/protocol/opcua/OpcUaConnector.java | 4 +-
.../thrift/async/IoTDBThriftAsyncConnector.java | 42 +-
.../thrift/sync/IoTDBThriftSyncConnector.java | 82 +-
.../sync/IoTDBThriftSyncConnectorClient.java | 29 +-
.../protocol/websocket/WebSocketConnector.java | 4 +-
.../protocol/writeback/WriteBackConnector.java | 176 ++
.../apache/iotdb/db/pipe/event/EnrichedEvent.java | 2 +
.../event/common/heartbeat/PipeHeartbeatEvent.java | 18 +-
.../event/common/row/PipeDataTypeTransformer.java | 7 +-
.../iotdb/db/pipe/event/common/row/PipeRow.java | 5 +-
.../common/tablet/PipeRawTabletInsertionEvent.java | 6 +
.../tablet/TabletInsertionDataContainer.java | 4 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 9 +-
.../tsfile/TsFileInsertionDataContainer.java | 14 +-
.../tsfile/TsFileInsertionDataTabletIterator.java | 2 +-
.../tsfile/TsFileListInsertionDataContainer.java | 2 +-
.../pipe/extractor/IoTDBDataRegionExtractor.java | 12 +-
.../PipeHistoricalDataRegionTsFileExtractor.java | 5 +-
.../PipeRealtimeDataRegionHybridExtractor.java | 15 +-
.../realtime/assigner/DisruptorQueue.java | 19 +-
.../db/pipe/resource/memory/PipeMemoryBlock.java | 4 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 142 +-
.../resource/tsfile/PipeTsFileResourceManager.java | 8 +-
.../db/pipe/task/stage/PipeTaskExtractorStage.java | 12 +-
.../connector/PipeConnectorSubtaskManager.java | 28 +
.../iotdb/db/protocol/client/ConfigNodeClient.java | 65 +-
.../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 2 +-
.../org/apache/iotdb/db/protocol/mqtt/Message.java | 2 +-
.../db/protocol/rest/utils/InsertRowDataUtils.java | 2 +-
.../rest/v1/handler/QueryDataSetHandler.java | 9 +-
.../v1/handler/StatementConstructionHandler.java | 2 +-
.../rest/v2/handler/QueryDataSetHandler.java | 9 +-
.../v2/handler/StatementConstructionHandler.java | 2 +-
.../handler/MLNodeRPCServiceThriftHandler.java | 60 -
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 18 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 61 +-
.../protocol/thrift/impl/MLNodeRPCServiceImpl.java | 217 --
.../db/queryengine/common/header/ColumnHeader.java | 2 +-
.../common/header/ColumnHeaderConstant.java | 30 +-
.../queryengine/common/header/DatasetHeader.java | 2 +-
.../common/header/DatasetHeaderFactory.java | 8 -
.../execution/aggregation/Accumulator.java | 6 +-
.../execution/aggregation/AccumulatorFactory.java | 2 +-
.../execution/aggregation/Aggregator.java | 6 +-
.../execution/aggregation/AvgAccumulator.java | 8 +-
.../execution/aggregation/CountAccumulator.java | 6 +-
.../execution/aggregation/CountIfAccumulator.java | 6 +-
.../aggregation/CountTimeAccumulator.java | 6 +-
.../execution/aggregation/ExtremeAccumulator.java | 8 +-
.../aggregation/FirstValueAccumulator.java | 8 +-
.../aggregation/FirstValueDescAccumulator.java | 4 +-
.../aggregation/LastValueAccumulator.java | 8 +-
.../aggregation/LastValueDescAccumulator.java | 4 +-
.../execution/aggregation/MaxTimeAccumulator.java | 6 +-
.../aggregation/MaxTimeDescAccumulator.java | 2 +-
.../execution/aggregation/MaxValueAccumulator.java | 8 +-
.../execution/aggregation/MinTimeAccumulator.java | 6 +-
.../aggregation/MinTimeDescAccumulator.java | 2 +-
.../execution/aggregation/MinValueAccumulator.java | 8 +-
.../execution/aggregation/SumAccumulator.java | 8 +-
.../aggregation/TimeDurationAccumulator.java | 6 +-
.../MonotonicQueueSlidingWindowAggregator.java | 2 +-
.../slidingwindow/SlidingWindowAggregator.java | 8 +-
.../SlidingWindowAggregatorFactory.java | 4 +-
.../timerangeiterator/AggrWindowIterator.java | 91 +-
.../PreAggrWindowWithNaturalMonthIterator.java | 17 +-
.../TimeRangeIteratorFactory.java | 41 +-
.../execution/exchange/sink/SinkChannel.java | 5 +-
.../fragment/FragmentInstanceManager.java | 11 -
.../execution/load/AlignedChunkData.java | 4 +-
.../execution/load/LoadTsFileManager.java | 29 +-
.../execution/load/MergedTsFileSplitter.java | 2 +-
.../execution/load/NonAlignedChunkData.java | 2 +-
.../execution/load/TsFileSplitSender.java | 8 +-
.../queryengine/execution/load/TsFileSplitter.java | 2 +-
.../nodesplit/ClusteringMeasurementSplitter.java | 2 +-
.../execution/operator/AggregationUtil.java | 9 +-
.../operator/process/AbstractIntoOperator.java | 11 +-
.../operator/process/AggregationOperator.java | 2 +-
.../operator/process/DeviceViewIntoOperator.java | 13 +-
.../operator/process/DeviceViewOperator.java | 9 +-
.../execution/operator/process/FillOperator.java | 2 +-
.../operator/process/FilterAndProjectOperator.java | 6 +-
.../execution/operator/process/IntoOperator.java | 11 +-
.../operator/process/LinearFillOperator.java | 2 +-
.../operator/process/MergeSortOperator.java | 4 +-
.../process/RawDataAggregationOperator.java | 2 +-
.../operator/process/SingleDeviceViewOperator.java | 9 +-
.../process/SlidingWindowAggregationOperator.java | 2 +-
.../execution/operator/process/SortOperator.java | 11 +-
.../operator/process/TagAggregationOperator.java | 9 +-
.../execution/operator/process/TopKOperator.java | 8 +-
.../operator/process/TransformOperator.java | 6 +-
.../execution/operator/process/fill/IFill.java | 2 +-
.../operator/process/fill/ILinearFill.java | 2 +-
.../process/fill/identity/IdentityFill.java | 2 +-
.../process/fill/identity/IdentityLinearFill.java | 2 +-
.../operator/process/fill/linear/LinearFill.java | 2 +-
.../process/join/HorizontallyConcatOperator.java | 6 +-
.../process/join/RowBasedTimeJoinOperator.java | 2 +-
.../operator/process/join/merge/ColumnMerger.java | 2 +-
.../process/join/merge/MergeSortComparator.java | 2 +-
.../process/join/merge/MultiColumnMerger.java | 4 +-
.../join/merge/NonOverlappedMultiColumnMerger.java | 2 +-
.../process/join/merge/SingleColumnMerger.java | 4 +-
.../last/AbstractUpdateLastCacheOperator.java | 2 +-
.../operator/process/last/LastQueryOperator.java | 4 +-
.../process/last/LastQuerySortOperator.java | 4 +-
.../operator/process/last/LastQueryUtil.java | 13 +-
.../process/last/UpdateLastCacheOperator.java | 2 +-
.../last/UpdateViewPathLastCacheOperator.java | 2 +-
.../operator/process/ml/ForecastOperator.java | 241 --
.../schema/CountGroupByLevelMergeOperator.java | 10 +-
.../schema/CountGroupByLevelScanOperator.java | 12 +-
.../operator/schema/CountMergeOperator.java | 2 +-
.../schema/NodeManageMemoryMergeOperator.java | 16 +-
.../operator/schema/NodePathsConvertOperator.java | 7 +-
.../operator/schema/NodePathsCountOperator.java | 7 +-
.../operator/schema/SchemaCountOperator.java | 2 +-
.../operator/schema/SchemaFetchScanOperator.java | 6 +-
.../schema/SchemaQueryOrderByHeatOperator.java | 7 +-
.../operator/schema/SchemaQueryScanOperator.java | 7 +-
.../operator/schema/SchemaTsBlockUtil.java | 9 +-
.../operator/schema/source/DeviceSchemaSource.java | 15 +-
.../operator/schema/source/NodeSchemaSource.java | 10 +-
.../schema/source/PathsUsingTemplateSource.java | 5 +-
.../AbstractSeriesAggregationScanOperator.java | 2 +-
.../operator/source/AlignedSeriesScanOperator.java | 4 +-
.../operator/source/AlignedSeriesScanUtil.java | 2 +-
.../operator/source/ExchangeOperator.java | 6 +-
.../operator/source/SeriesScanOperator.java | 4 +-
.../execution/operator/source/SeriesScanUtil.java | 4 +-
.../operator/source/ShowQueriesOperator.java | 16 +-
.../operator/window/AbstractVariationWindow.java | 2 +-
.../execution/operator/window/ConditionWindow.java | 2 +-
.../operator/window/ConditionWindowManager.java | 4 +-
.../execution/operator/window/CountWindow.java | 2 +-
.../operator/window/CountWindowManager.java | 4 +-
.../execution/operator/window/IWindow.java | 2 +-
.../execution/operator/window/IWindowManager.java | 4 +-
.../execution/operator/window/SessionWindow.java | 2 +-
.../operator/window/SessionWindowManager.java | 2 +-
.../execution/operator/window/TimeWindow.java | 2 +-
.../operator/window/TimeWindowManager.java | 2 +-
.../operator/window/VariationWindowManager.java | 2 +-
.../operator/window/VariationWindowParameter.java | 2 +-
.../operator/window/WindowManagerFactory.java | 2 +-
.../db/queryengine/plan/analyze/Analysis.java | 13 +-
.../queryengine/plan/analyze/AnalyzeVisitor.java | 125 +-
.../plan/analyze/ExpressionAnalyzer.java | 4 +-
.../plan/analyze/ExpressionTypeAnalyzer.java | 4 +-
.../queryengine/plan/analyze/ExpressionUtils.java | 2 +-
.../db/queryengine/plan/analyze/IModelFetcher.java | 27 -
.../plan/analyze/LoadTsfileAnalyzer.java | 24 +-
.../db/queryengine/plan/analyze/ModelFetcher.java | 75 -
.../queryengine/plan/analyze/SelectIntoUtils.java | 2 +-
.../db/queryengine/plan/analyze/TypeProvider.java | 2 +-
.../analyze/cache/schema/SchemaCacheEntry.java | 2 +-
.../analyze/schema/AutoCreateSchemaExecutor.java | 2 +-
.../analyze/schema/ClusterSchemaFetchExecutor.java | 2 +-
.../plan/analyze/schema/ClusterSchemaFetcher.java | 2 +-
.../plan/analyze/schema/ISchemaAutoCreation.java | 2 +-
.../plan/analyze/schema/ISchemaFetcher.java | 2 +-
.../plan/analyze/schema/NormalSchemaFetcher.java | 2 +-
.../plan/analyze/schema/SchemaValidator.java | 2 +-
.../plan/analyze/schema/TemplateSchemaFetcher.java | 2 +-
.../plan/execution/config/ConfigTaskVisitor.java | 33 -
.../config/executor/ClusterConfigTaskExecutor.java | 184 +-
.../config/executor/IConfigTaskExecutor.java | 10 -
.../config/metadata/CountDatabaseTask.java | 2 +-
.../config/metadata/CountTimeSlotListTask.java | 2 +-
.../execution/config/metadata/GetRegionIdTask.java | 2 +-
.../config/metadata/GetSeriesSlotListTask.java | 2 +-
.../config/metadata/GetTimeSlotListTask.java | 8 +-
.../config/metadata/ShowClusterDetailsTask.java | 75 +-
.../execution/config/metadata/ShowClusterTask.java | 18 +-
.../config/metadata/ShowConfigNodesTask.java | 10 +-
.../config/metadata/ShowContinuousQueriesTask.java | 12 +-
.../config/metadata/ShowDataNodesTask.java | 9 +-
.../config/metadata/ShowFunctionsTask.java | 21 +-
.../config/metadata/ShowPipePluginsTask.java | 21 +-
.../execution/config/metadata/ShowRegionTask.java | 21 +-
.../execution/config/metadata/ShowTTLTask.java | 7 +-
.../config/metadata/ShowTriggersTask.java | 20 +-
.../config/metadata/ShowVariablesTask.java | 90 +-
.../config/metadata/model/CreateModelTask.java | 45 -
.../config/metadata/model/DropModelTask.java | 41 -
.../config/metadata/model/ShowModelsTask.java | 100 -
.../config/metadata/model/ShowTrialsTask.java | 90 -
.../template/ShowNodesInSchemaTemplateTask.java | 18 +-
.../metadata/template/ShowPathSetTemplateTask.java | 7 +-
.../metadata/template/ShowSchemaTemplateTask.java | 7 +-
.../execution/config/sys/pipe/ShowPipeTask.java | 31 +-
.../config/sys/quota/ShowSpaceQuotaTask.java | 32 +-
.../config/sys/quota/ShowThrottleQuotaTask.java | 36 +-
.../memory/StatementMemorySourceVisitor.java | 25 +-
.../db/queryengine/plan/expression/Expression.java | 2 +-
.../plan/expression/ExpressionFactory.java | 2 +-
.../plan/expression/binary/BinaryExpression.java | 2 +-
.../plan/expression/leaf/ConstantOperand.java | 2 +-
.../plan/expression/leaf/LeafOperand.java | 2 +-
.../plan/expression/leaf/TimeSeriesOperand.java | 2 +-
.../plan/expression/multi/FunctionExpression.java | 12 +-
.../plan/expression/multi/FunctionType.java | 3 +-
.../multi/builtin/BuiltInScalarFunctionHelper.java | 2 +-
.../multi/builtin/helper/CastFunctionHelper.java | 2 +-
.../multi/builtin/helper/DiffFunctionHelper.java | 2 +-
.../builtin/helper/ReplaceFunctionHelper.java | 2 +-
.../multi/builtin/helper/RoundFunctionHelper.java | 2 +-
.../builtin/helper/SubStringFunctionHelper.java | 2 +-
.../expression/other/CaseWhenThenExpression.java | 2 +-
.../plan/expression/ternary/TernaryExpression.java | 2 +-
.../plan/expression/unary/UnaryExpression.java | 2 +-
.../visitor/ColumnTransformerVisitor.java | 6 +-
.../visitor/IntermediateLayerVisitor.java | 2 +-
.../plan/optimization/LimitOffsetPushDown.java | 15 +-
.../db/queryengine/plan/parser/ASTVisitor.java | 122 +-
.../plan/parser/StatementGenerator.java | 52 +-
.../plan/planner/LocalExecutionPlanContext.java | 6 +-
.../plan/planner/LogicalPlanBuilder.java | 12 +-
.../plan/planner/LogicalPlanVisitor.java | 20 -
.../plan/planner/OperatorTreeGenerator.java | 51 +-
.../plan/planner/plan/node/PlanGraphPrinter.java | 92 +-
.../plan/planner/plan/node/PlanNodeType.java | 7 +-
.../plan/planner/plan/node/PlanVisitor.java | 5 -
.../plan/node/load/LoadSingleTsFileNode.java | 19 +-
.../planner/plan/node/load/LoadTsFileNode.java | 10 +-
.../write/CreateAlignedTimeSeriesNode.java | 2 +-
.../metedata/write/CreateMultiTimeSeriesNode.java | 2 +-
.../node/metedata/write/CreateTimeSeriesNode.java | 2 +-
.../plan/node/metedata/write/MeasurementGroup.java | 2 +-
.../planner/plan/node/process/ml/ForecastNode.java | 122 -
.../plan/planner/plan/node/write/InsertNode.java | 2 +-
.../planner/plan/node/write/InsertRowNode.java | 4 +-
.../plan/node/write/InsertRowsOfOneDeviceNode.java | 2 +-
.../planner/plan/node/write/InsertTabletNode.java | 4 +-
.../plan/node/write/PipeEnrichedInsertNode.java | 2 +-
.../parameter/DeviceViewIntoPathDescriptor.java | 2 +-
.../plan/parameter/GroupByTimeParameter.java | 85 +-
.../planner/plan/parameter/IntoPathDescriptor.java | 2 +-
.../model/ForecastModelInferenceDescriptor.java | 176 --
.../parameter/model/ModelInferenceDescriptor.java | 111 -
.../plan/scheduler/load/LoadTsFileScheduler.java | 27 +
.../plan/statement/StatementVisitor.java | 21 -
.../statement/component/GroupByTimeComponent.java | 52 +-
.../plan/statement/component/ResultColumn.java | 3 +-
.../plan/statement/component/SelectComponent.java | 8 -
.../plan/statement/crud/InsertBaseStatement.java | 2 +-
.../crud/InsertMultiTabletsStatement.java | 2 +-
.../plan/statement/crud/InsertRowStatement.java | 2 +-
.../crud/InsertRowsOfOneDeviceStatement.java | 2 +-
.../plan/statement/crud/InsertRowsStatement.java | 2 +-
.../plan/statement/crud/InsertTabletStatement.java | 4 +-
.../plan/statement/crud/LoadTsFileStatement.java | 11 +
.../crud/PipeEnrichedInsertBaseStatement.java | 2 +-
.../crud/PipeEnrichedLoadTsFileStatement.java | 10 +
.../plan/statement/crud/QueryStatement.java | 50 -
.../InternalCreateTimeSeriesStatement.java | 2 +-
.../plan/statement/literal/BooleanLiteral.java | 5 +-
.../plan/statement/literal/DoubleLiteral.java | 5 +-
.../plan/statement/literal/Literal.java | 2 +-
.../plan/statement/literal/LongLiteral.java | 5 +-
.../plan/statement/literal/NullLiteral.java | 5 +-
.../plan/statement/literal/StringLiteral.java | 5 +-
.../metadata/CreateAlignedTimeSeriesStatement.java | 2 +-
.../metadata/CreateMultiTimeSeriesStatement.java | 2 +-
.../metadata/CreateTimeSeriesStatement.java | 2 +-
.../statement/metadata/ShowDatabaseStatement.java | 7 +-
.../metadata/model/CreateModelStatement.java | 128 --
.../metadata/model/DropModelStatement.java | 57 -
.../metadata/model/ShowModelsStatement.java | 51 -
.../metadata/model/ShowTrialsStatement.java | 57 -
.../template/AlterSchemaTemplateStatement.java | 2 +-
.../template/CreateSchemaTemplateStatement.java | 2 +-
.../transformation/api/LayerPointReader.java | 2 +-
.../transformation/api/LayerRowReader.java | 2 +-
.../transformation/api/LayerRowWindowReader.java | 2 +-
...ializableRowRecordListBackedMultiColumnRow.java | 6 +-
...izableRowRecordListBackedMultiColumnWindow.java | 2 +-
.../LayerPointReaderBackedSingleColumnRow.java | 3 +-
.../dag/builder/EvaluationDAGBuilder.java | 2 +-
.../dag/column/CaseWhenThenColumnTransformer.java | 4 +-
.../transformation/dag/column/ColumnCache.java | 2 +-
.../dag/column/ColumnTransformer.java | 2 +-
.../binary/ArithmeticBinaryColumnTransformer.java | 4 +-
.../dag/column/binary/BinaryColumnTransformer.java | 4 +-
.../binary/CompareBinaryColumnTransformer.java | 15 +-
.../column/binary/LogicAndColumnTransformer.java | 4 +-
.../column/binary/LogicOrColumnTransformer.java | 4 +-
.../dag/column/leaf/ConstantColumnTransformer.java | 2 +-
.../column/multi/MappableUDFColumnTransformer.java | 48 +-
.../column/ternary/BetweenColumnTransformer.java | 13 +-
.../ternary/CompareTernaryColumnTransformer.java | 4 +-
.../unary/ArithmeticNegationColumnTransformer.java | 4 +-
.../dag/column/unary/InColumnTransformer.java | 8 +-
.../dag/column/unary/IsNullColumnTransformer.java | 4 +-
.../column/unary/LogicNotColumnTransformer.java | 4 +-
.../dag/column/unary/RegularColumnTransformer.java | 11 +-
.../dag/column/unary/UnaryColumnTransformer.java | 4 +-
.../scalar/CastFunctionColumnTransformer.java | 18 +-
.../scalar/DiffFunctionColumnTransformer.java | 4 +-
.../scalar/ReplaceFunctionColumnTransformer.java | 11 +-
.../scalar/RoundFunctionColumnTransformer.java | 4 +-
.../scalar/SubStringFunctionColumnTransformer.java | 11 +-
.../dag/input/ConstantInputReader.java | 2 +-
.../transformation/dag/input/IUDFInputDataSet.java | 2 +-
.../dag/input/QueryDataSetInputLayer.java | 2 +-
.../dag/input/TsBlockInputDataSet.java | 2 +-
.../MultiInputColumnIntermediateLayer.java | 4 +-
...InputColumnMultiReferenceIntermediateLayer.java | 2 +-
...nputColumnSingleReferenceIntermediateLayer.java | 2 +-
.../binary/ArithmeticBinaryTransformer.java | 4 +-
.../dag/transformer/binary/BinaryTransformer.java | 2 +-
.../binary/CompareBinaryTransformer.java | 4 +-
.../binary/CompareEqualToTransformer.java | 5 +-
.../binary/CompareGreaterEqualTransformer.java | 5 +-
.../binary/CompareGreaterThanTransformer.java | 5 +-
.../binary/CompareLessEqualTransformer.java | 5 +-
.../binary/CompareLessThanTransformer.java | 5 +-
.../binary/CompareNonEqualTransformer.java | 7 +-
.../transformer/binary/LogicBinaryTransformer.java | 4 +-
.../multi/MappableUDFQueryRowTransformer.java | 2 +-
.../dag/transformer/multi/UDFQueryTransformer.java | 2 +-
.../multi/UniversalUDFQueryTransformer.java | 2 +-
.../transformer/ternary/BetweenTransformer.java | 17 +-
.../ternary/CompareTernaryTransformer.java | 4 +-
.../transformer/ternary/TernaryTransformer.java | 2 +-
.../unary/ArithmeticNegationTransformer.java | 2 +-
.../dag/transformer/unary/InTransformer.java | 5 +-
.../dag/transformer/unary/IsNullTransformer.java | 2 +-
.../dag/transformer/unary/LogicNotTransformer.java | 4 +-
.../dag/transformer/unary/RegularTransformer.java | 7 +-
.../transformer/unary/TransparentTransformer.java | 2 +-
.../dag/transformer/unary/UnaryTransformer.java | 2 +-
.../unary/scalar/CastFunctionTransformer.java | 18 +-
.../unary/scalar/DiffFunctionTransformer.java | 2 +-
.../unary/scalar/ReplaceFunctionTransformer.java | 11 +-
.../unary/scalar/RoundFunctionTransformer.java | 2 +-
.../unary/scalar/SubStringFunctionTransformer.java | 10 +-
.../transformation/dag/udf/UDTFExecutor.java | 12 +-
.../dag/udf/UDTFInformationInferrer.java | 2 +-
.../transformation/dag/util/LayerCacheUtils.java | 2 +-
.../transformation/dag/util/TransformUtils.java | 6 +-
.../row/ElasticSerializableRowRecordList.java | 2 +-
.../row/SerializableRowRecordList.java | 4 +-
.../tv/ElasticSerializableBinaryTVList.java | 8 +-
.../tv/ElasticSerializableTVList.java | 12 +-
.../datastructure/tv/SerializableBinaryTVList.java | 2 +-
.../tv/SerializableBooleanTVList.java | 2 +-
.../datastructure/tv/SerializableDoubleTVList.java | 2 +-
.../datastructure/tv/SerializableFloatTVList.java | 2 +-
.../datastructure/tv/SerializableIntTVList.java | 2 +-
.../datastructure/tv/SerializableLongTVList.java | 2 +-
.../datastructure/tv/SerializableTVList.java | 4 +-
.../apache/iotdb/db/schemaengine/SchemaEngine.java | 66 +-
.../rescon/CachedSchemaEngineStatistics.java | 4 +
.../rescon/DataNodeSchemaQuotaManager.java | 85 +-
.../rescon/ISchemaEngineStatistics.java | 2 +
.../rescon/MemSchemaEngineStatistics.java | 39 +-
.../schemaregion/impl/SchemaRegionMemoryImpl.java | 13 +-
.../schemaregion/impl/SchemaRegionPBTreeImpl.java | 87 +-
.../visitor/SchemaRegionPlanDeserializer.java | 2 +-
.../visitor/SchemaRegionPlanSerializer.java | 2 +-
.../visitor/SchemaRegionPlanTxtSerializer.java | 2 +-
.../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 7 +-
.../impl/mem/mnode/factory/MemMNodeFactory.java | 3 +-
.../mtree/impl/mem/mnode/info/LogicalViewInfo.java | 2 +-
.../mtree/impl/mem/mnode/info/MeasurementInfo.java | 2 +-
.../impl/mem/snapshot/MemMTreeSnapshotUtil.java | 3 +-
.../mtree/impl/pbtree/CachedMTreeStore.java | 78 +-
.../mtree/impl/pbtree/MTreeBelowSGCachedImpl.java | 170 +-
.../impl/pbtree/StampedWriterPreferredLock.java | 10 +-
.../mtree/impl/pbtree/cache/CacheManager.java | 285 ++-
.../mtree/impl/pbtree/cache/ICacheManager.java | 10 +-
.../impl/pbtree/flush/PBTreeFlushExecutor.java | 104 +
.../mnode/container/CachedMNodeContainer.java | 5 +
.../mnode/container/ICachedMNodeContainer.java | 4 +-
.../pbtree/mnode/factory/CacheMNodeFactory.java | 3 +-
.../mtree/impl/pbtree/schemafile/RecordUtils.java | 99 +-
.../impl/pbtree/schemafile/WrappedSegment.java | 2 +-
.../read/resp/info/impl/ShowTimeSeriesResult.java | 2 +-
.../reader/impl/TimeseriesReaderWithViewFetch.java | 2 +-
.../schemaregion/utils/ResourceByPathUtils.java | 2 +-
.../write/req/ICreateAlignedTimeSeriesPlan.java | 2 +-
.../write/req/ICreateTimeSeriesPlan.java | 2 +-
.../write/req/SchemaRegionWritePlanFactory.java | 2 +-
.../req/impl/CreateAlignedTimeSeriesPlanImpl.java | 2 +-
.../write/req/impl/CreateTimeSeriesPlanImpl.java | 2 +-
.../template/ClusterTemplateManager.java | 6 +-
.../iotdb/db/schemaengine/template/Template.java | 2 +-
.../template/alter/TemplateExtendInfo.java | 2 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 8 +-
.../apache/iotdb/db/service/IoTDBShutdownHook.java | 7 +-
.../apache/iotdb/db/service/MLNodeRPCService.java | 92 -
.../org/apache/iotdb/db/service/RPCService.java | 40 +-
.../db/service/metrics/CompactionMetrics.java | 101 +-
.../metrics/IoTDBInternalLocalReporter.java | 2 +-
.../iotdb/db/storageengine/StorageEngine.java | 6 +-
.../db/storageengine/dataregion/DataRegion.java | 171 +-
.../dataregion/HashLastFlushTimeMap.java | 10 +-
.../compaction/constant/CompactionTaskType.java | 3 +-
.../execute/recover/CompactionRecoverManager.java | 16 +-
.../execute/task/AbstractCompactionTask.java | 53 +-
...skType.java => CompactionTaskPriorityType.java} | 2 +-
.../execute/task/CrossSpaceCompactionTask.java | 18 +-
.../execute/task/InnerSpaceCompactionTask.java | 20 +-
.../task/InsertionCrossSpaceCompactionTask.java | 363 +++
.../compaction/execute/utils/CompactionUtils.java | 167 +-
.../execute/utils/log/CompactionLogger.java | 13 +-
.../execute/utils/log/SimpleCompactionLogger.java | 4 +
.../execute/utils/log/TsFileIdentifier.java | 32 +-
.../utils/validator/CompactionValidator.java | 51 -
.../ResourceAndTsfileCompactionValidator.java | 57 -
.../utils/writer/AbstractCompactionWriter.java | 2 +-
.../writer/AbstractCrossCompactionWriter.java | 2 +-
.../writer/AbstractInnerCompactionWriter.java | 2 +-
.../utils/writer/FastCrossCompactionWriter.java | 2 +-
.../utils/writer/FastInnerCompactionWriter.java | 2 +-
.../writer/ReadPointCrossCompactionWriter.java | 2 +-
.../writer/ReadPointInnerCompactionWriter.java | 2 +-
.../compaction/io/CompactionTsFileWriter.java | 2 +-
.../compaction/schedule/CompactionScheduler.java | 56 +
.../compaction/schedule/CompactionTaskManager.java | 20 +-
.../compaction/schedule/CompactionWorker.java | 27 +-
.../DefaultCompactionTaskComparatorImpl.java | 15 +-
.../impl/RewriteCrossSpaceCompactionSelector.java | 206 +-
.../impl/SizeTieredCompactionSelector.java | 10 +-
.../utils/CrossSpaceCompactionCandidate.java | 116 -
.../utils/DeviceInfo.java} | 16 +-
.../InsertionCrossCompactionTaskResource.java} | 33 +-
.../selector/utils/TsFileResourceCandidate.java | 130 ++
.../dataregion/flush/FlushManager.java | 7 +-
.../dataregion/memtable/AbstractMemTable.java | 4 +-
.../memtable/AlignedReadOnlyMemChunk.java | 2 +-
.../memtable/AlignedWritableMemChunk.java | 4 +-
.../dataregion/memtable/IWritableMemChunk.java | 2 +-
.../dataregion/memtable/ReadOnlyMemChunk.java | 2 +-
.../dataregion/memtable/TsFileProcessor.java | 28 +-
.../dataregion/memtable/WritableMemChunk.java | 4 +-
.../read/reader/chunk/MemAlignedPageReader.java | 6 +-
.../read/reader/chunk/MemPageReader.java | 6 +-
.../reader/common/CachedPriorityMergeReader.java | 2 +-
.../dataregion/snapshot/SnapshotLoader.java | 20 +-
.../dataregion/tsfile/TsFileManager.java | 20 +-
.../dataregion/tsfile/TsFileResource.java | 22 +-
.../dataregion/utils/TsFileResourceUtils.java | 378 ++++
.../validate/TsFileResourceAndDataValidator.java | 58 +
.../validate/TsFileResourceValidator.java} | 43 +-
.../validate/TsFileValidator.java} | 36 +-
.../storageengine/dataregion/wal/WALManager.java | 40 +-
.../dataregion/wal/utils/WALInsertNodeCache.java | 21 +-
.../dataregion/wal/utils/WALWriteUtils.java | 2 +-
.../db/storageengine/rescon/disk/TierManager.java | 4 +-
.../rescon/memory/PrimitiveArrayManager.java | 4 +-
.../rescon/quotas/DefaultOperationQuota.java | 2 +-
.../iotdb/db/tools/TsFileSplitByPartitionTool.java | 4 +-
.../org/apache/iotdb/db/tools/TsFileSplitTool.java | 4 +-
.../db/tools/validate/TsFileValidationTool.java | 2 +-
.../org/apache/iotdb/db/utils/CommonUtils.java | 12 +-
.../org/apache/iotdb/db/utils/DateTimeUtils.java | 99 +-
.../iotdb/db/utils/EncodingInferenceUtils.java | 4 +-
.../java/org/apache/iotdb/db/utils/MemUtils.java | 5 +-
.../apache/iotdb/db/utils/QueryDataSetUtils.java | 6 +-
.../org/apache/iotdb/db/utils/SchemaUtils.java | 2 +-
.../apache/iotdb/db/utils/TimeValuePairUtils.java | 7 +-
.../iotdb/db/utils/TimestampPrecisionUtils.java | 19 +-
.../apache/iotdb/db/utils/TypeInferenceUtils.java | 2 +-
.../iotdb/db/utils/constant/SqlConstant.java | 3 -
.../db/utils/datastructure/AlignedTVList.java | 4 +-
.../db/utils/datastructure/BackAlignedTVList.java | 2 +-
.../db/utils/datastructure/BackBinaryTVList.java | 2 +-
.../db/utils/datastructure/BackBooleanTVList.java | 2 +-
.../db/utils/datastructure/BackDoubleTVList.java | 2 +-
.../db/utils/datastructure/BackFloatTVList.java | 2 +-
.../db/utils/datastructure/BackIntTVList.java | 2 +-
.../db/utils/datastructure/BackLongTVList.java | 2 +-
.../iotdb/db/utils/datastructure/BinaryTVList.java | 2 +-
.../db/utils/datastructure/BooleanTVList.java | 2 +-
.../iotdb/db/utils/datastructure/DoubleTVList.java | 2 +-
.../iotdb/db/utils/datastructure/FloatTVList.java | 2 +-
.../iotdb/db/utils/datastructure/IntTVList.java | 2 +-
.../iotdb/db/utils/datastructure/LongTVList.java | 2 +-
.../db/utils/datastructure/QuickAlignedTVList.java | 2 +-
.../iotdb/db/utils/datastructure/TVList.java | 2 +-
.../db/utils/datastructure/TimAlignedTVList.java | 2 +-
.../db/utils/datastructure/TimBinaryTVList.java | 2 +-
.../db/utils/datastructure/TimBooleanTVList.java | 2 +-
.../db/utils/datastructure/TimDoubleTVList.java | 2 +-
.../db/utils/datastructure/TimFloatTVList.java | 2 +-
.../iotdb/db/utils/datastructure/TimIntTVList.java | 2 +-
.../db/utils/datastructure/TimLongTVList.java | 2 +-
.../apache/iotdb/db/utils/sort/DiskSpiller.java | 4 +-
.../iotdb/db/utils/sort/FileSpillerReader.java | 6 +-
.../iotdb/db/utils/sort/SortBufferManager.java | 5 +-
.../iotdb/db/utils/windowing/api/Window.java | 2 +-
.../windowing/configuration/Configuration.java | 2 +-
.../SlidingSizeWindowConfiguration.java | 2 +-
.../SlidingTimeWindowConfiguration.java | 2 +-
.../handler/SlidingWindowEvaluationHandler.java | 3 +-
.../utils/windowing/window/EvictableBatchList.java | 2 +-
.../db/utils/windowing/window/WindowImpl.java | 4 +-
.../apache/iotdb/db/metadata/MetaUtilsTest.java | 2 +-
.../db/metadata/cache/DataNodeSchemaCacheTest.java | 2 +-
.../cache/dualkeycache/DualKeyCacheTest.java | 2 +-
.../mtree/lock/StampedWriterPreferredLockTest.java | 43 +
.../metadata/mtree/schemafile/RecordUtilTests.java | 2 +-
.../metadata/mtree/schemafile/SchemaFileTest.java | 2 +-
.../metadata/mtree/schemafile/SchemaPageTest.java | 2 +-
.../mtree/schemafile/WrappedSegmentTest.java | 2 +-
.../db/metadata/path/MeasurementPathTest.java | 2 +-
.../schemaRegion/SchemaRegionAliasAndTagTest.java | 2 +-
.../schemaRegion/SchemaRegionBasicTest.java | 2 +-
.../schemaRegion/SchemaRegionManagementTest.java | 2 +-
.../schemaRegion/SchemaRegionTemplateTest.java | 2 +-
.../schemaRegion/SchemaRegionTestUtil.java | 2 +-
.../schemaRegion/SchemaStatisticsTest.java | 2 +-
.../metadata/view/ViewExpressionToStringTest.java | 2 +-
.../apache/iotdb/db/metric/MetricServiceTest.java | 48 +-
.../iotdb/db/pipe/connector/PipeConnectorTest.java | 76 +-
.../iotdb/db/pipe/connector/PipeReceiverTest.java | 2 +-
.../db/pipe/connector/PipeThriftRequestTest.java | 2 +-
.../db/pipe/event/PipeDataTypeTransformerTest.java | 2 +-
.../pipe/event/PipeTabletInsertionEventTest.java | 11 +-
.../db/pipe/extractor/PipeRealtimeExtractTest.java | 2 +-
.../resource/PipeTsFileResourceManagerTest.java | 4 +-
.../apache/iotdb/db/query/udf/example/Adder.java | 2 +-
.../common/schematree/ClusterSchemaTreeTest.java | 2 +-
.../queryengine/common/schematree/NodeRefTest.java | 2 +-
.../queryengine/execution/ConfigExecutionTest.java | 2 +-
.../db/queryengine/execution/DataDriverTest.java | 2 +-
.../execution/aggregation/AccumulatorTest.java | 6 +-
.../aggregation/TimeRangeIteratorTest.java | 183 +-
.../execution/exchange/SharedTsBlockQueueTest.java | 2 +-
.../execution/exchange/SinkChannelTest.java | 5 +-
.../executor/RegionWriteExecutorTest.java | 2 +-
.../execution/load/LoadTsFileSchedulerTest.java | 2 +-
.../db/queryengine/execution/load/TestBase.java | 2 +-
.../operator/AggregationOperatorTest.java | 14 +-
.../AlignedSeriesAggregationScanOperatorTest.java | 29 +-
.../operator/AlignedSeriesScanOperatorTest.java | 2 +-
.../execution/operator/AlignedSeriesTestUtil.java | 2 +-
.../execution/operator/DeviceViewOperatorTest.java | 5 +-
.../execution/operator/FillOperatorTest.java | 2 +-
.../operator/HorizontallyConcatOperatorTest.java | 11 +-
.../execution/operator/LastQueryOperatorTest.java | 7 +-
.../operator/LastQuerySortOperatorTest.java | 7 +-
.../execution/operator/LimitOperatorTest.java | 2 +-
.../execution/operator/LinearFillOperatorTest.java | 2 +-
.../execution/operator/MergeSortOperatorTest.java | 4 +-
.../execution/operator/MultiColumnMergerTest.java | 6 +-
.../NonOverlappedMultiColumnMergerTest.java | 6 +-
.../execution/operator/OffsetOperatorTest.java | 2 +-
.../execution/operator/OperatorMemoryTest.java | 27 +-
.../operator/RawDataAggregationOperatorTest.java | 23 +-
.../SeriesAggregationScanOperatorTest.java | 29 +-
.../execution/operator/SeriesScanOperatorTest.java | 2 +-
.../execution/operator/SingleColumnMergerTest.java | 6 +-
.../operator/SingleDeviceViewOperatorTest.java | 2 +-
.../SlidingWindowAggregationOperatorTest.java | 10 +-
.../execution/operator/SortOperatorTest.java | 2 +-
.../execution/operator/TopKOperatorTest.java | 4 +-
.../operator/UpdateLastCacheOperatorTest.java | 7 +-
.../schema/CountGroupByLevelMergeOperatorTest.java | 4 +-
.../operator/schema/SchemaCountOperatorTest.java | 3 +-
.../schema/SchemaFetchScanOperatorTest.java | 2 +-
.../schema/SchemaQueryScanOperatorTest.java | 2 +-
.../operator/sink/IdentitySinkOperatorTest.java | 2 +-
.../plan/analyze/AggregationDescriptorTest.java | 2 +-
.../db/queryengine/plan/analyze/AnalyzeTest.java | 2 +-
.../plan/analyze/FakeSchemaFetcherImpl.java | 2 +-
.../plan/optimization/LimitOffsetPushDownTest.java | 2 +-
.../plan/parser/StatementGeneratorTest.java | 34 +-
.../db/queryengine/plan/plan/ExpressionTest.java | 2 +-
.../queryengine/plan/plan/LogicalPlannerTest.java | 2 +-
.../queryengine/plan/plan/PipelineBuilderTest.java | 2 +-
.../plan/plan/QueryLogicalPlanUtil.java | 2 +-
.../distribution/AggregationDistributionTest.java | 2 +-
.../distribution/DistributionPlannerBasicTest.java | 2 +-
.../queryengine/plan/plan/distribution/Util.java | 2 +-
.../queryengine/plan/plan/distribution/Util2.java | 2 +-
.../plan/plan/node/PlanGraphPrinterTest.java | 70 +
.../plan/plan/node/load/LoadTsFileNodeTest.java | 2 +-
.../metadata/write/MetadataWriteNodeSerDeTest.java | 2 +-
.../node/process/AggregationNodeSerdeTest.java | 5 +-
.../plan/node/process/FilterNodeSerdeTest.java | 2 +-
.../node/process/GroupByLevelNodeSerdeTest.java | 5 +-
.../plan/node/process/GroupByTagNodeSerdeTest.java | 5 +-
.../plan/plan/node/process/LimitNodeSerdeTest.java | 2 +-
.../plan/node/process/OffsetNodeSerdeTest.java | 2 +-
.../plan/plan/node/process/SortNodeSerdeTest.java | 2 +-
.../plan/node/process/TimeJoinNodeSerdeTest.java | 2 +-
.../source/SeriesAggregationScanNodeSerdeTest.java | 5 +-
.../plan/node/source/SeriesScanNodeSerdeTest.java | 2 +-
.../write/InsertMultiTabletsNodeSerdeTest.java | 5 +-
.../plan/node/write/InsertRowNodeSerdeTest.java | 2 +-
.../plan/node/write/InsertRowsNodeSerdeTest.java | 2 +-
.../write/InsertRowsOfOneDeviceNodeSerdeTest.java | 2 +-
.../plan/node/write/InsertTabletNodeSerdeTest.java | 2 +-
.../plan/node/write/WritePlanNodeSplitTest.java | 2 +-
...meticAndCompareBinaryColumnTransformerTest.java | 6 +-
.../binary/LogicBinaryColumnTransformerTest.java | 6 +-
.../column/unary/UnaryColumnTransformerTest.java | 6 +-
.../ElasticSerializableRowRecordListTest.java | 10 +-
.../ElasticSerializableTVListTest.java | 7 +-
.../SerializableBinaryTVListTest.java | 5 +-
.../SerializableBooleanTVListTest.java | 2 +-
.../SerializableDoubleTVListTest.java | 2 +-
.../datastructure/SerializableFloatTVListTest.java | 2 +-
.../datastructure/SerializableIntTVListTest.java | 2 +-
.../datastructure/SerializableLongTVListTest.java | 2 +-
.../SerializableRowRecordListTest.java | 6 +-
.../DataNodeInternalRPCServiceImplTest.java | 2 +-
.../storageengine/buffer/BloomFilterCacheTest.java | 2 +-
.../db/storageengine/buffer/ChunkCacheTest.java | 2 +-
.../storageengine/dataregion/DataRegionTest.java | 2 +-
.../dataregion/LastFlushTimeMapTest.java | 86 +-
.../iotdb/db/storageengine/dataregion/TTLTest.java | 2 +-
.../compaction/AbstractCompactionTest.java | 83 +-
.../compaction/CompactionTaskComparatorTest.java | 21 +-
.../compaction/CompactionValidationTest.java | 47 +-
.../CrossSpaceCompactionWithUnusualCasesTest.java | 122 +-
.../compaction/FastAlignedCrossCompactionTest.java | 2 +-
.../FastCrossCompactionPerformerTest.java | 6 +-
.../FastInnerCompactionPerformerTest.java | 4 +-
.../FastNonAlignedCrossCompactionTest.java | 2 +-
.../compaction/ReadChunkInnerCompactionTest.java | 2 +-
.../ReadPointAlignedCrossCompactionTest.java | 2 +-
.../ReadPointCompactionPerformerTest.java | 2 +-
.../ReadPointNonAlignedCrossCompactionTest.java | 2 +-
.../compaction/TestUtilsForAlignedSeries.java | 2 +-
.../TsFileValidationCorrectnessTests.java | 296 +++
...eCompactionWithFastPerformerValidationTest.java | 12 +-
...actionWithReadPointPerformerValidationTest.java | 2 +-
.../InsertionCrossSpaceCompactionRecoverTest.java | 372 +++
.../InsertionCrossSpaceCompactionSelectorTest.java | 2372 ++++++++++++++++++++
.../cross/InsertionCrossSpaceCompactionTest.java | 510 +++++
.../dataregion/compaction/cross/MergeTest.java | 2 +-
.../compaction/cross/MergeUpgradeTest.java | 2 +-
...eCrossSpaceCompactionWithFastPerformerTest.java | 14 +-
...sSpaceCompactionWithReadPointPerformerTest.java | 6 +-
.../inner/AbstractInnerSpaceCompactionTest.java | 2 +-
.../inner/FastCompactionPerformerAlignedTest.java | 2 +-
.../FastCompactionPerformerNoAlignedTest.java | 2 +-
.../inner/InnerCompactionEmptyTsFileTest.java | 2 +-
.../inner/InnerCompactionMoreDataTest.java | 2 +-
.../compaction/inner/InnerCompactionTest.java | 2 +-
.../ReadChunkCompactionPerformerAlignedTest.java | 5 +-
.../ReadChunkCompactionPerformerNoAlignedTest.java | 2 +-
.../inner/sizetiered/SizeTieredCompactionTest.java | 2 +-
.../recover/SizeTieredCompactionRecoverTest.java | 2 +-
.../settle/SettleRequestHandlerTest.java | 2 +-
.../compaction/utils/CompactionCheckerUtils.java | 2 +-
.../utils/CompactionFileGeneratorUtils.java | 2 +-
.../compaction/utils/CompactionTestFileWriter.java | 6 +-
.../utils/MultiTsFileDeviceIteratorTest.java | 2 +-
.../compaction/utils/TsFileGeneratorUtils.java | 134 +-
.../dataregion/memtable/MemTableFlushTaskTest.java | 2 +-
.../dataregion/memtable/MemTableTestUtils.java | 2 +-
.../dataregion/memtable/MemtableBenchmark.java | 2 +-
.../dataregion/memtable/PrimitiveMemTableTest.java | 8 +-
.../dataregion/memtable/TsFileProcessorTest.java | 2 +-
.../reader/chunk/MemAlignedChunkLoaderTest.java | 5 +-
.../read/reader/chunk/MemChunkLoaderTest.java | 7 +-
.../read/reader/common/FakedSeriesReader.java | 2 +-
.../AlignedSeriesScanLimitOffsetPushDownTest.java | 2 +-
.../read/reader/series/SeriesReaderTestUtil.java | 2 +-
.../series/SeriesScanLimitOffsetPushDownTest.java | 2 +-
.../dataregion/wal/WALManagerTest.java | 7 +-
.../wal/allocation/ElasticStrategyTest.java | 5 +-
.../wal/allocation/FirstCreateStrategyTest.java | 5 +-
.../wal/allocation/RoundRobinStrategyTest.java | 5 +-
.../dataregion/wal/buffer/WALBufferCommonTest.java | 5 +-
.../dataregion/wal/io/WALFileTest.java | 7 +-
.../wal/node/ConsensusReqReaderTest.java | 7 +-
.../dataregion/wal/node/WALEntryHandlerTest.java | 5 +-
.../dataregion/wal/node/WALNodeTest.java | 5 +-
.../wal/recover/WALRecoverManagerTest.java | 5 +-
.../wal/recover/WALRecoverWriterTest.java | 5 +-
.../file/SealedTsFileRecoverPerformerTest.java | 2 +-
.../wal/recover/file/TsFilePlanRedoerTest.java | 20 +-
.../file/UnsealedTsFileRecoverPerformerTest.java | 2 +-
.../wal/utils/WALInsertNodeCacheTest.java | 5 +-
.../rescon/memory/ResourceManagerTest.java | 2 +-
.../org/apache/iotdb/db/tools/MLogParserTest.java | 2 +-
.../iotdb/db/tools/PBTreeFileSketchTest.java | 2 +-
.../iotdb/db/tools/TsFileAndModSettleToolTest.java | 2 +-
.../iotdb/db/tools/TsFileSelfCheckToolTest.java | 2 +-
.../iotdb/db/tools/TsFileSketchToolTest.java | 2 +-
.../db/trigger/executor/TriggerExecuteTest.java | 2 +-
.../DateTimeUtilsTest.java} | 33 +-
.../iotdb/db/utils/EncodingInferenceUtilsTest.java | 2 +-
.../org/apache/iotdb/db/utils/MemUtilsTest.java | 8 +-
.../iotdb/db/utils/QueryDataSetUtilsTest.java | 8 +-
.../org/apache/iotdb/db/utils/SchemaTestUtils.java | 2 +-
.../org/apache/iotdb/db/utils/SchemaUtilsTest.java | 2 +-
.../apache/iotdb/db/utils/SerializeUtilsTest.java | 70 +-
.../iotdb/db/utils/TypeInferenceUtilsTest.java | 2 +-
.../db/utils/datastructure/AlignedTVListTest.java | 9 +-
.../db/utils/datastructure/BinaryTVListTest.java | 15 +-
.../db/utils/datastructure/MergeSortHeapTest.java | 19 +-
.../datastructure/PrimitiveArrayManagerTest.java | 2 +-
.../apache/iotdb/db/utils/sort/SortUtilTest.java | 4 +-
.../SlidingSizeWindowEvaluationHandlerTest.java | 2 +-
.../SlidingTimeWindowEvaluationHandlerTest.java | 2 +-
.../metrics/{dropwizard-metrics => core}/pom.xml | 16 +-
.../iotdb/metrics/core/IoTDBMetricManager.java | 156 ++
.../metrics/core/reporter/IoTDBJmxReporter.java | 183 ++
.../iotdb/metrics/core/type/IoTDBAutoGauge.java} | 18 +-
.../metrics/core/type/IoTDBAutoGaugeMBean.java} | 14 +-
.../iotdb/metrics/core/type/IoTDBCounter.java} | 21 +-
.../metrics/core/type/IoTDBCounterMBean.java} | 14 +-
.../iotdb/metrics/core/type/IoTDBGauge.java} | 16 +-
.../iotdb/metrics/core/type/IoTDBGaugeMBean.java} | 14 +-
.../iotdb/metrics/core/type/IoTDBHistogram.java} | 37 +-
.../metrics/core/type/IoTDBHistogramMBean.java} | 23 +-
.../metrics/core/type/IoTDBHistogramSnapshot.java} | 13 +-
.../apache/iotdb/metrics/core/type/IoTDBRate.java} | 27 +-
.../iotdb/metrics/core/type/IoTDBRateMBean.java} | 17 +-
.../iotdb/metrics/core/type/IoTDBTimer.java} | 39 +-
.../core/type/IoTDBTimerHistogramSnapshot.java} | 13 +-
.../iotdb/metrics/core/type/IoTDBTimerMBean.java} | 26 +-
.../iotdb/metrics/core/uitls/IoTDBCachedGauge.java | 111 +
.../metrics/core/uitls/IoTDBCachedGaugeMBean.java} | 10 +-
.../core/uitls/IoTDBMetricObjNameFactory.java | 104 +
.../metrics/core/uitls/IoTDBMovingAverage.java | 86 +
.../metrics/core/uitls/ObjectNameFactory.java} | 20 +-
.../org.apache.iotdb.metrics.AbstractMetricManager | 2 +-
.../org.apache.iotdb.metrics.reporter.JmxReporter} | 2 +-
.../dropwizard/DropwizardMetricManager.java | 129 --
.../dropwizard/DropwizardMetricNameTool.java | 82 -
.../dropwizard/reporter/DropwizardJmxReporter.java | 83 -
.../metrics/dropwizard/type/DropwizardGauge.java | 72 -
.../dropwizard/type/DropwizardHistogram.java | 47 -
.../type/DropwizardHistogramSnapshot.java | 58 -
.../metrics/dropwizard/type/DropwizardRate.java | 94 -
.../metrics/dropwizard/type/DropwizardTimer.java | 48 -
.../org.apache.iotdb.metrics.reporter.JmxReporter | 18 -
iotdb-core/metrics/interface/pom.xml | 15 +-
.../iotdb/metrics/AbstractMetricManager.java | 69 +-
.../iotdb/metrics/AbstractMetricService.java | 71 +-
.../iotdb/metrics/DoNothingMetricService.java | 10 +
.../apache/iotdb/metrics/config/MetricConfig.java | 17 +-
.../metrics/config/MetricConfigDescriptor.java | 9 +-
.../iotdb/metrics/impl/DoNothingAutoGauge.java | 5 +-
.../iotdb/metrics/impl/DoNothingCounter.java | 5 +-
.../apache/iotdb/metrics/impl/DoNothingGauge.java | 5 +-
.../iotdb/metrics/impl/DoNothingHistogram.java | 5 +-
.../metrics/impl/DoNothingHistogramSnapshot.java | 4 +-
.../iotdb/metrics/impl/DoNothingMetricManager.java | 8 +-
.../apache/iotdb/metrics/impl/DoNothingRate.java | 13 +-
.../apache/iotdb/metrics/impl/DoNothingTimer.java | 3 +-
.../metrics/metricsets/cpu/CpuUsageMetrics.java | 2 +-
.../apache/iotdb/metrics/reporter/JmxReporter.java | 7 +-
.../metrics/reporter/iotdb/IoTDBReporter.java | 2 +-
.../reporter/iotdb/IoTDBSessionReporter.java | 2 +-
.../reporter/prometheus/PrometheusReporter.java | 12 +-
.../org/apache/iotdb/metrics/type/AutoGauge.java | 4 +-
.../org/apache/iotdb/metrics/type/Counter.java | 4 +-
.../java/org/apache/iotdb/metrics/type/Gauge.java | 4 +-
.../org/apache/iotdb/metrics/type/Histogram.java | 4 +-
.../org/apache/iotdb/metrics/type/IMetric.java | 4 +
.../java/org/apache/iotdb/metrics/type/Rate.java | 8 -
...tricFrameType.java => AbstractMetricMBean.java} | 17 +-
.../iotdb/metrics/config/MetricConfigTest.java | 3 -
iotdb-core/metrics/micrometer-metrics/pom.xml | 54 -
.../micrometer/MicrometerMetricManager.java | 153 --
.../micrometer/reporter/MicrometerJmxReporter.java | 90 -
.../micrometer/type/MicrometerAutoGauge.java | 52 -
.../metrics/micrometer/type/MicrometerCounter.java | 45 -
.../org.apache.iotdb.metrics.reporter.JmxReporter | 18 -
iotdb-core/metrics/pom.xml | 5 +-
iotdb-core/mlnode/.flake8 | 33 -
iotdb-core/mlnode/.gitignore | 9 -
iotdb-core/mlnode/README.md | 47 -
iotdb-core/mlnode/iotdb/__init__.py | 17 -
iotdb-core/mlnode/iotdb/mlnode/__init__.py | 17 -
.../mlnode/iotdb/mlnode/algorithm/factory.py | 66 -
.../iotdb/mlnode/algorithm/hyperparameter.py | 407 ----
iotdb-core/mlnode/iotdb/mlnode/algorithm/metric.py | 80 -
.../mlnode/algorithm/models/forecast/__init__.py | 17 -
.../mlnode/algorithm/models/forecast/dlinear.py | 133 --
.../mlnode/algorithm/models/forecast/nbeats.py | 131 --
.../mlnode/iotdb/mlnode/algorithm/validator.py | 45 -
iotdb-core/mlnode/iotdb/mlnode/client.py | 274 ---
iotdb-core/mlnode/iotdb/mlnode/config.py | 168 --
iotdb-core/mlnode/iotdb/mlnode/constant.py | 99 -
iotdb-core/mlnode/iotdb/mlnode/dataset/dataset.py | 74 -
iotdb-core/mlnode/iotdb/mlnode/dataset/factory.py | 50 -
iotdb-core/mlnode/iotdb/mlnode/dataset/source.py | 80 -
.../mlnode/iotdb/mlnode/dataset/utils/__init__.py | 17 -
.../iotdb/mlnode/dataset/utils/time_features.py | 149 --
iotdb-core/mlnode/iotdb/mlnode/exception.py | 66 -
iotdb-core/mlnode/iotdb/mlnode/handler.py | 95 -
iotdb-core/mlnode/iotdb/mlnode/log.py | 34 -
iotdb-core/mlnode/iotdb/mlnode/parser.py | 99 -
iotdb-core/mlnode/iotdb/mlnode/process/__init__.py | 17 -
iotdb-core/mlnode/iotdb/mlnode/process/manager.py | 124 -
iotdb-core/mlnode/iotdb/mlnode/process/task.py | 265 ---
iotdb-core/mlnode/iotdb/mlnode/process/trial.py | 224 --
iotdb-core/mlnode/iotdb/mlnode/script.py | 36 -
iotdb-core/mlnode/iotdb/mlnode/serde.py | 508 -----
iotdb-core/mlnode/iotdb/mlnode/service.py | 58 -
iotdb-core/mlnode/iotdb/mlnode/storage.py | 110 -
iotdb-core/mlnode/iotdb/mlnode/util.py | 80 -
iotdb-core/mlnode/pom.xml | 175 --
iotdb-core/mlnode/pyproject.toml | 55 -
iotdb-core/mlnode/requirements.txt | 30 -
iotdb-core/mlnode/requirements_dev.txt | 28 -
iotdb-core/mlnode/resources/conf/iotdb-mlnode.toml | 70 -
.../mlnode/resources/conf/logging_config.ini | 40 -
.../mlnode/test/test_create_forecast_model.py | 100 -
iotdb-core/mlnode/test/test_model_storage.py | 100 -
iotdb-core/mlnode/test/test_serde.py | 171 --
iotdb-core/node-commons/pom.xml | 33 +-
.../resources/conf/iotdb-common.properties | 101 +-
.../commons/auth/entity/PriPrivilegeType.java | 1 +
.../iotdb/commons/auth/entity/PrivilegeType.java | 2 +
.../org/apache/iotdb/commons/auth/entity/Role.java | 4 +
.../iotdb/commons/client/ClientPoolFactory.java | 26 -
.../iotdb/commons/client/mlnode/MLNodeClient.java | 212 --
.../commons/client/mlnode/MLNodeClientManager.java | 40 -
.../iotdb/commons/client/mlnode/MLNodeInfo.java | 29 -
.../iotdb/commons/concurrent/ThreadName.java | 2 -
.../apache/iotdb/commons/conf/CommonConfig.java | 83 +-
.../iotdb/commons/conf/CommonDescriptor.java | 43 +-
.../apache/iotdb/commons/conf/IoTDBConstant.java | 4 +-
.../exception/pipe/PipeRuntimeExceptionType.java | 5 +
.../PipeRuntimeOutOfMemoryCriticalException.java | 100 +
.../commons/model/ForecastModeInformation.java | 153 --
.../iotdb/commons/model/ModelHyperparameter.java | 81 -
.../iotdb/commons/model/ModelInformation.java | 261 ---
.../org/apache/iotdb/commons/model/ModelTable.java | 91 -
.../iotdb/commons/model/TrialInformation.java | 108 -
.../model/exception/ModelManagementException.java | 26 -
.../org/apache/iotdb/commons/path/AlignedPath.java | 10 +-
.../apache/iotdb/commons/path/MeasurementPath.java | 2 +-
.../org/apache/iotdb/commons/path/PartialPath.java | 2 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 24 +
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 37 +
.../builtin/connector/WebSocketConnector.java | 4 +-
.../builtin/connector/WriteBackConnector.java} | 14 +-
.../commons/schema/filter/SchemaFilterFactory.java | 2 +-
.../commons/schema/filter/impl/DataTypeFilter.java | 2 +-
.../node/common/AbstractMeasurementMNode.java | 2 +-
.../commons/schema/node/info/IMeasurementInfo.java | 2 +-
.../schema/node/role/IMeasurementMNode.java | 2 +-
.../commons/schema/node/utils/IMNodeFactory.java | 3 +-
.../commons/schema/view/LogicalViewSchema.java | 2 +-
.../viewExpression/leaf/ConstantViewOperand.java | 2 +-
.../service/AbstractThriftServiceThread.java | 36 +
.../apache/iotdb/commons/service/ServiceType.java | 1 -
.../iotdb/commons/service/ThriftServiceThread.java | 28 +
.../commons/service/metric/MetricService.java | 49 +
.../iotdb/commons/service/metric/enums/Metric.java | 1 +
.../commons/udf/builtin/String/UDTFConcat.java | 63 +-
.../commons/udf/builtin/String/UDTFEndsWith.java | 20 +
.../commons/udf/builtin/String/UDTFLower.java | 23 +-
.../commons/udf/builtin/String/UDTFStartsWith.java | 36 +-
.../commons/udf/builtin/String/UDTFStrCompare.java | 40 +-
.../commons/udf/builtin/String/UDTFStrLength.java | 36 +-
.../commons/udf/builtin/String/UDTFStrLocate.java | 37 +-
.../iotdb/commons/udf/builtin/String/UDTFTrim.java | 36 +-
.../commons/udf/builtin/String/UDTFUpper.java | 23 +-
.../apache/iotdb/commons/udf/builtin/UDTFAbs.java | 85 +
.../iotdb/commons/udf/builtin/UDTFConst.java | 132 +-
.../iotdb/commons/udf/builtin/UDTFConstE.java | 34 +-
.../iotdb/commons/udf/builtin/UDTFConstPi.java | 34 +-
.../iotdb/commons/udf/builtin/UDTFContains.java | 36 +-
.../udf/builtin/UDTFContinuouslySatisfy.java | 2 +-
.../udf/builtin/UDTFEqualSizeBucketAggSample.java | 2 +-
.../udf/builtin/UDTFEqualSizeBucketSample.java | 2 +-
.../iotdb/commons/udf/builtin/UDTFInRange.java | 123 +-
.../apache/iotdb/commons/udf/builtin/UDTFJexl.java | 2 +-
.../apache/iotdb/commons/udf/builtin/UDTFM4.java | 2 +-
.../iotdb/commons/udf/builtin/UDTFMatches.java | 35 +-
.../apache/iotdb/commons/udf/builtin/UDTFMath.java | 87 +-
.../iotdb/commons/udf/builtin/UDTFOnOff.java | 87 +-
.../iotdb/commons/udf/builtin/UDTFSelectK.java | 2 +-
.../iotdb/commons/udf/builtin/UDTFValueTrend.java | 2 +-
.../commons/udf/service/UDFManagementService.java | 19 -
.../commons/udf/utils/UDFDataTypeTransformer.java | 7 +-
.../apache/iotdb/commons/utils/NodeUrlUtils.java | 65 +-
.../apache/iotdb/commons/utils/SerializeUtils.java | 2 +-
iotdb-core/pom.xml | 3 +-
iotdb-core/tsfile/pom.xml | 9 +-
.../iotdb/tsfile/common/conf/TSFileConfig.java | 2 +-
.../iotdb/tsfile/encoding/decoder/Decoder.java | 2 +-
.../tsfile/encoding/decoder/FloatDecoder.java | 2 +-
.../tsfile/encoding/encoder/FloatEncoder.java | 2 +-
.../tsfile/encoding/encoder/PlainEncoder.java | 2 +-
.../tsfile/encoding/encoder/TSEncodingBuilder.java | 4 +-
.../write/UnSupportedDataTypeException.java | 31 -
.../iotdb/tsfile/file/header/ChunkHeader.java | 2 +-
.../iotdb/tsfile/file/header/PageHeader.java | 2 +-
.../tsfile/file/metadata/AlignedChunkMetadata.java | 2 +-
.../iotdb/tsfile/file/metadata/ChunkMetadata.java | 2 +-
.../iotdb/tsfile/file/metadata/IChunkMetadata.java | 2 +-
.../tsfile/file/metadata/TimeseriesMetadata.java | 2 +-
.../file/metadata/statistics/BinaryStatistics.java | 7 +-
.../metadata/statistics/BooleanStatistics.java | 2 +-
.../file/metadata/statistics/DoubleStatistics.java | 2 +-
.../file/metadata/statistics/FloatStatistics.java | 2 +-
.../metadata/statistics/IntegerStatistics.java | 2 +-
.../file/metadata/statistics/LongStatistics.java | 2 +-
.../file/metadata/statistics/Statistics.java | 2 +-
.../file/metadata/statistics/TimeStatistics.java | 2 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 2 +-
.../apache/iotdb/tsfile/read/common/BatchData.java | 4 +-
.../iotdb/tsfile/read/common/BatchDataFactory.java | 2 +-
.../tsfile/read/common/DescReadBatchData.java | 2 +-
.../tsfile/read/common/DescReadWriteBatchData.java | 4 +-
.../org/apache/iotdb/tsfile/read/common/Field.java | 4 +-
.../apache/iotdb/tsfile/read/common/RowRecord.java | 2 +-
.../iotdb/tsfile/read/common/block/TsBlock.java | 4 +-
.../tsfile/read/common/block/TsBlockBuilder.java | 14 +-
.../block/column/BinaryArrayColumnEncoder.java | 3 +-
.../read/common/block/column/BinaryColumn.java | 4 +-
.../common/block/column/BinaryColumnBuilder.java | 7 +-
.../read/common/block/column/BooleanColumn.java | 4 +-
.../common/block/column/BooleanColumnBuilder.java | 7 +-
.../block/column/ByteArrayColumnEncoder.java | 3 +-
.../read/common/block/column/ColumnEncoder.java | 3 +-
.../common/block/column/ColumnEncoderFactory.java | 2 +
.../read/common/block/column/DoubleColumn.java | 4 +-
.../common/block/column/DoubleColumnBuilder.java | 7 +-
.../read/common/block/column/FloatColumn.java | 4 +-
.../common/block/column/FloatColumnBuilder.java | 7 +-
.../block/column/Int32ArrayColumnEncoder.java | 3 +-
.../block/column/Int64ArrayColumnEncoder.java | 3 +-
.../tsfile/read/common/block/column/IntColumn.java | 4 +-
.../read/common/block/column/IntColumnBuilder.java | 7 +-
.../read/common/block/column/LongColumn.java | 4 +-
.../common/block/column/LongColumnBuilder.java | 7 +-
.../read/common/block/column/NullColumn.java | 4 +-
.../block/column/RunLengthColumnEncoder.java | 4 +-
.../block/column/RunLengthEncodedColumn.java | 4 +-
.../read/common/block/column/TimeColumn.java | 4 +-
.../common/block/column/TimeColumnBuilder.java | 7 +-
.../read/common/block/column/TsBlockSerde.java | 4 +-
.../iotdb/tsfile/read/common/type/BinaryType.java | 4 +-
.../iotdb/tsfile/read/common/type/BooleanType.java | 4 +-
.../iotdb/tsfile/read/common/type/DoubleType.java | 4 +-
.../iotdb/tsfile/read/common/type/FloatType.java | 4 +-
.../iotdb/tsfile/read/common/type/IntType.java | 4 +-
.../iotdb/tsfile/read/common/type/LongType.java | 4 +-
.../apache/iotdb/tsfile/read/common/type/Type.java | 4 +-
.../iotdb/tsfile/read/common/type/TypeFactory.java | 2 +-
.../tsfile/read/controller/IMetadataQuerier.java | 2 +-
.../read/controller/MetadataQuerierByFileImpl.java | 2 +-
.../tsfile/read/expression/QueryExpression.java | 2 +-
.../iotdb/tsfile/read/filter/GroupByFilter.java | 5 +
.../tsfile/read/filter/GroupByMonthFilter.java | 183 +-
.../iotdb/tsfile/read/filter/operator/Between.java | 2 +-
.../iotdb/tsfile/read/filter/operator/Eq.java | 2 +-
.../iotdb/tsfile/read/filter/operator/Gt.java | 2 +-
.../iotdb/tsfile/read/filter/operator/GtEq.java | 2 +-
.../iotdb/tsfile/read/filter/operator/Lt.java | 2 +-
.../iotdb/tsfile/read/filter/operator/LtEq.java | 2 +-
.../iotdb/tsfile/read/filter/operator/NotEq.java | 2 +-
.../query/dataset/DataSetWithTimeGenerator.java | 2 +-
.../query/dataset/DataSetWithoutTimeGenerator.java | 4 +-
.../tsfile/read/query/dataset/QueryDataSet.java | 2 +-
.../query/executor/ExecutorWithTimeGenerator.java | 2 +-
.../tsfile/read/query/executor/TsFileExecutor.java | 2 +-
.../read/query/timegenerator/TimeGenerator.java | 2 +-
.../iotdb/tsfile/read/reader/IPageReader.java | 2 +-
.../read/reader/chunk/AlignedChunkReader.java | 2 +-
.../tsfile/read/reader/chunk/ChunkReader.java | 2 +-
.../tsfile/read/reader/page/AlignedPageReader.java | 2 +-
.../iotdb/tsfile/read/reader/page/PageReader.java | 6 +-
.../tsfile/read/reader/page/ValuePageReader.java | 6 +-
.../reader/series/FileSeriesReaderByTimestamp.java | 2 +-
.../org/apache/iotdb/tsfile/utils/BytesUtils.java | 4 +
.../iotdb/tsfile/utils/ReadWriteIOUtils.java | 2 +-
.../apache/iotdb/tsfile/utils/TimeDuration.java | 194 ++
.../iotdb/tsfile/utils/TsFileGeneratorUtils.java | 11 +-
.../write/chunk/AlignedChunkGroupWriterImpl.java | 4 +-
.../tsfile/write/chunk/AlignedChunkWriterImpl.java | 4 +-
.../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 2 +-
.../chunk/NonAlignedChunkGroupWriterImpl.java | 4 +-
.../iotdb/tsfile/write/chunk/TimeChunkWriter.java | 2 +-
.../iotdb/tsfile/write/chunk/ValueChunkWriter.java | 2 +-
.../apache/iotdb/tsfile/write/page/PageWriter.java | 2 +-
.../iotdb/tsfile/write/page/ValuePageWriter.java | 2 +-
.../apache/iotdb/tsfile/write/record/Tablet.java | 10 +-
.../write/record/datapoint/BooleanDataPoint.java | 2 +-
.../tsfile/write/record/datapoint/DataPoint.java | 8 +-
.../write/record/datapoint/DoubleDataPoint.java | 2 +-
.../write/record/datapoint/FloatDataPoint.java | 2 +-
.../write/record/datapoint/IntDataPoint.java | 2 +-
.../write/record/datapoint/LongDataPoint.java | 2 +-
.../write/record/datapoint/StringDataPoint.java | 2 +-
.../tsfile/write/schema/IMeasurementSchema.java | 2 +-
.../tsfile/write/schema/MeasurementSchema.java | 2 +-
.../tsfile/write/schema/TimeseriesSchema.java | 2 +-
.../write/schema/VectorMeasurementSchema.java | 2 +-
.../write/writer/RestorableTsFileIOWriter.java | 2 +-
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 2 +-
.../write/writer/TsFileIOWriterEndFileTest.java | 2 +-
.../write/writer/tsmiterator/DiskTSMIterator.java | 2 +-
.../write/writer/tsmiterator/TSMIterator.java | 2 +-
.../common/block/BinaryArrayColumnEncoderTest.java | 4 +-
.../common/block/ByteArrayColumnEncoderTest.java | 4 +-
.../tsfile/common/block/ColumnEncoderTest.java | 2 +-
.../common/block/Int32ArrayColumnEncoderTest.java | 4 +-
.../common/block/Int64ArrayColumnEncoderTest.java | 4 +-
.../tsfile/common/block/NullColumnUnitTest.java | 4 +-
.../common/block/RunLengthColumnEncoderTest.java | 9 +-
.../tsfile/common/block/TsBlockSerdeTest.java | 9 +-
.../iotdb/tsfile/common/block/TsBlockTest.java | 19 +-
.../encoding/decoder/DictionaryDecoderTest.java | 5 +-
.../tsfile/encoding/decoder/FloatDecoderTest.java | 2 +-
.../tsfile/exception/TsFileExceptionTest.java | 1 -
.../iotdb/tsfile/file/header/PageHeaderTest.java | 2 +-
.../metadata/statistics/StringStatisticsTest.java | 37 +-
.../tsfile/file/metadata/utils/TestHelper.java | 2 +-
.../org/apache/iotdb/tsfile/read/ReadTest.java | 3 +-
.../iotdb/tsfile/read/TimePlainEncodeReadTest.java | 9 +-
.../apache/iotdb/tsfile/read/TsFileReaderTest.java | 4 +-
.../tsfile/read/TsFileSequenceReaderTest.java | 2 +-
.../iotdb/tsfile/read/common/BatchDataTest.java | 2 +-
.../iotdb/tsfile/read/common/ColumnTest.java | 6 +-
.../tsfile/read/filter/FilterSerializeTest.java | 18 +-
.../tsfile/read/filter/GroupByMonthFilterTest.java | 60 +-
.../iotdb/tsfile/read/filter/OperatorTest.java | 9 +-
.../tsfile/read/filter/StatisticsFilterTest.java | 2 +-
.../read/query/executor/QueryExecutorTest.java | 3 +-
.../read/query/timegenerator/ReadWriteTest.java | 2 +-
.../timegenerator/TimeGeneratorReadEmptyTest.java | 2 +-
.../timegenerator/TimeGeneratorReadWriteTest.java | 2 +-
.../query/timegenerator/TimeGeneratorTest.java | 3 +-
.../TsFileGeneratorForSeriesReaderByTimestamp.java | 2 +-
.../iotdb/tsfile/read/reader/FakedBatchReader.java | 2 +-
.../tsfile/read/reader/FakedMultiBatchReader.java | 2 +-
.../iotdb/tsfile/read/reader/PageReaderTest.java | 5 +-
.../apache/iotdb/tsfile/utils/FileGenerator.java | 2 +-
.../org/apache/iotdb/tsfile/utils/RecordUtils.java | 4 +-
.../apache/iotdb/tsfile/utils/RecordUtilsTest.java | 2 +-
.../iotdb/tsfile/utils/TimeDurationTest.java | 67 +
.../iotdb/tsfile/utils/TsFileGeneratorForTest.java | 2 +-
.../iotdb/tsfile/utils/TsPrimitiveTypeTest.java | 10 +-
.../tsfile/write/DefaultSchemaTemplateTest.java | 2 +-
.../tsfile/write/MetadataIndexConstructorTest.java | 2 +-
.../org/apache/iotdb/tsfile/write/PerfTest.java | 2 +-
.../iotdb/tsfile/write/ReadPageInMemTest.java | 2 +-
...SameMeasurementsWithDifferentDataTypesTest.java | 2 +-
.../iotdb/tsfile/write/TsFileIOWriterTest.java | 2 +-
.../tsfile/write/TsFileIntegrityCheckingTool.java | 2 +-
.../iotdb/tsfile/write/TsFileReadWriteTest.java | 2 +-
.../iotdb/tsfile/write/TsFileWriteApiTest.java | 7 +-
.../iotdb/tsfile/write/TsFileWriterTest.java | 2 +-
.../org/apache/iotdb/tsfile/write/WriteTest.java | 2 +-
.../iotdb/tsfile/write/record/TabletTest.java | 2 +-
.../write/schema/converter/SchemaBuilderTest.java | 2 +-
.../write/writer/AlignedChunkWriterImplTest.java | 2 +-
.../write/writer/ForceAppendTsFileWriterTest.java | 2 +-
.../writer/MeasurementSchemaSerializeTest.java | 2 +-
.../iotdb/tsfile/write/writer/PageWriterTest.java | 7 +-
.../write/writer/RestorableTsFileIOWriterTest.java | 2 +-
.../tsfile/write/writer/TimeChunkWriterTest.java | 2 +-
.../tsfile/write/writer/TimePageWriterTest.java | 2 +-
.../writer/TsFileIOWriterMemoryControlTest.java | 8 +-
.../tsfile/write/writer/ValueChunkWriterTest.java | 2 +-
.../tsfile/write/writer/ValuePageWriterTest.java | 2 +-
.../write/writer/VectorMeasurementSchemaStub.java | 2 +-
iotdb-protocol/openapi/pom.xml | 2 +-
iotdb-protocol/pom.xml | 3 +-
iotdb-protocol/thrift-commons/pom.xml | 2 +-
.../thrift-commons/src/main/thrift/common.thrift | 25 +-
iotdb-protocol/thrift-confignode/pom.xml | 4 +-
.../src/main/thrift/confignode.thrift | 126 +-
iotdb-protocol/thrift-consensus/pom.xml | 4 +-
iotdb-protocol/thrift-datanode/pom.xml | 4 +-
.../src/main/thrift/datanode.thrift | 115 +-
iotdb-protocol/thrift-mlnode/pom.xml | 69 -
.../thrift-mlnode/src/main/thrift/mlnode.thrift | 60 -
library-udf/pom.xml | 9 +-
.../apache/iotdb/library/anomaly/UDTFRange.java | 9 +-
.../apache/iotdb/library/dprofile/UDTFSample.java | 8 +-
.../iotdb/library/string/UDTFRegexMatch.java | 44 +-
.../iotdb/library/string/UDTFRegexReplace.java | 46 +-
.../iotdb/library/string/UDTFRegexSplit.java | 64 +-
.../iotdb/library/string/UDTFStrReplace.java | 45 +-
pom.xml | 19 +-
1376 files changed, 15945 insertions(+), 16278 deletions(-)
diff --cc iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
index 843065269d8,3c084e62fa3..b96a5d5839b
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
@@@ -115,9 -114,6 +115,7 @@@ import org.apache.iotdb.confignode.rpc.
import org.apache.iotdb.confignode.rpc.thrift.TGetJarInListReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetJarInListResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetLocationForTriggerResp;
- import org.apache.iotdb.confignode.rpc.thrift.TGetModelInfoReq;
- import org.apache.iotdb.confignode.rpc.thrift.TGetModelInfoResp;
+import org.apache.iotdb.confignode.rpc.thrift.TGetPartitionParameterResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetPathsSetTemplatesReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetPathsSetTemplatesResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetPipePluginTableResp;
@@@ -166,11 -158,8 +160,9 @@@ import org.apache.iotdb.confignode.rpc.
import org.apache.iotdb.confignode.rpc.thrift.TSystemConfigurationResp;
import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp;
import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq;
- import org.apache.iotdb.confignode.rpc.thrift.TUpdateModelInfoReq;
- import org.apache.iotdb.confignode.rpc.thrift.TUpdateModelStateReq;
import org.apache.iotdb.confignode.service.ConfigNode;
import org.apache.iotdb.consensus.exception.ConsensusException;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.queryengine.plan.statement.AuthorType;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 568ca69f642,f0ec678e894..d7b9958cfbf
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@@ -29,8 -29,6 +29,7 @@@ import org.apache.iotdb.db.audit.AuditL
import org.apache.iotdb.db.audit.AuditLogStorage;
import org.apache.iotdb.db.exception.LoadConfigurationException;
import org.apache.iotdb.db.protocol.thrift.impl.ClientRPCServiceImpl;
+import org.apache.iotdb.db.queryengine.plan.analyze.partition.ClusterPartitionFetcher;
- import org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionValidationLevel;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.constant.CrossCompactionPerformer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.constant.InnerSeqCompactionPerformer;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.constant.InnerUnseqCompactionPerformer;
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/config/constant/PipeExtractorConstant.java
index 19d62b569b5,2c9a72d58d7..ad55fb17904
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/config/constant/PipeExtractorConstant.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/config/constant/PipeExtractorConstant.java
@@@ -55,13 -53,8 +55,15 @@@ public class PipeExtractorConstant
public static final String EXTRACTOR_REALTIME_MODE_FILE_VALUE = "file";
public static final String EXTRACTOR_REALTIME_MODE_LOG_VALUE = "log";
public static final String EXTRACTOR_REALTIME_MODE_FORCED_LOG_VALUE = "forced-log";
+ public static final String EXTRACTOR_REALTIME_MODE_STREAM_MODE_VALUE = "stream";
+ public static final String EXTRACTOR_REALTIME_MODE_BATCH_MODE_VALUE = "batch";
+ public static final String EXTRACTOR_LOCAL_SPLIT_ENABLE_KEY = "extractor.local-split.enable";
+ public static final String EXTRACTOR_SPLIT_MAX_CONCURRENT_FILE_KEY =
+ "extractor.split.max-concurrent-file";
+ public static final int EXTRACTOR_SPLIT_MAX_CONCURRENT_FILE_DEFAULT_VALUE = 16;
+ public static final String EXTRACTOR_SPLIT_MAX_FILE_BATCH_SIZE_KEY =
+ "extractor.split.max-file-batch-size";
+ public static final long EXTRACTOR_SPLIT_MAX_FILE_BATCH_SIZE_DEFAULT_VALUE = 4 * GB;
private PipeExtractorConstant() {
throw new IllegalStateException("Utility class");
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBThriftAsyncConnector.java
index 09c387b51be,fdfaca268ee..5e365a4d3f0
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBThriftAsyncConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/async/IoTDBThriftAsyncConnector.java
@@@ -88,16 -71,9 +88,19 @@@ import java.util.concurrent.atomic.Atom
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
+import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_EXTERNAL_CONFIG_NODES_KEY;
+import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_EXTERNAL_USER_NAME_DEFAULT_VALUE;
+import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_EXTERNAL_USER_NAME_KEY;
+import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_EXTERNAL_USER_PASSWORD_DEFAULT_VALUE;
+import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_EXTERNAL_USER_PASSWORD_KEY;
+ import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_IOTDB_BATCH_MODE_ENABLE_KEY;
+import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_LOCAL_SPLIT_ENABLE_KEY;
+import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_SPLIT_MAX_CONCURRENT_FILE_DEFAULT_VALUE;
+import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_SPLIT_MAX_CONCURRENT_FILE_KEY;
+import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_SPLIT_MAX_SIZE_DEFAULT_VALUE;
+import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.CONNECTOR_SPLIT_MAX_SIZE_KEY;
+ import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.SINK_IOTDB_BATCH_MODE_ENABLE_KEY;
+ import static org.apache.iotdb.db.pipe.config.constant.PipeConnectorConstant.SINK_IOTDB_SSL_ENABLE_KEY;
public class IoTDBThriftAsyncConnector extends IoTDBConnector {
@@@ -171,35 -130,12 +180,39 @@@
throws Exception {
super.customize(parameters, configuration);
- retryConnector.customize(parameters, configuration);
+ // Disable batch mode for retry connector, in case retry events are never sent again
+ PipeParameters retryParameters = new PipeParameters(new HashMap<>(parameters.getAttribute()));
+ retryParameters.getAttribute().put(SINK_IOTDB_BATCH_MODE_ENABLE_KEY, "false");
+ retryParameters.getAttribute().put(CONNECTOR_IOTDB_BATCH_MODE_ENABLE_KEY, "false");
+ retryConnector.customize(retryParameters, configuration);
+ useLocalSplit = parameters.getBooleanOrDefault(CONNECTOR_LOCAL_SPLIT_ENABLE_KEY, false);
+
+ if (useLocalSplit) {
+ String configNodeUrls =
+ parameters.getStringOrDefault(CONNECTOR_EXTERNAL_CONFIG_NODES_KEY, "");
+ String[] urlSplits = configNodeUrls.split(",");
+ for (String urlSplit : urlSplits) {
+ targetConfigNodes.add(SessionUtils.parseNodeUrl(urlSplit));
+ }
+
+ queryTargetPartitionParameters();
+
+ splitMaxSize =
+ parameters.getIntOrDefault(
+ CONNECTOR_SPLIT_MAX_SIZE_KEY, CONNECTOR_SPLIT_MAX_SIZE_DEFAULT_VALUE);
+ maxConcurrentFileNum =
+ parameters.getIntOrDefault(
+ CONNECTOR_SPLIT_MAX_CONCURRENT_FILE_KEY,
+ CONNECTOR_SPLIT_MAX_CONCURRENT_FILE_DEFAULT_VALUE);
+ targetUserName =
+ parameters.getStringOrDefault(
+ CONNECTOR_EXTERNAL_USER_NAME_KEY, CONNECTOR_EXTERNAL_USER_NAME_DEFAULT_VALUE);
+ targetPassword =
+ parameters.getStringOrDefault(
+ CONNECTOR_EXTERNAL_USER_PASSWORD_KEY, CONNECTOR_EXTERNAL_USER_PASSWORD_DEFAULT_VALUE);
+ }
+
if (isTabletBatchModeEnabled) {
tabletBatchBuilder = new IoTDBThriftAsyncPipeTransferBatchReqBuilder(parameters);
}
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/writeback/WriteBackConnector.java
index 00000000000,eb262704c68..1225a15d786
mode 000000,100644..100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/writeback/WriteBackConnector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/writeback/WriteBackConnector.java
@@@ -1,0 -1,176 +1,176 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ package org.apache.iotdb.db.pipe.connector.protocol.writeback;
+
+ import org.apache.iotdb.common.rpc.thrift.TSStatus;
+ import org.apache.iotdb.db.auth.AuthorityChecker;
+ import org.apache.iotdb.db.conf.IoTDBDescriptor;
+ import org.apache.iotdb.db.pipe.agent.PipeAgent;
+ import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTabletBinaryReq;
+ import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTabletInsertNodeReq;
+ import org.apache.iotdb.db.pipe.connector.payload.evolvable.request.PipeTransferTabletRawReq;
+ import org.apache.iotdb.db.pipe.event.EnrichedEvent;
+ import org.apache.iotdb.db.pipe.event.common.heartbeat.PipeHeartbeatEvent;
+ import org.apache.iotdb.db.pipe.event.common.tablet.PipeInsertNodeTabletInsertionEvent;
+ import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
+ import org.apache.iotdb.db.protocol.session.SessionManager;
+ import org.apache.iotdb.db.queryengine.common.SessionInfo;
+ import org.apache.iotdb.db.queryengine.plan.Coordinator;
-import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher;
++import org.apache.iotdb.db.queryengine.plan.analyze.partition.ClusterPartitionFetcher;
+ import org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher;
+ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement;
+ import org.apache.iotdb.db.queryengine.plan.statement.crud.PipeEnrichedInsertBaseStatement;
+ import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALPipeException;
+ import org.apache.iotdb.pipe.api.PipeConnector;
+ import org.apache.iotdb.pipe.api.customizer.configuration.PipeConnectorRuntimeConfiguration;
+ import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator;
+ import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
+ import org.apache.iotdb.pipe.api.event.Event;
+ import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+ import org.apache.iotdb.pipe.api.exception.PipeException;
+ import org.apache.iotdb.rpc.TSStatusCode;
+
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ import java.time.ZoneId;
+
+ public class WriteBackConnector implements PipeConnector {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(WriteBackConnector.class);
+
+ @Override
+ public void validate(PipeParameterValidator validator) throws Exception {
+ // Do nothing
+ }
+
+ @Override
+ public void customize(PipeParameters parameters, PipeConnectorRuntimeConfiguration configuration)
+ throws Exception {
+ // Do nothing
+ }
+
+ @Override
+ public void handshake() throws Exception {
+ // Do nothing
+ }
+
+ @Override
+ public void heartbeat() throws Exception {
+ // Do nothing
+ }
+
+ @Override
+ public void transfer(TabletInsertionEvent tabletInsertionEvent) throws Exception {
+ // PipeProcessor can change the type of TabletInsertionEvent
+ if (!(tabletInsertionEvent instanceof PipeInsertNodeTabletInsertionEvent)
+ && !(tabletInsertionEvent instanceof PipeRawTabletInsertionEvent)) {
+ LOGGER.warn(
+ "WriteBackConnector only support "
+ + "PipeInsertNodeTabletInsertionEvent and PipeRawTabletInsertionEvent. "
+ + "Ignore {}.",
+ tabletInsertionEvent);
+ return;
+ }
+
+ if (((EnrichedEvent) tabletInsertionEvent).shouldParsePatternOrTime()) {
+ if (tabletInsertionEvent instanceof PipeInsertNodeTabletInsertionEvent) {
+ transfer(
+ ((PipeInsertNodeTabletInsertionEvent) tabletInsertionEvent).parseEventWithPattern());
+ } else { // tabletInsertionEvent instanceof PipeRawTabletInsertionEvent
+ transfer(((PipeRawTabletInsertionEvent) tabletInsertionEvent).parseEventWithPattern());
+ }
+ return;
+ }
+
+ if (tabletInsertionEvent instanceof PipeInsertNodeTabletInsertionEvent) {
+ doTransfer((PipeInsertNodeTabletInsertionEvent) tabletInsertionEvent);
+ } else {
+ doTransfer((PipeRawTabletInsertionEvent) tabletInsertionEvent);
+ }
+ }
+
+ @Override
+ public void transfer(Event event) throws Exception {
+ if (!(event instanceof PipeHeartbeatEvent)) {
+ LOGGER.warn("WriteBackConnector does not support transferring generic event: {}.", event);
+ }
+ }
+
+ private void doTransfer(PipeInsertNodeTabletInsertionEvent pipeInsertNodeTabletInsertionEvent)
+ throws PipeException, WALPipeException {
+ final TSStatus status =
+ pipeInsertNodeTabletInsertionEvent.getInsertNodeViaCacheIfPossible() == null
+ ? PipeAgent.receiver()
+ .thrift()
+ .receive(
+ PipeTransferTabletBinaryReq.toTPipeTransferReq(
+ pipeInsertNodeTabletInsertionEvent.getByteBuffer()),
+ ClusterPartitionFetcher.getInstance(),
+ ClusterSchemaFetcher.getInstance())
+ .getStatus()
+ : executeStatement(
+ PipeTransferTabletInsertNodeReq.toTPipeTransferRawReq(
+ pipeInsertNodeTabletInsertionEvent.getInsertNode())
+ .constructStatement());
+
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ throw new PipeException(
+ String.format(
+ "Transfer PipeInsertNodeTabletInsertionEvent %s error, result status %s",
+ pipeInsertNodeTabletInsertionEvent, status));
+ }
+ }
+
+ private void doTransfer(PipeRawTabletInsertionEvent pipeRawTabletInsertionEvent)
+ throws PipeException {
+ final TSStatus status =
+ executeStatement(
+ PipeTransferTabletRawReq.toTPipeTransferRawReq(
+ pipeRawTabletInsertionEvent.convertToTablet(),
+ pipeRawTabletInsertionEvent.isAligned())
+ .constructStatement());
+
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ throw new PipeException(
+ String.format(
+ "Transfer PipeRawTabletInsertionEvent %s error, result status %s",
+ pipeRawTabletInsertionEvent, status));
+ }
+ }
+
+ private TSStatus executeStatement(InsertBaseStatement statement) {
+ return Coordinator.getInstance()
+ .execute(
+ new PipeEnrichedInsertBaseStatement(statement),
+ SessionManager.getInstance().requestQueryId(),
+ new SessionInfo(0, AuthorityChecker.SUPER_USER, ZoneId.systemDefault().getId()),
+ "",
+ ClusterPartitionFetcher.getInstance(),
+ ClusterSchemaFetcher.getInstance(),
+ IoTDBDescriptor.getInstance().getConfig().getQueryTimeoutThreshold())
+ .status;
+ }
+
+ @Override
+ public void close() throws Exception {
+ // Do nothing
+ }
+ }
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/TsFileListInsertionDataContainer.java
index e163243cc1e,00000000000..032e6b9cacf
mode 100644,000000..100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/TsFileListInsertionDataContainer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/TsFileListInsertionDataContainer.java
@@@ -1,260 -1,0 +1,260 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.event.common.tsfile;
+
+import org.apache.iotdb.commons.pipe.task.meta.PipeTaskMeta;
+import org.apache.iotdb.db.pipe.event.EnrichedEvent;
+import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+import org.apache.iotdb.pipe.api.exception.PipeException;
+import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
- import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
++import org.apache.iotdb.tsfile.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.TsFileDeviceIterator;
+import org.apache.iotdb.tsfile.read.TsFileReader;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.expression.IExpression;
+import org.apache.iotdb.tsfile.read.expression.impl.BinaryExpression;
+import org.apache.iotdb.tsfile.read.expression.impl.GlobalTimeExpression;
+import org.apache.iotdb.tsfile.read.filter.TimeFilter;
+import org.apache.iotdb.tsfile.utils.Pair;
+import org.apache.iotdb.tsfile.write.record.Tablet;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+public class TsFileListInsertionDataContainer implements AutoCloseable {
+
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(TsFileListInsertionDataContainer.class);
+
+ private final String pattern; // used to filter data
+ private final IExpression timeFilterExpression; // used to filter data
+
+ private final PipeTaskMeta pipeTaskMeta; // used to report progress
+ private final EnrichedEvent sourceEvent; // used to report progress
+
+ private final List<TsFileSequenceReader> tsFileSequenceReaders;
+ private final List<TsFileReader> tsFileReaders;
+ private int currFileIndex = 0;
+
+ private final List<Iterator<Map.Entry<String, List<String>>>> deviceMeasurementsMapIterators;
+ private final List<Map<String, Boolean>> deviceIsAlignedMaps;
+ private final List<Map<String, TSDataType>> measurementDataTypeMaps;
+
+ public TsFileListInsertionDataContainer(
+ List<File> tsFiles, String pattern, long startTime, long endTime) throws IOException {
+ this(tsFiles, pattern, startTime, endTime, null, null);
+ }
+
+ public TsFileListInsertionDataContainer(
+ List<File> tsFiles,
+ String pattern,
+ long startTime,
+ long endTime,
+ PipeTaskMeta pipeTaskMeta,
+ EnrichedEvent sourceEvent)
+ throws IOException {
+ this.pattern = pattern;
+ timeFilterExpression =
+ (startTime == Long.MIN_VALUE && endTime == Long.MAX_VALUE)
+ ? null
+ : BinaryExpression.and(
+ new GlobalTimeExpression(TimeFilter.gtEq(startTime)),
+ new GlobalTimeExpression(TimeFilter.ltEq(endTime)));
+
+ this.pipeTaskMeta = pipeTaskMeta;
+ this.sourceEvent = sourceEvent;
+
+ try {
+ tsFileSequenceReaders = new ArrayList<>();
+ tsFileReaders = new ArrayList<>();
+ for (File tsFile : tsFiles) {
+ TsFileSequenceReader tsFileSequenceReader =
+ new TsFileSequenceReader(tsFile.getAbsolutePath(), true, true);
+ tsFileSequenceReaders.add(tsFileSequenceReader);
+ tsFileReaders.add(new TsFileReader(tsFileSequenceReader));
+ }
+
+ deviceMeasurementsMapIterators = new ArrayList<>();
+ deviceIsAlignedMaps = new ArrayList<>();
+ for (int i = 0; i < tsFiles.size(); i++) {
+ deviceMeasurementsMapIterators.add(
+ filterDeviceMeasurementsMapByPattern(i).entrySet().iterator());
+ deviceIsAlignedMaps.add(readDeviceIsAlignedMap(i));
+ }
+ measurementDataTypeMaps = new ArrayList<>();
+ for (TsFileSequenceReader tsFileSequenceReader : tsFileSequenceReaders) {
+ measurementDataTypeMaps.add(tsFileSequenceReader.getFullPathDataTypeMap());
+ }
+ } catch (Exception e) {
+ close();
+ throw e;
+ }
+ }
+
+ private Map<String, List<String>> filterDeviceMeasurementsMapByPattern(int i) throws IOException {
+ final Map<String, List<String>> filteredDeviceMeasurementsMap = new HashMap<>();
+
+ for (Map.Entry<String, List<String>> entry :
+ tsFileSequenceReaders.get(i).getDeviceMeasurementsMap().entrySet()) {
+ final String deviceId = entry.getKey();
+
+ // case 1: for example, pattern is root.a.b or pattern is null and device is root.a.b.c
+ // in this case, all data can be matched without checking the measurements
+ if (pattern == null
+ || pattern.length() <= deviceId.length() && deviceId.startsWith(pattern)) {
+ if (!entry.getValue().isEmpty()) {
+ filteredDeviceMeasurementsMap.put(deviceId, entry.getValue());
+ }
+ }
+
+ // case 2: for example, pattern is root.a.b.c and device is root.a.b
+ // in this case, we need to check the full path
+ else if (pattern.length() > deviceId.length() && pattern.startsWith(deviceId)) {
+ final List<String> filteredMeasurements = new ArrayList<>();
+
+ for (final String measurement : entry.getValue()) {
+ // low cost check comes first
+ if (pattern.length() == deviceId.length() + measurement.length() + 1
+ // high cost check comes later
+ && pattern.endsWith(TsFileConstant.PATH_SEPARATOR + measurement)) {
+ filteredMeasurements.add(measurement);
+ }
+ }
+
+ if (!filteredMeasurements.isEmpty()) {
+ filteredDeviceMeasurementsMap.put(deviceId, filteredMeasurements);
+ }
+ }
+ }
+
+ return filteredDeviceMeasurementsMap;
+ }
+
+ private Map<String, Boolean> readDeviceIsAlignedMap(int i) throws IOException {
+ final Map<String, Boolean> deviceIsAlignedResultMap = new HashMap<>();
+ final TsFileDeviceIterator deviceIsAlignedIterator =
+ tsFileSequenceReaders.get(i).getAllDevicesIteratorWithIsAligned();
+ while (deviceIsAlignedIterator.hasNext()) {
+ final Pair<String, Boolean> deviceIsAlignedPair = deviceIsAlignedIterator.next();
+ deviceIsAlignedResultMap.put(deviceIsAlignedPair.getLeft(), deviceIsAlignedPair.getRight());
+ }
+ return deviceIsAlignedResultMap;
+ }
+
+ /** @return TabletInsertionEvent in a streaming way */
+ public Iterable<TabletInsertionEvent> toTabletInsertionEvents() {
+ return () ->
+ new Iterator<TabletInsertionEvent>() {
+
+ private TsFileInsertionDataTabletIterator tabletIterator = null;
+
+ @Override
+ public boolean hasNext() {
+ while (tabletIterator == null || !tabletIterator.hasNext()) {
+ if (!deviceMeasurementsMapIterators.get(currFileIndex).hasNext()) {
+ if (currFileIndex >= tsFileReaders.size()) {
+ close();
+ return false;
+ } else {
+ currFileIndex++;
+ continue;
+ }
+ }
+
+ final Map.Entry<String, List<String>> entry =
+ deviceMeasurementsMapIterators.get(currFileIndex).next();
+
+ try {
+ tabletIterator =
+ new TsFileInsertionDataTabletIterator(
+ tsFileReaders.get(currFileIndex),
+ measurementDataTypeMaps.get(currFileIndex),
+ entry.getKey(),
+ entry.getValue(),
+ timeFilterExpression);
+ } catch (IOException e) {
+ close();
+ throw new PipeException("failed to create TsFileInsertionDataTabletIterator", e);
+ }
+ }
+
+ return true;
+ }
+
+ @Override
+ public TabletInsertionEvent next() {
+ if (!hasNext()) {
+ close();
+ throw new NoSuchElementException();
+ }
+
+ final Tablet tablet = tabletIterator.next();
+ final boolean isAligned =
+ deviceIsAlignedMaps.get(currFileIndex).getOrDefault(tablet.deviceId, false);
+
+ final TabletInsertionEvent next;
+ if (!hasNext()) {
+ next =
+ new PipeRawTabletInsertionEvent(
+ tablet, isAligned, pipeTaskMeta, sourceEvent, true);
+ close();
+ } else {
+ next =
+ new PipeRawTabletInsertionEvent(
+ tablet, isAligned, pipeTaskMeta, sourceEvent, false);
+ }
+ return next;
+ }
+ };
+ }
+
+ @Override
+ public void close() {
+ for (TsFileReader tsFileReader : tsFileReaders) {
+ try {
+ if (tsFileReader != null) {
+ tsFileReader.close();
+ }
+ } catch (IOException e) {
+ LOGGER.warn("Failed to close TsFileReader", e);
+ }
+ }
+
+ for (TsFileSequenceReader tsFileSequenceReader : tsFileSequenceReaders) {
+ try {
+ if (tsFileSequenceReader != null) {
+ tsFileSequenceReader.close();
+ }
+ } catch (IOException e) {
+ LOGGER.warn("Failed to close TsFileSequenceReader", e);
+ }
+ }
+ }
+}
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java
index 94ada46cd08,15f6696118d..a7539be6059
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/IoTDBDataRegionExtractor.java
@@@ -51,18 -50,15 +51,20 @@@ import java.util.concurrent.atomic.Atom
import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_ENABLE_DEFAULT_VALUE;
import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_HISTORY_ENABLE_KEY;
+import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_LOCAL_SPLIT_ENABLE_KEY;
import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_ENABLE_DEFAULT_VALUE;
import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_ENABLE_KEY;
+ import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_BATCH_MODE_VALUE;
import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_FILE_VALUE;
import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_FORCED_LOG_VALUE;
import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_HYBRID_VALUE;
import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_KEY;
import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_LOG_VALUE;
+ import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_REALTIME_MODE_STREAM_MODE_VALUE;
+import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_SPLIT_MAX_CONCURRENT_FILE_DEFAULT_VALUE;
+import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_SPLIT_MAX_CONCURRENT_FILE_KEY;
+import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_SPLIT_MAX_FILE_BATCH_SIZE_DEFAULT_VALUE;
+import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_SPLIT_MAX_FILE_BATCH_SIZE_KEY;
import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.SOURCE_HISTORY_ENABLE_KEY;
import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.SOURCE_REALTIME_ENABLE_KEY;
import static org.apache.iotdb.db.pipe.config.constant.PipeExtractorConstant.SOURCE_REALTIME_MODE_KEY;
@@@ -123,10 -119,12 +125,12 @@@ public class IoTDBDataRegionExtractor i
EXTRACTOR_REALTIME_MODE_FILE_VALUE,
EXTRACTOR_REALTIME_MODE_HYBRID_VALUE,
EXTRACTOR_REALTIME_MODE_LOG_VALUE,
- EXTRACTOR_REALTIME_MODE_FORCED_LOG_VALUE);
+ EXTRACTOR_REALTIME_MODE_FORCED_LOG_VALUE,
+ EXTRACTOR_REALTIME_MODE_STREAM_MODE_VALUE,
+ EXTRACTOR_REALTIME_MODE_BATCH_MODE_VALUE);
}
- constructHistoricalExtractor();
+ constructHistoricalExtractor(validator.getParameters());
constructRealtimeExtractor(validator.getParameters());
historicalExtractor.validate(validator);
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java
index e52f8ff0a63,7ee76d0d97a..9cc24da6a9e
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java
@@@ -83,9 -82,6 +83,7 @@@ import org.apache.iotdb.confignode.rpc.
import org.apache.iotdb.confignode.rpc.thrift.TGetJarInListReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetJarInListResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetLocationForTriggerResp;
- import org.apache.iotdb.confignode.rpc.thrift.TGetModelInfoReq;
- import org.apache.iotdb.confignode.rpc.thrift.TGetModelInfoResp;
+import org.apache.iotdb.confignode.rpc.thrift.TGetPartitionParameterResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetPathsSetTemplatesReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetPathsSetTemplatesResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetPipePluginTableResp;
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
index 68b21458c26,9260e39d22b..06edeb358e7
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
@@@ -215,14 -200,10 +212,14 @@@ import org.apache.iotdb.rpc.RpcUtils
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.trigger.api.enums.FailureStrategy;
import org.apache.iotdb.trigger.api.enums.TriggerEvent;
+ import org.apache.iotdb.tsfile.enums.TSDataType;
import org.apache.iotdb.tsfile.exception.NotImplementedException;
+import org.apache.iotdb.tsfile.file.header.ChunkHeader;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
- import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+import org.apache.iotdb.tsfile.utils.TsFileUtils;
import org.apache.iotdb.tsfile.write.record.Tablet;
import com.google.common.collect.ImmutableList;
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/LoadTsFileManager.java
index 04b8fdb7c6b,00aa3bf9b52..3ba4be77ad7
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/LoadTsFileManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/LoadTsFileManager.java
@@@ -297,13 -263,22 +301,23 @@@ public class LoadTsFileManager
if (writer.isWritingChunkGroup()) {
writer.endChunkGroup();
}
- writer.endFile();
-
- DataRegion dataRegion = entry.getKey().getDataRegion();
- dataRegion.loadNewTsFile(generateResource(writer), true, isGeneratedByPipe);
-
- MetricService.getInstance()
- .count(
- getTsFileWritePointCount(writer),
- Metric.QUANTITY.toString(),
- MetricLevel.CORE,
- Tag.NAME.toString(),
- Metric.POINTS_IN.toString(),
- Tag.DATABASE.toString(),
- dataRegion.getDatabaseName(),
- Tag.REGION.toString(),
- dataRegion.getDataRegionId());
+ if (writer.canWrite()) {
+ writer.endFile();
- entry
- .getKey()
- .getDataRegion()
- .loadNewTsFile(generateResource(writer), true, isGeneratedByPipe);
++ DataRegion dataRegion = entry.getKey().getDataRegion();
++ dataRegion.loadNewTsFile(generateResource(writer), true, isGeneratedByPipe);
++
++ MetricService.getInstance()
++ .count(
++ getTsFileWritePointCount(writer),
++ Metric.QUANTITY.toString(),
++ MetricLevel.CORE,
++ Tag.NAME.toString(),
++ Metric.POINTS_IN.toString(),
++ Tag.DATABASE.toString(),
++ dataRegion.getDatabaseName(),
++ Tag.REGION.toString(),
++ dataRegion.getDataRegionId());
+ }
}
}
@@@ -313,7 -288,14 +327,14 @@@
return tsFileResource;
}
+ private long getTsFileWritePointCount(TsFileIOWriter writer) {
+ return writer.getChunkGroupMetadataList().stream()
+ .flatMap(chunkGroupMetadata -> chunkGroupMetadata.getChunkMetadataList().stream())
+ .mapToLong(chunkMetadata -> chunkMetadata.getStatistics().getCount())
+ .sum();
+ }
+
- private void close() {
+ private synchronized void close() {
if (isClosed) {
return;
}
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/MergedTsFileSplitter.java
index 08cfc649d3b,00000000000..4e4fd8020d8
mode 100644,000000..100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/MergedTsFileSplitter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/MergedTsFileSplitter.java
@@@ -1,765 -1,0 +1,765 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.execution.load;
+
+import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
+import org.apache.iotdb.commons.utils.TimePartitionUtils;
+import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
+import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
+import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
+import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
++import org.apache.iotdb.tsfile.enums.TSDataType;
+import org.apache.iotdb.tsfile.exception.TsFileRuntimeException;
+import org.apache.iotdb.tsfile.file.MetaMarker;
+import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader;
+import org.apache.iotdb.tsfile.file.header.ChunkHeader;
+import org.apache.iotdb.tsfile.file.header.PageHeader;
+import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
- import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.BatchData;
+import org.apache.iotdb.tsfile.read.reader.page.PageReader;
+import org.apache.iotdb.tsfile.read.reader.page.TimePageReader;
+import org.apache.iotdb.tsfile.read.reader.page.ValuePageReader;
+import org.apache.iotdb.tsfile.utils.Pair;
+import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
+import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.PriorityQueue;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+
+public class MergedTsFileSplitter {
+
+ private static final Logger logger = LoggerFactory.getLogger(TsFileSplitter.class);
+
+ private final List<File> tsFiles;
+ private final Function<TsFileData, Boolean> consumer;
+ private final PriorityQueue<SplitTask> taskPriorityQueue;
+ private final int maxConcurrentFileNum;
+ private ExecutorService asyncExecutor;
+ private long timePartitionInterval;
+ private AtomicInteger splitIdGenerator = new AtomicInteger();
+ private Statistic statistic = new Statistic();
+
+ public MergedTsFileSplitter(
+ List<File> tsFiles,
+ Function<TsFileData, Boolean> consumer,
+ ExecutorService asyncExecutor,
+ long timePartitionInterval,
+ int maxConcurrentFileNum) {
+ this.tsFiles = tsFiles;
+ this.consumer = consumer;
+ this.asyncExecutor = asyncExecutor;
+ this.timePartitionInterval = timePartitionInterval;
+ this.maxConcurrentFileNum = maxConcurrentFileNum;
+ taskPriorityQueue = new PriorityQueue<>();
+ }
+
+ public void splitTsFileByDataPartition() throws IOException, IllegalStateException {
+ long startTime = System.nanoTime();
+ int i = 0;
+ for (; i < tsFiles.size(); i++) {
+ // only allow at most maxConcurrentFileNum files to be merged at the same time
+ if (taskPriorityQueue.size() > maxConcurrentFileNum) {
+ break;
+ }
+
+ File tsFile = tsFiles.get(i);
+ SplitTask splitTask = new SplitTask(tsFile, asyncExecutor, i);
+ logger.info("Start to split {}", tsFiles.get(i));
+ if (splitTask.hasNext()) {
+ taskPriorityQueue.add(splitTask);
+ }
+ }
+ statistic.initTime = System.nanoTime() - startTime;
+
+ while (!taskPriorityQueue.isEmpty()) {
+ startTime = System.nanoTime();
+ SplitTask task = taskPriorityQueue.poll();
+ TsFileData tsFileData = task.removeNext();
+ statistic.fetchDataTime += System.nanoTime() - startTime;
+ tsFileData.setSplitId(splitIdGenerator.incrementAndGet());
+
+ startTime = System.nanoTime();
+ consumer.apply(tsFileData);
+ statistic.consumeTime += System.nanoTime() - startTime;
+
+ startTime = System.nanoTime();
+ if (task.hasNext()) {
+ taskPriorityQueue.add(task);
+ } else {
+ // when a file is exhausted, add the next non-empty file
+ for (; i < tsFiles.size(); i++) {
+ SplitTask splitTask = new SplitTask(tsFiles.get(i), asyncExecutor, i);
+ logger.info("Start to split {}", tsFiles.get(i));
+ if (splitTask.hasNext()) {
+ taskPriorityQueue.add(splitTask);
+ i++;
+ break;
+ }
+ }
+ }
+ statistic.enqueueTime += System.nanoTime() - startTime;
+ }
+ }
+
+ public void close() throws IOException {
+ logger.info(
+ "Init/FetchData/Consume/Enqueue Time: {}/{}/{}/{}ms",
+ statistic.initTime / 1_000_000L,
+ statistic.fetchDataTime / 1_000_000L,
+ statistic.consumeTime / 1_000_000L,
+ statistic.enqueueTime / 1_000_000L);
+ for (SplitTask task : taskPriorityQueue) {
+ task.close();
+ }
+ taskPriorityQueue.clear();
+ logger.info("Splitter closed.");
+ }
+
+ private class Statistic {
+ private long initTime;
+ private long fetchDataTime;
+ private long consumeTime;
+ private long enqueueTime;
+ }
+
+ private class SplitTask implements Comparable<SplitTask> {
+
+ private final File tsFile;
+ private final int fileId;
+ private TsFileSequenceReader reader;
+ private final TreeMap<Long, List<Deletion>> offset2Deletions;
+
+ private String curDevice;
+ private boolean isTimeChunkNeedDecode;
+ private Map<Integer, List<AlignedChunkData>> pageIndex2ChunkData;
+ private Map<Integer, long[]> pageIndex2Times;
+ private Map<Long, IChunkMetadata> offset2ChunkMetadata;
+
+ private BlockingQueue<TsFileData> nextSplits;
+ private TsFileData nextSplit;
+ private byte marker = -1;
+
+ private ExecutorService asyncExecutor;
+ private Future<Void> asyncTask;
+
+ public SplitTask(File tsFile, ExecutorService asyncExecutor, int fileId) throws IOException {
+ this.tsFile = tsFile;
+ this.fileId = fileId;
+ this.asyncExecutor = asyncExecutor;
+ offset2Deletions = new TreeMap<>();
+ init();
+ }
+
+ @Override
+ public int compareTo(SplitTask o) {
+ try {
+ TsFileData thisNext = showNext();
+ TsFileData thatNext = o.showNext();
+ // out put modification first
+ if (thisNext.isModification() && thatNext.isModification()) {
+ return 0;
+ }
+ if (thisNext.isModification()) {
+ return 1;
+ }
+ if (thatNext.isModification()) {
+ return -1;
+ }
+
+ ChunkData thisChunk = (ChunkData) thisNext;
+ ChunkData thatChunk = ((ChunkData) thatNext);
+ Comparator<ChunkData> chunkDataComparator =
+ Comparator.comparing(ChunkData::getDevice, String::compareTo)
+ .thenComparing(ChunkData::firstMeasurement, String::compareTo);
+ int chunkCompare = chunkDataComparator.compare(thisChunk, thatChunk);
+ if (chunkCompare != 0) {
+ return chunkCompare;
+ }
+ return Integer.compare(this.fileId, o.fileId);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void init() throws IOException {
+ this.reader = new TsFileSequenceReader(tsFile.getAbsolutePath());
+ getAllModification(offset2Deletions);
+
+ if (!checkMagic(reader)) {
+ throw new TsFileRuntimeException(
+ String.format("Magic String check error when parsing TsFile %s.", tsFile.getPath()));
+ }
+ reader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1);
+
+ curDevice = null;
+ isTimeChunkNeedDecode = true;
+ pageIndex2ChunkData = new HashMap<>();
+ pageIndex2Times = null;
+ offset2ChunkMetadata = new HashMap<>();
+ getChunkMetadata(reader, offset2ChunkMetadata);
+
+ nextSplits = new LinkedBlockingDeque<>(64);
+ if (asyncExecutor != null) {
+ asyncTask =
+ asyncExecutor.submit(
+ () -> {
+ try {
+ asyncComputeNext();
+ } catch (Throwable e) {
+ logger.info("Exception during splitting", e);
+ throw e;
+ }
+ return null;
+ });
+ }
+ }
+
+ private void asyncComputeNext() throws IOException {
+ while (reader != null && !Thread.interrupted()) {
+ computeNext();
+ }
+ logger.info("{} ends splitting", tsFile);
+ }
+
+ public boolean hasNext() throws IOException {
+ if (nextSplit != null && !(nextSplit instanceof EmptyTsFileData)) {
+ return true;
+ }
+ if (reader == null && nextSplits.isEmpty()) {
+ return false;
+ }
+
+ if (asyncExecutor == null) {
+ computeNext();
+ if (!nextSplits.isEmpty()) {
+ nextSplit = nextSplits.poll();
+ return true;
+ } else {
+ return false;
+ }
+ } else {
+ try {
+ nextSplit = nextSplits.take();
+ } catch (InterruptedException e) {
+ return false;
+ }
+ return !(nextSplit instanceof EmptyTsFileData);
+ }
+ }
+
+ public TsFileData removeNext() throws IOException {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ TsFileData split = nextSplit;
+ nextSplit = null;
+ return split;
+ }
+
+ public TsFileData showNext() throws IOException {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ return nextSplit;
+ }
+
+ public void close() throws IOException {
+ if (asyncTask != null) {
+ asyncTask.cancel(true);
+ try {
+ asyncTask.get();
+ } catch (CancellationException ignored) {
+
+ } catch (InterruptedException | ExecutionException e) {
+ throw new IOException(e);
+ }
+ }
+ if (reader != null) {
+ reader.close();
+ reader = null;
+ }
+ }
+
+ private byte nextMarker() throws IOException {
+ if (marker != -1) {
+ // inherit the marker from the previous breakpoint
+ // e.g. the marker after processing a chunk
+ return marker;
+ }
+ return marker = reader.readMarker();
+ }
+
+ private void insertNewChunk(ChunkData chunkData) throws IOException {
+ if (asyncExecutor == null) {
+ nextSplits.add(chunkData);
+ } else {
+ try {
+ nextSplits.put(chunkData);
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ }
+ }
+
+ @SuppressWarnings({"squid:S3776", "squid:S6541"})
+ private void computeNext() throws IOException, IllegalStateException {
+ if (reader == null) {
+ return;
+ }
+
+ while (nextMarker() != MetaMarker.SEPARATOR) {
+ switch (marker) {
+ case MetaMarker.CHUNK_HEADER:
+ case MetaMarker.TIME_CHUNK_HEADER:
+ case MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER:
+ case MetaMarker.ONLY_ONE_PAGE_TIME_CHUNK_HEADER:
+ long chunkOffset = reader.position();
+ boolean chunkDataGenerated =
+ consumeAllAlignedChunkData(chunkOffset, pageIndex2ChunkData);
+ handleModification(offset2Deletions, chunkOffset);
+ if (chunkDataGenerated) {
+ return;
+ }
+
+ ChunkHeader header = reader.readChunkHeader(marker);
+ if (header.getDataSize() == 0) {
+ throw new TsFileRuntimeException(
+ String.format(
+ "Empty Nonaligned Chunk or Time Chunk with offset %d in TsFile %s.",
+ chunkOffset, tsFile.getPath()));
+ }
+
+ boolean isAligned =
+ ((header.getChunkType() & TsFileConstant.TIME_COLUMN_MASK)
+ == TsFileConstant.TIME_COLUMN_MASK);
+ IChunkMetadata chunkMetadata = offset2ChunkMetadata.get(chunkOffset - Byte.BYTES);
+ TTimePartitionSlot timePartitionSlot =
+ TimePartitionUtils.getTimePartitionSlot(
+ chunkMetadata.getStartTime(), timePartitionInterval);
+ ChunkData chunkData =
+ ChunkData.createChunkData(isAligned, curDevice, header, timePartitionSlot);
+
+ if (!needDecodeChunk(chunkMetadata)) {
+ chunkData.setNotDecode();
+ chunkData.writeEntireChunk(reader.readChunk(-1, header.getDataSize()), chunkMetadata);
+ if (isAligned) {
+ isTimeChunkNeedDecode = false;
+ pageIndex2ChunkData
+ .computeIfAbsent(1, o -> new ArrayList<>())
+ .add((AlignedChunkData) chunkData);
+ } else {
+ insertNewChunk(chunkData);
+ }
+ break;
+ }
+
+ Decoder defaultTimeDecoder =
+ Decoder.getDecoderByType(
+ TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getTimeEncoder()),
+ TSDataType.INT64);
+ Decoder valueDecoder =
+ Decoder.getDecoderByType(header.getEncodingType(), header.getDataType());
+ int dataSize = header.getDataSize();
+ int pageIndex = 0;
+ if (isAligned) {
+ isTimeChunkNeedDecode = true;
+ pageIndex2Times = new HashMap<>();
+ }
+
+ while (dataSize > 0) {
+ PageHeader pageHeader =
+ reader.readPageHeader(
+ header.getDataType(),
+ (header.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER);
+ long pageDataSize = pageHeader.getSerializedPageSize();
+ if (!needDecodePage(pageHeader, chunkMetadata)) { // an entire page
+ long startTime =
+ pageHeader.getStatistics() == null
+ ? chunkMetadata.getStartTime()
+ : pageHeader.getStartTime();
+ TTimePartitionSlot pageTimePartitionSlot =
+ TimePartitionUtils.getTimePartitionSlot(startTime, timePartitionInterval);
+ if (!timePartitionSlot.equals(pageTimePartitionSlot)) {
+ if (!isAligned) {
+ insertNewChunk(chunkData);
+ }
+ timePartitionSlot = pageTimePartitionSlot;
+ chunkData =
+ ChunkData.createChunkData(isAligned, curDevice, header, timePartitionSlot);
+ }
+ if (isAligned) {
+ pageIndex2ChunkData
+ .computeIfAbsent(pageIndex, o -> new ArrayList<>())
+ .add((AlignedChunkData) chunkData);
+ }
+ chunkData.writeEntirePage(pageHeader, reader.readCompressedPage(pageHeader));
+ } else { // split page
+ ByteBuffer pageData = reader.readPage(pageHeader, header.getCompressionType());
+ Pair<long[], Object[]> tvArray =
+ decodePage(
+ isAligned, pageData, pageHeader, defaultTimeDecoder, valueDecoder, header);
+ long[] times = tvArray.left;
+ Object[] values = tvArray.right;
+ if (isAligned) {
+ pageIndex2Times.put(pageIndex, times);
+ }
+
+ int start = 0;
+ long endTime = timePartitionSlot.getStartTime() + timePartitionInterval;
+ for (int i = 0; i < times.length; i++) {
+ if (times[i] >= endTime) {
+ chunkData.writeDecodePage(times, values, start, i);
+ if (isAligned) {
+ pageIndex2ChunkData
+ .computeIfAbsent(pageIndex, o -> new ArrayList<>())
+ .add((AlignedChunkData) chunkData);
+ } else {
+ insertNewChunk(chunkData);
+ }
+
+ timePartitionSlot =
+ TimePartitionUtils.getTimePartitionSlot(times[i], timePartitionInterval);
+ endTime = timePartitionSlot.getStartTime() + timePartitionInterval;
+ chunkData =
+ ChunkData.createChunkData(isAligned, curDevice, header, timePartitionSlot);
+ start = i;
+ }
+ }
+ chunkData.writeDecodePage(times, values, start, times.length);
+ if (isAligned) {
+ pageIndex2ChunkData
+ .computeIfAbsent(pageIndex, o -> new ArrayList<>())
+ .add((AlignedChunkData) chunkData);
+ }
+ }
+
+ pageIndex += 1;
+ dataSize -= pageDataSize;
+ }
+
+ if (!isAligned) {
+ insertNewChunk(chunkData);
+ }
+ break;
+ case MetaMarker.VALUE_CHUNK_HEADER:
+ case MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER:
+ chunkOffset = reader.position();
+ chunkMetadata = offset2ChunkMetadata.get(chunkOffset - Byte.BYTES);
+ header = reader.readChunkHeader(marker);
+ if (header.getDataSize() == 0) {
+ handleEmptyValueChunk(
+ header, pageIndex2ChunkData, chunkMetadata, isTimeChunkNeedDecode);
+ break;
+ }
+
+ if (!isTimeChunkNeedDecode) {
+ AlignedChunkData alignedChunkData = pageIndex2ChunkData.get(1).get(0);
+ alignedChunkData.addValueChunk(header);
+ alignedChunkData.writeEntireChunk(
+ reader.readChunk(-1, header.getDataSize()), chunkMetadata);
+ break;
+ }
+
+ Set<ChunkData> allChunkData = new HashSet<>();
+ dataSize = header.getDataSize();
+ pageIndex = 0;
+ valueDecoder = Decoder.getDecoderByType(header.getEncodingType(), header.getDataType());
+
+ while (dataSize > 0) {
+ PageHeader pageHeader =
+ reader.readPageHeader(
+ header.getDataType(),
+ (header.getChunkType() & 0x3F) == MetaMarker.CHUNK_HEADER);
+ List<AlignedChunkData> alignedChunkDataList = pageIndex2ChunkData.get(pageIndex);
+ for (AlignedChunkData alignedChunkData : alignedChunkDataList) {
+ if (!allChunkData.contains(alignedChunkData)) {
+ alignedChunkData.addValueChunk(header);
+ allChunkData.add(alignedChunkData);
+ }
+ }
+ if (alignedChunkDataList.size() == 1) { // write entire page
+ // write the entire page if it's not an empty page.
+ alignedChunkDataList
+ .get(0)
+ .writeEntirePage(pageHeader, reader.readCompressedPage(pageHeader));
+ } else { // decode page
+ long[] times = pageIndex2Times.get(pageIndex);
+ TsPrimitiveType[] values =
+ decodeValuePage(reader, header, pageHeader, times, valueDecoder);
+ for (AlignedChunkData alignedChunkData : alignedChunkDataList) {
+ alignedChunkData.writeDecodeValuePage(times, values, header.getDataType());
+ }
+ }
+ long pageDataSize = pageHeader.getSerializedPageSize();
+ pageIndex += 1;
+ dataSize -= pageDataSize;
+ }
+ break;
+ case MetaMarker.CHUNK_GROUP_HEADER:
+ ChunkGroupHeader chunkGroupHeader = reader.readChunkGroupHeader();
+ curDevice = chunkGroupHeader.getDeviceID();
+ break;
+ case MetaMarker.OPERATION_INDEX_RANGE:
+ reader.readPlanIndex();
+ break;
+ default:
+ MetaMarker.handleUnexpectedMarker(marker);
+ }
+ marker = -1;
+ if (!nextSplits.isEmpty()) {
+ return;
+ }
+ }
+
+ consumeAllAlignedChunkData(reader.position(), pageIndex2ChunkData);
+ handleModification(offset2Deletions, Long.MAX_VALUE);
+ close();
+ if (asyncExecutor != null) {
+ try {
+ nextSplits.put(new EmptyTsFileData());
+ } catch (InterruptedException e) {
+ asyncTask.cancel(true);
+ }
+ }
+ }
+
+ private class EmptyTsFileData implements TsFileData {
+
+ @Override
+ public long getDataSize() {
+ return 0;
+ }
+
+ @Override
+ public void writeToFileWriter(TsFileIOWriter writer) throws IOException {}
+
+ @Override
+ public boolean isModification() {
+ return false;
+ }
+
+ @Override
+ public void serialize(DataOutputStream stream) throws IOException {}
+
+ @Override
+ public int getSplitId() {
+ return 0;
+ }
+
+ @Override
+ public void setSplitId(int sid) {}
+ }
+
+ private void getAllModification(Map<Long, List<Deletion>> offset2Deletions) throws IOException {
+ try (ModificationFile modificationFile =
+ new ModificationFile(tsFile.getAbsolutePath() + ModificationFile.FILE_SUFFIX)) {
+ for (Modification modification : modificationFile.getModifications()) {
+ offset2Deletions
+ .computeIfAbsent(modification.getFileOffset(), o -> new ArrayList<>())
+ .add((Deletion) modification);
+ }
+ }
+ }
+
+ private boolean checkMagic(TsFileSequenceReader reader) throws IOException {
+ String magic = reader.readHeadMagic();
+ if (!magic.equals(TSFileConfig.MAGIC_STRING)) {
+ logger.error("the file's MAGIC STRING is incorrect, file path: {}", reader.getFileName());
+ return false;
+ }
+
+ byte versionNumber = reader.readVersionNumber();
+ if (versionNumber != TSFileConfig.VERSION_NUMBER) {
+ logger.error("the file's Version Number is incorrect, file path: {}", reader.getFileName());
+ return false;
+ }
+
+ if (!reader.readTailMagic().equals(TSFileConfig.MAGIC_STRING)) {
+ logger.error("the file is not closed correctly, file path: {}", reader.getFileName());
+ return false;
+ }
+ return true;
+ }
+
+ private void getChunkMetadata(
+ TsFileSequenceReader reader, Map<Long, IChunkMetadata> offset2ChunkMetadata)
+ throws IOException {
+ Map<String, List<TimeseriesMetadata>> device2Metadata = reader.getAllTimeseriesMetadata(true);
+ for (Map.Entry<String, List<TimeseriesMetadata>> entry : device2Metadata.entrySet()) {
+ for (TimeseriesMetadata timeseriesMetadata : entry.getValue()) {
+ for (IChunkMetadata chunkMetadata : timeseriesMetadata.getChunkMetadataList()) {
+ offset2ChunkMetadata.put(chunkMetadata.getOffsetOfChunkHeader(), chunkMetadata);
+ }
+ }
+ }
+ }
+
+ private void handleModification(
+ TreeMap<Long, List<Deletion>> offset2Deletions, long chunkOffset) {
+ while (!offset2Deletions.isEmpty() && offset2Deletions.firstEntry().getKey() <= chunkOffset) {
+ offset2Deletions
+ .pollFirstEntry()
+ .getValue()
+ .forEach(o -> nextSplits.add(new DeletionData(o)));
+ }
+ }
+
+ private boolean consumeAllAlignedChunkData(
+ long offset, Map<Integer, List<AlignedChunkData>> pageIndex2ChunkData) {
+ if (pageIndex2ChunkData.isEmpty()) {
+ return false;
+ }
+
+ Set<ChunkData> allChunkData = new HashSet<>();
+ for (Map.Entry<Integer, List<AlignedChunkData>> entry : pageIndex2ChunkData.entrySet()) {
+ allChunkData.addAll(entry.getValue());
+ }
+ for (ChunkData chunkData : allChunkData) {
+ try {
+ nextSplits.put(chunkData);
+ } catch (InterruptedException e) {
+ return false;
+ }
+ }
+ pageIndex2ChunkData.clear();
+ return true;
+ }
+
+ private boolean needDecodeChunk(IChunkMetadata chunkMetadata) {
+ return !TimePartitionUtils.getTimePartitionSlot(
+ chunkMetadata.getStartTime(), timePartitionInterval)
+ .equals(
+ TimePartitionUtils.getTimePartitionSlot(
+ chunkMetadata.getEndTime(), timePartitionInterval));
+ }
+
+ private boolean needDecodePage(PageHeader pageHeader, IChunkMetadata chunkMetadata) {
+ if (pageHeader.getStatistics() == null) {
+ return !TimePartitionUtils.getTimePartitionSlot(
+ chunkMetadata.getStartTime(), timePartitionInterval)
+ .equals(
+ TimePartitionUtils.getTimePartitionSlot(
+ chunkMetadata.getEndTime(), timePartitionInterval));
+ }
+ return !TimePartitionUtils.getTimePartitionSlot(
+ pageHeader.getStartTime(), timePartitionInterval)
+ .equals(
+ TimePartitionUtils.getTimePartitionSlot(
+ pageHeader.getEndTime(), timePartitionInterval));
+ }
+
+ private Pair<long[], Object[]> decodePage(
+ boolean isAligned,
+ ByteBuffer pageData,
+ PageHeader pageHeader,
+ Decoder timeDecoder,
+ Decoder valueDecoder,
+ ChunkHeader chunkHeader)
+ throws IOException {
+ if (isAligned) {
+ TimePageReader timePageReader = new TimePageReader(pageHeader, pageData, timeDecoder);
+ long[] times = timePageReader.getNextTimeBatch();
+ return new Pair<>(times, new Object[times.length]);
+ }
+
+ valueDecoder.reset();
+ PageReader pageReader =
+ new PageReader(pageData, chunkHeader.getDataType(), valueDecoder, timeDecoder, null);
+ BatchData batchData = pageReader.getAllSatisfiedPageData();
+ long[] times = new long[batchData.length()];
+ Object[] values = new Object[batchData.length()];
+ int index = 0;
+ while (batchData.hasCurrent()) {
+ times[index] = batchData.currentTime();
+ values[index++] = batchData.currentValue();
+ batchData.next();
+ }
+ return new Pair<>(times, values);
+ }
+
+ private void handleEmptyValueChunk(
+ ChunkHeader header,
+ Map<Integer, List<AlignedChunkData>> pageIndex2ChunkData,
+ IChunkMetadata chunkMetadata,
+ boolean isTimeChunkNeedDecode)
+ throws IOException {
+ Set<ChunkData> allChunkData = new HashSet<>();
+ for (Map.Entry<Integer, List<AlignedChunkData>> entry : pageIndex2ChunkData.entrySet()) {
+ for (AlignedChunkData alignedChunkData : entry.getValue()) {
+ if (!allChunkData.contains(alignedChunkData)) {
+ alignedChunkData.addValueChunk(header);
+ if (!isTimeChunkNeedDecode) {
+ alignedChunkData.writeEntireChunk(ByteBuffer.allocate(0), chunkMetadata);
+ }
+ allChunkData.add(alignedChunkData);
+ }
+ }
+ }
+ }
+
+ private TsPrimitiveType[] decodeValuePage(
+ TsFileSequenceReader reader,
+ ChunkHeader chunkHeader,
+ PageHeader pageHeader,
+ long[] times,
+ Decoder valueDecoder)
+ throws IOException {
+ if (pageHeader.getSerializedPageSize() == 0) {
+ return new TsPrimitiveType[times.length];
+ }
+
+ valueDecoder.reset();
+ ByteBuffer pageData = reader.readPage(pageHeader, chunkHeader.getCompressionType());
+ ValuePageReader valuePageReader =
+ new ValuePageReader(pageHeader, pageData, chunkHeader.getDataType(), valueDecoder);
+ return valuePageReader.nextValueBatch(
+ times); // should be origin time, so recording satisfied length is necessary
+ }
+ }
+}
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/TsFileSplitSender.java
index 50686d37b8d,00000000000..a17e5e2c18e
mode 100644,000000..100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/TsFileSplitSender.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/TsFileSplitSender.java
@@@ -1,493 -1,0 +1,493 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.execution.load;
+
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.client.IClientManager;
+import org.apache.iotdb.commons.client.exception.ClientManagerException;
+import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient;
+import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory;
+import org.apache.iotdb.commons.concurrent.IoTThreadFactory;
+import org.apache.iotdb.db.exception.mpp.FragmentInstanceDispatchException;
+import org.apache.iotdb.db.queryengine.execution.load.locseq.LocationSequencer;
+import org.apache.iotdb.db.queryengine.execution.load.locseq.LocationStatistics;
+import org.apache.iotdb.db.queryengine.execution.load.locseq.ThroughputBasedLocationSequencer;
+import org.apache.iotdb.db.queryengine.execution.load.nodesplit.ClusteringMeasurementSplitter;
+import org.apache.iotdb.db.queryengine.execution.load.nodesplit.PieceNodeSplitter;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFileNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode;
+import org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileScheduler.LoadCommand;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.mpp.rpc.thrift.TLoadCommandReq;
+import org.apache.iotdb.mpp.rpc.thrift.TLoadResp;
+import org.apache.iotdb.mpp.rpc.thrift.TTsFilePieceReq;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.compress.ICompressor;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.utils.Pair;
+
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+
+import static org.apache.iotdb.commons.conf.IoTDBConstant.MB;
+import static org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileDispatcherImpl.NODE_CONNECTION_ERROR;
+
+public class TsFileSplitSender {
+
+ private static final int MAX_RETRY = 5;
+ private static final long RETRY_INTERVAL_MS = 6_000L;
+ private static final int MAX_PENDING_PIECE_NODE = 5;
+ private static final Logger logger = LoggerFactory.getLogger(TsFileSplitSender.class);
+
+ private LoadTsFileNode loadTsFileNode;
+ private DataPartitionBatchFetcher targetPartitionFetcher;
+ private long targetPartitionInterval;
+ private final IClientManager<TEndPoint, SyncDataNodeInternalServiceClient>
+ internalServiceClientManager;
+ // All consensus groups accessed in Phase1 should be notified in Phase2
+ private final Set<TDataNodeLocation> allReplicaSets = new ConcurrentSkipListSet<>();
+ private String uuid;
+ private LocationStatistics locationStatistics = new LocationStatistics();
+ private boolean isGeneratedByPipe;
+ private Map<Pair<LoadTsFilePieceNode, TRegionReplicaSet>, Exception> phaseOneFailures =
+ new ConcurrentHashMap<>();
+ private Map<TDataNodeLocation, Exception> phaseTwoFailures = new HashMap<>();
+ private long maxSplitSize;
+ private PieceNodeSplitter pieceNodeSplitter = new ClusteringMeasurementSplitter(1.0, 10);
+ // private PieceNodeSplitter pieceNodeSplitter = new OrderedMeasurementSplitter();
+ private CompressionType compressionType = CompressionType.LZ4;
+ private Statistic statistic = new Statistic();
+ private ExecutorService splitNodeService;
+ private Queue<Pair<Future<List<LoadTsFilePieceNode>>, TRegionReplicaSet>> splitFutures;
+ private int maxConcurrentFileNum;
+ private String userName;
+ private String password;
+
+ public TsFileSplitSender(
+ LoadTsFileNode loadTsFileNode,
+ DataPartitionBatchFetcher targetPartitionFetcher,
+ long targetPartitionInterval,
+ IClientManager<TEndPoint, SyncDataNodeInternalServiceClient> internalServiceClientManager,
+ boolean isGeneratedByPipe,
+ long maxSplitSize,
+ int maxConcurrentFileNum,
+ String userName,
+ String password) {
+ this.loadTsFileNode = loadTsFileNode;
+ this.targetPartitionFetcher = targetPartitionFetcher;
+ this.targetPartitionInterval = targetPartitionInterval;
+ this.internalServiceClientManager = internalServiceClientManager;
+ this.isGeneratedByPipe = isGeneratedByPipe;
+ this.maxSplitSize = maxSplitSize;
+ this.splitNodeService = IoTDBThreadPoolFactory.newCachedThreadPool("SplitLoadTsFilePieceNode");
+ this.splitFutures = new ArrayDeque<>(MAX_PENDING_PIECE_NODE);
+ this.maxConcurrentFileNum = maxConcurrentFileNum;
+ this.userName = userName;
+ this.password = password;
+
+ this.statistic.totalSize = loadTsFileNode.getTotalSize();
+ }
+
+ public void start() throws IOException {
+ statistic.taskStartTime = System.currentTimeMillis();
+ // skip files without data
+ loadTsFileNode.getResources().removeIf(f -> f.getDevices().isEmpty());
+ uuid = UUID.randomUUID().toString();
+ logger.info("Start to split {}", loadTsFileNode);
+
+ boolean isFirstPhaseSuccess = firstPhase(loadTsFileNode);
+ boolean isSecondPhaseSuccess = secondPhase(isFirstPhaseSuccess);
+ if (isFirstPhaseSuccess && isSecondPhaseSuccess) {
+ logger.info("Load TsFiles {} Successfully", loadTsFileNode.getResources());
+ } else {
+ logger.warn("Can not Load TsFiles {}", loadTsFileNode.getResources());
+ }
+ statistic.taskEndTime = System.currentTimeMillis();
+ locationStatistics.logLocationStatistics();
+ statistic.logStatistic();
+ }
+
+ private boolean firstPhase(LoadTsFileNode node) throws IOException {
+ long start = System.currentTimeMillis();
+ TsFileDataManager tsFileDataManager =
+ new DeviceBatchTsFileDataManager(
+ this::dispatchOnePieceNode,
+ node.getPlanNodeId(),
+ node.lastResource().getTsFile(),
+ targetPartitionFetcher,
+ maxSplitSize,
+ userName);
+
+ ExecutorService executorService =
+ IoTDBThreadPoolFactory.newThreadPool(
+ 32,
+ Integer.MAX_VALUE,
+ 20,
+ TimeUnit.SECONDS,
+ new SynchronousQueue<>(),
+ new IoTThreadFactory("MergedTsFileSplitter"),
+ "MergedTsFileSplitter");
+ MergedTsFileSplitter splitter =
+ new MergedTsFileSplitter(
+ node.getResources().stream()
+ .map(TsFileResource::getTsFile)
+ .collect(Collectors.toList()),
+ tsFileDataManager::addOrSendTsFileData,
+ executorService,
+ targetPartitionInterval,
+ maxConcurrentFileNum);
+ splitter.splitTsFileByDataPartition();
+ splitter.close();
+ logger.info("Split ends after {}ms", System.currentTimeMillis() - start);
+ boolean success =
+ tsFileDataManager.sendAllTsFileData()
+ && processRemainingPieceNodes()
+ && phaseOneFailures.isEmpty();
- statistic.p1TimeMs = System.currentTimeMillis() - start;
- logger.info("Cleanup ends after {}ms", statistic.p1TimeMs);
++ statistic.p1TimeMS = System.currentTimeMillis() - start;
++ logger.info("Cleanup ends after {}ms", statistic.p1TimeMS);
+ return success;
+ }
+
+ private boolean secondPhase(boolean isFirstPhaseSuccess) {
+ TLoadCommandReq loadCommandReq =
+ new TLoadCommandReq(
+ (isFirstPhaseSuccess ? LoadCommand.EXECUTE : LoadCommand.ROLLBACK).ordinal(), uuid);
+ loadCommandReq.setIsGeneratedByPipe(isGeneratedByPipe);
+
+ long p2StartMS = System.currentTimeMillis();
+ List<Pair<TDataNodeLocation, Future<Void>>> loadFutures = new ArrayList<>();
+ AtomicBoolean hasTimeout = new AtomicBoolean();
+ for (TDataNodeLocation dataNodeLocation : allReplicaSets) {
+ loadFutures.add(
+ new Pair<>(
+ dataNodeLocation,
+ splitNodeService.submit(
+ () -> {
+ logger.info(
+ "Start dispatching Load command for uuid {} to {}", uuid, dataNodeLocation);
+ TEndPoint endPoint = dataNodeLocation.getInternalEndPoint();
+ Exception locationException = null;
+
+ for (int i = 0; i < MAX_RETRY; i++) {
+ long timeout = 0;
+ try (SyncDataNodeInternalServiceClient client =
+ internalServiceClientManager.borrowClient(endPoint)) {
+ timeout = client.getTimeout();
+ TLoadResp loadResp = client.sendLoadCommand(loadCommandReq);
+ if (!loadResp.isAccepted()) {
+ logger.warn(loadResp.message);
+ locationException =
+ new FragmentInstanceDispatchException(loadResp.status);
+ } else {
+ locationException = null;
+ }
+ break;
+ } catch (ClientManagerException | TException e) {
+ TSStatus status = new TSStatus();
+ status.setCode(TSStatusCode.DISPATCH_ERROR.getStatusCode());
+ status.setMessage(
+ "can't connect to node {}, please reset longer dn_connection_timeout_ms "
+ + "in iotdb-common.properties and restart iotdb."
+ + endPoint);
+ locationException = new FragmentInstanceDispatchException(status);
+ hasTimeout.set(true);
+ statistic.p2Timeout = timeout;
+ }
+ try {
+ Thread.sleep(RETRY_INTERVAL_MS);
+ } catch (InterruptedException e) {
+ locationException = e;
+ break;
+ }
+ }
+
+ if (locationException != null) {
+ phaseTwoFailures.put(dataNodeLocation, locationException);
+ }
+ return null;
+ })));
+ }
+ for (Pair<TDataNodeLocation, Future<Void>> loadFuture : loadFutures) {
+ try {
+ loadFuture.right.get();
+ } catch (InterruptedException | ExecutionException e) {
+ phaseTwoFailures.put(loadFuture.left, e);
+ }
+ }
+ statistic.p2TimeMS = System.currentTimeMillis() - p2StartMS;
+ statistic.hasP2Timeout = hasTimeout.get();
+
+ return phaseTwoFailures.isEmpty();
+ }
+
+ public LocationSequencer createLocationSequencer(TRegionReplicaSet replicaSet) {
+ // return new FixedLocationSequencer(replicaSet);
+ // return new RandomLocationSequencer(replicaSet);
+ return new ThroughputBasedLocationSequencer(replicaSet, locationStatistics);
+ }
+
+ private ByteBuffer compressBuffer(ByteBuffer buffer) throws IOException {
+ statistic.rawSize.addAndGet(buffer.remaining());
+ if (compressionType.equals(CompressionType.UNCOMPRESSED)) {
+ statistic.compressedSize.addAndGet(buffer.remaining());
+ return buffer;
+ }
+ ICompressor compressor = ICompressor.getCompressor(compressionType);
+ int maxBytesForCompression = compressor.getMaxBytesForCompression(buffer.remaining()) + 1;
+ ByteBuffer compressed = ByteBuffer.allocate(maxBytesForCompression);
+ int compressLength =
+ compressor.compress(
+ buffer.array(),
+ buffer.arrayOffset() + buffer.position(),
+ buffer.remaining(),
+ compressed.array());
+ compressed.limit(compressLength);
+ statistic.compressedSize.addAndGet(compressLength);
+ return compressed;
+ }
+
+ private Future<List<LoadTsFilePieceNode>> submitSplitPieceNode(LoadTsFilePieceNode pieceNode) {
+ return splitNodeService.submit(() -> pieceNodeSplitter.split(pieceNode));
+ }
+
+ private boolean processRemainingPieceNodes() {
+ List<LoadTsFilePieceNode> subNodes;
+ for (Pair<Future<List<LoadTsFilePieceNode>>, TRegionReplicaSet> pair : splitFutures) {
+ try {
+ subNodes = pair.left.get();
+ } catch (InterruptedException | ExecutionException e) {
+ logger.error("Unexpected error during splitting node", e);
+ return false;
+ }
+ if (!dispatchPieceNodes(subNodes, pair.right)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private boolean dispatchPieceNodes(
+ List<LoadTsFilePieceNode> subNodes, TRegionReplicaSet replicaSet) {
+
+ long start = System.nanoTime();
+ List<Boolean> subNodeResults =
+ subNodes.stream()
+ .parallel()
+ .map(
+ node -> {
+ ByteBuffer buffer;
+ long startTime = System.nanoTime();
+ int uncompressedLength;
+ try {
+ buffer = node.serializeToByteBuffer();
+ uncompressedLength = buffer.remaining();
+ buffer = compressBuffer(buffer);
+ } catch (IOException e) {
+ phaseOneFailures.put(new Pair<>(node, replicaSet), e);
+ return false;
+ }
+ long compressingTime = System.nanoTime() - startTime;
+ statistic.compressingTimeNs.addAndGet(compressingTime);
+
+ TTsFilePieceReq loadTsFileReq =
+ new TTsFilePieceReq(buffer, uuid, replicaSet.getRegionId());
+ loadTsFileReq.setUsername(userName);
+ loadTsFileReq.setPassword(password);
+ loadTsFileReq.setCompressionType(compressionType.serialize());
+ loadTsFileReq.setUncompressedLength(uncompressedLength);
+ LocationSequencer locationSequencer = createLocationSequencer(replicaSet);
+
+ boolean loadSucceed = false;
+ Exception lastConnectionError = null;
+ TDataNodeLocation currLocation = null;
+ for (TDataNodeLocation location : locationSequencer) {
+ TRegionReplicaSet relaySet = new TRegionReplicaSet(replicaSet);
+ relaySet.getDataNodeLocations().remove(location);
+ loadTsFileReq.setRelayTargets(relaySet);
+ loadTsFileReq.setNeedSchemaRegistration(true);
+
+ if (location.getDataNodeId() == 0 && logger.isDebugEnabled()) {
+ locationStatistics.logLocationStatistics();
+ logger.info("Chose location {}", location.getDataNodeId());
+ }
+ currLocation = location;
+ startTime = System.nanoTime();
+ for (int i = 0; i < MAX_RETRY; i++) {
+ try (SyncDataNodeInternalServiceClient client =
+ internalServiceClientManager.borrowClient(
+ currLocation.internalEndPoint)) {
+ TLoadResp loadResp = client.sendTsFilePieceNode(loadTsFileReq);
+ logger.debug("Response from {}: {}", location.getDataNodeId(), loadResp);
+ if (!loadResp.isAccepted()) {
+ logger.warn(loadResp.message);
+ phaseOneFailures.put(
+ new Pair<>(node, replicaSet),
+ new FragmentInstanceDispatchException(loadResp.status));
+ return false;
+ }
+ loadSucceed = true;
+ break;
+ } catch (ClientManagerException | TException e) {
+ lastConnectionError = e;
+ }
+
+ try {
+ Thread.sleep(RETRY_INTERVAL_MS);
+ } catch (InterruptedException e) {
+ return false;
+ }
+ }
+
+ if (loadSucceed) {
+ break;
+ }
+ }
+
+ if (!loadSucceed) {
+ String warning = NODE_CONNECTION_ERROR;
+ logger.warn(warning, currLocation, lastConnectionError);
+ TSStatus status = new TSStatus();
+ status.setCode(TSStatusCode.DISPATCH_ERROR.getStatusCode());
+ status.setMessage(warning + currLocation);
+ phaseOneFailures.put(
+ new Pair<>(node, replicaSet),
+ new FragmentInstanceDispatchException(status));
+ return false;
+ }
+ long timeConsumption = System.nanoTime() - startTime;
+ logger.debug("Time consumption: {}", timeConsumption);
+ locationStatistics.updateThroughput(
+ currLocation, node.getDataSize(), timeConsumption);
+
+ return true;
+ })
+ .collect(Collectors.toList());
+ long elapsedTime = System.nanoTime() - start;
+ statistic.dispatchNodesTimeNS.addAndGet(elapsedTime);
+ return !subNodeResults.contains(false);
+ }
+
+ public boolean dispatchOnePieceNode(LoadTsFilePieceNode pieceNode, TRegionReplicaSet replicaSet) {
+ long allStart = System.nanoTime();
+ allReplicaSets.addAll(replicaSet.dataNodeLocations);
+
+ List<LoadTsFilePieceNode> subNodes;
+ if (splitFutures.size() < MAX_PENDING_PIECE_NODE) {
+ splitFutures.add(new Pair<>(submitSplitPieceNode(pieceNode), replicaSet));
+ statistic.dispatchNodeTimeNS.addAndGet(System.nanoTime() - allStart);
+ return true;
+ } else {
+ long start = System.nanoTime();
+ Pair<Future<List<LoadTsFilePieceNode>>, TRegionReplicaSet> pair = splitFutures.poll();
+ try {
+ subNodes = pair.left.get();
+ long elapsedTime = System.nanoTime() - start;
+ statistic.splitTime.addAndGet(elapsedTime);
+ statistic.pieceNodeNum.incrementAndGet();
+ logger.debug(
+ "{} splits are generated after {}ms", subNodes.size(), elapsedTime / 1_000_000L);
+
+ splitFutures.add(new Pair<>(submitSplitPieceNode(pieceNode), replicaSet));
+ } catch (InterruptedException | ExecutionException e) {
+ logger.error("Unexpected error during splitting node", e);
+ return false;
+ }
+ boolean success = dispatchPieceNodes(subNodes, pair.right);
+ statistic.dispatchNodeTimeNS.addAndGet(System.nanoTime() - allStart);
+ return success;
+ }
+ }
+
+ public static class Statistic {
+
+ public long taskStartTime;
+ public long taskEndTime;
+ public AtomicLong rawSize = new AtomicLong();
+ public AtomicLong compressedSize = new AtomicLong();
+ public AtomicLong splitTime = new AtomicLong();
+ public AtomicLong pieceNodeNum = new AtomicLong();
+ public AtomicLong dispatchNodesTimeNS = new AtomicLong();
+ public AtomicLong dispatchNodeTimeNS = new AtomicLong();
+ public AtomicLong compressingTimeNs = new AtomicLong();
- public long p1TimeMs;
++ public long p1TimeMS;
+ public long p2TimeMS;
+ public long totalSize;
+ public boolean hasP2Timeout;
+ public long p2Timeout;
+
+ public void logStatistic() {
+ logger.info(
+ "Time consumption: {}ms, totalSize: {}MB",
+ taskEndTime - taskStartTime,
+ totalSize * 1.0 / MB);
+ logger.info(
+ "Generated {} piece nodes, splitTime: {}, dispatchSplitsTime: {}, dispatchNodeTime: {}",
+ pieceNodeNum.get(),
+ splitTime.get() / 1_000_000L,
+ dispatchNodesTimeNS.get() / 1_000_000L,
+ dispatchNodeTimeNS.get() / 1_000_000L);
+ logger.info(
+ "Transmission size: {}/{} ({}), compressionTime: {}ms",
+ compressedSize.get(),
+ rawSize.get(),
+ compressedSize.get() * 1.0 / rawSize.get(),
+ compressingTimeNs.get() / 1_000_000L);
- logger.info("Sync TsFile time: {}ms ({})", p1TimeMs, p1ThroughputMBPS());
++ logger.info("Sync TsFile time: {}ms ({})", p1TimeMS, p1ThroughputMBPS());
+ logger.info("Load command execution time: {}ms ({})", p2TimeMS, p2ThroughputMBPS());
+ }
+
+ public double p2ThroughputMBPS() {
+ return totalSize * 1.0 / MB / (p2TimeMS / 1000.0);
+ }
+
+ public double p1ThroughputMBPS() {
+ return totalSize * 1.0 / MB / (p1TimeMS / 1000.0);
+ }
+ }
+
+ public Statistic getStatistic() {
+ return statistic;
+ }
+}
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/nodesplit/ClusteringMeasurementSplitter.java
index 9a4fdf3b96f,00000000000..1516fddebe0
mode 100644,000000..100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/nodesplit/ClusteringMeasurementSplitter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/nodesplit/ClusteringMeasurementSplitter.java
@@@ -1,616 -1,0 +1,616 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.execution.load.nodesplit;
+
+import org.apache.iotdb.db.queryengine.execution.load.AlignedChunkData;
+import org.apache.iotdb.db.queryengine.execution.load.ChunkData;
+import org.apache.iotdb.db.queryengine.execution.load.NonAlignedChunkData;
+import org.apache.iotdb.db.queryengine.execution.load.TsFileData;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode;
++import org.apache.iotdb.tsfile.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.header.ChunkHeader;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
- import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.common.Chunk;
+
+import net.ricecode.similarity.LevenshteinDistanceStrategy;
+import net.ricecode.similarity.SimilarityStrategy;
+import net.ricecode.similarity.StringSimilarityService;
+import net.ricecode.similarity.StringSimilarityServiceImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+public class ClusteringMeasurementSplitter implements PieceNodeSplitter {
+
+ private static final Logger logger = LoggerFactory.getLogger(ClusteringMeasurementSplitter.class);
+ private double groupFactor;
+ private int maxIteration;
+ private int dataSampleLength = 32;
+ private double stdErrThreshold = 1;
+ private long splitStartTime;
+ private long splitTimeBudget = 5;
+ private static final int MAX_CLUSTER_NUM = 500;
+
+ public ClusteringMeasurementSplitter(double groupFactor, int maxIteration) {
+ this.groupFactor = groupFactor;
+ this.maxIteration = maxIteration;
+ }
+
+ @Override
+ public List<LoadTsFilePieceNode> split(LoadTsFilePieceNode pieceNode) {
+ splitStartTime = System.currentTimeMillis();
+ if (pieceNode.isHasModification()) {
+ // the order of modifications should be preserved, so with modifications clustering cannot be
+ // used
+ return new OrderedMeasurementSplitter().split(pieceNode);
+ }
+
+ // split by measurement first
+ Map<String, LoadTsFilePieceNode> measurementPieceNodeMap =
+ new HashMap<>(pieceNode.getAllTsFileData().size());
+ for (TsFileData tsFileData : pieceNode.getAllTsFileData()) {
+ ChunkData chunkData = (ChunkData) tsFileData;
+ String currMeasurement = chunkData.firstMeasurement();
+ LoadTsFilePieceNode pieceNodeSplit =
+ measurementPieceNodeMap.computeIfAbsent(
+ currMeasurement,
+ m -> new LoadTsFilePieceNode(pieceNode.getPlanNodeId(), pieceNode.getTsFile()));
+ pieceNodeSplit.addTsFileData(chunkData);
+ }
+ // use clustering to merge similar measurements
+ List<LoadTsFilePieceNode> loadTsFilePieceNodes = clusterPieceNode(measurementPieceNodeMap);
+ if (logger.isDebugEnabled()) {
+ logger.debug(
+ "Split distribution before refinement: {}",
+ loadTsFilePieceNodes.stream()
+ .map(l -> l.getAllTsFileData().size())
+ .collect(Collectors.toList()));
+ refineClusters(loadTsFilePieceNodes);
+ logger.debug(
+ "Split distribution after refinement: {}",
+ loadTsFilePieceNodes.stream()
+ .map(l -> l.getAllTsFileData().size())
+ .collect(Collectors.toList()));
+ }
+ return loadTsFilePieceNodes;
+ }
+
+ private void refineClusters(List<LoadTsFilePieceNode> pieceNodes) {
+ double average =
+ pieceNodes.stream().mapToLong(LoadTsFilePieceNode::getDataSize).average().orElse(0.0);
+ double finalAverage1 = average;
+ double stderr =
+ Math.sqrt(
+ pieceNodes.stream()
+ .mapToLong(LoadTsFilePieceNode::getDataSize)
+ .mapToDouble(s -> (s - finalAverage1) * (s - finalAverage1))
+ .sum()
+ / pieceNodes.size());
+ logger.debug(
+ "{} splits before refinement, average/stderr: {}/{}", pieceNodes.size(), average, stderr);
+
+ while (stderr > average * stdErrThreshold
+ && System.currentTimeMillis() - splitStartTime < splitTimeBudget) {
+ pieceNodes.sort(Comparator.comparingLong(LoadTsFilePieceNode::getDataSize));
+ LoadTsFilePieceNode smallestPiece = pieceNodes.get(0);
+ LoadTsFilePieceNode largestPiece = pieceNodes.get(pieceNodes.size() - 1);
+ double smallDiff = average - smallestPiece.getDataSize();
+ double largeDiff = largestPiece.getDataSize() - average;
+ if (smallDiff > largeDiff) {
+ mergeSmallPiece(pieceNodes);
+ } else {
+ if (!splitLargePiece(pieceNodes)) {
+ // the largest node may not be splittable (only one series), merge the smallest instead
+ mergeSmallPiece(pieceNodes);
+ }
+ }
+
+ average =
+ pieceNodes.stream().mapToLong(LoadTsFilePieceNode::getDataSize).average().orElse(0.0);
+ double finalAverage = average;
+ stderr =
+ Math.sqrt(
+ pieceNodes.stream()
+ .mapToLong(LoadTsFilePieceNode::getDataSize)
+ .mapToDouble(s -> (s - finalAverage) * (s - finalAverage))
+ .sum()
+ / pieceNodes.size());
+ logger.debug("{} splits, average/stderr: {}/{}", pieceNodes.size(), average, stderr);
+ }
+ logger.debug(
+ "{} splits after refinement, average/stderr: {}/{}", pieceNodes.size(), average, stderr);
+ }
+
+ private void mergeSmallPiece(List<LoadTsFilePieceNode> pieceNodes) {
+ LoadTsFilePieceNode pieceA = pieceNodes.remove(0);
+ LoadTsFilePieceNode pieceB = pieceNodes.remove(0);
+ LoadTsFilePieceNode newPiece =
+ new LoadTsFilePieceNode(pieceA.getPlanNodeId(), pieceA.getTsFile());
+ pieceA.getAllTsFileData().forEach(newPiece::addTsFileData);
+ pieceB.getAllTsFileData().forEach(newPiece::addTsFileData);
+ pieceNodes.add(newPiece);
+ pieceNodes.stream().mapToLong(LoadTsFilePieceNode::getDataSize).average().orElse(0.0);
+ pieceNodes.stream().mapToLong(LoadTsFilePieceNode::getDataSize).sum();
+ }
+
+ private boolean splitLargePiece(List<LoadTsFilePieceNode> pieceNodes) {
+ LoadTsFilePieceNode oldPiece = pieceNodes.remove(pieceNodes.size() - 1);
+ LoadTsFilePieceNode newNodeA =
+ new LoadTsFilePieceNode(oldPiece.getPlanNodeId(), oldPiece.getTsFile());
+ LoadTsFilePieceNode newNodeB =
+ new LoadTsFilePieceNode(oldPiece.getPlanNodeId(), oldPiece.getTsFile());
+ long sizeTarget = oldPiece.getDataSize() / 2;
+
+ // cannot break a series into two pieces since the chunk order must be preserved
+ String currMeasurement = null;
+ List<TsFileData> allTsFileData = oldPiece.getAllTsFileData();
+ int i = 0;
+ for (; i < allTsFileData.size(); i++) {
+ TsFileData tsFileData = allTsFileData.get(i);
+ if (tsFileData.isModification()) {
+ // modifications follows previous chunk data
+ newNodeA.addTsFileData(tsFileData);
+ } else {
+ ChunkData chunkData = (ChunkData) tsFileData;
+ if (currMeasurement == null || currMeasurement.equals(chunkData.firstMeasurement())) {
+ // the first chunk or chunk of the same series, add it to A
+ currMeasurement = chunkData.firstMeasurement();
+ newNodeA.addTsFileData(tsFileData);
+ sizeTarget -= chunkData.getDataSize();
+ } else {
+ // chunk of the next series
+ if (sizeTarget < 0) {
+ // splitA is full, break to fill splitB
+ break;
+ } else {
+ // splitA is not full, also add this series to A
+ currMeasurement = chunkData.firstMeasurement();
+ newNodeA.addTsFileData(tsFileData);
+ sizeTarget -= chunkData.getDataSize();
+ }
+ }
+ }
+ }
+ // add remaining series to B
+ for (; i < allTsFileData.size(); i++) {
+ TsFileData tsFileData = allTsFileData.get(i);
+ newNodeB.addTsFileData(tsFileData);
+ }
+
+ pieceNodes.add(newNodeA);
+ if (!newNodeB.getAllTsFileData().isEmpty()) {
+ pieceNodes.add(newNodeB);
+ return true;
+ }
+ return false;
+ }
+
+ private List<LoadTsFilePieceNode> clusterPieceNode(
+ Map<String, LoadTsFilePieceNode> measurementPieceNodeMap) {
+ // convert to feature vector
+ Map<String, SeriesFeatureVector> measurementVectorMap =
+ new HashMap<>(measurementPieceNodeMap.size());
+ for (Entry<String, LoadTsFilePieceNode> entry : measurementPieceNodeMap.entrySet()) {
+ measurementVectorMap.put(entry.getKey(), convertToFeature(entry.getValue()));
+ }
+ // normalize
+ normalize(measurementVectorMap.values());
+ Map<String, double[]> doubleVectors = new HashMap<>(measurementPieceNodeMap.size());
+ for (Entry<String, SeriesFeatureVector> e : measurementVectorMap.entrySet()) {
+ doubleVectors.put(e.getKey(), e.getValue().numericVector);
+ }
+ // clustering
+ int numCluster = Math.min((int) (doubleVectors.size() / groupFactor), MAX_CLUSTER_NUM);
+ if (numCluster < 1) {
+ numCluster = 1;
+ }
+ VectorDistance distance = new EuclideanDistance();
+ Clustering clustering = new KMeans(numCluster, maxIteration);
+ List<List<String>> clusterResult = clustering.cluster(doubleVectors, distance);
+ // collect result
+ List<LoadTsFilePieceNode> clusteredNodes = new ArrayList<>();
+ for (List<String> cluster : clusterResult) {
+ if (cluster.isEmpty()) {
+ continue;
+ }
+ LoadTsFilePieceNode pieceNode = measurementPieceNodeMap.get(cluster.get(0));
+ LoadTsFilePieceNode clusterNode =
+ new LoadTsFilePieceNode(pieceNode.getPlanNodeId(), pieceNode.getTsFile());
+ for (String measurementId : cluster) {
+ pieceNode = measurementPieceNodeMap.get(measurementId);
+ for (TsFileData tsFileData : pieceNode.getAllTsFileData()) {
+ clusterNode.addTsFileData(tsFileData);
+ }
+ }
+ clusteredNodes.add(clusterNode);
+ }
+
+ return clusteredNodes;
+ }
+
+ private void normalize(Collection<SeriesFeatureVector> vectors) {
+ String firstMeasurementId = null;
+ int minDataSize = Integer.MAX_VALUE;
+ int maxDataSize = Integer.MIN_VALUE;
+ int minDataType = Integer.MAX_VALUE;
+ int maxDataType = Integer.MIN_VALUE;
+ int minCompressionType = Integer.MAX_VALUE;
+ int maxCompressionType = Integer.MIN_VALUE;
+ int minEncodingType = Integer.MAX_VALUE;
+ int maxEncodingType = Integer.MIN_VALUE;
+ int minNumOfPages = Integer.MAX_VALUE;
+ int maxNumOfPages = Integer.MIN_VALUE;
+ String firstDataSample = null;
+ for (SeriesFeatureVector vector : vectors) {
+ if (firstMeasurementId == null) {
+ firstMeasurementId = vector.measurementId;
+ firstDataSample = vector.dataSample;
+ }
+ minDataSize = Math.min(minDataSize, vector.dataSize);
+ maxDataSize = Math.max(maxDataSize, vector.dataSize);
+ minDataType = Math.min(minDataType, vector.dataType.ordinal());
+ maxDataType = Math.max(maxDataType, vector.dataType.ordinal());
+ minCompressionType = Math.min(minCompressionType, vector.compressionType.ordinal());
+ maxCompressionType = Math.max(maxCompressionType, vector.compressionType.ordinal());
+ minEncodingType = Math.min(minEncodingType, vector.encodingType.ordinal());
+ maxEncodingType = Math.max(maxEncodingType, vector.encodingType.ordinal());
+ minNumOfPages = Math.min(minNumOfPages, vector.numOfPages);
+ maxNumOfPages = Math.max(maxNumOfPages, vector.numOfPages);
+ }
+
+ SimilarityStrategy strategy = new LevenshteinDistanceStrategy();
+ StringSimilarityService service = new StringSimilarityServiceImpl(strategy);
+ int finalMinDataSize = minDataSize;
+ String finalFirstMeasurementId = firstMeasurementId;
+ int finalMaxDataSize = maxDataSize;
+ int finalMinDataType = minDataType;
+ int finalMaxDataType = maxDataType;
+ int finalMinCompressionType = minCompressionType;
+ int finalMaxCompressionType = maxCompressionType;
+ int finalMinEncodingType = minEncodingType;
+ int finalMaxEncodingType = maxEncodingType;
+ int finalMinNumOfPages = minNumOfPages;
+ int finalMaxNumOfPages = maxNumOfPages;
+ String finalFirstDataSample = firstDataSample;
+ vectors.stream()
+ .parallel()
+ .forEach(
+ vector -> {
+ vector.numericVector[0] =
+ service.score(finalFirstMeasurementId, vector.measurementId);
+ vector.numericVector[1] =
+ (vector.dataSize - finalMinDataSize)
+ * 1.0
+ / (finalMaxDataSize - finalMinDataSize);
+ vector.numericVector[2] =
+ (vector.dataType.ordinal() - finalMinDataType)
+ * 1.0
+ / (finalMaxDataType - finalMinDataType);
+ vector.numericVector[3] =
+ (vector.compressionType.ordinal() - finalMinCompressionType)
+ * 1.0
+ / (finalMaxCompressionType - finalMinCompressionType);
+ vector.numericVector[4] =
+ (vector.encodingType.ordinal() - finalMinEncodingType)
+ * 1.0
+ / (finalMaxEncodingType - finalMinEncodingType);
+ vector.numericVector[5] =
+ (vector.numOfPages - finalMinNumOfPages)
+ * 1.0
+ / (finalMaxNumOfPages - finalMinNumOfPages);
+ vector.numericVector[6] = service.score(finalFirstDataSample, vector.dataSample);
+ double[] numericVector = vector.numericVector;
+ for (int i = 0; i < numericVector.length; i++) {
+ if (Double.isNaN(numericVector[i])) {
+ numericVector[i] = 0.0;
+ } else if (Double.isInfinite(numericVector[i])) {
+ numericVector[i] = 1.0;
+ }
+ }
+ });
+ }
+
+ private SeriesFeatureVector convertToFeature(LoadTsFilePieceNode pieceNode) {
+ List<TsFileData> allTsFileData = pieceNode.getAllTsFileData();
+ SeriesFeatureVector vector = null;
+ for (TsFileData tsFileData : allTsFileData) {
+ ChunkData chunkData = (ChunkData) tsFileData;
+ if (vector == null) {
+ vector = SeriesFeatureVector.fromChunkData(chunkData, dataSampleLength);
+ } else {
+ vector.mergeChunkData(chunkData);
+ }
+ }
+ return vector;
+ }
+
+ public static class SeriesFeatureVector {
+
+ private String measurementId;
+ private int dataSize;
+ private TSDataType dataType;
+ private CompressionType compressionType;
+ private TSEncoding encodingType;
+ private int numOfPages;
+ private String dataSample;
+ private double[] numericVector = new double[7];
+
+ public static SeriesFeatureVector fromChunkData(ChunkData data, int dataSampleLength) {
+ ChunkHeader chunkHeader;
+ Chunk chunk;
+ ByteBuffer chunkBuffer;
+ // sample a buffer from the chunk data
+ if (data.isAligned()) {
+ AlignedChunkData alignedChunkData = (AlignedChunkData) data;
+ chunkHeader = alignedChunkData.getChunkHeaderList().get(0);
+ if (!alignedChunkData.getChunkList().isEmpty()) {
+ chunk = alignedChunkData.getChunkList().get(0);
+ ByteBuffer buffer = chunk.getData();
+ int sampleLength = Math.min(dataSampleLength, buffer.remaining());
+ chunkBuffer = buffer.slice();
+ chunkBuffer.limit(sampleLength);
+ } else {
+ chunkBuffer = ByteBuffer.wrap(alignedChunkData.getByteStream().getBuf());
+ int sampleLength = Math.min(dataSampleLength, chunkBuffer.remaining());
+ chunkBuffer.limit(sampleLength);
+ }
+ } else {
+ NonAlignedChunkData nonAlignedChunkData = (NonAlignedChunkData) data;
+ chunkHeader = nonAlignedChunkData.getChunkHeader();
+ chunk = nonAlignedChunkData.getChunk();
+ if (chunk != null) {
+ ByteBuffer buffer = chunk.getData();
+ int sampleLength = Math.min(dataSampleLength, buffer.remaining());
+ chunkBuffer = buffer.slice();
+ chunkBuffer.limit(sampleLength);
+ } else {
+ chunkBuffer = ByteBuffer.wrap(nonAlignedChunkData.getByteStream().getBuf());
+ int sampleLength = Math.min(dataSampleLength, chunkBuffer.remaining());
+ chunkBuffer.limit(sampleLength);
+ }
+ }
+
+ SeriesFeatureVector vector = new SeriesFeatureVector();
+ vector.measurementId = chunkHeader.getMeasurementID();
+ vector.dataSize = chunkHeader.getDataSize();
+ vector.dataType = chunkHeader.getDataType();
+ vector.compressionType = chunkHeader.getCompressionType();
+ vector.encodingType = chunkHeader.getEncodingType();
+ vector.numOfPages = chunkHeader.getNumOfPages();
+ vector.dataSample =
+ new String(
+ chunkBuffer.array(),
+ chunkBuffer.arrayOffset() + chunkBuffer.position(),
+ chunkBuffer.remaining());
+ return vector;
+ }
+
+ public void mergeChunkData(ChunkData data) {
+ ChunkHeader chunkHeader;
+ if (data.isAligned()) {
+ AlignedChunkData alignedChunkData = (AlignedChunkData) data;
+ chunkHeader = alignedChunkData.getChunkHeaderList().get(0);
+ } else {
+ NonAlignedChunkData nonAlignedChunkData = (NonAlignedChunkData) data;
+ chunkHeader = nonAlignedChunkData.getChunkHeader();
+ }
+ dataSize += chunkHeader.getDataSize();
+ numOfPages += chunkHeader.getNumOfPages();
+ }
+ }
+
+ public interface VectorDistance {
+
+ double calculate(double[] v1, double[] v2);
+ }
+
+ public static class EuclideanDistance implements VectorDistance {
+
+ @Override
+ public double calculate(double[] v1, double[] v2) {
+ double sum = 0;
+ for (int i = 0; i < v1.length; i++) {
+ sum += (v1[i] - v2[i]) * (v1[i] - v2[i]);
+ }
+ return Math.sqrt(sum) + 1;
+ }
+ }
+
+ public interface Clustering {
+
+ List<List<String>> cluster(Map<String, double[]> tagVectorMap, VectorDistance distance);
+ }
+
+ public class KMeans implements Clustering {
+
+ private int k;
+ private int maxIteration;
+ private double[][] centroids;
+ private AtomicInteger[] centroidCounters;
+ private Random random;
+ private Map<Entry<String, double[]>, Integer> recordCentroidMapping;
+ private int vecLength = 0;
+
+ public KMeans(int k, int maxIteration) {
+ this.k = k;
+ this.maxIteration = maxIteration;
+ this.centroids = new double[k][];
+ this.centroidCounters = new AtomicInteger[k];
+ for (int i = 0; i < centroidCounters.length; i++) {
+ centroidCounters[i] = new AtomicInteger();
+ }
+ this.random = new Random();
+ this.recordCentroidMapping = new ConcurrentHashMap<>();
+ }
+
+ private void clearCentroidCounter() {
+ for (AtomicInteger centroidCounter : centroidCounters) {
+ centroidCounter.set(0);
+ }
+ }
+
+ @Override
+ public List<List<String>> cluster(Map<String, double[]> tagVectorMap, VectorDistance distance) {
+ recordCentroidMapping.clear();
+ if (k > tagVectorMap.size()) {
+ k = tagVectorMap.size();
+ this.centroids = new double[k][];
+ }
+
+ for (Entry<String, double[]> entry : tagVectorMap.entrySet()) {
+ vecLength = entry.getValue().length;
+ }
+
+ randomCentroid(vecLength, tagVectorMap);
+
+ for (int i = 0; i < maxIteration; i++) {
+ if (!assignCentroid(tagVectorMap, distance)) {
+ break;
+ }
+ newCentroid();
+ clearCentroidCounter();
+ if (System.currentTimeMillis() - splitStartTime > splitTimeBudget) {
+ break;
+ }
+ }
+
+ Map<Integer, List<Entry<String, double[]>>> centroidRecordMap =
+ collectCentroidRecordMapping();
+ return centroidRecordMap.values().stream()
+ .map(l -> l.stream().map(Entry::getKey).collect(Collectors.toList()))
+ .collect(Collectors.toList());
+ }
+
+ private Map<Integer, List<Entry<String, double[]>>> collectCentroidRecordMapping() {
+ Map<Integer, List<Entry<String, double[]>>> centroidRecordMapping = new ConcurrentHashMap<>();
+ recordCentroidMapping.entrySet().stream()
+ .parallel()
+ .forEach(
+ e ->
+ centroidRecordMapping.compute(
+ e.getValue(),
+ (key, oldV) -> {
+ if (oldV == null) {
+ oldV = new ArrayList<>();
+ }
+ oldV.add(e.getKey());
+ return oldV;
+ }));
+ return centroidRecordMapping;
+ }
+
+ private void newCentroid() {
+ Map<Integer, List<Entry<String, double[]>>> centroidRecordMapping =
+ collectCentroidRecordMapping();
+ centroidRecordMapping.entrySet().stream()
+ .parallel()
+ .forEach(
+ e -> {
+ Integer centroidId = e.getKey();
+ List<Entry<String, double[]>> records = e.getValue();
+ int recordNum = records.size();
+ double[] sumVec = new double[vecLength];
+ for (Entry<String, double[]> record : records) {
+ for (int i = 0; i < sumVec.length; i++) {
+ sumVec[i] += record.getValue()[i];
+ }
+ }
+ for (int i = 0; i < sumVec.length; i++) {
+ sumVec[i] = sumVec[i] / recordNum;
+ }
+ centroids[centroidId] = sumVec;
+ });
+ }
+
+ private boolean assignCentroid(Map<String, double[]> tagVectorMap, VectorDistance distance) {
+ AtomicBoolean centroidUpdated = new AtomicBoolean(false);
+ tagVectorMap.entrySet().stream()
+ .parallel()
+ .forEach(
+ e -> {
+ double[] vector = e.getValue();
+ double minDist = Double.MAX_VALUE;
+ int nearestCentroid = 0;
+ for (int i = 0; i < centroids.length; i++) {
+ double dist =
+ distance.calculate(vector, centroids[i])
+ * (1 + centroidCounters[i].get() * 0.1);
+ if (dist < minDist) {
+ minDist = dist;
+ nearestCentroid = i;
+ }
+ }
+
+ centroidCounters[nearestCentroid].incrementAndGet();
+ int finalNearestCentroid = nearestCentroid;
+ recordCentroidMapping.put(e, finalNearestCentroid);
+ recordCentroidMapping.compute(
+ e,
+ (t, oc) -> {
+ if (oc == null || oc != finalNearestCentroid) {
+ centroidUpdated.set(true);
+ return finalNearestCentroid;
+ } else {
+ return oc;
+ }
+ });
+ });
+ return centroidUpdated.get();
+ }
+
+ private void randomCentroid(int vecLength, Map<String, double[]> tagVectorMap) {
+ pickRandomCentroid(tagVectorMap);
+ // genRandomCentroid(vecLength);
+ }
+
+ private void pickRandomCentroid(Map<String, double[]> tagVectorMap) {
+ List<double[]> recordVectors = new ArrayList<>(tagVectorMap.values());
+ Collections.shuffle(recordVectors);
+ for (int i = 0; i < k; i++) {
+ centroids[i] = recordVectors.get(i);
+ }
+ }
+
+ private void genRandomCentroid(int vecLength) {
+ for (int i = 0; i < k; i++) {
+ double[] centroid = new double[vecLength];
+ for (int j = 0; j < vecLength; j++) {
+ centroid[j] = random.nextDouble();
+ }
+ centroids[i] = centroid;
+ }
+ }
+ }
+}
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java
index 78bf3023943,1fa30dbea7c..6ee39dd696c
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/SchemaValidator.java
@@@ -28,11 -28,10 +28,11 @@@ import org.apache.iotdb.db.queryengine.
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertMultiTabletsStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsOfOneDeviceStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement;
+ import org.apache.iotdb.tsfile.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
- import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import java.util.ArrayList;
import java.util.List;
public class SchemaValidator {
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
index c06defac208,5ad9b4ea985..10bebb01483
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
@@@ -111,8 -106,7 +106,7 @@@ import org.apache.iotdb.db.queryengine.
import org.apache.iotdb.db.queryengine.plan.Coordinator;
import org.apache.iotdb.db.queryengine.plan.analyze.Analysis;
import org.apache.iotdb.db.queryengine.plan.analyze.Analyzer;
- import org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer;
-import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher;
+import org.apache.iotdb.db.queryengine.plan.analyze.partition.ClusterPartitionFetcher;
import org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher;
import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult;
import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult;
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
index 48423aa5f92,d960a86ba23..df818ee63c3
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/load/LoadTsFileScheduler.java
@@@ -25,7 -25,15 +25,12 @@@ import org.apache.iotdb.common.rpc.thri
import org.apache.iotdb.commons.client.IClientManager;
import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient;
import org.apache.iotdb.commons.conf.CommonDescriptor;
+ import org.apache.iotdb.commons.consensus.ConsensusGroupId;
+ import org.apache.iotdb.commons.consensus.DataRegionId;
-import org.apache.iotdb.commons.partition.DataPartition;
-import org.apache.iotdb.commons.partition.DataPartitionQueryParam;
import org.apache.iotdb.commons.partition.StorageExecutor;
+ import org.apache.iotdb.commons.service.metric.MetricService;
+ import org.apache.iotdb.commons.service.metric.enums.Metric;
+ import org.apache.iotdb.commons.service.metric.enums.Tag;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.mpp.FragmentInstanceDispatchException;
@@@ -43,8 -52,12 +48,11 @@@ import org.apache.iotdb.db.queryengine.
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode;
import org.apache.iotdb.db.queryengine.plan.scheduler.FragInstanceDispatchResult;
import org.apache.iotdb.db.queryengine.plan.scheduler.IScheduler;
+ import org.apache.iotdb.db.storageengine.StorageEngine;
+ import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
+ import org.apache.iotdb.metrics.utils.MetricLevel;
import org.apache.iotdb.mpp.rpc.thrift.TLoadCommandReq;
import org.apache.iotdb.rpc.TSStatusCode;
-import org.apache.iotdb.tsfile.utils.Pair;
import io.airlift.units.Duration;
import org.slf4j.Logger;
diff --cc iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/load/LoadTsFileSchedulerTest.java
index 9f7a713b15d,00000000000..9e226eb2dca
mode 100644,000000..100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/load/LoadTsFileSchedulerTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/load/LoadTsFileSchedulerTest.java
@@@ -1,189 -1,0 +1,189 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.execution.load;
+
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory;
+import org.apache.iotdb.commons.consensus.ConsensusGroupId;
+import org.apache.iotdb.db.queryengine.common.FragmentInstanceId;
+import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
+import org.apache.iotdb.db.queryengine.common.PlanFragmentId;
+import org.apache.iotdb.db.queryengine.common.QueryId;
+import org.apache.iotdb.db.queryengine.execution.QueryStateMachine;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.DistributedQueryPlan;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.PlanFragment;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.SubPlan;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadSingleTsFileNode;
+import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode;
+import org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileScheduler;
+import org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileScheduler.LoadCommand;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.mpp.rpc.thrift.TLoadCommandReq;
+import org.apache.iotdb.mpp.rpc.thrift.TLoadResp;
+import org.apache.iotdb.mpp.rpc.thrift.TTsFilePieceReq;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.junit.Test;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.ConcurrentSkipListSet;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+
+public class LoadTsFileSchedulerTest extends TestBase {
+
+ protected Map<TEndPoint, Map<ConsensusGroupId, Map<String, Map<File, Set<Integer>>>>>
+ phaseOneResults = new ConcurrentSkipListMap<>();
+ // the third key is UUid, the value is command type
+ protected Map<TEndPoint, Map<String, Integer>> phaseTwoResults = new ConcurrentSkipListMap<>();
+
+ @Test
+ public void test() {
+ MPPQueryContext context = new MPPQueryContext(QueryId.MOCK_QUERY_ID);
+
+ PlanFragmentId fragmentId = new PlanFragmentId("load_tsfile_scheduler_test", 0);
+ SubPlan subPlan = new SubPlan(new PlanFragment(fragmentId, null));
+ List<FragmentInstance> fragmentInstanceList = new ArrayList<>();
+ for (int i = 0; i < tsFileResources.size(); i++) {
+ TsFileResource tsFileResource = tsFileResources.get(i);
+ LoadSingleTsFileNode singleTsFileNode =
+ new LoadSingleTsFileNode(
- new PlanNodeId("load_tsfile_scheduler_test" + (i + 1)), tsFileResource, false);
++ new PlanNodeId("load_tsfile_scheduler_test" + (i + 1)), tsFileResource, false, 0);
+ fragmentInstanceList.add(
+ new FragmentInstance(
+ new PlanFragment(fragmentId, singleTsFileNode),
+ new FragmentInstanceId(fragmentId, "" + i),
+ null,
+ null,
+ 0,
+ null));
+ }
+ DistributedQueryPlan queryPlan =
+ new DistributedQueryPlan(context, subPlan, null, fragmentInstanceList);
+
+ LoadTsFileScheduler scheduler =
+ new LoadTsFileScheduler(
+ queryPlan,
+ context,
+ new QueryStateMachine(
+ context.getQueryId(),
+ IoTDBThreadPoolFactory.newCachedThreadPool("LoadTsFileSchedulerTest")),
+ internalServiceClientManager,
+ dummyDataPartitionBatchFetcher(),
+ false);
+
+ long start = System.currentTimeMillis();
+ scheduler.start();
+ long timeConsumption = System.currentTimeMillis() - start;
+
+ printPhaseResult();
+ System.out.printf("Split ends after %dms", timeConsumption);
+ }
+
+ public TLoadResp handleTsFilePieceNode(TTsFilePieceReq req, TEndPoint tEndpoint) {
+ ConsensusGroupId groupId =
+ ConsensusGroupId.Factory.createFromTConsensusGroupId(req.consensusGroupId);
+ LoadTsFilePieceNode pieceNode =
+ (LoadTsFilePieceNode) PlanNodeType.deserialize(req.body.slice());
+ Set<Integer> splitIds =
+ phaseOneResults
+ .computeIfAbsent(
+ tEndpoint,
+ e -> new ConcurrentSkipListMap<>(Comparator.comparingInt(ConsensusGroupId::getId)))
+ .computeIfAbsent(groupId, g -> new ConcurrentSkipListMap<>())
+ .computeIfAbsent(req.uuid, id -> new ConcurrentSkipListMap<>())
+ .computeIfAbsent(pieceNode.getTsFile(), f -> new ConcurrentSkipListSet<>());
+ splitIds.addAll(
+ pieceNode.getAllTsFileData().stream()
+ .map(TsFileData::getSplitId)
+ .collect(Collectors.toList()));
+
+ return new TLoadResp()
+ .setAccepted(true)
+ .setStatus(new TSStatus().setCode(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
+ }
+
+ public TLoadResp handleTsLoadCommand(TLoadCommandReq req, TEndPoint tEndpoint) {
+ phaseTwoResults
+ .computeIfAbsent(tEndpoint, e -> new ConcurrentSkipListMap<>())
+ .computeIfAbsent(req.uuid, id -> req.commandType);
+
+ return new TLoadResp()
+ .setAccepted(true)
+ .setStatus(new TSStatus().setCode(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
+ }
+
+ public void printPhaseResult() {
+ System.out.print("Phase one:\n");
+ for (Entry<TEndPoint, Map<ConsensusGroupId, Map<String, Map<File, Set<Integer>>>>>
+ tEndPointMapEntry : phaseOneResults.entrySet()) {
+ TEndPoint endPoint = tEndPointMapEntry.getKey();
+ for (Entry<ConsensusGroupId, Map<String, Map<File, Set<Integer>>>> consensusGroupIdMapEntry :
+ tEndPointMapEntry.getValue().entrySet()) {
+ ConsensusGroupId consensusGroupId = consensusGroupIdMapEntry.getKey();
+ int chunkNum = 0;
+ int fileNum = 0;
+ int taskNum = 0;
+ for (Entry<String, Map<File, Set<Integer>>> stringMapEntry :
+ consensusGroupIdMapEntry.getValue().entrySet()) {;
+ taskNum += 1;
+ for (Entry<File, Set<Integer>> fileListEntry : stringMapEntry.getValue().entrySet()) {
+ Set<Integer> chunks = fileListEntry.getValue();
+ chunkNum += chunks.size();
+ fileNum += 1;
+ }
+ }
+ System.out.printf(
+ "%s - %s - %s tasks - %s files - %s chunks\n",
+ endPoint, consensusGroupId, taskNum, fileNum, chunkNum);
+ // if (consensusGroupId.getId() == 0) {
+ // // d1, non-aligned series
+ // assertEquals(expectedChunkNum() / 2, chunkNum);
+ // } else {
+ // // d2, aligned series
+ // assertEquals(expectedChunkNum() / 2 / seriesNum, chunkNum);
+ // }
+ }
+ }
+
+ System.out.print("Phase two:\n");
+ for (Entry<TEndPoint, Map<String, Integer>> tEndPointMapEntry : phaseTwoResults.entrySet()) {
+ TEndPoint endPoint = tEndPointMapEntry.getKey();
+ for (Entry<String, Integer> stringMapEntry : tEndPointMapEntry.getValue().entrySet()) {
+ String uuid = stringMapEntry.getKey();
+ int command = stringMapEntry.getValue();
+ System.out.printf("%s - %s - %s\n", endPoint, uuid, LoadCommand.values()[command]);
+ assertEquals(LoadCommand.EXECUTE.ordinal(), command);
+ }
+ }
+ }
+}
diff --cc iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/load/TestBase.java
index de0bf3c9590,00000000000..2902d64cd61
mode 100644,000000..100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/load/TestBase.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/load/TestBase.java
@@@ -1,431 -1,0 +1,431 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.execution.load;
+
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
+import org.apache.iotdb.commons.client.ClientManager;
+import org.apache.iotdb.commons.client.ClientPoolFactory.SyncDataNodeInternalServiceClientPoolFactory;
+import org.apache.iotdb.commons.client.IClientManager;
+import org.apache.iotdb.commons.client.property.ThriftClientProperty;
+import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient;
+import org.apache.iotdb.commons.concurrent.ExceptionalCountDownLatch;
+import org.apache.iotdb.commons.consensus.ConsensusGroupId;
+import org.apache.iotdb.commons.consensus.ConsensusGroupId.Factory;
+import org.apache.iotdb.commons.partition.DataPartition;
+import org.apache.iotdb.commons.partition.DataPartitionQueryParam;
+import org.apache.iotdb.commons.partition.SchemaNodeManagementPartition;
+import org.apache.iotdb.commons.partition.SchemaPartition;
+import org.apache.iotdb.commons.path.PathPatternTree;
+import org.apache.iotdb.commons.utils.FileUtils;
+import org.apache.iotdb.commons.utils.TimePartitionUtils;
+import org.apache.iotdb.db.exception.DataRegionException;
+import org.apache.iotdb.db.exception.LoadFileException;
+import org.apache.iotdb.db.queryengine.plan.analyze.IPartitionFetcher;
+import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
+import org.apache.iotdb.db.storageengine.dataregion.flush.TsFileFlushPolicy.DirectFlushPolicy;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.db.storageengine.dataregion.wal.recover.WALRecoverManager;
+import org.apache.iotdb.db.utils.SequenceUtils.DoubleSequenceGenerator;
+import org.apache.iotdb.db.utils.SequenceUtils.DoubleSequenceGeneratorFactory;
+import org.apache.iotdb.db.utils.SequenceUtils.GaussianDoubleSequenceGenerator;
+import org.apache.iotdb.db.utils.SequenceUtils.SimpleDoubleSequenceGenerator;
+import org.apache.iotdb.db.utils.SequenceUtils.UniformDoubleSequenceGenerator;
+import org.apache.iotdb.mpp.rpc.thrift.TLoadCommandReq;
+import org.apache.iotdb.mpp.rpc.thrift.TLoadResp;
+import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq;
+import org.apache.iotdb.mpp.rpc.thrift.TTsFilePieceReq;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
++import org.apache.iotdb.tsfile.enums.TSDataType;
+import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
- import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.utils.Pair;
+import org.apache.iotdb.tsfile.utils.TsFileGeneratorUtils;
+import org.apache.iotdb.tsfile.write.TsFileWriter;
+import org.apache.iotdb.tsfile.write.record.Tablet;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.protocol.TProtocol;
+import org.apache.thrift.transport.TByteBuffer;
+import org.apache.thrift.transport.TTransportException;
+import org.junit.After;
+import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static org.apache.iotdb.commons.conf.IoTDBConstant.GB;
+
+public class TestBase {
+
+ private static final Logger logger = LoggerFactory.getLogger(TestBase.class);
+ public static final String BASE_OUTPUT_PATH = "target".concat(File.separator).concat("loadTest");
+ public static final String PARTIAL_PATH_STRING =
+ "%s" + File.separator + "%d" + File.separator + "%d" + File.separator;
+ public static final String TEST_TSFILE_PATH =
+ BASE_OUTPUT_PATH + File.separator + "testTsFile".concat(File.separator) + PARTIAL_PATH_STRING;
+
+ protected int fileNum = 100;
+ // series number of each file, sn non-aligned series and 1 aligned series with sn measurements
+ protected int seriesNum = 1000;
+ protected int deviceNum = 100;
+ // number of chunks of each series in a file, each series has only one chunk in a file
+ protected double chunkTimeRangeRatio = 0.001;
+ // the interval between two consecutive points of a series
+ protected long pointInterval = 50_000;
+ protected List<Pair<String, DoubleSequenceGeneratorFactory>> measurementSequenceGeneratorPairs =
+ Arrays.asList(
+ new Pair<>("Simple_", new SimpleDoubleSequenceGenerator.Factory()),
+ new Pair<>("UniformA_", new UniformDoubleSequenceGenerator.Factory(1.0)),
+ new Pair<>("GaussianA_", new GaussianDoubleSequenceGenerator.Factory(1.0, 1.0)),
+ new Pair<>("UniformB_", new UniformDoubleSequenceGenerator.Factory(10.0)),
+ new Pair<>("GaussianB_", new GaussianDoubleSequenceGenerator.Factory(15.0, 3.0)));
+ protected final List<File> files = new ArrayList<>();
+ protected final List<TsFileResource> tsFileResources = new ArrayList<>();
+ protected IPartitionFetcher partitionFetcher;
+ // the key is deviceId, not partitioned by time in the simple test
+ protected Map<String, TRegionReplicaSet> partitionTable = new HashMap<>();
+ protected Map<TConsensusGroupId, DataRegion> dataRegionMap = new HashMap<>();
+ protected Map<ConsensusGroupId, TRegionReplicaSet> groupId2ReplicaSetMap = new HashMap<>();
+ protected IClientManager<TEndPoint, SyncDataNodeInternalServiceClient>
+ internalServiceClientManager;
+ // the third key is UUid, the forth key is targetFile
+
+ private int groupSizeInByte;
+
+ @Before
+ public void setup() throws IOException, WriteProcessException, DataRegionException {
+ setupFiles();
+ logger.info("{} files set up", files.size());
+ partitionFetcher = dummyPartitionFetcher();
+ setupPartitionTable();
+ setupClientManager();
+ groupSizeInByte = TSFileDescriptor.getInstance().getConfig().getGroupSizeInByte();
+ TSFileDescriptor.getInstance().getConfig().setGroupSizeInByte((int) (GB));
+ }
+
+ @After
+ public void cleanup() {
+ FileUtils.deleteDirectory(new File(BASE_OUTPUT_PATH));
+ FileUtils.deleteDirectory(new File("target" + File.separator + "data"));
+ TSFileDescriptor.getInstance().getConfig().setGroupSizeInByte(groupSizeInByte);
+ }
+
+ public int expectedChunkNum() {
+ double totalTimeRange = chunkTimeRangeRatio * fileNum;
+ int splitChunkNum = 0;
+ // if the boundary of the ith partition does not overlap a chunk, it introduces an additional
+ // split
+ // TODO: due to machine precision, the calculation may have error. Also, if the data amount is
+ // too large, there could be more than one chunk for each series in the file.
+ for (int i = 0; i <= totalTimeRange; i++) {
+ if (i * 1.0 % chunkTimeRangeRatio > 0.00001) {
+ splitChunkNum += 1;
+ }
+ }
+ return (splitChunkNum + fileNum) * seriesNum * 2;
+ }
+
+ public TLoadResp handleTsFilePieceNode(TTsFilePieceReq req, TEndPoint tEndpoint)
+ throws TException, IOException {
+ return new TLoadResp()
+ .setAccepted(true)
+ .setStatus(new TSStatus().setCode(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
+ }
+
+ public TLoadResp handleTsLoadCommand(TLoadCommandReq req, TEndPoint tEndpoint)
+ throws LoadFileException, IOException {
+ return new TLoadResp()
+ .setAccepted(true)
+ .setStatus(new TSStatus().setCode(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
+ }
+
+ public TProtocol dummyProtocol() throws TTransportException {
+ return new TBinaryProtocol(new TByteBuffer(ByteBuffer.allocate(0)));
+ }
+
+ public void setupClientManager() {
+ SyncDataNodeInternalServiceClientPoolFactory poolFactory =
+ new SyncDataNodeInternalServiceClientPoolFactory();
+ internalServiceClientManager =
+ new ClientManager<TEndPoint, SyncDataNodeInternalServiceClient>(poolFactory) {
+ @Override
+ public SyncDataNodeInternalServiceClient borrowClient(TEndPoint node) {
+ try {
+ return new SyncDataNodeInternalServiceClient(
+ dummyProtocol(), new ThriftClientProperty.Builder().build(), node, this) {
+ @Override
+ public TLoadResp sendTsFilePieceNode(TTsFilePieceReq req) throws TException {
+ try {
+ return handleTsFilePieceNode(req, getTEndpoint());
+ } catch (IOException e) {
+ throw new TException(e);
+ }
+ }
+
+ @Override
+ public TLoadResp sendLoadCommand(TLoadCommandReq req) throws TException {
+ try {
+ return handleTsLoadCommand(req, getTEndpoint());
+ } catch (LoadFileException | IOException e) {
+ throw new TException(e);
+ }
+ }
+
+ @Override
+ public void close() {}
+ };
+ } catch (TTransportException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void clear(TEndPoint node) {}
+
+ @Override
+ public void close() {}
+ };
+ }
+
+ public void setupPartitionTable() throws DataRegionException {
+ ConsensusGroupId d1GroupId = Factory.create(TConsensusGroupType.DataRegion.getValue(), 0);
+ TRegionReplicaSet d1Replicas =
+ new TRegionReplicaSet(
+ d1GroupId.convertToTConsensusGroupId(),
+ Arrays.asList(
+ new TDataNodeLocation()
+ .setDataNodeId(0)
+ .setInternalEndPoint(new TEndPoint("localhost", 10000)),
+ new TDataNodeLocation()
+ .setDataNodeId(1)
+ .setInternalEndPoint(new TEndPoint("localhost", 10001)),
+ new TDataNodeLocation()
+ .setDataNodeId(2)
+ .setInternalEndPoint(new TEndPoint("localhost", 10002))));
+
+ WALRecoverManager.getInstance().setAllDataRegionScannedLatch(new ExceptionalCountDownLatch(0));
+ DataRegion dataRegion =
+ new DataRegion(
+ BASE_OUTPUT_PATH, d1GroupId.toString(), new DirectFlushPolicy(), "root.loadTest");
+ for (int i = 0; i < deviceNum; i++) {
+ partitionTable.put("d" + i, d1Replicas);
+ dataRegionMap.put(d1GroupId.convertToTConsensusGroupId(), dataRegion);
+ }
+
+ groupId2ReplicaSetMap.put(d1GroupId, d1Replicas);
+ ConsensusGroupId d2GroupId = Factory.create(TConsensusGroupType.DataRegion.getValue(), 1);
+ TRegionReplicaSet d2Replicas =
+ new TRegionReplicaSet(
+ d2GroupId.convertToTConsensusGroupId(),
+ Arrays.asList(
+ new TDataNodeLocation()
+ .setDataNodeId(3)
+ .setInternalEndPoint(new TEndPoint("localhost", 10003)),
+ new TDataNodeLocation()
+ .setDataNodeId(4)
+ .setInternalEndPoint(new TEndPoint("localhost", 10004)),
+ new TDataNodeLocation()
+ .setDataNodeId(5)
+ .setInternalEndPoint(new TEndPoint("localhost", 10005))));
+ partitionTable.put("dd1", d2Replicas);
+ groupId2ReplicaSetMap.put(d2GroupId, d2Replicas);
+ }
+
+ public IPartitionFetcher dummyPartitionFetcher() {
+ return new IPartitionFetcher() {
+ @Override
+ public SchemaPartition getSchemaPartition(PathPatternTree patternTree) {
+ return null;
+ }
+
+ @Override
+ public SchemaPartition getOrCreateSchemaPartition(
+ PathPatternTree patternTree, String userName) {
+ return null;
+ }
+
+ @Override
+ public DataPartition getDataPartition(
+ Map<String, List<DataPartitionQueryParam>> sgNameToQueryParamsMap) {
+ return null;
+ }
+
+ @Override
+ public DataPartition getDataPartitionWithUnclosedTimeRange(
+ Map<String, List<DataPartitionQueryParam>> sgNameToQueryParamsMap) {
+ return null;
+ }
+
+ @Override
+ public DataPartition getOrCreateDataPartition(
+ Map<String, List<DataPartitionQueryParam>> sgNameToQueryParamsMap) {
+ return null;
+ }
+
+ @Override
+ public DataPartition getOrCreateDataPartition(
+ List<DataPartitionQueryParam> dataPartitionQueryParams, String userName) {
+ return null;
+ }
+
+ @Override
+ public SchemaNodeManagementPartition getSchemaNodeManagementPartitionWithLevel(
+ PathPatternTree patternTree, PathPatternTree scope, Integer level) {
+ return null;
+ }
+
+ @Override
+ public boolean updateRegionCache(TRegionRouteReq req) {
+ return false;
+ }
+
+ @Override
+ public void invalidAllCache() {}
+ };
+ }
+
+ public DataPartitionBatchFetcher dummyDataPartitionBatchFetcher() {
+ return new DataPartitionBatchFetcher(partitionFetcher) {
+ @Override
+ public List<TRegionReplicaSet> queryDataPartition(
+ List<Pair<String, TTimePartitionSlot>> slotList, String userName) {
+ return slotList.stream().map(p -> partitionTable.get(p.left)).collect(Collectors.toList());
+ }
+ };
+ }
+
+ public void setupFiles() {
+ measurementSequenceGeneratorPairs.sort(Comparator.comparing(Pair::getLeft));
+ List<Pair<MeasurementSchema, DoubleSequenceGeneratorFactory>> schemaGeneratorPairs =
+ new ArrayList<>();
+ for (int sn = 0; sn < seriesNum; sn++) {
+ Pair<String, DoubleSequenceGeneratorFactory> measurementGeneratorPair =
+ measurementSequenceGeneratorPairs.get(sn % measurementSequenceGeneratorPairs.size());
+ MeasurementSchema measurementSchema =
+ new MeasurementSchema(measurementGeneratorPair.left + sn, TSDataType.DOUBLE);
+ measurementSchema.setCompressor(CompressionType.LZ4.serialize());
+ measurementSchema.setEncoding(TSEncoding.PLAIN.serialize());
+ schemaGeneratorPairs.add(new Pair<>(measurementSchema, measurementGeneratorPair.right));
+ }
+ schemaGeneratorPairs.sort(Comparator.comparing(s -> s.left.getMeasurementId()));
+ List<MeasurementSchema> measurementSchemas =
+ schemaGeneratorPairs.stream().map(m -> m.left).collect(Collectors.toList());
+ IntStream.range(0, fileNum)
+ .parallel()
+ .forEach(
+ i -> {
+ try {
+ File file = new File(getTestTsFilePath("root.sg1", 0, 0, i));
+ TsFileResource tsFileResource = new TsFileResource(file);
+ synchronized (files) {
+ files.add(file);
+ }
+
+ try (TsFileWriter writer = new TsFileWriter(file)) {
+ // sn non-aligned series under d1 and 1 aligned series with sn measurements under
+ // dd2
+ for (int sn = 0; sn < seriesNum; sn++) {
+ for (int dn = 0; dn < deviceNum; dn++) {
+ writer.registerTimeseries(
+ new Path("d" + dn), schemaGeneratorPairs.get(sn).left);
+ }
+ }
+ writer.registerAlignedTimeseries(new Path("dd1"), measurementSchemas);
+
+ // one chunk for each series
+ long timePartitionInterval = TimePartitionUtils.getTimePartitionInterval();
+ long chunkTimeRange = (long) (timePartitionInterval * chunkTimeRangeRatio);
+ int chunkPointNum = (int) (chunkTimeRange / pointInterval);
+
+ Tablet tablet = new Tablet("d0", measurementSchemas, chunkPointNum);
+ for (int sn = 0; sn < seriesNum; sn++) {
+ DoubleSequenceGenerator sequenceGenerator =
+ schemaGeneratorPairs.get(sn).right.create();
+ for (int pn = 0; pn < chunkPointNum; pn++) {
+ if (sn == 0) {
+ long currTime = chunkTimeRange * i + pointInterval * pn;
+ tablet.addTimestamp(pn, currTime);
+ }
+ tablet.addValue(
+ schemaGeneratorPairs.get(sn).left.getMeasurementId(),
+ pn,
+ sequenceGenerator.gen(pn));
+ }
+ }
+
+ tablet.rowSize = chunkPointNum;
+ for (int dn = 0; dn < deviceNum; dn++) {
+ tablet.deviceId = "d" + dn;
+ writer.write(tablet);
+ }
+ tablet.deviceId = "d2";
+ // writer.writeAligned(tablet);
+
+ writer.flushAllChunkGroups();
+ for (int dn = 0; dn < deviceNum; dn++) {
+ tsFileResource.updateStartTime("d" + dn, chunkTimeRange * i);
+ tsFileResource.updateEndTime("d" + dn, chunkTimeRange * (i + 1));
+ }
+ }
+
+ tsFileResource.close();
+ synchronized (tsFileResources) {
+ tsFileResources.add(tsFileResource);
+ }
+ } catch (IOException | WriteProcessException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ tsFileResources.sort(Comparator.comparingLong(TsFileResource::getVersion));
+ }
+
+ public static String getTestTsFilePath(
+ String logicalStorageGroupName,
+ long VirtualStorageGroupId,
+ long TimePartitionId,
+ long tsFileVersion) {
+ String filePath =
+ String.format(
+ TEST_TSFILE_PATH, logicalStorageGroupName, VirtualStorageGroupId, TimePartitionId);
+ return TsFileGeneratorUtils.getTsFilePath(filePath, tsFileVersion);
+ }
+}
diff --cc iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index ba4f8f1fe97,6d8c534af48..829e6578e2e
--- a/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/iotdb-core/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@@ -22,7 -22,9 +22,8 @@@ package org.apache.iotdb.tsfile.read
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
-import org.apache.iotdb.tsfile.compress.IUnCompressor;
import org.apache.iotdb.tsfile.encoding.decoder.Decoder;
+ import org.apache.iotdb.tsfile.enums.TSDataType;
import org.apache.iotdb.tsfile.exception.TsFileRuntimeException;
import org.apache.iotdb.tsfile.exception.TsFileStatisticsMistakesException;
import org.apache.iotdb.tsfile.file.MetaMarker;
diff --cc iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
index 8b43d56a229,c7629909bea..588c9937a26
--- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
+++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
@@@ -833,11 -797,15 +797,20 @@@ struct TShowThrottleReq
1: optional string userName;
}
+struct TGetPartitionParameterResp{
+ 1: required i64 partitionInterval;
+ 2: required i32 seriesPartitionSlotNum;
+}
+
+ // ====================================================
+ // Activation
+ // ====================================================
+ struct TLicenseContentResp {
+ 1: required common.TSStatus status
+ 2: optional common.TLicense licenseContent
+ }
+
+
service IConfigNodeRPCService {
// ======================================================