You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by sp...@apache.org on 2023/09/09 12:20:25 UTC
[iotdb] 01/01: Merge branch 'master' into feature/metric-detail
This is an automated email from the ASF dual-hosted git repository.
spricoder pushed a commit to branch feature/metric-detail
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8be8650d743a5f8954f86cf48c5d26a6c133329e
Merge: 32133de04e2 4f1fe0638cd
Author: spricoder <sp...@qq.com>
AuthorDate: Sat Sep 9 20:20:08 2023 +0800
Merge branch 'master' into feature/metric-detail
.../{sonar-coveralls.yml => sonar-codecov.yml} | 21 +-
LICENSE | 17 +
NOTICE | 2 +-
NOTICE-binary | 2 +-
README.md | 4 +-
codecov.yml | 6 +-
....apache.iotdb.db.protocol.mqtt.PayloadFormatter | 19 +
.../apache/iotdb/cli/it/StartClientScriptIT.java | 32 --
.../apache/iotdb/db/it/IoTDBInsertWithQueryIT.java | 23 +-
.../iotdb/session/it/IoTDBSessionSimpleIT.java | 45 +++
.../cli/src/assembly/resources/sbin/start-cli.sh | 2 +-
.../java/org/apache/iotdb/cli/AbstractCli.java | 56 +--
.../src/main/java/org/apache/iotdb/cli/Cli.java | 3 -
.../java/org/apache/iotdb/cli/AbstractCliIT.java | 39 --
.../main/java/org/apache/iotdb/rpc/RpcUtils.java | 29 +-
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 +
.../java/org/apache/iotdb/rpc/RpcUtilsTest.java | 67 ++++
iotdb-connector/flink-sql-iotdb-connector/pom.xml | 6 +
.../org/apache/iotdb/flink/sql/common/Options.java | 8 +-
.../org/apache/iotdb/flink/sql/common/Utils.java | 29 +-
.../sql/factory/IoTDBDynamicTableFactory.java | 74 ++--
.../sql/function/IoTDBBoundedScanFunction.java | 35 +-
.../flink/sql/function/IoTDBCDCSourceFunction.java | 93 +++--
.../flink/sql/function/IoTDBLookupFunction.java | 24 +-
.../flink/sql/function/IoTDBSinkFunction.java | 74 ++--
iotdb-connector/grafana-plugin/backend-compile.sh | 2 +-
iotdb-connector/grafana-plugin/go.mod | 2 +-
iotdb-connector/grafana-plugin/pom.xml | 7 +-
.../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 7 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 35 +-
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 20 +-
.../resources/conf/iotdb-confignode.properties | 2 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 44 ++-
.../confignode/conf/ConfigNodeDescriptor.java | 48 ++-
.../consensus/request/ConfigPhysicalPlan.java | 10 +-
.../consensus/request/ConfigPhysicalPlanType.java | 3 +-
.../{ShowTrailPlan.java => GetModelInfoPlan.java} | 38 +-
.../{ShowTrailPlan.java => ShowTrialPlan.java} | 38 +-
.../request/write/model/UpdateModelInfoPlan.java | 16 +-
.../request/write/model/UpdateModelStatePlan.java | 18 +-
.../response/model/GetModelInfoResp.java} | 27 +-
.../response/{ => model}/ModelTableResp.java | 2 +-
.../TrialTableResp.java} | 28 +-
.../iotdb/confignode/manager/ConfigManager.java | 20 +-
.../apache/iotdb/confignode/manager/IManager.java | 13 +-
.../iotdb/confignode/manager/ModelManager.java | 99 +++--
.../iotdb/confignode/manager/ProcedureManager.java | 5 +-
.../manager/consensus/ConsensusManager.java | 3 +
.../iotdb/confignode/manager/node/NodeManager.java | 6 +-
.../manager/partition/PartitionManager.java | 5 +-
.../iotdb/confignode/persistence/ModelInfo.java | 147 +++++---
.../persistence/executor/ConfigPlanExecutor.java | 9 +-
.../procedure/impl/model/CreateModelProcedure.java | 125 +++----
.../procedure/impl/model/DropModelProcedure.java | 131 +++----
.../procedure/state/model/CreateModelState.java | 2 -
.../thrift/ConfigNodeRPCServiceProcessor.java | 15 +-
.../apache/iotdb/consensus/ratis/utils/Utils.java | 25 ++
.../consensus/ratis/{ => utils}/UtilsTest.java | 32 +-
iotdb-core/datanode/pom.xml | 5 +
.../resources/conf/iotdb-datanode.properties | 2 +-
.../check-overlap-sequence-files-and-repair.bat | 62 ++++
.../check-overlap-sequence-files-and-repair.sh | 46 ++-
.../tools/tsfile/overlap-statistic-tool.bat | 62 ++++
.../tools/tsfile/overlap-statistic-tool.sh | 46 ++-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 23 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 5 +
.../db/consensus/SchemaRegionConsensusImpl.java | 4 +
.../{ => runtime}/IntoProcessException.java | 2 +-
.../ModelInferenceProcessException.java} | 6 +-
.../{ => runtime}/WriteLockFailedException.java | 2 +-
.../config/constant/PipeConnectorConstant.java | 6 +
.../connector/protocol/opcua/OpcUaConnector.java | 246 +++++++++++++
.../protocol/opcua/OpcUaKeyStoreLoader.java | 120 ++++++
.../protocol/opcua/OpcUaServerBuilder.java | 287 +++++++++++++++
...ocketConnector.java => WebSocketConnector.java} | 34 +-
.../websocket/WebSocketConnectorServer.java | 84 ++---
.../apache/iotdb/db/pipe/event/EnrichedEvent.java | 26 +-
.../db/pipe/event/realtime/PipeRealtimeEvent.java | 10 +-
.../PipeHistoricalDataRegionTsFileExtractor.java | 7 +-
.../realtime/PipeRealtimeDataRegionExtractor.java | 49 ++-
.../PipeRealtimeDataRegionFakeExtractor.java | 2 +-
.../PipeRealtimeDataRegionHybridExtractor.java | 17 +-
.../PipeRealtimeDataRegionLogExtractor.java | 17 +-
.../PipeRealtimeDataRegionTsFileExtractor.java | 17 +-
.../matcher/CachedSchemaPatternMatcher.java | 6 +-
.../pipe/task/connection/BlockingPendingQueue.java | 5 +
.../pipe/task/connection/PipeEventCollector.java | 12 +-
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 7 +-
.../subtask/connector/PipeConnectorSubtask.java | 15 +-
.../connector/PipeConnectorSubtaskManager.java | 7 +-
.../subtask/processor/PipeProcessorSubtask.java | 31 +-
.../iotdb/db/protocol/client/ConfigNodeClient.java | 26 +-
.../db/protocol/client/DataNodeInternalClient.java | 2 +-
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 2 +-
.../protocol/thrift/impl/MLNodeRPCServiceImpl.java | 14 +-
.../common/header/ColumnHeaderConstant.java | 9 +-
.../common/header/DatasetHeaderFactory.java | 4 +-
.../fragment/FragmentInstanceManager.java | 11 +
.../execution/load/AlignedChunkData.java | 2 +-
.../execution/load/NonAlignedChunkData.java | 2 +-
.../queryengine/execution/load/TsFileSplitter.java | 18 +-
.../execution/operator/AggregationUtil.java | 2 +-
.../operator/process/AbstractIntoOperator.java | 2 +-
.../operator/process/ml/ForecastOperator.java | 241 ++++++++++++
.../db/queryengine/plan/analyze/Analysis.java | 11 +
.../queryengine/plan/analyze/AnalyzeVisitor.java | 147 ++++++--
.../plan/analyze/ExpressionAnalyzer.java | 7 +
.../plan/analyze/ExpressionTypeAnalyzer.java | 2 +
.../plan/analyze/IModelFetcher.java} | 10 +-
.../db/queryengine/plan/analyze/ModelFetcher.java | 75 ++++
.../plan/execution/config/ConfigTaskVisitor.java | 8 +-
.../config/executor/ClusterConfigTaskExecutor.java | 259 ++++++++-----
.../config/executor/IConfigTaskExecutor.java | 2 +-
.../config/metadata/model/ShowModelsTask.java | 12 +-
.../{ShowTrailsTask.java => ShowTrialsTask.java} | 10 +-
.../plan/expression/ExpressionFactory.java | 9 +
.../plan/expression/multi/FunctionExpression.java | 10 +
.../plan/expression/multi/FunctionType.java | 3 +-
.../visitor/RemoveRootPrefixVisitor.java | 45 +++
.../db/queryengine/plan/parser/ASTVisitor.java | 84 +++--
.../plan/parser/StatementGenerator.java | 50 +--
.../plan/planner/LogicalPlanBuilder.java | 10 +
.../plan/planner/LogicalPlanVisitor.java | 20 +
.../plan/planner/OperatorTreeGenerator.java | 49 +++
.../plan/planner/plan/node/PlanGraphPrinter.java | 12 +
.../plan/planner/plan/node/PlanNodeType.java | 5 +-
.../plan/planner/plan/node/PlanVisitor.java | 5 +
.../plan/node/load/LoadSingleTsFileNode.java | 4 +-
.../planner/plan/node/process/ml/ForecastNode.java | 122 ++++++
.../plan/planner/plan/node/write/InsertNode.java | 1 +
.../planner/plan/node/write/InsertRowNode.java | 4 +-
.../planner/plan/node/write/InsertRowsNode.java | 2 +-
.../plan/node/write/InsertRowsOfOneDeviceNode.java | 2 +-
.../planner/plan/node/write/InsertTabletNode.java | 31 +-
.../model/ForecastModelInferenceDescriptor.java | 176 +++++++++
.../parameter/model/ModelInferenceDescriptor.java | 111 ++++++
.../plan/statement/StatementVisitor.java | 6 +-
.../plan/statement/component/ResultColumn.java | 5 +-
.../plan/statement/component/SelectComponent.java | 10 +-
.../plan/statement/crud/InsertRowStatement.java | 2 +-
.../crud/InsertRowsOfOneDeviceStatement.java | 3 +-
.../plan/statement/crud/InsertTabletStatement.java | 15 +-
.../plan/statement/crud/QueryStatement.java | 52 ++-
.../metadata/model/CreateModelStatement.java | 71 ++--
...ailsStatement.java => ShowTrialsStatement.java} | 6 +-
.../apache/iotdb/db/service/MLNodeRPCService.java | 8 +-
.../iotdb/db/storageengine/StorageEngine.java | 4 +-
.../execute/task/CrossSpaceCompactionTask.java | 8 +-
.../execute/task/InnerSpaceCompactionTask.java | 7 +-
.../compaction/execute/utils/CompactionUtils.java | 59 +--
.../compaction/io/CompactionTsFileWriter.java | 34 +-
.../dataregion/compaction/tool/ITimeRange.java} | 13 +-
.../dataregion/compaction/tool/Interval.java} | 30 +-
.../compaction/tool/ListTimeRangeImpl.java | 89 +++++
.../compaction/tool/OverlapStatistic.java | 91 +++++
.../compaction/tool/OverlapStatisticTool.java | 248 +++++++++++++
.../dataregion/compaction/tool/PrintUtil.java | 209 +++++++++++
.../tool/SequenceFileSubTaskThreadExecutor.java} | 22 +-
.../compaction/tool/SequenceFileTaskSummary.java | 58 +++
.../compaction/tool/SingleSequenceFileTask.java | 104 ++++++
.../compaction/tool/TimePartitionProcessTask.java | 143 ++++++++
.../tool/TimePartitionProcessWorker.java | 58 +++
.../compaction/tool/TsFileStatisticReader.java | 96 +++++
.../compaction/tool/UnseqSpaceStatistics.java | 84 +++++
.../storageengine/dataregion/wal/WALManager.java | 27 +-
.../storageengine/dataregion/wal/node/WALNode.java | 8 +-
.../storageengine/rescon/disk/FolderManager.java | 2 +
.../TsFileOverlapValidationAndRepairTool.java | 276 ++++++++++++++
.../apache/iotdb/db/utils/TimePartitionUtils.java | 24 +-
.../iotdb/db/utils/TimestampPrecisionUtils.java | 10 +-
.../iotdb/db/utils/constant/SqlConstant.java | 3 +
.../extractor/CachedSchemaPatternMatcherTest.java | 2 +-
.../plan/parser/StatementGeneratorTest.java | 9 +-
.../plan/node/write/WritePlanNodeSplitTest.java | 28 +-
.../iotdb/db/storageengine/StorageEngineTest.java | 13 +
.../compaction/tools/ListTimeRangeImplTest.java | 138 +++++++
.../compaction/tools/UnseqSpaceStatisticsTest.java | 63 ++++
.../utils/CompactionUpdateFileCountTest.java | 123 +++++++
.../TsFileOverlapValidationAndRepairToolTest.java | 193 ++++++++++
iotdb-core/mlnode/.gitignore | 3 +-
iotdb-core/mlnode/README.md | 4 +-
.../mlnode/iotdb/mlnode/algorithm/factory.py | 66 ++++
.../iotdb/mlnode/algorithm/hyperparameter.py | 407 +++++++++++++++++++++
iotdb-core/mlnode/iotdb/mlnode/algorithm/metric.py | 80 ++++
.../mlnode/algorithm/models/forecast/__init__.py} | 10 -
.../mlnode/algorithm/models/forecast/dlinear.py | 133 +++++++
.../mlnode/algorithm/models/forecast/nbeats.py | 131 +++++++
.../mlnode/iotdb/mlnode/algorithm/validator.py | 45 +++
iotdb-core/mlnode/iotdb/mlnode/client.py | 111 ++----
iotdb-core/mlnode/iotdb/mlnode/config.py | 36 ++
iotdb-core/mlnode/iotdb/mlnode/constant.py | 68 +++-
iotdb-core/mlnode/iotdb/mlnode/dataset/dataset.py | 74 ++++
iotdb-core/mlnode/iotdb/mlnode/dataset/factory.py | 50 +++
iotdb-core/mlnode/iotdb/mlnode/dataset/source.py | 80 ++++
.../mlnode/dataset/utils/__init__.py} | 10 -
.../iotdb/mlnode/dataset/utils/time_features.py | 149 ++++++++
iotdb-core/mlnode/iotdb/mlnode/exception.py | 39 +-
iotdb-core/mlnode/iotdb/mlnode/handler.py | 63 +++-
iotdb-core/mlnode/iotdb/mlnode/parser.py | 99 +++++
.../mlnode/process/__init__.py} | 10 -
iotdb-core/mlnode/iotdb/mlnode/process/manager.py | 124 +++++++
iotdb-core/mlnode/iotdb/mlnode/process/task.py | 265 ++++++++++++++
iotdb-core/mlnode/iotdb/mlnode/process/trial.py | 224 ++++++++++++
iotdb-core/mlnode/iotdb/mlnode/serde.py | 147 ++++++--
iotdb-core/mlnode/iotdb/mlnode/service.py | 4 -
iotdb-core/mlnode/iotdb/mlnode/storage.py | 28 +-
iotdb-core/mlnode/iotdb/mlnode/util.py | 25 +-
iotdb-core/mlnode/requirements.txt | 7 +-
iotdb-core/mlnode/resources/conf/iotdb-mlnode.toml | 14 +-
.../mlnode/test/test_create_forecast_model.py | 100 +++++
iotdb-core/mlnode/test/test_model_storage.py | 17 +-
iotdb-core/mlnode/test/test_serde.py | 8 +-
.../resources/conf/iotdb-common.properties | 10 +
.../iotdb/commons/client/ClientPoolFactory.java | 26 ++
.../iotdb/commons/client/mlnode}/MLNodeClient.java | 144 ++++++--
.../client/mlnode/MLNodeClientManager.java} | 31 +-
.../iotdb/commons/client/mlnode/MLNodeInfo.java} | 12 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 13 +-
.../iotdb/commons/conf/CommonDescriptor.java | 5 +
.../commons/model/ForecastModeInformation.java | 153 ++++++++
.../iotdb/commons/model/ModelInformation.java | 235 ++++++------
...TrailInformation.java => TrialInformation.java} | 34 +-
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 2 +
.../plugin/builtin/connector/OpcUaConnector.java} | 15 +-
.../udf/builtin/ModelInferenceFunction.java} | 46 ++-
.../commons/udf/service/UDFManagementService.java | 19 +
.../tsfile/file/metadata/enums/TSDataType.java | 10 +
.../iotdb/tsfile/read/common/block/TsBlock.java | 20 +-
.../tsfile/read/common/block/TsBlockBuilder.java | 4 +-
.../thrift-commons/src/main/thrift/common.thrift | 5 +-
.../src/main/thrift/confignode.thrift | 43 ++-
.../src/main/thrift/datanode.thrift | 7 +-
.../thrift-mlnode/src/main/thrift/mlnode.thrift | 12 +-
.../library/series/UDTFConsecutiveSequences.java | 23 +-
.../library/series/UDTFConsecutiveWindows.java | 23 +-
.../iotdb/library/series/util/ConsecutiveUtil.java | 23 +-
pom.xml | 2 +
237 files changed, 8848 insertions(+), 1752 deletions(-)