You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2023/10/31 00:23:08 UTC
(iotdb) branch master updated: Remove IoTDB-ML Module
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 100c7b9afc2 Remove IoTDB-ML Module
100c7b9afc2 is described below
commit 100c7b9afc2bd42c089bce8c6866ec32e4e8ca90
Author: YangCaiyin <yc...@gmail.com>
AuthorDate: Tue Oct 31 08:23:02 2023 +0800
Remove IoTDB-ML Module
---
.github/workflows/iotdb-ml.yml | 61 ---
.gitignore | 3 -
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 6 +-
.../org/apache/iotdb/db/qp/sql/IdentifierParser.g4 | 3 -
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 44 --
.../antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 | 12 -
.../confignode/client/DataNodeRequestType.java | 3 -
.../client/sync/SyncDataNodeClientPool.java | 3 -
.../consensus/request/ConfigPhysicalPlan.java | 28 --
.../consensus/request/ConfigPhysicalPlanType.java | 9 +-
.../request/read/model/GetModelInfoPlan.java | 79 ----
.../request/read/model/ShowModelPlan.java | 89 ----
.../request/read/model/ShowTrialPlan.java | 97 ----
.../request/write/model/CreateModelPlan.java | 78 ----
.../request/write/model/DropModelPlan.java | 81 ----
.../request/write/model/UpdateModelInfoPlan.java | 98 ----
.../request/write/model/UpdateModelStatePlan.java | 105 -----
.../consensus/response/model/GetModelInfoResp.java | 46 --
.../consensus/response/model/ModelTableResp.java | 55 ---
.../consensus/response/model/TrialTableResp.java | 55 ---
.../iotdb/confignode/manager/ConfigManager.java | 78 ----
.../apache/iotdb/confignode/manager/IManager.java | 38 --
.../iotdb/confignode/manager/ModelManager.java | 204 ---------
.../iotdb/confignode/manager/ProcedureManager.java | 31 --
.../iotdb/confignode/persistence/ModelInfo.java | 278 -----------
.../persistence/executor/ConfigPlanExecutor.java | 28 --
.../procedure/impl/model/CreateModelProcedure.java | 223 ---------
.../procedure/impl/model/DropModelProcedure.java | 209 ---------
.../procedure/state/model/CreateModelState.java | 25 -
.../procedure/state/model/DropModelState.java | 28 --
.../procedure/store/ProcedureFactory.java | 12 -
.../confignode/procedure/store/ProcedureType.java | 4 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 45 --
iotdb-core/datanode/pom.xml | 5 -
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 22 -
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 13 -
.../runtime/ModelInferenceProcessException.java | 27 --
.../iotdb/db/protocol/client/ConfigNodeClient.java | 60 ---
.../handler/MLNodeRPCServiceThriftHandler.java | 60 ---
.../impl/DataNodeInternalRPCServiceImpl.java | 32 --
.../thrift/impl/IMLNodeRPCServiceWithHandler.java | 26 --
.../protocol/thrift/impl/MLNodeRPCServiceImpl.java | 217 ---------
.../common/header/ColumnHeaderConstant.java | 24 -
.../common/header/DatasetHeaderFactory.java | 8 -
.../fragment/FragmentInstanceManager.java | 11 -
.../operator/process/ml/ForecastOperator.java | 241 ----------
.../db/queryengine/plan/analyze/Analysis.java | 11 -
.../queryengine/plan/analyze/AnalyzeVisitor.java | 107 -----
.../plan/analyze/ExpressionAnalyzer.java | 2 -
.../plan/analyze/ExpressionTypeAnalyzer.java | 2 -
.../db/queryengine/plan/analyze/IModelFetcher.java | 27 --
.../db/queryengine/plan/analyze/ModelFetcher.java | 75 ---
.../plan/execution/config/ConfigTaskVisitor.java | 33 --
.../config/executor/ClusterConfigTaskExecutor.java | 166 -------
.../config/executor/IConfigTaskExecutor.java | 10 -
.../config/metadata/model/CreateModelTask.java | 45 --
.../config/metadata/model/DropModelTask.java | 41 --
.../config/metadata/model/ShowModelsTask.java | 100 ----
.../config/metadata/model/ShowTrialsTask.java | 90 ----
.../plan/expression/multi/FunctionExpression.java | 10 -
.../plan/expression/multi/FunctionType.java | 3 +-
.../db/queryengine/plan/parser/ASTVisitor.java | 67 ---
.../plan/parser/StatementGenerator.java | 43 --
.../plan/planner/LogicalPlanBuilder.java | 10 -
.../plan/planner/LogicalPlanVisitor.java | 20 -
.../plan/planner/OperatorTreeGenerator.java | 49 --
.../plan/planner/plan/node/PlanGraphPrinter.java | 12 -
.../plan/planner/plan/node/PlanNodeType.java | 7 +-
.../plan/planner/plan/node/PlanVisitor.java | 5 -
.../planner/plan/node/process/ml/ForecastNode.java | 122 -----
.../model/ForecastModelInferenceDescriptor.java | 176 -------
.../parameter/model/ModelInferenceDescriptor.java | 111 -----
.../plan/statement/StatementVisitor.java | 21 -
.../plan/statement/component/ResultColumn.java | 3 +-
.../plan/statement/component/SelectComponent.java | 8 -
.../plan/statement/crud/QueryStatement.java | 50 --
.../metadata/model/CreateModelStatement.java | 128 ------
.../metadata/model/DropModelStatement.java | 57 ---
.../metadata/model/ShowModelsStatement.java | 51 ---
.../metadata/model/ShowTrialsStatement.java | 57 ---
.../java/org/apache/iotdb/db/service/DataNode.java | 4 -
.../apache/iotdb/db/service/MLNodeRPCService.java | 92 ----
.../iotdb/db/service/MLNodeRPCServiceMBean.java | 22 -
.../iotdb/db/utils/constant/SqlConstant.java | 3 -
.../plan/parser/StatementGeneratorTest.java | 32 --
iotdb-core/mlnode/.flake8 | 33 --
iotdb-core/mlnode/.gitignore | 9 -
iotdb-core/mlnode/README.md | 47 --
iotdb-core/mlnode/iotdb/__init__.py | 17 -
iotdb-core/mlnode/iotdb/mlnode/__init__.py | 17 -
.../mlnode/iotdb/mlnode/algorithm/factory.py | 66 ---
.../iotdb/mlnode/algorithm/hyperparameter.py | 407 -----------------
iotdb-core/mlnode/iotdb/mlnode/algorithm/metric.py | 80 ----
.../mlnode/algorithm/models/forecast/__init__.py | 17 -
.../mlnode/algorithm/models/forecast/dlinear.py | 133 ------
.../mlnode/algorithm/models/forecast/nbeats.py | 131 ------
.../mlnode/iotdb/mlnode/algorithm/validator.py | 45 --
iotdb-core/mlnode/iotdb/mlnode/client.py | 274 -----------
iotdb-core/mlnode/iotdb/mlnode/config.py | 168 -------
iotdb-core/mlnode/iotdb/mlnode/constant.py | 99 ----
iotdb-core/mlnode/iotdb/mlnode/dataset/dataset.py | 74 ---
iotdb-core/mlnode/iotdb/mlnode/dataset/factory.py | 50 --
iotdb-core/mlnode/iotdb/mlnode/dataset/source.py | 80 ----
.../mlnode/iotdb/mlnode/dataset/utils/__init__.py | 17 -
.../iotdb/mlnode/dataset/utils/time_features.py | 149 ------
iotdb-core/mlnode/iotdb/mlnode/exception.py | 66 ---
iotdb-core/mlnode/iotdb/mlnode/handler.py | 95 ----
iotdb-core/mlnode/iotdb/mlnode/log.py | 34 --
iotdb-core/mlnode/iotdb/mlnode/parser.py | 99 ----
iotdb-core/mlnode/iotdb/mlnode/process/__init__.py | 17 -
iotdb-core/mlnode/iotdb/mlnode/process/manager.py | 124 -----
iotdb-core/mlnode/iotdb/mlnode/process/task.py | 265 -----------
iotdb-core/mlnode/iotdb/mlnode/process/trial.py | 224 ---------
iotdb-core/mlnode/iotdb/mlnode/script.py | 36 --
iotdb-core/mlnode/iotdb/mlnode/serde.py | 508 ---------------------
iotdb-core/mlnode/iotdb/mlnode/service.py | 58 ---
iotdb-core/mlnode/iotdb/mlnode/storage.py | 110 -----
iotdb-core/mlnode/iotdb/mlnode/util.py | 80 ----
iotdb-core/mlnode/pom.xml | 175 -------
iotdb-core/mlnode/pyproject.toml | 55 ---
iotdb-core/mlnode/requirements.txt | 30 --
iotdb-core/mlnode/requirements_dev.txt | 28 --
iotdb-core/mlnode/resources/conf/iotdb-mlnode.toml | 70 ---
.../mlnode/resources/conf/logging_config.ini | 40 --
.../mlnode/test/test_create_forecast_model.py | 100 ----
iotdb-core/mlnode/test/test_model_storage.py | 100 ----
iotdb-core/mlnode/test/test_serde.py | 171 -------
iotdb-core/node-commons/pom.xml | 5 -
.../iotdb/commons/client/ClientPoolFactory.java | 26 --
.../iotdb/commons/client/mlnode/MLNodeClient.java | 212 ---------
.../commons/client/mlnode/MLNodeClientManager.java | 40 --
.../iotdb/commons/client/mlnode/MLNodeInfo.java | 29 --
.../iotdb/commons/concurrent/ThreadName.java | 2 -
.../apache/iotdb/commons/conf/CommonConfig.java | 23 -
.../iotdb/commons/conf/CommonDescriptor.java | 13 -
.../commons/model/ForecastModeInformation.java | 153 -------
.../iotdb/commons/model/ModelHyperparameter.java | 81 ----
.../iotdb/commons/model/ModelInformation.java | 261 -----------
.../org/apache/iotdb/commons/model/ModelTable.java | 91 ----
.../iotdb/commons/model/TrialInformation.java | 108 -----
.../model/exception/ModelManagementException.java | 26 --
.../apache/iotdb/commons/service/ServiceType.java | 1 -
.../udf/builtin/ModelInferenceFunction.java | 49 --
.../commons/udf/service/UDFManagementService.java | 19 -
iotdb-core/pom.xml | 1 -
iotdb-protocol/pom.xml | 1 -
.../thrift-commons/src/main/thrift/common.thrift | 14 -
.../src/main/thrift/confignode.thrift | 99 ----
.../src/main/thrift/datanode.thrift | 98 ----
iotdb-protocol/thrift-mlnode/pom.xml | 69 ---
.../thrift-mlnode/src/main/thrift/mlnode.thrift | 60 ---
pom.xml | 2 +-
152 files changed, 9 insertions(+), 10648 deletions(-)
diff --git a/.github/workflows/iotdb-ml.yml b/.github/workflows/iotdb-ml.yml
deleted file mode 100644
index a107e8b1444..00000000000
--- a/.github/workflows/iotdb-ml.yml
+++ /dev/null
@@ -1,61 +0,0 @@
-# This workflow is just for checking whether modifications works for the MLNode
-
-name: MLNode
-
-on:
- push:
- branches:
- - master
- - 'rel/*'
- - "new_*"
- paths-ignore:
- - 'docs/**'
- - 'site/**'
- pull_request:
- branches:
- - master
- - 'rel/*'
- - "new_*"
- paths-ignore:
- - 'docs/**'
- - 'site/**'
- # allow manually run the action:
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
- GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
-
-jobs:
- unix:
- strategy:
- fail-fast: false
- max-parallel: 20
- runs-on: ubuntu-latest
-
- steps:
- - uses: actions/checkout@v3
- - name: Cache Maven packages
- uses: actions/cache@v3
- with:
- path: ~/.m2
- key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
- restore-keys: ${{ runner.os }}-m2-
- - name: Build IoTDB server distribution zip
- run: mvn -B clean install -pl distribution -am -DskipTests
- - name: Build IoTDB server docker image
- run: |
- docker build . -f docker/src/main/Dockerfile-1c1d -t "iotdb:dev"
- docker images
- - name: Install IoTDB mlnode requirements
- run: pip3 install -r iotdb-core/mlnode/requirements_dev.txt
- - name: Build MLNode
- run: mvn clean package -DskipUTs -pl iotdb-core/mlnode -am && pip3 install poetry && cd iotdb-core/mlnode && poetry build && pip install dist/apache_iotdb_mlnode-1.0.0-py3-none-any.whl --force-reinstall
- - name: Integration test
- shell: bash
- run: |
- pytest iotdb-core/mlnode
diff --git a/.gitignore b/.gitignore
index 779ea5cfd62..b33d12e672b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,8 +44,6 @@ tsfile-jdbc/src/main/resources/output/queryRes.csv
!iotdb-client/client-cpp/src/test/CMakeLists.txt
!example/rest-client-c-example/CMakeLists.txt
!example/client-cpp-example/src/CMakeLists.txt
-!iotdb-core/mlnode/requirements.txt
-!iotdb-core/mlnode/requirements_dev.txt
*.jar
@@ -121,7 +119,6 @@ Makefile
# Exclude copied license
iotdb-client/client-py/LICENSE
-iotdb-core/mlnode/LICENSE
# ANTLR
iotdb-core/antlr/gen/
diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
index 475a4ec0c1e..0283acd556b 100644
--- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
+++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
@@ -189,11 +189,7 @@ public enum TSStatusCode {
CQ_ALREADY_EXIST(1402),
CQ_UPDATE_LAST_EXEC_TIME_ERROR(1403),
- // ML Model
- CREATE_MODEL_ERROR(1500),
- DROP_MODEL_ERROR(1501),
- MODEL_EXIST_ERROR(1502),
- MLNODE_INTERNAL_ERROR(1510),
+ // code 1500-1599 are used by IoTDB-ML
// Pipe Plugin
CREATE_PIPE_PLUGIN_ERROR(1600),
diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4
index e92b7b99dfa..5c293ab8728 100644
--- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4
+++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IdentifierParser.g4
@@ -132,8 +132,6 @@ keyWords
| MERGE
| METADATA
| MIGRATE
- | MODEL
- | MODELS
| NAN
| NODEID
| NODES
@@ -227,7 +225,6 @@ keyWords
| TOLERANCE
| TOP
| TRACING
- | TRIALS
| TRIGGER
| TRIGGERS
| TRUE
diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
index fefed050b27..9524a64c7f3 100644
--- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
+++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4
@@ -61,8 +61,6 @@ ddlStatement
// Cluster
| showVariables | showCluster | showRegions | showDataNodes | showConfigNodes
| getRegionId | getTimeSlotList | countTimeSlotList | getSeriesSlotList | migrateRegion
- // ML Model
- | createModel | dropModel | showModels | showTrials
// Quota
| setSpaceQuota | showSpaceQuota | setThrottleQuota | showThrottleQuota
// View
@@ -582,48 +580,6 @@ showPipePlugins
: SHOW PIPEPLUGINS
;
-// ML Model =========================================================================================
-// ---- Create Model
-createModel
- : CREATE MODEL modelId=identifier
- OPTIONS LR_BRACKET attributePair (COMMA attributePair)* RR_BRACKET
- (WITH HYPERPARAMETERS LR_BRACKET hparamPair (COMMA hparamPair)* RR_BRACKET)?
- ON DATASET LR_BRACKET selectStatement RR_BRACKET
- ;
-
-hparamPair
- : hparamKey=attributeKey operator_eq hparamValue
- ;
-
-hparamValue
- : attributeValue
- | hparamRange
- | hparamCandidates
- ;
-
-hparamRange
- : LR_BRACKET hparamRangeStart=attributeValue COMMA hparamRangeEnd=attributeValue RR_BRACKET
- ;
-
-hparamCandidates
- : LS_BRACKET attributeValue (COMMA attributeValue)* RS_BRACKET
- ;
-
-// ---- Drop Model
-dropModel
- : DROP MODEL modelId=identifier
- ;
-
-// ---- Show Models
-showModels
- : SHOW MODELS
- ;
-
-// ---- Show Trials
-showTrials
- : SHOW TRIALS modelId=identifier
- ;
-
// Create Logical View
createLogicalView
: CREATE VIEW viewTargetPaths AS viewSourcePaths
diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
index d98d9b1cbcd..6f66d77bc30 100644
--- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
+++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/SqlLexer.g4
@@ -434,14 +434,6 @@ MIGRATE
: M I G R A T E
;
-MODEL
- : M O D E L
- ;
-
-MODELS
- : M O D E L S
- ;
-
NAN
: N A N
;
@@ -806,10 +798,6 @@ TRACING
: T R A C I N G
;
-TRIALS
- : T R I A L S
- ;
-
TRIGGER
: T R I G G E R
;
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java
index 6f5743cf826..a0f73b456e5 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java
@@ -102,9 +102,6 @@ public enum DataNodeRequestType {
// TODO Need to migrate to Node Maintenance
KILL_QUERY_INSTANCE,
- // ML Model
- DELETE_MODEL_METRICS,
-
// Quota
SET_SPACE_QUOTA,
SET_THROTTLE_QUOTA,
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java
index 5b6de0c0189..7b1549d44cf 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java
@@ -31,7 +31,6 @@ import org.apache.iotdb.confignode.client.DataNodeRequestType;
import org.apache.iotdb.mpp.rpc.thrift.TCreateDataRegionReq;
import org.apache.iotdb.mpp.rpc.thrift.TCreatePeerReq;
import org.apache.iotdb.mpp.rpc.thrift.TCreateSchemaRegionReq;
-import org.apache.iotdb.mpp.rpc.thrift.TDeleteModelMetricsReq;
import org.apache.iotdb.mpp.rpc.thrift.TDisableDataNodeReq;
import org.apache.iotdb.mpp.rpc.thrift.TInvalidateCacheReq;
import org.apache.iotdb.mpp.rpc.thrift.TInvalidatePermissionCacheReq;
@@ -135,8 +134,6 @@ public class SyncDataNodeClientPool {
return client.removeRegionPeer((TMaintainPeerReq) req);
case DELETE_OLD_REGION_PEER:
return client.deleteOldRegionPeer((TMaintainPeerReq) req);
- case DELETE_MODEL_METRICS:
- return client.deleteModelMetrics((TDeleteModelMetricsReq) req);
default:
return RpcUtils.getStatus(
TSStatusCode.EXECUTE_STATEMENT_ERROR, "Unknown request type: " + requestType);
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java
index 02cf6d8c47a..1604740dea8 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java
@@ -26,9 +26,6 @@ import org.apache.iotdb.confignode.consensus.request.read.database.GetDatabasePl
import org.apache.iotdb.confignode.consensus.request.read.datanode.GetDataNodeConfigurationPlan;
import org.apache.iotdb.confignode.consensus.request.read.function.GetFunctionTablePlan;
import org.apache.iotdb.confignode.consensus.request.read.function.GetUDFJarPlan;
-import org.apache.iotdb.confignode.consensus.request.read.model.GetModelInfoPlan;
-import org.apache.iotdb.confignode.consensus.request.read.model.ShowModelPlan;
-import org.apache.iotdb.confignode.consensus.request.read.model.ShowTrialPlan;
import org.apache.iotdb.confignode.consensus.request.read.partition.CountTimeSlotListPlan;
import org.apache.iotdb.confignode.consensus.request.read.partition.GetDataPartitionPlan;
import org.apache.iotdb.confignode.consensus.request.read.partition.GetNodePathsPartitionPlan;
@@ -73,10 +70,6 @@ import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNo
import org.apache.iotdb.confignode.consensus.request.write.datanode.UpdateDataNodePlan;
import org.apache.iotdb.confignode.consensus.request.write.function.CreateFunctionPlan;
import org.apache.iotdb.confignode.consensus.request.write.function.DropFunctionPlan;
-import org.apache.iotdb.confignode.consensus.request.write.model.CreateModelPlan;
-import org.apache.iotdb.confignode.consensus.request.write.model.DropModelPlan;
-import org.apache.iotdb.confignode.consensus.request.write.model.UpdateModelInfoPlan;
-import org.apache.iotdb.confignode.consensus.request.write.model.UpdateModelStatePlan;
import org.apache.iotdb.confignode.consensus.request.write.partition.CreateDataPartitionPlan;
import org.apache.iotdb.confignode.consensus.request.write.partition.CreateSchemaPartitionPlan;
import org.apache.iotdb.confignode.consensus.request.write.partition.UpdateRegionLocationPlan;
@@ -464,27 +457,6 @@ public abstract class ConfigPhysicalPlan implements IConsensusRequest {
case GetFunctionJar:
plan = new GetUDFJarPlan();
break;
- case CreateModel:
- plan = new CreateModelPlan();
- break;
- case UpdateModelInfo:
- plan = new UpdateModelInfoPlan();
- break;
- case UpdateModelState:
- plan = new UpdateModelStatePlan();
- break;
- case DropModel:
- plan = new DropModelPlan();
- break;
- case ShowModel:
- plan = new ShowModelPlan();
- break;
- case ShowTrial:
- plan = new ShowTrialPlan();
- break;
- case GetModelInfo:
- plan = new GetModelInfoPlan();
- break;
case CreatePipePlugin:
plan = new CreatePipePluginPlan();
break;
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java
index 75647e378dd..c2f281d310a 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java
@@ -179,14 +179,7 @@ public enum ConfigPhysicalPlanType {
UPDATE_CQ_LAST_EXEC_TIME((short) 1103),
SHOW_CQ((short) 1104),
- /** Ml model. */
- CreateModel((short) 1200),
- UpdateModelInfo((short) 1201),
- UpdateModelState((short) 1202),
- DropModel((short) 1203),
- ShowModel((short) 1204),
- ShowTrial((short) 1205),
- GetModelInfo((short) 1206),
+ // 1200-1299 planId is used by IoTDB-ML.
/** Pipe Plugin. */
CreatePipePlugin((short) 1300),
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/GetModelInfoPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/GetModelInfoPlan.java
deleted file mode 100644
index 79c7e9e0142..00000000000
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/GetModelInfoPlan.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.consensus.request.read.model;
-
-import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan;
-import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType;
-import org.apache.iotdb.confignode.rpc.thrift.TGetModelInfoReq;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Objects;
-
-public class GetModelInfoPlan extends ConfigPhysicalPlan {
-
- private String modelId;
-
- public GetModelInfoPlan() {
- super(ConfigPhysicalPlanType.GetModelInfo);
- }
-
- public GetModelInfoPlan(TGetModelInfoReq getModelInfoReq) {
- super(ConfigPhysicalPlanType.GetModelInfo);
- this.modelId = getModelInfoReq.getModelId();
- }
-
- public String getModelId() {
- return modelId;
- }
-
- @Override
- protected void serializeImpl(DataOutputStream stream) throws IOException {
- stream.writeShort(getType().getPlanType());
- ReadWriteIOUtils.write(modelId, stream);
- }
-
- @Override
- protected void deserializeImpl(ByteBuffer buffer) throws IOException {
- this.modelId = ReadWriteIOUtils.readString(buffer);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- if (!super.equals(o)) {
- return false;
- }
- GetModelInfoPlan that = (GetModelInfoPlan) o;
- return Objects.equals(modelId, that.modelId);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(super.hashCode(), modelId);
- }
-}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/ShowModelPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/ShowModelPlan.java
deleted file mode 100644
index 3fe4bf0af7e..00000000000
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/ShowModelPlan.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.consensus.request.read.model;
-
-import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan;
-import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType;
-import org.apache.iotdb.confignode.rpc.thrift.TShowModelReq;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Objects;
-
-public class ShowModelPlan extends ConfigPhysicalPlan {
-
- private String modelId;
-
- public ShowModelPlan() {
- super(ConfigPhysicalPlanType.ShowModel);
- }
-
- public ShowModelPlan(TShowModelReq showModelReq) {
- super(ConfigPhysicalPlanType.ShowModel);
- if (showModelReq.isSetModelId()) {
- this.modelId = showModelReq.getModelId();
- }
- }
-
- public boolean isSetModelId() {
- return modelId != null;
- }
-
- public String getModelId() {
- return modelId;
- }
-
- @Override
- protected void serializeImpl(DataOutputStream stream) throws IOException {
- stream.writeShort(getType().getPlanType());
- ReadWriteIOUtils.write(modelId != null, stream);
- ReadWriteIOUtils.write(modelId, stream);
- }
-
- @Override
- protected void deserializeImpl(ByteBuffer buffer) throws IOException {
- boolean isSetModelId = ReadWriteIOUtils.readBool(buffer);
- if (isSetModelId) {
- this.modelId = ReadWriteIOUtils.readString(buffer);
- }
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- if (!super.equals(o)) {
- return false;
- }
- ShowModelPlan that = (ShowModelPlan) o;
- return Objects.equals(modelId, that.modelId);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(super.hashCode(), modelId);
- }
-}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/ShowTrialPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/ShowTrialPlan.java
deleted file mode 100644
index 136e1c2408d..00000000000
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/ShowTrialPlan.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.consensus.request.read.model;
-
-import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan;
-import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType;
-import org.apache.iotdb.confignode.rpc.thrift.TShowTrialReq;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Objects;
-
-public class ShowTrialPlan extends ConfigPhysicalPlan {
-
- private String modelId;
- private String trialId;
-
- public ShowTrialPlan() {
- super(ConfigPhysicalPlanType.ShowTrial);
- }
-
- public ShowTrialPlan(TShowTrialReq showTrailReq) {
- super(ConfigPhysicalPlanType.ShowTrial);
- this.modelId = showTrailReq.getModelId();
- if (showTrailReq.isSetTrialId()) {
- this.trialId = showTrailReq.getTrialId();
- }
- }
-
- public String getModelId() {
- return modelId;
- }
-
- public boolean isSetTrialId() {
- return trialId != null;
- }
-
- public String getTrialId() {
- return trialId;
- }
-
- @Override
- protected void serializeImpl(DataOutputStream stream) throws IOException {
- stream.writeShort(getType().getPlanType());
- ReadWriteIOUtils.write(modelId, stream);
- ReadWriteIOUtils.write(trialId != null, stream);
- ReadWriteIOUtils.write(trialId, stream);
- }
-
- @Override
- protected void deserializeImpl(ByteBuffer buffer) throws IOException {
- this.modelId = ReadWriteIOUtils.readString(buffer);
- boolean isSetTrailId = ReadWriteIOUtils.readBool(buffer);
- if (isSetTrailId) {
- this.trialId = ReadWriteIOUtils.readString(buffer);
- }
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- if (!super.equals(o)) {
- return false;
- }
- ShowTrialPlan that = (ShowTrialPlan) o;
- return modelId.equals(that.modelId) && Objects.equals(trialId, that.trialId);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(super.hashCode(), modelId, trialId);
- }
-}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/CreateModelPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/CreateModelPlan.java
deleted file mode 100644
index 29422ef998b..00000000000
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/CreateModelPlan.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.consensus.request.write.model;
-
-import org.apache.iotdb.commons.model.ModelInformation;
-import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan;
-import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Objects;
-
-public class CreateModelPlan extends ConfigPhysicalPlan {
-
- private ModelInformation modelInformation;
-
- public CreateModelPlan() {
- super(ConfigPhysicalPlanType.CreateModel);
- }
-
- public CreateModelPlan(ModelInformation modelInformation) {
- super(ConfigPhysicalPlanType.CreateModel);
- this.modelInformation = modelInformation;
- }
-
- public ModelInformation getModelInformation() {
- return modelInformation;
- }
-
- @Override
- protected void serializeImpl(DataOutputStream stream) throws IOException {
- stream.writeShort(getType().getPlanType());
- modelInformation.serialize(stream);
- }
-
- @Override
- protected void deserializeImpl(ByteBuffer buffer) throws IOException {
- modelInformation = ModelInformation.deserialize(buffer);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- if (!super.equals(o)) {
- return false;
- }
- CreateModelPlan that = (CreateModelPlan) o;
- return Objects.equals(modelInformation, that.modelInformation);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(super.hashCode(), modelInformation);
- }
-}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/DropModelPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/DropModelPlan.java
deleted file mode 100644
index c94d7e786a1..00000000000
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/DropModelPlan.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.consensus.request.write.model;
-
-import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan;
-import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
-import org.apache.commons.lang3.Validate;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Objects;
-
-public class DropModelPlan extends ConfigPhysicalPlan {
-
- private String modelId;
-
- public DropModelPlan() {
- super(ConfigPhysicalPlanType.DropModel);
- }
-
- public DropModelPlan(String modelId) {
- super(ConfigPhysicalPlanType.DropModel);
- Validate.notNull(modelId);
- this.modelId = modelId;
- }
-
- public String getModelId() {
- return modelId;
- }
-
- @Override
- protected void serializeImpl(DataOutputStream stream) throws IOException {
- stream.writeShort(getType().getPlanType());
- ReadWriteIOUtils.write(modelId, stream);
- }
-
- @Override
- protected void deserializeImpl(ByteBuffer buffer) throws IOException {
- modelId = ReadWriteIOUtils.readString(buffer);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- if (!super.equals(o)) {
- return false;
- }
- DropModelPlan that = (DropModelPlan) o;
- return modelId.equals(that.modelId);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(super.hashCode(), modelId);
- }
-}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/UpdateModelInfoPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/UpdateModelInfoPlan.java
deleted file mode 100644
index 5261472668d..00000000000
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/UpdateModelInfoPlan.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.consensus.request.write.model;
-
-import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan;
-import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType;
-import org.apache.iotdb.confignode.rpc.thrift.TUpdateModelInfoReq;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Map;
-import java.util.Objects;
-
-public class UpdateModelInfoPlan extends ConfigPhysicalPlan {
-
- private String modelId;
- private String trialId;
- private Map<String, String> modelInfo;
-
- public UpdateModelInfoPlan() {
- super(ConfigPhysicalPlanType.UpdateModelInfo);
- }
-
- public UpdateModelInfoPlan(TUpdateModelInfoReq updateModelInfoReq) {
- super(ConfigPhysicalPlanType.UpdateModelInfo);
- this.modelId = updateModelInfoReq.getModelId();
- this.trialId = updateModelInfoReq.getTrialId();
- this.modelInfo = updateModelInfoReq.getModelInfo();
- }
-
- public String getModelId() {
- return modelId;
- }
-
- public String getTrialId() {
- return trialId;
- }
-
- public Map<String, String> getModelInfo() {
- return modelInfo;
- }
-
- @Override
- protected void serializeImpl(DataOutputStream stream) throws IOException {
- stream.writeShort(getType().getPlanType());
- ReadWriteIOUtils.write(modelId, stream);
- ReadWriteIOUtils.write(trialId, stream);
- ReadWriteIOUtils.write(modelInfo, stream);
- }
-
- @Override
- protected void deserializeImpl(ByteBuffer buffer) throws IOException {
- this.modelId = ReadWriteIOUtils.readString(buffer);
- this.trialId = ReadWriteIOUtils.readString(buffer);
- this.modelInfo = ReadWriteIOUtils.readMap(buffer);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- if (!super.equals(o)) {
- return false;
- }
- UpdateModelInfoPlan that = (UpdateModelInfoPlan) o;
- return modelId.equals(that.modelId)
- && trialId.equals(that.trialId)
- && modelInfo.equals(that.modelInfo);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(super.hashCode(), modelId, trialId, modelInfo);
- }
-}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/UpdateModelStatePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/UpdateModelStatePlan.java
deleted file mode 100644
index 6ac3fc1a5c1..00000000000
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/model/UpdateModelStatePlan.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.consensus.request.write.model;
-
-import org.apache.iotdb.common.rpc.thrift.TrainingState;
-import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan;
-import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType;
-import org.apache.iotdb.confignode.rpc.thrift.TUpdateModelStateReq;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Objects;
-
-public class UpdateModelStatePlan extends ConfigPhysicalPlan {
-
- private String modelId;
- private TrainingState state;
- private String bestTrialId;
-
- public UpdateModelStatePlan() {
- super(ConfigPhysicalPlanType.UpdateModelState);
- }
-
- public UpdateModelStatePlan(TUpdateModelStateReq updateModelStateReq) {
- super(ConfigPhysicalPlanType.UpdateModelState);
- this.modelId = updateModelStateReq.getModelId();
- this.state = updateModelStateReq.getState();
- this.bestTrialId = updateModelStateReq.getBestTrialId();
- }
-
- public String getModelId() {
- return modelId;
- }
-
- public TrainingState getState() {
- return state;
- }
-
- public String getBestTrialId() {
- return bestTrialId;
- }
-
- @Override
- protected void serializeImpl(DataOutputStream stream) throws IOException {
- stream.writeShort(getType().getPlanType());
- ReadWriteIOUtils.write(modelId, stream);
- ReadWriteIOUtils.write(state.getValue(), stream);
- boolean isNull = bestTrialId == null;
- ReadWriteIOUtils.write(isNull, stream);
- if (!isNull) {
- ReadWriteIOUtils.write(bestTrialId, stream);
- }
- }
-
- @Override
- protected void deserializeImpl(ByteBuffer buffer) throws IOException {
- this.modelId = ReadWriteIOUtils.readString(buffer);
- this.state = TrainingState.findByValue(ReadWriteIOUtils.readInt(buffer));
- boolean isNull = ReadWriteIOUtils.readBool(buffer);
- if (!isNull) {
- this.bestTrialId = ReadWriteIOUtils.readString(buffer);
- }
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- if (!super.equals(o)) {
- return false;
- }
- UpdateModelStatePlan that = (UpdateModelStatePlan) o;
- return modelId.equals(that.modelId)
- && state == that.state
- && Objects.equals(bestTrialId, that.bestTrialId);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(super.hashCode(), modelId, state, bestTrialId);
- }
-}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/model/GetModelInfoResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/model/GetModelInfoResp.java
deleted file mode 100644
index dd4a842d6ec..00000000000
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/model/GetModelInfoResp.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.consensus.response.model;
-
-import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.confignode.rpc.thrift.TGetModelInfoResp;
-import org.apache.iotdb.consensus.common.DataSet;
-
-import java.nio.ByteBuffer;
-
-public class GetModelInfoResp implements DataSet {
-
- private final TSStatus status;
- private ByteBuffer serializedModelInformation;
-
- public GetModelInfoResp(TSStatus status) {
- this.status = status;
- }
-
- public void setModelInfo(ByteBuffer serializedModelInformation) {
- this.serializedModelInformation = serializedModelInformation;
- }
-
- public TGetModelInfoResp convertToThriftResponse() {
- TGetModelInfoResp resp = new TGetModelInfoResp(status);
- resp.setModelInfo(serializedModelInformation);
- return resp;
- }
-}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/model/ModelTableResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/model/ModelTableResp.java
deleted file mode 100644
index 9a23d9ed713..00000000000
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/model/ModelTableResp.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.consensus.response.model;
-
-import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.commons.model.ModelInformation;
-import org.apache.iotdb.confignode.rpc.thrift.TShowModelResp;
-import org.apache.iotdb.consensus.common.DataSet;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-public class ModelTableResp implements DataSet {
-
- private final TSStatus status;
- private final List<ByteBuffer> serializedAllModelInformation;
-
- public ModelTableResp(TSStatus status) {
- this.status = status;
- this.serializedAllModelInformation = new ArrayList<>();
- }
-
- public void addModelInformation(List<ModelInformation> modelInformationList) throws IOException {
- for (ModelInformation modelInformation : modelInformationList) {
- this.serializedAllModelInformation.add(modelInformation.serializeShowModelResult());
- }
- }
-
- public void addModelInformation(ModelInformation modelInformation) throws IOException {
- this.serializedAllModelInformation.add(modelInformation.serializeShowModelResult());
- }
-
- public TShowModelResp convertToThriftResponse() throws IOException {
- return new TShowModelResp(status, serializedAllModelInformation);
- }
-}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/model/TrialTableResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/model/TrialTableResp.java
deleted file mode 100644
index 279703efa11..00000000000
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/model/TrialTableResp.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.consensus.response.model;
-
-import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.commons.model.TrialInformation;
-import org.apache.iotdb.confignode.rpc.thrift.TShowTrialResp;
-import org.apache.iotdb.consensus.common.DataSet;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-public class TrialTableResp implements DataSet {
-
- private final TSStatus status;
- private final List<ByteBuffer> serializedAllTrialInformation;
-
- public TrialTableResp(TSStatus status) {
- this.status = status;
- this.serializedAllTrialInformation = new ArrayList<>();
- }
-
- public void addTrialInformation(TrialInformation trialInformation) throws IOException {
- this.serializedAllTrialInformation.add(trialInformation.serializeShowTrialResult());
- }
-
- public void addTrialInformation(List<TrialInformation> trialInformationList) throws IOException {
- for (TrialInformation trialInformation : trialInformationList) {
- this.serializedAllTrialInformation.add(trialInformation.serializeShowTrialResult());
- }
- }
-
- public TShowTrialResp convertToThriftResponse() throws IOException {
- return new TShowTrialResp(status, serializedAllTrialInformation);
- }
-}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
index 511f2f9a65c..ba26a7e951a 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
@@ -91,7 +91,6 @@ import org.apache.iotdb.confignode.manager.pipe.PipeManager;
import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager;
import org.apache.iotdb.confignode.manager.schema.ClusterSchemaQuotaStatistics;
import org.apache.iotdb.confignode.persistence.AuthorInfo;
-import org.apache.iotdb.confignode.persistence.ModelInfo;
import org.apache.iotdb.confignode.persistence.ProcedureInfo;
import org.apache.iotdb.confignode.persistence.TriggerInfo;
import org.apache.iotdb.confignode.persistence.UDFInfo;
@@ -112,7 +111,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListReq;
import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListResp;
import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreateFunctionReq;
-import org.apache.iotdb.confignode.rpc.thrift.TCreateModelReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreatePipePluginReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreateSchemaTemplateReq;
@@ -126,7 +124,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TDeactivateSchemaTemplateReq;
import org.apache.iotdb.confignode.rpc.thrift.TDeleteLogicalViewReq;
import org.apache.iotdb.confignode.rpc.thrift.TDeleteTimeSeriesReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq;
-import org.apache.iotdb.confignode.rpc.thrift.TDropModelReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropTriggerReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetAllPipeInfoResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetAllTemplatesResp;
@@ -135,8 +132,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetJarInListReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetJarInListResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetLocationForTriggerResp;
-import org.apache.iotdb.confignode.rpc.thrift.TGetModelInfoReq;
-import org.apache.iotdb.confignode.rpc.thrift.TGetModelInfoResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetPathsSetTemplatesReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetPathsSetTemplatesResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetPipePluginTableResp;
@@ -163,20 +158,14 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowConfigNodesResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp;
-import org.apache.iotdb.confignode.rpc.thrift.TShowModelReq;
-import org.apache.iotdb.confignode.rpc.thrift.TShowModelResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq;
import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowThrottleReq;
-import org.apache.iotdb.confignode.rpc.thrift.TShowTrialReq;
-import org.apache.iotdb.confignode.rpc.thrift.TShowTrialResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp;
import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp;
import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp;
import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList;
import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq;
-import org.apache.iotdb.confignode.rpc.thrift.TUpdateModelInfoReq;
-import org.apache.iotdb.confignode.rpc.thrift.TUpdateModelStateReq;
import org.apache.iotdb.consensus.common.DataSet;
import org.apache.iotdb.consensus.exception.ConsensusException;
import org.apache.iotdb.db.schemaengine.template.Template;
@@ -243,9 +232,6 @@ public class ConfigManager implements IManager {
/** CQ. */
private final CQManager cqManager;
- /** ML Model. */
- private final ModelManager modelManager;
-
/** Pipe */
private final PipeManager pipeManager;
@@ -268,7 +254,6 @@ public class ConfigManager implements IManager {
UDFInfo udfInfo = new UDFInfo();
TriggerInfo triggerInfo = new TriggerInfo();
CQInfo cqInfo = new CQInfo();
- ModelInfo modelInfo = new ModelInfo();
PipeInfo pipeInfo = new PipeInfo();
QuotaInfo quotaInfo = new QuotaInfo();
@@ -283,7 +268,6 @@ public class ConfigManager implements IManager {
udfInfo,
triggerInfo,
cqInfo,
- modelInfo,
pipeInfo,
quotaInfo);
this.stateMachine = new ConfigRegionStateMachine(this, executor);
@@ -298,7 +282,6 @@ public class ConfigManager implements IManager {
this.udfManager = new UDFManager(this, udfInfo);
this.triggerManager = new TriggerManager(this, triggerInfo);
this.cqManager = new CQManager(this);
- this.modelManager = new ModelManager(this, modelInfo);
this.pipeManager = new PipeManager(this, pipeInfo);
// 1. keep PipeManager initialization before LoadManager initialization, because
@@ -945,11 +928,6 @@ public class ConfigManager implements IManager {
return triggerManager;
}
- @Override
- public ModelManager getModelManager() {
- return modelManager;
- }
-
@Override
public PipeManager getPipeManager() {
return pipeManager;
@@ -1896,62 +1874,6 @@ public class ConfigManager implements IManager {
return transferResult;
}
- @Override
- public TSStatus createModel(TCreateModelReq req) {
- TSStatus status = confirmLeader();
- return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
- ? modelManager.createModel(req)
- : status;
- }
-
- @Override
- public TSStatus dropModel(TDropModelReq req) {
- TSStatus status = confirmLeader();
- return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
- ? modelManager.dropModel(req)
- : status;
- }
-
- @Override
- public TShowModelResp showModel(TShowModelReq req) {
- TSStatus status = confirmLeader();
- return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
- ? modelManager.showModel(req)
- : new TShowModelResp(status, Collections.emptyList());
- }
-
- @Override
- public TShowTrialResp showTrial(TShowTrialReq req) {
- TSStatus status = confirmLeader();
- return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
- ? modelManager.showTrial(req)
- : new TShowTrialResp(status, Collections.emptyList());
- }
-
- @Override
- public TSStatus updateModelInfo(TUpdateModelInfoReq req) {
- TSStatus status = confirmLeader();
- return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
- ? modelManager.updateModelInfo(req)
- : status;
- }
-
- @Override
- public TSStatus updateModelState(TUpdateModelStateReq req) {
- TSStatus status = confirmLeader();
- return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
- ? modelManager.updateModelState(req)
- : status;
- }
-
- @Override
- public TGetModelInfoResp getModelInfo(TGetModelInfoReq req) {
- TSStatus status = confirmLeader();
- return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()
- ? modelManager.getModelInfo(req)
- : new TGetModelInfoResp(status);
- }
-
@Override
public TSStatus setSpaceQuota(TSetSpaceQuotaReq req) {
TSStatus status = confirmLeader();
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
index 409eeba5499..f0629a5f61a 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
@@ -55,7 +55,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListReq;
import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListResp;
import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreateFunctionReq;
-import org.apache.iotdb.confignode.rpc.thrift.TCreateModelReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreatePipePluginReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreateSchemaTemplateReq;
@@ -68,7 +67,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TDeactivateSchemaTemplateReq;
import org.apache.iotdb.confignode.rpc.thrift.TDeleteLogicalViewReq;
import org.apache.iotdb.confignode.rpc.thrift.TDeleteTimeSeriesReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq;
-import org.apache.iotdb.confignode.rpc.thrift.TDropModelReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropTriggerReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetAllPipeInfoResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetAllTemplatesResp;
@@ -77,8 +75,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetJarInListReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetJarInListResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetLocationForTriggerResp;
-import org.apache.iotdb.confignode.rpc.thrift.TGetModelInfoReq;
-import org.apache.iotdb.confignode.rpc.thrift.TGetModelInfoResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetPathsSetTemplatesReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetPathsSetTemplatesResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetPipePluginTableResp;
@@ -104,16 +100,10 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowConfigNodesResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp;
-import org.apache.iotdb.confignode.rpc.thrift.TShowModelReq;
-import org.apache.iotdb.confignode.rpc.thrift.TShowModelResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq;
import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp;
-import org.apache.iotdb.confignode.rpc.thrift.TShowTrialReq;
-import org.apache.iotdb.confignode.rpc.thrift.TShowTrialResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp;
import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq;
-import org.apache.iotdb.confignode.rpc.thrift.TUpdateModelInfoReq;
-import org.apache.iotdb.confignode.rpc.thrift.TUpdateModelStateReq;
import org.apache.iotdb.consensus.common.DataSet;
import java.util.List;
@@ -187,13 +177,6 @@ public interface IManager {
*/
CQManager getCQManager();
- /**
- * Get ModelManager.
- *
- * @return ModelManager instance
- */
- ModelManager getModelManager();
-
/**
* Get PipeManager.
*
@@ -649,27 +632,6 @@ public interface IManager {
TSStatus transfer(List<TDataNodeLocation> newUnknownDataList);
- /** Create a model. */
- TSStatus createModel(TCreateModelReq req);
-
- /** Drop a model. */
- TSStatus dropModel(TDropModelReq req);
-
- /** Return the model table. */
- TShowModelResp showModel(TShowModelReq req);
-
- /** Return the trial table. */
- TShowTrialResp showTrial(TShowTrialReq req);
-
- /** Update the model info. */
- TSStatus updateModelInfo(TUpdateModelInfoReq req);
-
- /** Update the model state. */
- TSStatus updateModelState(TUpdateModelStateReq req);
-
- /** Update the model state */
- TGetModelInfoResp getModelInfo(TGetModelInfoReq req);
-
/** Set space quota. */
TSStatus setSpaceQuota(TSetSpaceQuotaReq req);
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ModelManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ModelManager.java
deleted file mode 100644
index 288e1c2263c..00000000000
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ModelManager.java
+++ /dev/null
@@ -1,204 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.manager;
-
-import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.common.rpc.thrift.TaskType;
-import org.apache.iotdb.commons.model.ForecastModeInformation;
-import org.apache.iotdb.commons.model.ModelInformation;
-import org.apache.iotdb.confignode.consensus.request.read.model.GetModelInfoPlan;
-import org.apache.iotdb.confignode.consensus.request.read.model.ShowModelPlan;
-import org.apache.iotdb.confignode.consensus.request.read.model.ShowTrialPlan;
-import org.apache.iotdb.confignode.consensus.request.write.model.UpdateModelInfoPlan;
-import org.apache.iotdb.confignode.consensus.request.write.model.UpdateModelStatePlan;
-import org.apache.iotdb.confignode.consensus.response.model.GetModelInfoResp;
-import org.apache.iotdb.confignode.consensus.response.model.ModelTableResp;
-import org.apache.iotdb.confignode.consensus.response.model.TrialTableResp;
-import org.apache.iotdb.confignode.persistence.ModelInfo;
-import org.apache.iotdb.confignode.rpc.thrift.TCreateModelReq;
-import org.apache.iotdb.confignode.rpc.thrift.TDropModelReq;
-import org.apache.iotdb.confignode.rpc.thrift.TGetModelInfoReq;
-import org.apache.iotdb.confignode.rpc.thrift.TGetModelInfoResp;
-import org.apache.iotdb.confignode.rpc.thrift.TShowModelReq;
-import org.apache.iotdb.confignode.rpc.thrift.TShowModelResp;
-import org.apache.iotdb.confignode.rpc.thrift.TShowTrialReq;
-import org.apache.iotdb.confignode.rpc.thrift.TShowTrialResp;
-import org.apache.iotdb.confignode.rpc.thrift.TUpdateModelInfoReq;
-import org.apache.iotdb.confignode.rpc.thrift.TUpdateModelStateReq;
-import org.apache.iotdb.consensus.common.DataSet;
-import org.apache.iotdb.consensus.exception.ConsensusException;
-import org.apache.iotdb.rpc.TSStatusCode;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import static org.apache.iotdb.commons.model.ForecastModeInformation.DEFAULT_INPUT_LENGTH;
-import static org.apache.iotdb.commons.model.ForecastModeInformation.DEFAULT_PREDICT_LENGTH;
-import static org.apache.iotdb.commons.model.ForecastModeInformation.INPUT_LENGTH;
-import static org.apache.iotdb.commons.model.ForecastModeInformation.INPUT_TYPE_LIST;
-import static org.apache.iotdb.commons.model.ForecastModeInformation.PREDICT_INDEX_LIST;
-import static org.apache.iotdb.commons.model.ForecastModeInformation.PREDICT_LENGTH;
-
-public class ModelManager {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ModelManager.class);
-
- private final ConfigManager configManager;
- private final ModelInfo modelInfo;
-
- public ModelManager(ConfigManager configManager, ModelInfo modelInfo) {
- this.configManager = configManager;
- this.modelInfo = modelInfo;
- }
-
- public ModelInfo getModelInfo() {
- return modelInfo;
- }
-
- public TSStatus createModel(TCreateModelReq req) {
- TaskType taskType = req.getTaskType();
-
- Map<String, String> options = req.getOptions();
- ModelInformation modelInformation;
- if (taskType == TaskType.FORECAST) {
- String inputTypeListStr = options.get(INPUT_TYPE_LIST);
- List<TSDataType> inputTypeList =
- Arrays.stream(inputTypeListStr.substring(1, inputTypeListStr.length() - 1).split(","))
- .map(s -> TSDataType.valueOf(s.toUpperCase().trim()))
- .collect(Collectors.toList());
-
- String predictIndexListStr = options.get(PREDICT_INDEX_LIST);
- List<Integer> predictIndexList =
- Arrays.stream(
- predictIndexListStr.substring(1, predictIndexListStr.length() - 1).split(","))
- .map(s -> Integer.valueOf(s.trim()))
- .collect(Collectors.toList());
-
- modelInformation =
- new ForecastModeInformation(
- req.getModelId(),
- req.getOptions(),
- req.getDatasetFetchSQL().replace("\n", "").replace("\t", " "),
- inputTypeList,
- predictIndexList,
- Integer.parseInt(options.getOrDefault(INPUT_LENGTH, DEFAULT_INPUT_LENGTH)),
- Integer.parseInt(options.getOrDefault(PREDICT_LENGTH, DEFAULT_PREDICT_LENGTH)));
- } else {
- throw new IllegalArgumentException("Invalid task type: " + taskType);
- }
- return configManager
- .getProcedureManager()
- .createModel(modelInformation, req.getHyperparameters());
- }
-
- public TSStatus dropModel(TDropModelReq req) {
- return configManager.getProcedureManager().dropModel(req.getModelId());
- }
-
- public TSStatus updateModelInfo(TUpdateModelInfoReq req) {
- try {
- return configManager.getConsensusManager().write(new UpdateModelInfoPlan(req));
- } catch (ConsensusException e) {
- LOGGER.warn(
- String.format("Unexpected error happened while updating model %s: ", req.getModelId()),
- e);
- // consensus layer related errors
- TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
- res.setMessage(e.getMessage());
- return res;
- }
- }
-
- public TSStatus updateModelState(TUpdateModelStateReq req) {
- try {
- return configManager.getConsensusManager().write(new UpdateModelStatePlan(req));
- } catch (ConsensusException e) {
- LOGGER.warn(
- String.format(
- "Unexpected error happened while updating state of model %s: ", req.getModelId()),
- e);
- // consensus layer related errors
- TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
- res.setMessage(e.getMessage());
- return res;
- }
- }
-
- public TShowModelResp showModel(TShowModelReq req) {
- try {
- DataSet response = configManager.getConsensusManager().read(new ShowModelPlan(req));
- return ((ModelTableResp) response).convertToThriftResponse();
- } catch (ConsensusException e) {
- LOGGER.warn(
- String.format("Unexpected error happened while showing model %s: ", req.getModelId()), e);
- // consensus layer related errors
- TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
- res.setMessage(e.getMessage());
- return new TShowModelResp(res, Collections.emptyList());
- } catch (IOException e) {
- LOGGER.warn("Fail to get ModelTable", e);
- return new TShowModelResp(
- new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode())
- .setMessage(e.getMessage()),
- Collections.emptyList());
- }
- }
-
- public TShowTrialResp showTrial(TShowTrialReq req) {
- try {
- DataSet response = configManager.getConsensusManager().read(new ShowTrialPlan(req));
- return ((TrialTableResp) response).convertToThriftResponse();
- } catch (ConsensusException e) {
- LOGGER.warn(
- String.format("Unexpected error happened while showing trial %s: ", req.getModelId()), e);
- // consensus layer related errors
- TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
- res.setMessage(e.getMessage());
- return new TShowTrialResp(res, Collections.emptyList());
- } catch (IOException e) {
- LOGGER.warn("Fail to get TrailTable", e);
- return new TShowTrialResp(
- new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode())
- .setMessage(e.getMessage()),
- Collections.emptyList());
- }
- }
-
- public TGetModelInfoResp getModelInfo(TGetModelInfoReq req) {
- try {
- DataSet response = configManager.getConsensusManager().read(new GetModelInfoPlan(req));
- return ((GetModelInfoResp) response).convertToThriftResponse();
- } catch (ConsensusException e) {
- LOGGER.warn("Unexpected error happened while getting model: ", e);
- // consensus layer related errors
- TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
- res.setMessage(e.getMessage());
- return new TGetModelInfoResp(res);
- }
- }
-}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
index d96238c0828..290f89edbaf 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
@@ -27,7 +27,6 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.IoTDBException;
-import org.apache.iotdb.commons.model.ModelInformation;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PathDeserializeUtil;
import org.apache.iotdb.commons.path.PathPatternTree;
@@ -48,8 +47,6 @@ import org.apache.iotdb.confignode.procedure.Procedure;
import org.apache.iotdb.confignode.procedure.ProcedureExecutor;
import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
import org.apache.iotdb.confignode.procedure.impl.cq.CreateCQProcedure;
-import org.apache.iotdb.confignode.procedure.impl.model.CreateModelProcedure;
-import org.apache.iotdb.confignode.procedure.impl.model.DropModelProcedure;
import org.apache.iotdb.confignode.procedure.impl.node.AddConfigNodeProcedure;
import org.apache.iotdb.confignode.procedure.impl.node.RemoveConfigNodeProcedure;
import org.apache.iotdb.confignode.procedure.impl.node.RemoveDataNodeProcedure;
@@ -706,34 +703,6 @@ public class ProcedureManager {
return statusList.get(0);
}
- public TSStatus createModel(
- ModelInformation modelInformation, Map<String, String> hyperparameters) {
- long procedureId =
- executor.submitProcedure(new CreateModelProcedure(modelInformation, hyperparameters));
- List<TSStatus> statusList = new ArrayList<>();
- boolean isSucceed =
- waitingProcedureFinished(Collections.singletonList(procedureId), statusList);
- if (isSucceed) {
- return RpcUtils.SUCCESS_STATUS;
- } else {
- return new TSStatus(TSStatusCode.CREATE_MODEL_ERROR.getStatusCode())
- .setMessage(statusList.get(0).getMessage());
- }
- }
-
- public TSStatus dropModel(String modelId) {
- long procedureId = executor.submitProcedure(new DropModelProcedure(modelId));
- List<TSStatus> statusList = new ArrayList<>();
- boolean isSucceed =
- waitingProcedureFinished(Collections.singletonList(procedureId), statusList);
- if (isSucceed) {
- return RpcUtils.SUCCESS_STATUS;
- } else {
- return new TSStatus(TSStatusCode.DROP_MODEL_ERROR.getStatusCode())
- .setMessage(statusList.get(0).getMessage());
- }
- }
-
public TSStatus createPipePlugin(PipePluginMeta pipePluginMeta, byte[] jarFile) {
final CreatePipePluginProcedure createPipePluginProcedure =
new CreatePipePluginProcedure(pipePluginMeta, jarFile);
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/ModelInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/ModelInfo.java
deleted file mode 100644
index fe8c2b7b103..00000000000
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/ModelInfo.java
+++ /dev/null
@@ -1,278 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.persistence;
-
-import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.commons.model.ModelInformation;
-import org.apache.iotdb.commons.model.ModelTable;
-import org.apache.iotdb.commons.model.TrialInformation;
-import org.apache.iotdb.commons.snapshot.SnapshotProcessor;
-import org.apache.iotdb.confignode.consensus.request.read.model.GetModelInfoPlan;
-import org.apache.iotdb.confignode.consensus.request.read.model.ShowModelPlan;
-import org.apache.iotdb.confignode.consensus.request.read.model.ShowTrialPlan;
-import org.apache.iotdb.confignode.consensus.request.write.model.CreateModelPlan;
-import org.apache.iotdb.confignode.consensus.request.write.model.DropModelPlan;
-import org.apache.iotdb.confignode.consensus.request.write.model.UpdateModelInfoPlan;
-import org.apache.iotdb.confignode.consensus.request.write.model.UpdateModelStatePlan;
-import org.apache.iotdb.confignode.consensus.response.model.GetModelInfoResp;
-import org.apache.iotdb.confignode.consensus.response.model.ModelTableResp;
-import org.apache.iotdb.confignode.consensus.response.model.TrialTableResp;
-import org.apache.iotdb.rpc.TSStatusCode;
-import org.apache.iotdb.tsfile.utils.PublicBAOS;
-
-import org.apache.thrift.TException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.concurrent.ThreadSafe;
-
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-@ThreadSafe
-public class ModelInfo implements SnapshotProcessor {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ModelInfo.class);
-
- private static final String SNAPSHOT_FILENAME = "model_info.snapshot";
-
- private ModelTable modelTable;
-
- private final ReadWriteLock modelTableLock = new ReentrantReadWriteLock();
-
- public ModelInfo() {
- this.modelTable = new ModelTable();
- }
-
- public void acquireModelTableReadLock() {
- LOGGER.info("acquire ModelTableReadLock");
- modelTableLock.readLock().lock();
- }
-
- public void releaseModelTableReadLock() {
- LOGGER.info("release ModelTableReadLock");
- modelTableLock.readLock().unlock();
- }
-
- public void acquireModelTableWriteLock() {
- LOGGER.info("acquire ModelTableWriteLock");
- modelTableLock.writeLock().lock();
- }
-
- public void releaseModelTableWriteLock() {
- LOGGER.info("release ModelTableWriteLock");
- modelTableLock.writeLock().unlock();
- }
-
- public TSStatus createModel(CreateModelPlan plan) {
- try {
- acquireModelTableWriteLock();
- ModelInformation modelInformation = plan.getModelInformation();
- if (modelTable.containsModel(modelInformation.getModelId())) {
- return new TSStatus(TSStatusCode.MODEL_EXIST_ERROR.getStatusCode())
- .setMessage(
- String.format(
- "model [%s] has already been created.", modelInformation.getModelId()));
- } else {
- modelTable.addModel(modelInformation);
- return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
- }
-
- } catch (Exception e) {
- final String errorMessage =
- String.format(
- "Failed to add model [%s] in ModelTable on Config Nodes, because of %s",
- plan.getModelInformation().getModelId(), e);
- LOGGER.warn(errorMessage, e);
- return new TSStatus(TSStatusCode.CREATE_MODEL_ERROR.getStatusCode()).setMessage(errorMessage);
- } finally {
- releaseModelTableWriteLock();
- }
- }
-
- public TSStatus dropModel(DropModelPlan plan) {
- acquireModelTableWriteLock();
- String modelId = plan.getModelId();
- TSStatus status;
- if (modelTable.containsModel(modelId)) {
- modelTable.removeModel(modelId);
- status = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
- } else {
- status =
- new TSStatus(TSStatusCode.DROP_MODEL_ERROR.getStatusCode())
- .setMessage(String.format("model [%s] has not been created.", modelId));
- }
- releaseModelTableWriteLock();
- return status;
- }
-
- public ModelTableResp showModel(ShowModelPlan plan) {
- acquireModelTableReadLock();
- try {
- ModelTableResp modelTableResp =
- new ModelTableResp(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
- if (plan.isSetModelId()) {
- ModelInformation modelInformation = modelTable.getModelInformationById(plan.getModelId());
- if (modelInformation != null) {
- modelTableResp.addModelInformation(modelInformation);
- }
- } else {
- modelTableResp.addModelInformation(modelTable.getAllModelInformation());
- }
- return modelTableResp;
- } catch (IOException e) {
- LOGGER.warn("Fail to get ModelTable", e);
- return new ModelTableResp(
- new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode())
- .setMessage(e.getMessage()));
- } finally {
- releaseModelTableReadLock();
- }
- }
-
- public TrialTableResp showTrial(ShowTrialPlan plan) {
- acquireModelTableReadLock();
- try {
- String modelId = plan.getModelId();
- ModelInformation modelInformation = modelTable.getModelInformationById(modelId);
- if (modelInformation == null) {
- return new TrialTableResp(
- new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode())
- .setMessage(
- String.format(
- "Failed to show trails of model [%s], this model has not been created.",
- modelId)));
- }
-
- TrialTableResp trialTableResp =
- new TrialTableResp(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
- if (plan.isSetTrialId()) {
- TrialInformation trialInformation =
- modelInformation.getTrialInformationById(plan.getTrialId());
- if (trialInformation != null) {
- trialTableResp.addTrialInformation(trialInformation);
- }
- } else {
- trialTableResp.addTrialInformation(modelInformation.getAllTrialInformation());
- }
- return trialTableResp;
- } catch (IOException e) {
- LOGGER.warn("Fail to get TrailTable", e);
- return new TrialTableResp(
- new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode())
- .setMessage(e.getMessage()));
- } finally {
- releaseModelTableReadLock();
- }
- }
-
- public GetModelInfoResp getModelInfo(GetModelInfoPlan plan) {
- acquireModelTableReadLock();
- try {
- GetModelInfoResp getModelInfoResp =
- new GetModelInfoResp(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
- ModelInformation modelInformation = modelTable.getModelInformationById(plan.getModelId());
- if (modelInformation != null) {
- PublicBAOS buffer = new PublicBAOS();
- DataOutputStream stream = new DataOutputStream(buffer);
- modelInformation.serialize(stream);
- getModelInfoResp.setModelInfo(ByteBuffer.wrap(buffer.getBuf(), 0, buffer.size()));
- }
- return getModelInfoResp;
- } catch (IOException e) {
- LOGGER.warn("Fail to get model info", e);
- return new GetModelInfoResp(
- new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode())
- .setMessage(e.getMessage()));
- } finally {
- releaseModelTableReadLock();
- }
- }
-
- public TSStatus updateModelInfo(UpdateModelInfoPlan plan) {
- acquireModelTableWriteLock();
- try {
- String modelId = plan.getModelId();
- if (modelTable.containsModel(modelId)) {
- modelTable.updateModel(modelId, plan.getTrialId(), plan.getModelInfo());
- }
- return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
- } finally {
- releaseModelTableWriteLock();
- }
- }
-
- public TSStatus updateModelState(UpdateModelStatePlan plan) {
- acquireModelTableWriteLock();
- try {
- String modelId = plan.getModelId();
- if (modelTable.containsModel(modelId)) {
- modelTable.updateState(modelId, plan.getState(), plan.getBestTrialId());
- }
- return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
- } finally {
- releaseModelTableWriteLock();
- }
- }
-
- @Override
- public boolean processTakeSnapshot(File snapshotDir) throws TException, IOException {
- File snapshotFile = new File(snapshotDir, SNAPSHOT_FILENAME);
- if (snapshotFile.exists() && snapshotFile.isFile()) {
- LOGGER.error(
- "Failed to take snapshot of ModelInfo, because snapshot file [{}] is already exist.",
- snapshotFile.getAbsolutePath());
- return false;
- }
-
- acquireModelTableReadLock();
- try (FileOutputStream fileOutputStream = new FileOutputStream(snapshotFile)) {
- modelTable.serialize(fileOutputStream);
- fileOutputStream.getFD().sync();
- return true;
- } finally {
- releaseModelTableReadLock();
- }
- }
-
- @Override
- public void processLoadSnapshot(File snapshotDir) throws TException, IOException {
- File snapshotFile = new File(snapshotDir, SNAPSHOT_FILENAME);
- if (!snapshotFile.exists() || !snapshotFile.isFile()) {
- LOGGER.error(
- "Failed to load snapshot of ModelInfo, snapshot file [{}] does not exist.",
- snapshotFile.getAbsolutePath());
- return;
- }
- acquireModelTableWriteLock();
- try (FileInputStream fileInputStream = new FileInputStream(snapshotFile)) {
- modelTable.clear();
- modelTable = ModelTable.deserialize(fileInputStream);
- } finally {
- releaseModelTableWriteLock();
- }
- }
-}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
index dbd7c70ceb2..3650724ac6a 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
@@ -31,9 +31,6 @@ import org.apache.iotdb.confignode.consensus.request.read.database.CountDatabase
import org.apache.iotdb.confignode.consensus.request.read.database.GetDatabasePlan;
import org.apache.iotdb.confignode.consensus.request.read.datanode.GetDataNodeConfigurationPlan;
import org.apache.iotdb.confignode.consensus.request.read.function.GetUDFJarPlan;
-import org.apache.iotdb.confignode.consensus.request.read.model.GetModelInfoPlan;
-import org.apache.iotdb.confignode.consensus.request.read.model.ShowModelPlan;
-import org.apache.iotdb.confignode.consensus.request.read.model.ShowTrialPlan;
import org.apache.iotdb.confignode.consensus.request.read.partition.CountTimeSlotListPlan;
import org.apache.iotdb.confignode.consensus.request.read.partition.GetDataPartitionPlan;
import org.apache.iotdb.confignode.consensus.request.read.partition.GetNodePathsPartitionPlan;
@@ -70,10 +67,6 @@ import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNo
import org.apache.iotdb.confignode.consensus.request.write.datanode.UpdateDataNodePlan;
import org.apache.iotdb.confignode.consensus.request.write.function.CreateFunctionPlan;
import org.apache.iotdb.confignode.consensus.request.write.function.DropFunctionPlan;
-import org.apache.iotdb.confignode.consensus.request.write.model.CreateModelPlan;
-import org.apache.iotdb.confignode.consensus.request.write.model.DropModelPlan;
-import org.apache.iotdb.confignode.consensus.request.write.model.UpdateModelInfoPlan;
-import org.apache.iotdb.confignode.consensus.request.write.model.UpdateModelStatePlan;
import org.apache.iotdb.confignode.consensus.request.write.partition.CreateDataPartitionPlan;
import org.apache.iotdb.confignode.consensus.request.write.partition.CreateSchemaPartitionPlan;
import org.apache.iotdb.confignode.consensus.request.write.partition.UpdateRegionLocationPlan;
@@ -108,7 +101,6 @@ import org.apache.iotdb.confignode.consensus.request.write.trigger.UpdateTrigger
import org.apache.iotdb.confignode.consensus.response.partition.SchemaNodeManagementResp;
import org.apache.iotdb.confignode.exception.physical.UnknownPhysicalPlanTypeException;
import org.apache.iotdb.confignode.persistence.AuthorInfo;
-import org.apache.iotdb.confignode.persistence.ModelInfo;
import org.apache.iotdb.confignode.persistence.ProcedureInfo;
import org.apache.iotdb.confignode.persistence.TriggerInfo;
import org.apache.iotdb.confignode.persistence.UDFInfo;
@@ -163,8 +155,6 @@ public class ConfigPlanExecutor {
private final CQInfo cqInfo;
- private final ModelInfo modelInfo;
-
private final PipeInfo pipeInfo;
private final QuotaInfo quotaInfo;
@@ -178,7 +168,6 @@ public class ConfigPlanExecutor {
UDFInfo udfInfo,
TriggerInfo triggerInfo,
CQInfo cqInfo,
- ModelInfo modelInfo,
PipeInfo pipeInfo,
QuotaInfo quotaInfo) {
@@ -205,9 +194,6 @@ public class ConfigPlanExecutor {
this.cqInfo = cqInfo;
this.snapshotProcessorList.add(cqInfo);
- this.modelInfo = modelInfo;
- this.snapshotProcessorList.add(modelInfo);
-
this.pipeInfo = pipeInfo;
this.snapshotProcessorList.add(pipeInfo);
@@ -278,12 +264,6 @@ public class ConfigPlanExecutor {
return udfInfo.getUDFTable();
case GetFunctionJar:
return udfInfo.getUDFJar((GetUDFJarPlan) req);
- case ShowModel:
- return modelInfo.showModel((ShowModelPlan) req);
- case ShowTrial:
- return modelInfo.showTrial((ShowTrialPlan) req);
- case GetModelInfo:
- return modelInfo.getModelInfo((GetModelInfoPlan) req);
case GetPipePluginTable:
return pipeInfo.getPipePluginInfo().showPipePlugins();
case GetPipePluginJar:
@@ -439,14 +419,6 @@ public class ConfigPlanExecutor {
return cqInfo.activeCQ((ActiveCQPlan) physicalPlan);
case UPDATE_CQ_LAST_EXEC_TIME:
return cqInfo.updateCQLastExecutionTime((UpdateCQLastExecTimePlan) physicalPlan);
- case CreateModel:
- return modelInfo.createModel((CreateModelPlan) physicalPlan);
- case UpdateModelInfo:
- return modelInfo.updateModelInfo((UpdateModelInfoPlan) physicalPlan);
- case UpdateModelState:
- return modelInfo.updateModelState((UpdateModelStatePlan) physicalPlan);
- case DropModel:
- return modelInfo.dropModel((DropModelPlan) physicalPlan);
case CreatePipePlugin:
return pipeInfo.getPipePluginInfo().createPipePlugin((CreatePipePluginPlan) physicalPlan);
case DropPipePlugin:
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/model/CreateModelProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/model/CreateModelProcedure.java
deleted file mode 100644
index d118f7382f6..00000000000
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/model/CreateModelProcedure.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.procedure.impl.model;
-
-import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.commons.client.mlnode.MLNodeClient;
-import org.apache.iotdb.commons.client.mlnode.MLNodeClientManager;
-import org.apache.iotdb.commons.client.mlnode.MLNodeInfo;
-import org.apache.iotdb.commons.model.ModelInformation;
-import org.apache.iotdb.commons.model.exception.ModelManagementException;
-import org.apache.iotdb.confignode.consensus.request.write.model.CreateModelPlan;
-import org.apache.iotdb.confignode.consensus.request.write.model.DropModelPlan;
-import org.apache.iotdb.confignode.manager.ConfigManager;
-import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
-import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
-import org.apache.iotdb.confignode.procedure.impl.node.AbstractNodeProcedure;
-import org.apache.iotdb.confignode.procedure.state.model.CreateModelState;
-import org.apache.iotdb.confignode.procedure.store.ProcedureType;
-import org.apache.iotdb.consensus.exception.ConsensusException;
-import org.apache.iotdb.rpc.TSStatusCode;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
-import org.apache.thrift.TException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Map;
-import java.util.Objects;
-
-public class CreateModelProcedure extends AbstractNodeProcedure<CreateModelState> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(CreateModelProcedure.class);
- private static final int RETRY_THRESHOLD = 1;
-
- private ModelInformation modelInformation;
- private Map<String, String> hyperparameters;
-
- public CreateModelProcedure() {
- super();
- }
-
- public CreateModelProcedure(
- ModelInformation modelInformation, Map<String, String> hyperparameters) {
- super();
- this.modelInformation = modelInformation;
- this.hyperparameters = hyperparameters;
- }
-
- @Override
- protected Flow executeFromState(ConfigNodeProcedureEnv env, CreateModelState state) {
- if (modelInformation == null) {
- return Flow.NO_MORE_STATE;
- }
- try {
- switch (state) {
- case INIT:
- createModel(env);
- break;
- case ML_NODE_ACTIVE:
- trainModel();
- return Flow.NO_MORE_STATE;
- default:
- throw new UnsupportedOperationException(
- String.format("Unknown state during executing createModelProcedure, %s", state));
- }
- } catch (Exception e) {
- if (isRollbackSupported(state)) {
- LOGGER.error("Fail in CreateModelProcedure", e);
- setFailure(new ProcedureException(e.getMessage()));
- } else {
- LOGGER.error(
- "Retrievable error trying to create model [{}], state [{}]",
- modelInformation.getModelId(),
- state,
- e);
- if (getCycles() > RETRY_THRESHOLD) {
- setFailure(
- new ProcedureException(
- String.format(
- "Fail to create model [%s] at STATE [%s]",
- modelInformation.getModelId(), state)));
- }
- }
- }
- return Flow.HAS_MORE_STATE;
- }
-
- private void createModel(ConfigNodeProcedureEnv env) throws ConsensusException {
- LOGGER.info("Start to create model [{}]", modelInformation.getModelId());
-
- ConfigManager configManager = env.getConfigManager();
- TSStatus response =
- configManager.getConsensusManager().write(new CreateModelPlan(modelInformation));
- if (response.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- throw new ModelManagementException(
- String.format(
- "Failed to add model [%s] in ModelTable on Config Nodes: %s",
- modelInformation.getModelId(), response.getMessage()));
- }
- setNextState(CreateModelState.ML_NODE_ACTIVE);
- }
-
- private void trainModel() {
- LOGGER.info("Start to train model [{}] on ML Node", modelInformation.getModelId());
-
- try (MLNodeClient client =
- MLNodeClientManager.getInstance().borrowClient(MLNodeInfo.endPoint)) {
- TSStatus status = client.createTrainingTask(modelInformation, hyperparameters);
- if (status.code != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- throw new TException(status.getMessage());
- }
- } catch (Exception e) {
- throw new ModelManagementException(
- String.format(
- "Fail to start training model [%s] on ML Node: %s",
- modelInformation.getModelId(), e.getMessage()));
- }
- }
-
- @Override
- protected void rollbackState(ConfigNodeProcedureEnv env, CreateModelState state)
- throws IOException, InterruptedException, ProcedureException {
- switch (state) {
- case INIT:
- // do nothing
- break;
- case ML_NODE_ACTIVE:
- LOGGER.info("Start [VALIDATED] rollback of model [{}]", modelInformation.getModelId());
- try {
- env.getConfigManager()
- .getConsensusManager()
- .write(new DropModelPlan(modelInformation.getModelId()));
- } catch (ConsensusException e) {
- LOGGER.warn("Failed in the write API executing the consensus layer due to: ", e);
- }
- break;
- default:
- break;
- }
- if (Objects.requireNonNull(state) == CreateModelState.INIT) {
- LOGGER.info("Start [INIT] rollback of model [{}]", modelInformation.getModelId());
- try {
- env.getConfigManager()
- .getConsensusManager()
- .write(new DropModelPlan(modelInformation.getModelId()));
- } catch (ConsensusException e) {
- LOGGER.warn("Failed in the write API executing the consensus layer due to: ", e);
- }
- }
- }
-
- @Override
- protected boolean isRollbackSupported(CreateModelState state) {
- return true;
- }
-
- @Override
- protected CreateModelState getState(int stateId) {
- return CreateModelState.values()[stateId];
- }
-
- @Override
- protected int getStateId(CreateModelState createModelState) {
- return createModelState.ordinal();
- }
-
- @Override
- protected CreateModelState getInitialState() {
- return CreateModelState.INIT;
- }
-
- @Override
- public void serialize(DataOutputStream stream) throws IOException {
- stream.writeShort(ProcedureType.CREATE_MODEL_PROCEDURE.getTypeCode());
- super.serialize(stream);
- modelInformation.serialize(stream);
- ReadWriteIOUtils.write(hyperparameters, stream);
- }
-
- @Override
- public void deserialize(ByteBuffer byteBuffer) {
- super.deserialize(byteBuffer);
- modelInformation = ModelInformation.deserialize(byteBuffer);
- hyperparameters = ReadWriteIOUtils.readMap(byteBuffer);
- }
-
- @Override
- public boolean equals(Object that) {
- if (that instanceof CreateModelProcedure) {
- CreateModelProcedure thatProc = (CreateModelProcedure) that;
- return thatProc.getProcId() == this.getProcId()
- && thatProc.getState() == this.getState()
- && thatProc.modelInformation.equals(this.modelInformation)
- && thatProc.hyperparameters.equals(this.hyperparameters);
- }
- return false;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(getProcId(), getState(), modelInformation, hyperparameters);
- }
-}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/model/DropModelProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/model/DropModelProcedure.java
deleted file mode 100644
index 0cf7ea2dae3..00000000000
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/model/DropModelProcedure.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.procedure.impl.model;
-
-import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
-import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.commons.client.mlnode.MLNodeClient;
-import org.apache.iotdb.commons.client.mlnode.MLNodeClientManager;
-import org.apache.iotdb.commons.client.mlnode.MLNodeInfo;
-import org.apache.iotdb.commons.model.exception.ModelManagementException;
-import org.apache.iotdb.confignode.client.DataNodeRequestType;
-import org.apache.iotdb.confignode.client.sync.SyncDataNodeClientPool;
-import org.apache.iotdb.confignode.consensus.request.write.model.DropModelPlan;
-import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
-import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
-import org.apache.iotdb.confignode.procedure.impl.node.AbstractNodeProcedure;
-import org.apache.iotdb.confignode.procedure.state.model.DropModelState;
-import org.apache.iotdb.confignode.procedure.store.ProcedureType;
-import org.apache.iotdb.mpp.rpc.thrift.TDeleteModelMetricsReq;
-import org.apache.iotdb.rpc.TSStatusCode;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
-import org.apache.thrift.TException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Objects;
-import java.util.Optional;
-
-public class DropModelProcedure extends AbstractNodeProcedure<DropModelState> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(DropModelProcedure.class);
- private static final int RETRY_THRESHOLD = 1;
-
- private String modelId;
-
- public DropModelProcedure() {
- super();
- }
-
- public DropModelProcedure(String modelId) {
- super();
- this.modelId = modelId;
- }
-
- @Override
- protected Flow executeFromState(ConfigNodeProcedureEnv env, DropModelState state) {
- if (modelId == null) {
- return Flow.NO_MORE_STATE;
- }
- try {
- switch (state) {
- case INIT:
- LOGGER.info("Start to drop model [{}] on Config Nodes", modelId);
- TSStatus response =
- env.getConfigManager().getConsensusManager().write(new DropModelPlan(modelId));
- if (response.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- throw new ModelManagementException(
- String.format(
- "Failed to drop model [%s], fail to drop model on Config Nodes: %s",
- modelId, response.getMessage()));
- }
- setNextState(DropModelState.DATA_NODE_DROPPED);
- break;
- case DATA_NODE_DROPPED:
- deleteMetricsOnDataNode(env);
- break;
- case ML_NODE_DROPPED:
- dropModelOnMLNode();
- return Flow.NO_MORE_STATE;
- default:
- throw new UnsupportedOperationException(
- String.format("Unknown state during executing dropModelProcedure, %s", state));
- }
- } catch (Exception e) {
- if (isRollbackSupported(state)) {
- LOGGER.error("Fail in DropModelProcedure", e);
- setFailure(new ProcedureException(e.getMessage()));
- } else {
- LOGGER.error("Retrievable error trying to drop model [{}], state [{}]", modelId, state, e);
- if (getCycles() > RETRY_THRESHOLD) {
- setFailure(
- new ProcedureException(
- String.format(
- "Fail to drop model [%s] at STATE [%s], %s",
- modelId, state, e.getMessage())));
- }
- }
- }
- return Flow.HAS_MORE_STATE;
- }
-
- private void dropModelOnMLNode() {
- LOGGER.info("Start to drop model file [{}] on Ml Node", modelId);
- try (MLNodeClient client =
- MLNodeClientManager.getInstance().borrowClient(MLNodeInfo.endPoint)) {
- TSStatus status = client.deleteModel(modelId);
- if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- throw new TException(status.getMessage());
- }
- } catch (Exception e) {
- throw new ModelManagementException(
- String.format(
- "Failed to drop model [%s], fail to delete model on MLNode: %s",
- modelId, e.getMessage()));
- }
- }
-
- private void deleteMetricsOnDataNode(ConfigNodeProcedureEnv env) {
- LOGGER.info("Start to drop model metrics [{}] on Data Nodes", modelId);
-
- Optional<TDataNodeLocation> targetDataNode =
- env.getConfigManager().getNodeManager().getLowestLoadDataNode();
- if (!targetDataNode.isPresent()) {
- // no usable DataNode
- throw new ModelManagementException(
- String.format("Failed to drop model [%s], there is no RUNNING DataNode", modelId));
- }
-
- TSStatus status =
- SyncDataNodeClientPool.getInstance()
- .sendSyncRequestToDataNodeWithRetry(
- targetDataNode.get().getInternalEndPoint(),
- new TDeleteModelMetricsReq(modelId),
- DataNodeRequestType.DELETE_MODEL_METRICS);
- if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- // It's ok for the next state if the metric path doesn't exit
- if (status.getCode() != TSStatusCode.PATH_NOT_EXIST.getStatusCode()) {
- throw new ModelManagementException(
- String.format(
- "Failed to drop model [%s], fail to delete metrics: %s",
- modelId, status.getMessage()));
- }
- LOGGER.warn("The path of metrics does not exist on model {}", modelId);
- }
-
- setNextState(DropModelState.ML_NODE_DROPPED);
- }
-
- @Override
- protected void rollbackState(ConfigNodeProcedureEnv env, DropModelState state)
- throws IOException, InterruptedException, ProcedureException {
- // no need to rollback
- }
-
- @Override
- protected DropModelState getState(int stateId) {
- return DropModelState.values()[stateId];
- }
-
- @Override
- protected int getStateId(DropModelState dropModelState) {
- return dropModelState.ordinal();
- }
-
- @Override
- protected DropModelState getInitialState() {
- return DropModelState.INIT;
- }
-
- @Override
- public void serialize(DataOutputStream stream) throws IOException {
- stream.writeShort(ProcedureType.DROP_MODEL_PROCEDURE.getTypeCode());
- super.serialize(stream);
- ReadWriteIOUtils.write(modelId, stream);
- }
-
- @Override
- public void deserialize(ByteBuffer byteBuffer) {
- super.deserialize(byteBuffer);
- modelId = ReadWriteIOUtils.readString(byteBuffer);
- }
-
- @Override
- public boolean equals(Object that) {
- if (that instanceof DropModelProcedure) {
- DropModelProcedure thatProc = (DropModelProcedure) that;
- return thatProc.getProcId() == this.getProcId()
- && thatProc.getState() == this.getState()
- && (thatProc.modelId).equals(this.modelId);
- }
- return false;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(getProcId(), getState(), modelId);
- }
-}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/model/CreateModelState.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/model/CreateModelState.java
deleted file mode 100644
index aad03181187..00000000000
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/model/CreateModelState.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.procedure.state.model;
-
-public enum CreateModelState {
- INIT,
- ML_NODE_ACTIVE
-}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/model/DropModelState.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/model/DropModelState.java
deleted file mode 100644
index 54e32e86da1..00000000000
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/model/DropModelState.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.procedure.state.model;
-
-public enum DropModelState {
- INIT,
- VALIDATED,
- DATA_NODE_DROPPED,
- ML_NODE_DROPPED,
- CONFIG_NODE_DROPPED
-}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java
index 997bbbbb8a4..35b9b739150 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java
@@ -21,8 +21,6 @@ package org.apache.iotdb.confignode.procedure.store;
import org.apache.iotdb.confignode.procedure.Procedure;
import org.apache.iotdb.confignode.procedure.impl.cq.CreateCQProcedure;
-import org.apache.iotdb.confignode.procedure.impl.model.CreateModelProcedure;
-import org.apache.iotdb.confignode.procedure.impl.model.DropModelProcedure;
import org.apache.iotdb.confignode.procedure.impl.node.AddConfigNodeProcedure;
import org.apache.iotdb.confignode.procedure.impl.node.RemoveConfigNodeProcedure;
import org.apache.iotdb.confignode.procedure.impl.node.RemoveDataNodeProcedure;
@@ -160,12 +158,6 @@ public class ProcedureFactory implements IProcedureFactory {
case DROP_PIPE_PLUGIN_PROCEDURE:
procedure = new DropPipePluginProcedure();
break;
- case CREATE_MODEL_PROCEDURE:
- procedure = new CreateModelProcedure();
- break;
- case DROP_MODEL_PROCEDURE:
- procedure = new DropModelProcedure();
- break;
case AUTH_OPERATE_PROCEDURE:
procedure = new AuthOperationProcedure();
break;
@@ -216,10 +208,6 @@ public class ProcedureFactory implements IProcedureFactory {
return ProcedureType.CREATE_PIPE_PLUGIN_PROCEDURE;
} else if (procedure instanceof DropPipePluginProcedure) {
return ProcedureType.DROP_PIPE_PLUGIN_PROCEDURE;
- } else if (procedure instanceof CreateModelProcedure) {
- return ProcedureType.CREATE_MODEL_PROCEDURE;
- } else if (procedure instanceof DropModelProcedure) {
- return ProcedureType.DROP_MODEL_PROCEDURE;
} else if (procedure instanceof CreatePipeProcedureV2) {
return ProcedureType.CREATE_PIPE_PROCEDURE_V2;
} else if (procedure instanceof StartPipeProcedureV2) {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java
index 5c57c3bcff1..8c0c2087d60 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java
@@ -58,9 +58,7 @@ public enum ProcedureType {
UNSET_TEMPLATE_PROCEDURE((short) 701),
SET_TEMPLATE_PROCEDURE((short) 702),
- /** Ml Model */
- CREATE_MODEL_PROCEDURE((short) 800),
- DROP_MODEL_PROCEDURE((short) 801),
+ // ProcedureId 800-899 is used by IoTDB-Ml
/** Pipe Plugin */
CREATE_PIPE_PLUGIN_PROCEDURE((short) 900),
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
index 06c0b75bb8d..047629d477d 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java
@@ -80,7 +80,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListReq;
import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListResp;
import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreateFunctionReq;
-import org.apache.iotdb.confignode.rpc.thrift.TCreateModelReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreatePipePluginReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreateSchemaTemplateReq;
@@ -103,7 +102,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TDeleteLogicalViewReq;
import org.apache.iotdb.confignode.rpc.thrift.TDeleteTimeSeriesReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropFunctionReq;
-import org.apache.iotdb.confignode.rpc.thrift.TDropModelReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropPipePluginReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropPipeSinkReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropTriggerReq;
@@ -114,8 +112,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetJarInListReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetJarInListResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetLocationForTriggerResp;
-import org.apache.iotdb.confignode.rpc.thrift.TGetModelInfoReq;
-import org.apache.iotdb.confignode.rpc.thrift.TGetModelInfoResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetPathsSetTemplatesReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetPathsSetTemplatesResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetPipePluginTableResp;
@@ -150,22 +146,16 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowConfigNodesResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp;
-import org.apache.iotdb.confignode.rpc.thrift.TShowModelReq;
-import org.apache.iotdb.confignode.rpc.thrift.TShowModelResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq;
import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq;
import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowThrottleReq;
-import org.apache.iotdb.confignode.rpc.thrift.TShowTrialReq;
-import org.apache.iotdb.confignode.rpc.thrift.TShowTrialResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp;
import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp;
import org.apache.iotdb.confignode.rpc.thrift.TSystemConfigurationResp;
import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp;
import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq;
-import org.apache.iotdb.confignode.rpc.thrift.TUpdateModelInfoReq;
-import org.apache.iotdb.confignode.rpc.thrift.TUpdateModelStateReq;
import org.apache.iotdb.confignode.service.ConfigNode;
import org.apache.iotdb.consensus.exception.ConsensusException;
import org.apache.iotdb.db.queryengine.plan.statement.AuthorType;
@@ -1004,41 +994,6 @@ public class ConfigNodeRPCServiceProcessor implements IConfigNodeRPCService.Ifac
return configManager.showCQ();
}
- @Override
- public TSStatus createModel(TCreateModelReq req) {
- return configManager.createModel(req);
- }
-
- @Override
- public TSStatus dropModel(TDropModelReq req) {
- return configManager.dropModel(req);
- }
-
- @Override
- public TShowModelResp showModel(TShowModelReq req) {
- return configManager.showModel(req);
- }
-
- @Override
- public TShowTrialResp showTrial(TShowTrialReq req) {
- return configManager.showTrial(req);
- }
-
- @Override
- public TSStatus updateModelInfo(TUpdateModelInfoReq req) {
- return configManager.updateModelInfo(req);
- }
-
- @Override
- public TSStatus updateModelState(TUpdateModelStateReq req) {
- return configManager.updateModelState(req);
- }
-
- @Override
- public TGetModelInfoResp getModelInfo(TGetModelInfoReq req) {
- return configManager.getModelInfo(req);
- }
-
@Override
public TSStatus setSpaceQuota(TSetSpaceQuotaReq req) throws TException {
return configManager.setSpaceQuota(req);
diff --git a/iotdb-core/datanode/pom.xml b/iotdb-core/datanode/pom.xml
index 01b2dec2265..c875c27ad24 100644
--- a/iotdb-core/datanode/pom.xml
+++ b/iotdb-core/datanode/pom.xml
@@ -104,11 +104,6 @@
<artifactId>iotdb-thrift-confignode</artifactId>
<version>1.3.0-SNAPSHOT</version>
</dependency>
- <dependency>
- <groupId>org.apache.iotdb</groupId>
- <artifactId>iotdb-thrift-mlnode</artifactId>
- <version>1.3.0-SNAPSHOT</version>
- </dependency>
<dependency>
<groupId>org.apache.iotdb</groupId>
<artifactId>pipe-api</artifactId>
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 394ba55fe9a..7979c29d2e6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -430,9 +430,6 @@ public class IoTDBConfig {
/** Compact the unsequence files into the overlapped sequence files */
private boolean enableCrossSpaceCompaction = true;
- /** Enable the service for MLNode */
- private boolean enableMLNodeService = false;
-
/** The buffer for sort operation */
private long sortBufferSize = 1024 * 1024L;
@@ -863,9 +860,6 @@ public class IoTDBConfig {
/** Internal port for coordinator */
private int internalPort = 10730;
- /** Port for MLNode */
- private int mlNodePort = 10780;
-
/** Internal port for dataRegion consensus protocol */
private int dataRegionConsensusPort = 10760;
@@ -2690,14 +2684,6 @@ public class IoTDBConfig {
this.enableCrossSpaceCompaction = enableCrossSpaceCompaction;
}
- public boolean isEnableMLNodeService() {
- return enableMLNodeService;
- }
-
- public void setEnableMLNodeService(boolean enableMLNodeService) {
- this.enableMLNodeService = enableMLNodeService;
- }
-
public InnerSequenceCompactionSelector getInnerSequenceCompactionSelector() {
return innerSequenceCompactionSelector;
}
@@ -2931,14 +2917,6 @@ public class IoTDBConfig {
this.internalPort = internalPort;
}
- public int getMLNodePort() {
- return mlNodePort;
- }
-
- public void setMLNodePort(int mlNodePort) {
- this.mlNodePort = mlNodePort;
- }
-
public int getDataRegionConsensusPort() {
return dataRegionConsensusPort;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index e9c315cd8b1..9ee180e0a16 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -311,19 +311,6 @@ public class IoTDBDescriptor {
.getProperty(IoTDBConstant.DN_RPC_PORT, Integer.toString(conf.getRpcPort()))
.trim()));
- conf.setEnableMLNodeService(
- Boolean.parseBoolean(
- properties
- .getProperty(
- "enable_mlnode_rpc_service", Boolean.toString(conf.isEnableMLNodeService()))
- .trim()));
-
- conf.setMLNodePort(
- Integer.parseInt(
- properties
- .getProperty("mlnode_rpc_port", Integer.toString(conf.getMLNodePort()))
- .trim()));
-
conf.setBufferedArraysMemoryProportion(
Double.parseDouble(
properties
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/runtime/ModelInferenceProcessException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/runtime/ModelInferenceProcessException.java
deleted file mode 100644
index 586c624a8d3..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/runtime/ModelInferenceProcessException.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.exception.runtime;
-
-public class ModelInferenceProcessException extends RuntimeException {
-
- public ModelInferenceProcessException(String message) {
- super(message);
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java
index 2cf05ec7838..dba48f6e09c 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java
@@ -48,7 +48,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListReq;
import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListResp;
import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreateFunctionReq;
-import org.apache.iotdb.confignode.rpc.thrift.TCreateModelReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreatePipePluginReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreateSchemaTemplateReq;
@@ -71,7 +70,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TDeleteLogicalViewReq;
import org.apache.iotdb.confignode.rpc.thrift.TDeleteTimeSeriesReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropFunctionReq;
-import org.apache.iotdb.confignode.rpc.thrift.TDropModelReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropPipePluginReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropPipeSinkReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropTriggerReq;
@@ -82,8 +80,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetJarInListReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetJarInListResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetLocationForTriggerResp;
-import org.apache.iotdb.confignode.rpc.thrift.TGetModelInfoReq;
-import org.apache.iotdb.confignode.rpc.thrift.TGetModelInfoResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetPathsSetTemplatesReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetPathsSetTemplatesResp;
import org.apache.iotdb.confignode.rpc.thrift.TGetPipePluginTableResp;
@@ -118,22 +114,16 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowConfigNodesResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp;
-import org.apache.iotdb.confignode.rpc.thrift.TShowModelReq;
-import org.apache.iotdb.confignode.rpc.thrift.TShowModelResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq;
import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq;
import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowThrottleReq;
-import org.apache.iotdb.confignode.rpc.thrift.TShowTrialReq;
-import org.apache.iotdb.confignode.rpc.thrift.TShowTrialResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp;
import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp;
import org.apache.iotdb.confignode.rpc.thrift.TSystemConfigurationResp;
import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp;
import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq;
-import org.apache.iotdb.confignode.rpc.thrift.TUpdateModelInfoReq;
-import org.apache.iotdb.confignode.rpc.thrift.TUpdateModelStateReq;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.rpc.RpcTransportFactory;
@@ -955,56 +945,6 @@ public class ConfigNodeClient implements IConfigNodeRPCService.Iface, ThriftClie
() -> client.showCQ(), resp -> !updateConfigNodeLeader(resp.status));
}
- @Override
- public TSStatus createModel(TCreateModelReq req) throws TException {
- return executeRemoteCallWithRetry(
- () -> client.createModel(req), status -> !updateConfigNodeLeader(status));
- }
-
- @Override
- public TSStatus dropModel(TDropModelReq req) throws TException {
- return executeRemoteCallWithRetry(
- () -> client.dropModel(req), status -> !updateConfigNodeLeader(status));
- }
-
- @Override
- public TShowModelResp showModel(TShowModelReq req) throws TException {
- return executeRemoteCallWithRetry(
- () -> client.showModel(req), resp -> !updateConfigNodeLeader(resp.status));
- }
-
- @Override
- public TShowTrialResp showTrial(TShowTrialReq req) throws TException {
- return executeRemoteCallWithRetry(
- () -> client.showTrial(req), resp -> !updateConfigNodeLeader(resp.status));
- }
-
- @Override
- public TSStatus updateModelInfo(TUpdateModelInfoReq req) throws TException {
- throw new TException(new UnsupportedOperationException().getCause());
- }
-
- @Override
- public TSStatus updateModelState(TUpdateModelStateReq req) throws TException {
- throw new TException(new UnsupportedOperationException().getCause());
- }
-
- @Override
- public TGetModelInfoResp getModelInfo(TGetModelInfoReq req) throws TException {
- for (int i = 0; i < RETRY_NUM; i++) {
- try {
- TGetModelInfoResp resp = client.getModelInfo(req);
- if (!updateConfigNodeLeader(resp.getStatus())) {
- return resp;
- }
- } catch (TException e) {
- configLeader = null;
- }
- waitAndReconnect();
- }
- throw new TException(MSG_RECONNECTION_FAIL);
- }
-
@Override
public TSStatus setSpaceQuota(TSetSpaceQuotaReq req) throws TException {
return executeRemoteCallWithRetry(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/handler/MLNodeRPCServiceThriftHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/handler/MLNodeRPCServiceThriftHandler.java
deleted file mode 100644
index 34b165ae682..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/handler/MLNodeRPCServiceThriftHandler.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * <p>http://www.apache.org/licenses/LICENSE-2.0
- *
- * <p>Unless required by applicable law or agreed to in writing, software distributed under the
- * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.iotdb.db.protocol.thrift.handler;
-
-import org.apache.iotdb.db.protocol.thrift.impl.IMLNodeRPCServiceWithHandler;
-
-import org.apache.thrift.protocol.TProtocol;
-import org.apache.thrift.server.ServerContext;
-import org.apache.thrift.server.TServerEventHandler;
-import org.apache.thrift.transport.TTransport;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-public class MLNodeRPCServiceThriftHandler implements TServerEventHandler {
-
- private final AtomicLong thriftConnectionNumber = new AtomicLong(0);
- private final IMLNodeRPCServiceWithHandler eventHandler;
-
- public MLNodeRPCServiceThriftHandler(IMLNodeRPCServiceWithHandler eventHandler) {
- this.eventHandler = eventHandler;
- }
-
- @Override
- public ServerContext createContext(TProtocol in, TProtocol out) {
- thriftConnectionNumber.incrementAndGet();
- return null;
- }
-
- @Override
- public void deleteContext(ServerContext arg0, TProtocol in, TProtocol out) {
- thriftConnectionNumber.decrementAndGet();
- eventHandler.handleExit();
- }
-
- @Override
- public void preServe() {
- // do nothing
- }
-
- @Override
- public void processContext(
- ServerContext serverContext, TTransport tTransport, TTransport tTransport1) {
- // do nothing
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
index 884469be6cf..f2761c7ade7 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
@@ -110,7 +110,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNo
import org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileScheduler;
import org.apache.iotdb.db.queryengine.plan.statement.component.WhereCondition;
import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.metadata.DeleteTimeSeriesStatement;
import org.apache.iotdb.db.schemaengine.SchemaEngine;
import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion;
import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ITimeSeriesSchemaInfo;
@@ -126,7 +125,6 @@ import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeThrottleQuotaMana
import org.apache.iotdb.db.trigger.executor.TriggerExecutor;
import org.apache.iotdb.db.trigger.executor.TriggerFireResult;
import org.apache.iotdb.db.trigger.service.TriggerManagementService;
-import org.apache.iotdb.db.utils.ErrorHandlingUtils;
import org.apache.iotdb.db.utils.SetThreadName;
import org.apache.iotdb.metrics.type.AutoGauge;
import org.apache.iotdb.metrics.utils.MetricLevel;
@@ -153,7 +151,6 @@ import org.apache.iotdb.mpp.rpc.thrift.TCreateSchemaRegionReq;
import org.apache.iotdb.mpp.rpc.thrift.TCreateTriggerInstanceReq;
import org.apache.iotdb.mpp.rpc.thrift.TDeactivateTemplateReq;
import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataForDeleteSchemaReq;
-import org.apache.iotdb.mpp.rpc.thrift.TDeleteModelMetricsReq;
import org.apache.iotdb.mpp.rpc.thrift.TDeleteTimeSeriesReq;
import org.apache.iotdb.mpp.rpc.thrift.TDeleteViewSchemaReq;
import org.apache.iotdb.mpp.rpc.thrift.TDisableDataNodeReq;
@@ -225,7 +222,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.TimeZone;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
@@ -1095,34 +1091,6 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
}
}
- @Override
- public TSStatus deleteModelMetrics(TDeleteModelMetricsReq req) {
- IClientSession session = new InternalClientSession(req.getModelId());
- SESSION_MANAGER.registerSession(session);
- SESSION_MANAGER.supplySession(
- session, "MLNode", TimeZone.getDefault().getID(), ClientVersion.V_1_0);
-
- try {
- DeleteTimeSeriesStatement deleteTimeSeriesStatement = StatementGenerator.createStatement(req);
-
- long queryId = SESSION_MANAGER.requestQueryId();
- ExecutionResult result =
- COORDINATOR.execute(
- deleteTimeSeriesStatement,
- queryId,
- SESSION_MANAGER.getSessionInfo(session),
- "",
- partitionFetcher,
- schemaFetcher);
- return result.status;
- } catch (Exception e) {
- return ErrorHandlingUtils.onQueryException(e, OperationType.DELETE_TIMESERIES);
- } finally {
- SESSION_MANAGER.closeSession(session, COORDINATOR::cleanupQueryExecution);
- SESSION_MANAGER.removeCurrSession();
- }
- }
-
@Override
public TSStatus setSpaceQuota(TSetSpaceQuotaReq req) throws TException {
return spaceQuotaManager.setSpaceQuota(req);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/IMLNodeRPCServiceWithHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/IMLNodeRPCServiceWithHandler.java
deleted file mode 100644
index af489f31db2..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/IMLNodeRPCServiceWithHandler.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.protocol.thrift.impl;
-
-import org.apache.iotdb.mpp.rpc.thrift.IMLNodeInternalRPCService;
-
-public interface IMLNodeRPCServiceWithHandler extends IMLNodeInternalRPCService.Iface {
- void handleExit();
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/MLNodeRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/MLNodeRPCServiceImpl.java
deleted file mode 100644
index 46f1a966112..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/MLNodeRPCServiceImpl.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.protocol.thrift.impl;
-
-import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.commons.conf.IoTDBConstant.ClientVersion;
-import org.apache.iotdb.db.protocol.session.IClientSession;
-import org.apache.iotdb.db.protocol.session.InternalClientSession;
-import org.apache.iotdb.db.protocol.session.SessionManager;
-import org.apache.iotdb.db.protocol.thrift.OperationType;
-import org.apache.iotdb.db.queryengine.common.header.DatasetHeader;
-import org.apache.iotdb.db.queryengine.plan.Coordinator;
-import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher;
-import org.apache.iotdb.db.queryengine.plan.analyze.IPartitionFetcher;
-import org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher;
-import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaFetcher;
-import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult;
-import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution;
-import org.apache.iotdb.db.queryengine.plan.parser.StatementGenerator;
-import org.apache.iotdb.db.queryengine.plan.statement.Statement;
-import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement;
-import org.apache.iotdb.db.utils.ErrorHandlingUtils;
-import org.apache.iotdb.db.utils.QueryDataSetUtils;
-import org.apache.iotdb.db.utils.SetThreadName;
-import org.apache.iotdb.mpp.rpc.thrift.TFetchMoreDataReq;
-import org.apache.iotdb.mpp.rpc.thrift.TFetchMoreDataResp;
-import org.apache.iotdb.mpp.rpc.thrift.TFetchTimeseriesReq;
-import org.apache.iotdb.mpp.rpc.thrift.TFetchTimeseriesResp;
-import org.apache.iotdb.mpp.rpc.thrift.TFetchWindowBatchReq;
-import org.apache.iotdb.mpp.rpc.thrift.TFetchWindowBatchResp;
-import org.apache.iotdb.mpp.rpc.thrift.TRecordModelMetricsReq;
-import org.apache.iotdb.rpc.RpcUtils;
-import org.apache.iotdb.rpc.TSStatusCode;
-import org.apache.iotdb.tsfile.utils.Pair;
-
-import org.apache.thrift.TException;
-
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.TimeZone;
-
-public class MLNodeRPCServiceImpl implements IMLNodeRPCServiceWithHandler {
-
- public static final String ML_METRICS_PATH_PREFIX = "root.__system.ml.exp";
-
- private static final SessionManager SESSION_MANAGER = SessionManager.getInstance();
-
- private static final Coordinator COORDINATOR = Coordinator.getInstance();
-
- private final IPartitionFetcher partitionFetcher;
-
- private final ISchemaFetcher schemaFetcher;
-
- private final IClientSession session;
-
- public MLNodeRPCServiceImpl() {
- super();
- partitionFetcher = ClusterPartitionFetcher.getInstance();
- schemaFetcher = ClusterSchemaFetcher.getInstance();
- session = new InternalClientSession("MLNodeService");
- SESSION_MANAGER.registerSession(session);
- SESSION_MANAGER.supplySession(
- session, "MLNode", TimeZone.getDefault().getID(), ClientVersion.V_1_0);
- }
-
- @Override
- public TFetchTimeseriesResp fetchTimeseries(TFetchTimeseriesReq req) throws TException {
- boolean finished = false;
- TFetchTimeseriesResp resp = new TFetchTimeseriesResp();
- Throwable t = null;
- try {
-
- Statement s = StatementGenerator.createStatement(req, session.getZoneId());
-
- if (s == null) {
- resp.setStatus(
- RpcUtils.getStatus(
- TSStatusCode.SQL_PARSE_ERROR, "This operation type is not supported"));
- return resp;
- }
-
- long queryId =
- SESSION_MANAGER.requestQueryId(session, SESSION_MANAGER.requestStatementId(session));
- ExecutionResult result =
- COORDINATOR.execute(
- s,
- queryId,
- SESSION_MANAGER.getSessionInfo(session),
- "",
- partitionFetcher,
- schemaFetcher,
- req.getTimeout());
-
- if (result.status.code != TSStatusCode.SUCCESS_STATUS.getStatusCode()
- && result.status.code != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) {
- resp.setStatus(result.status);
- return resp;
- }
-
- IQueryExecution queryExecution = COORDINATOR.getQueryExecution(queryId);
-
- try (SetThreadName threadName = new SetThreadName(result.queryId.getId())) {
-
- DatasetHeader header = queryExecution.getDatasetHeader();
- resp.setStatus(result.status);
- resp.setColumnNameList(header.getRespColumns());
- resp.setColumnTypeList(header.getRespDataTypeList());
- resp.setColumnNameIndexMap(header.getColumnNameIndexMap());
- resp.setQueryId(queryId);
-
- Pair<List<ByteBuffer>, Boolean> pair =
- QueryDataSetUtils.convertQueryResultByFetchSize(queryExecution, req.fetchSize);
- resp.setTsDataset(pair.left);
- finished = pair.right;
- resp.setHasMoreData(!finished);
- return resp;
- }
- } catch (Exception e) {
- finished = true;
- t = e;
- resp.setStatus(ErrorHandlingUtils.onQueryException(e, OperationType.EXECUTE_STATEMENT));
- return resp;
- } catch (Error error) {
- t = error;
- throw error;
- } finally {
- if (finished) {
- COORDINATOR.cleanupQueryExecution(resp.queryId, t);
- }
- }
- }
-
- @Override
- public TFetchMoreDataResp fetchMoreData(TFetchMoreDataReq req) throws TException {
- TFetchMoreDataResp resp = new TFetchMoreDataResp();
- boolean finished = false;
- Throwable t = null;
- try {
- IQueryExecution queryExecution = COORDINATOR.getQueryExecution(req.queryId);
- resp.setStatus(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
-
- if (queryExecution == null) {
- resp.setHasMoreData(false);
- return resp;
- }
-
- try (SetThreadName queryName = new SetThreadName(queryExecution.getQueryId())) {
- Pair<List<ByteBuffer>, Boolean> pair =
- QueryDataSetUtils.convertQueryResultByFetchSize(queryExecution, req.fetchSize);
- List<ByteBuffer> result = pair.left;
- finished = pair.right;
- resp.setTsDataset(result);
- resp.setHasMoreData(!finished);
- return resp;
- }
- } catch (Exception e) {
- finished = true;
- t = e;
- resp.setStatus(ErrorHandlingUtils.onQueryException(e, OperationType.FETCH_RESULTS));
- return resp;
- } catch (Error error) {
- t = error;
- throw error;
- } finally {
- if (finished) {
- COORDINATOR.cleanupQueryExecution(req.queryId, t);
- }
- }
- }
-
- @Override
- public TSStatus recordModelMetrics(TRecordModelMetricsReq req) throws TException {
- try {
- InsertRowStatement insertRowStatement = StatementGenerator.createStatement(req);
-
- long queryId = SESSION_MANAGER.requestQueryId();
- ExecutionResult result =
- COORDINATOR.execute(
- insertRowStatement,
- queryId,
- SESSION_MANAGER.getSessionInfo(session),
- "",
- partitionFetcher,
- schemaFetcher);
- return result.status;
- } catch (Exception e) {
- return ErrorHandlingUtils.onQueryException(e, OperationType.INSERT_RECORD);
- }
- }
-
- @Override
- public TFetchWindowBatchResp fetchWindowBatch(TFetchWindowBatchReq req) throws TException {
- throw new TException(new UnsupportedOperationException().getCause());
- }
-
- @Override
- public void handleExit() {
- SESSION_MANAGER.closeSession(session, COORDINATOR::cleanupQueryExecution);
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java
index 6d90140312f..296a5bc8c1f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java
@@ -179,15 +179,6 @@ public class ColumnHeaderConstant {
public static final String USER = "User";
public static final String READ_WRITE = "Read/Write";
- // column names for show models/trials
- public static final String MODEL_ID = "ModelId";
- public static final String TRIAL_ID = "TrialId";
- public static final String MODEL_TASK = "ModelTask";
- public static final String MODEL_TYPE = "ModelType";
- public static final String QUERY_BODY = "QueryBody";
- public static final String HYPERPARAMETER = "Hyperparameter";
- public static final String MODEL_PATH = "ModelPath";
-
// column names for views (e.g. logical view)
public static final String VIEW_TYPE = "ViewType";
public static final String SOURCE = "Source";
@@ -452,21 +443,6 @@ public class ColumnHeaderConstant {
new ColumnHeader(LIMIT, TSDataType.TEXT),
new ColumnHeader(READ_WRITE, TSDataType.TEXT));
- public static final List<ColumnHeader> showModelsColumnHeaders =
- ImmutableList.of(
- new ColumnHeader(MODEL_ID, TSDataType.TEXT),
- new ColumnHeader(MODEL_TASK, TSDataType.TEXT),
- new ColumnHeader(MODEL_TYPE, TSDataType.TEXT),
- new ColumnHeader(QUERY_BODY, TSDataType.TEXT),
- new ColumnHeader(STATE, TSDataType.TEXT),
- new ColumnHeader(HYPERPARAMETER, TSDataType.TEXT));
-
- public static final List<ColumnHeader> showTrialsColumnHeaders =
- ImmutableList.of(
- new ColumnHeader(TRIAL_ID, TSDataType.TEXT),
- new ColumnHeader(MODEL_PATH, TSDataType.TEXT),
- new ColumnHeader(HYPERPARAMETER, TSDataType.TEXT));
-
public static final List<ColumnHeader> showLogicalViewColumnHeaders =
ImmutableList.of(
new ColumnHeader(TIMESERIES, TSDataType.TEXT),
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java
index 423626d227f..ef276ee7ca9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java
@@ -185,14 +185,6 @@ public class DatasetHeaderFactory {
return new DatasetHeader(ColumnHeaderConstant.showThrottleQuotaColumnHeaders, true);
}
- public static DatasetHeader getShowModelsHeader() {
- return new DatasetHeader(ColumnHeaderConstant.showModelsColumnHeaders, true);
- }
-
- public static DatasetHeader getShowTrialsHeader() {
- return new DatasetHeader(ColumnHeaderConstant.showTrialsColumnHeaders, true);
- }
-
public static DatasetHeader getShowLogicalViewHeader() {
return new DatasetHeader(ColumnHeaderConstant.showLogicalViewColumnHeaders, true);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java
index 4c8323b705b..b9ead85cbb5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.db.queryengine.execution.fragment;
import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory;
import org.apache.iotdb.commons.concurrent.ThreadName;
import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil;
-import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.queryengine.common.FragmentInstanceId;
import org.apache.iotdb.db.queryengine.common.QueryId;
@@ -79,7 +78,6 @@ public class FragmentInstanceManager {
private final CounterStat failedInstances = new CounterStat();
private final ExecutorService intoOperationExecutor;
- private final ExecutorService modelInferenceExecutor;
private final MPPDataExchangeManager exchangeManager =
MPPDataExchangeService.getInstance().getMPPDataExchangeManager();
@@ -117,11 +115,6 @@ public class FragmentInstanceManager {
IoTDBThreadPoolFactory.newFixedThreadPool(
IoTDBDescriptor.getInstance().getConfig().getIntoOperationExecutionThreadCount(),
"into-operation-executor");
-
- this.modelInferenceExecutor =
- IoTDBThreadPoolFactory.newFixedThreadPool(
- CommonDescriptor.getInstance().getConfig().getModelInferenceExecutionThreadCount(),
- "model-inference-executor");
}
public int getInstanceContextSize() {
@@ -367,10 +360,6 @@ public class FragmentInstanceManager {
return intoOperationExecutor;
}
- public ExecutorService getModelInferenceExecutor() {
- return modelInferenceExecutor;
- }
-
private static class InstanceHolder {
private InstanceHolder() {}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/ml/ForecastOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/ml/ForecastOperator.java
deleted file mode 100644
index 9df739ee40a..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/ml/ForecastOperator.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.queryengine.execution.operator.process.ml;
-
-import org.apache.iotdb.commons.client.mlnode.MLNodeClient;
-import org.apache.iotdb.commons.client.mlnode.MLNodeClientManager;
-import org.apache.iotdb.commons.client.mlnode.MLNodeInfo;
-import org.apache.iotdb.db.exception.runtime.ModelInferenceProcessException;
-import org.apache.iotdb.db.queryengine.execution.operator.Operator;
-import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext;
-import org.apache.iotdb.db.queryengine.execution.operator.process.ProcessOperator;
-import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
-import org.apache.iotdb.mlnode.rpc.thrift.TForecastResp;
-import org.apache.iotdb.rpc.TSStatusCode;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.read.common.block.TsBlock;
-import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
-import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
-import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder;
-import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
-
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import static com.google.common.util.concurrent.Futures.successfulAsList;
-
-public class ForecastOperator implements ProcessOperator {
-
- private final OperatorContext operatorContext;
- private final Operator child;
-
- private final String modelPath;
- private final List<TSDataType> inputTypeList;
- private final List<String> inputColumnNameList;
- private final int expectedPredictLength;
-
- private final TsBlockBuilder inputTsBlockBuilder;
-
- private final ExecutorService modelInferenceExecutor;
- private ListenableFuture<TForecastResp> forecastExecutionFuture;
-
- private boolean finished = false;
-
- private final long maxRetainedSize;
- private final long maxReturnSize;
-
- public ForecastOperator(
- OperatorContext operatorContext,
- Operator child,
- String modelPath,
- List<TSDataType> inputTypeList,
- List<String> inputColumnNameList,
- int expectedPredictLength,
- ExecutorService modelInferenceExecutor,
- long maxRetainedSize,
- long maxReturnSize) {
- this.operatorContext = operatorContext;
- this.child = child;
- this.modelPath = modelPath;
- this.inputTypeList = inputTypeList;
- this.inputColumnNameList = inputColumnNameList;
- this.expectedPredictLength = expectedPredictLength;
- this.inputTsBlockBuilder = new TsBlockBuilder(inputTypeList);
- this.modelInferenceExecutor = modelInferenceExecutor;
- this.maxRetainedSize = maxRetainedSize;
- this.maxReturnSize = maxReturnSize;
- }
-
- @Override
- public OperatorContext getOperatorContext() {
- return operatorContext;
- }
-
- @Override
- public ListenableFuture<?> isBlocked() {
- ListenableFuture<?> childBlocked = child.isBlocked();
- boolean executionDone = forecastExecutionDone();
- if (executionDone && childBlocked.isDone()) {
- return NOT_BLOCKED;
- } else if (childBlocked.isDone()) {
- return forecastExecutionFuture;
- } else if (executionDone) {
- return childBlocked;
- } else {
- return successfulAsList(Arrays.asList(forecastExecutionFuture, childBlocked));
- }
- }
-
- private boolean forecastExecutionDone() {
- if (forecastExecutionFuture == null) {
- return true;
- }
- return forecastExecutionFuture.isDone();
- }
-
- @Override
- public boolean hasNext() throws Exception {
- return !finished;
- }
-
- @Override
- public TsBlock next() throws Exception {
- if (forecastExecutionFuture == null) {
- if (child.hasNextWithTimer()) {
- TsBlock inputTsBlock = child.nextWithTimer();
- if (inputTsBlock != null) {
- appendTsBlockToBuilder(inputTsBlock);
- }
- } else {
- submitForecastTask();
- }
- return null;
- } else {
- try {
- if (!forecastExecutionFuture.isDone()) {
- throw new IllegalStateException(
- "The operator cannot continue until the forecast execution is done.");
- }
-
- TForecastResp forecastResp = forecastExecutionFuture.get();
- if (forecastResp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- String message =
- String.format(
- "Error occurred while executing forecast: %s",
- forecastResp.getStatus().getMessage());
- throw new ModelInferenceProcessException(message);
- }
-
- finished = true;
- TsBlock resultTsBlock =
- new TsBlockSerde().deserialize(forecastResp.bufferForForecastResult());
- resultTsBlock = modifyTimeColumn(resultTsBlock);
- return resultTsBlock;
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new ModelInferenceProcessException(e.getMessage());
- } catch (ExecutionException e) {
- throw new ModelInferenceProcessException(e.getMessage());
- }
- }
- }
-
- private TsBlock modifyTimeColumn(TsBlock resultTsBlock) {
- TsBlockBuilder newTsBlockBuilder = TsBlockBuilder.createWithOnlyTimeColumn();
- TimeColumnBuilder timeColumnBuilder = newTsBlockBuilder.getTimeColumnBuilder();
- for (int i = 0; i < resultTsBlock.getPositionCount(); i++) {
- timeColumnBuilder.writeLong(
- TimestampPrecisionUtils.convertToCurrPrecision(
- resultTsBlock.getTimeByIndex(i), TimeUnit.NANOSECONDS));
- newTsBlockBuilder.declarePosition();
- }
- return newTsBlockBuilder.build().appendValueColumns(resultTsBlock.getValueColumns());
- }
-
- private void appendTsBlockToBuilder(TsBlock inputTsBlock) {
- TimeColumnBuilder timeColumnBuilder = inputTsBlockBuilder.getTimeColumnBuilder();
- ColumnBuilder[] columnBuilders = inputTsBlockBuilder.getValueColumnBuilders();
-
- for (int i = 0; i < inputTsBlock.getPositionCount(); i++) {
- timeColumnBuilder.writeLong(inputTsBlock.getTimeByIndex(i));
- for (int columnIndex = 0; columnIndex < inputTsBlock.getValueColumnCount(); columnIndex++) {
- columnBuilders[columnIndex].write(inputTsBlock.getColumn(columnIndex), i);
- }
- inputTsBlockBuilder.declarePosition();
- }
- }
-
- private void submitForecastTask() {
-
- TsBlock inputTsBlock = inputTsBlockBuilder.build();
- inputTsBlock.reverse();
-
- forecastExecutionFuture =
- Futures.submit(
- () -> {
- try (MLNodeClient client =
- MLNodeClientManager.getInstance().borrowClient(MLNodeInfo.endPoint)) {
- return client.forecast(
- modelPath,
- inputTsBlock,
- inputTypeList,
- inputColumnNameList,
- expectedPredictLength);
- } catch (Exception e) {
- throw new ModelInferenceProcessException(e.getMessage());
- }
- },
- modelInferenceExecutor);
- }
-
- @Override
- public boolean isFinished() throws Exception {
- return finished;
- }
-
- @Override
- public void close() throws Exception {
- if (forecastExecutionFuture != null) {
- forecastExecutionFuture.cancel(true);
- }
- child.close();
- }
-
- @Override
- public long calculateMaxPeekMemory() {
- return maxReturnSize + maxRetainedSize + child.calculateMaxPeekMemory();
- }
-
- @Override
- public long calculateMaxReturnSize() {
- return maxReturnSize;
- }
-
- @Override
- public long calculateRetainedSizeAfterCallingNext() {
- return maxRetainedSize + child.calculateRetainedSizeAfterCallingNext();
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java
index aee45a16193..e9a541a9d79 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java
@@ -38,7 +38,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.GroupByParame
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.GroupByTimeParameter;
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.IntoPathDescriptor;
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.OrderByParameter;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.model.ModelInferenceDescriptor;
import org.apache.iotdb.db.queryengine.plan.statement.Statement;
import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering;
import org.apache.iotdb.db.queryengine.plan.statement.component.SortItem;
@@ -221,8 +220,6 @@ public class Analysis {
// indicate whether the Nodes produce source data are VirtualSourceNodes
private boolean isVirtualSource = false;
- private ModelInferenceDescriptor modelInferenceDescriptor;
-
/////////////////////////////////////////////////////////////////////////////////////////////////
// SELECT INTO Analysis
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -754,14 +751,6 @@ public class Analysis {
this.lastQueryNonWritableViewSourceExpressionMap = lastQueryNonWritableViewSourceExpressionMap;
}
- public ModelInferenceDescriptor getModelInferenceDescriptor() {
- return modelInferenceDescriptor;
- }
-
- public void setModelInferenceDescriptor(ModelInferenceDescriptor modelInferenceDescriptor) {
- this.modelInferenceDescriptor = modelInferenceDescriptor;
- }
-
public Map<String, List<String>> getOutputDeviceToQueriedDevicesMap() {
return outputDeviceToQueriedDevicesMap;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
index 6b47b26ad0e..1344cd593c2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java
@@ -26,8 +26,6 @@ import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.commons.exception.MetadataException;
-import org.apache.iotdb.commons.model.ForecastModeInformation;
-import org.apache.iotdb.commons.model.ModelInformation;
import org.apache.iotdb.commons.partition.DataPartition;
import org.apache.iotdb.commons.partition.DataPartitionQueryParam;
import org.apache.iotdb.commons.partition.SchemaNodeManagementPartition;
@@ -39,7 +37,6 @@ import org.apache.iotdb.commons.schema.SchemaConstant;
import org.apache.iotdb.commons.schema.view.LogicalViewSchema;
import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression;
import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics;
-import org.apache.iotdb.commons.udf.builtin.ModelInferenceFunction;
import org.apache.iotdb.commons.utils.TimePartitionUtils;
import org.apache.iotdb.confignode.rpc.thrift.TGetDataNodeLocationsResp;
import org.apache.iotdb.db.conf.IoTDBConfig;
@@ -79,8 +76,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.GroupByTimePa
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.GroupByVariationParameter;
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.IntoPathDescriptor;
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.OrderByParameter;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.model.ForecastModelInferenceDescriptor;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.model.ModelInferenceDescriptor;
import org.apache.iotdb.db.queryengine.plan.statement.Statement;
import org.apache.iotdb.db.queryengine.plan.statement.StatementNode;
import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor;
@@ -92,7 +87,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.component.GroupBySessionCo
import org.apache.iotdb.db.queryengine.plan.statement.component.GroupByTimeComponent;
import org.apache.iotdb.db.queryengine.plan.statement.component.GroupByVariationComponent;
import org.apache.iotdb.db.queryengine.plan.statement.component.IntoComponent;
-import org.apache.iotdb.db.queryengine.plan.statement.component.OrderByComponent;
import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering;
import org.apache.iotdb.db.queryengine.plan.statement.component.ResultColumn;
import org.apache.iotdb.db.queryengine.plan.statement.component.SortItem;
@@ -182,7 +176,6 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.ALLOWED_SCHEMA_PROPS;
import static org.apache.iotdb.commons.conf.IoTDBConstant.DEADBAND;
import static org.apache.iotdb.commons.conf.IoTDBConstant.LOSS;
import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD;
-import static org.apache.iotdb.commons.udf.builtin.ModelInferenceFunction.FORECAST;
import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.DEVICE;
import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.ENDTIME;
import static org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.PARTITION_FETCHER;
@@ -202,7 +195,6 @@ import static org.apache.iotdb.db.queryengine.plan.optimization.LimitOffsetPushD
import static org.apache.iotdb.db.queryengine.plan.optimization.LimitOffsetPushDown.pushDownLimitOffsetInGroupByTimeForDevice;
import static org.apache.iotdb.db.schemaengine.schemaregion.view.visitor.GetSourcePathsVisitor.getSourcePaths;
import static org.apache.iotdb.db.utils.constant.SqlConstant.COUNT_TIME_HEADER;
-import static org.apache.iotdb.db.utils.constant.SqlConstant.MODEL_ID;
/** This visitor is used to analyze each type of Statement and returns the {@link Analysis}. */
public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> {
@@ -222,7 +214,6 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
private final IPartitionFetcher partitionFetcher;
private final ISchemaFetcher schemaFetcher;
- private final IModelFetcher modelFetcher;
private static final PerformanceOverviewMetrics PERFORMANCE_OVERVIEW_METRICS =
PerformanceOverviewMetrics.getInstance();
@@ -230,7 +221,6 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
public AnalyzeVisitor(IPartitionFetcher partitionFetcher, ISchemaFetcher schemaFetcher) {
this.partitionFetcher = partitionFetcher;
this.schemaFetcher = schemaFetcher;
- this.modelFetcher = ModelFetcher.getInstance();
}
@Override
@@ -255,10 +245,6 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
// check for semantic errors
queryStatement.semanticCheck();
- if (queryStatement.isModelInferenceQuery()) {
- analyzeModelInference(analysis, queryStatement);
- }
-
ISchemaTree schemaTree = analyzeSchema(queryStatement, analysis, context);
// If there is no leaf node in the schema tree, the query should be completed immediately
if (schemaTree.isEmpty()) {
@@ -356,47 +342,6 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
return analysis;
}
- private void analyzeModelInference(Analysis analysis, QueryStatement queryStatement) {
- FunctionExpression modelInferenceExpression =
- (FunctionExpression)
- queryStatement.getSelectComponent().getResultColumns().get(0).getExpression();
- String modelId = modelInferenceExpression.getFunctionAttributes().get(MODEL_ID);
-
- ModelInformation modelInformation = modelFetcher.getModelInformation(modelId);
- if (modelInformation == null || !modelInformation.available()) {
- throw new SemanticException("Model " + modelId + " is not available");
- }
-
- ModelInferenceFunction functionType =
- ModelInferenceFunction.valueOf(modelInferenceExpression.getFunctionName().toUpperCase());
- if (functionType == FORECAST) {
- ForecastModelInferenceDescriptor modelInferenceDescriptor =
- new ForecastModelInferenceDescriptor(
- functionType, (ForecastModeInformation) modelInformation);
- Map<String, String> modelInferenceAttributes =
- modelInferenceExpression.getFunctionAttributes();
- if (modelInferenceAttributes.containsKey("predict_length")) {
- modelInferenceDescriptor.setExpectedPredictLength(
- Integer.parseInt(modelInferenceAttributes.get("predict_length")));
- }
- analysis.setModelInferenceDescriptor(modelInferenceDescriptor);
-
- List<ResultColumn> newResultColumns = new ArrayList<>();
- for (Expression inputExpression : modelInferenceExpression.getExpressions()) {
- newResultColumns.add(
- new ResultColumn(inputExpression, ResultColumn.ColumnType.MODEL_INFERENCE));
- }
- queryStatement.getSelectComponent().setResultColumns(newResultColumns);
-
- OrderByComponent descTimeOrder = new OrderByComponent();
- descTimeOrder.addSortItem(new SortItem("TIME", Ordering.DESC));
- queryStatement.setOrderByComponent(descTimeOrder);
- } else {
- throw new IllegalArgumentException(
- "Unsupported model inference function type " + functionType);
- }
- }
-
private ISchemaTree analyzeSchema(
QueryStatement queryStatement, Analysis analysis, MPPQueryContext context) {
// concat path and construct path pattern tree
@@ -1495,58 +1440,6 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext>
return;
}
- if (queryStatement.isModelInferenceQuery()) {
- List<ColumnHeader> columnHeaders = new ArrayList<>();
-
- ModelInferenceDescriptor modelInferenceDescriptor = analysis.getModelInferenceDescriptor();
- if (Objects.requireNonNull(modelInferenceDescriptor.getFunctionType()) == FORECAST) {
- ForecastModelInferenceDescriptor forecastModelInferenceDescriptor =
- (ForecastModelInferenceDescriptor) modelInferenceDescriptor;
-
- List<TSDataType> inputTypeList = forecastModelInferenceDescriptor.getInputTypeList();
- if (outputExpressions.size() != inputTypeList.size()) {
- throw new SemanticException(
- String.format(
- "The number of input expressions does not match the number of input types [%d] when training",
- inputTypeList.size()));
- }
- for (int i = 0; i < inputTypeList.size(); i++) {
- Expression inputExpression = outputExpressions.get(i).left;
- TSDataType inputDataType = analysis.getType(inputExpression);
- if (inputDataType != inputTypeList.get(i)) {
- throw new SemanticException(
- String.format(
- "The type of input expression [%s] does not match the type of input type [%s] when training",
- inputDataType, inputTypeList.get(i)));
- }
- }
-
- List<FunctionExpression> modelInferenceOutputExpressions = new ArrayList<>();
- for (int predictIndex : forecastModelInferenceDescriptor.getPredictIndexList()) {
- Expression inputExpression = outputExpressions.get(predictIndex).left;
- FunctionExpression modelInferenceOutputExpression =
- new FunctionExpression(
- FORECAST.getFunctionName(),
- forecastModelInferenceDescriptor.getOutputAttributes(),
- Collections.singletonList(inputExpression));
- analyzeExpression(analysis, modelInferenceOutputExpression);
- modelInferenceOutputExpressions.add(modelInferenceOutputExpression);
- columnHeaders.add(
- new ColumnHeader(
- modelInferenceOutputExpression.toString(),
- analysis.getType(modelInferenceOutputExpression)));
- }
- forecastModelInferenceDescriptor.setModelInferenceOutputExpressions(
- modelInferenceOutputExpressions);
- } else {
- throw new SemanticException(
- "Unsupported model inference function type "
- + modelInferenceDescriptor.getFunctionType());
- }
- analysis.setRespDatasetHeader(new DatasetHeader(columnHeaders, false));
- return;
- }
-
boolean isIgnoreTimestamp = queryStatement.isAggregationQuery() && !queryStatement.isGroupBy();
List<ColumnHeader> columnHeaders = new ArrayList<>();
if (queryStatement.isAlignByDevice()) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ExpressionAnalyzer.java
index 841f9254ab2..d79e06f34ca 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ExpressionAnalyzer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ExpressionAnalyzer.java
@@ -197,8 +197,6 @@ public class ExpressionAnalyzer {
}
}
return ResultColumn.ColumnType.AGGREGATION;
- } else if (((FunctionExpression) expression).isModelInferenceFunction()) {
- return ResultColumn.ColumnType.MODEL_INFERENCE;
} else {
ResultColumn.ColumnType checkedType = null;
int lastCheckedIndex = 0;
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ExpressionTypeAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ExpressionTypeAnalyzer.java
index 18eb59894b7..41cd1fbe882 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ExpressionTypeAnalyzer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ExpressionTypeAnalyzer.java
@@ -269,8 +269,6 @@ public class ExpressionTypeAnalyzer {
functionExpression,
TypeInferenceUtils.getBuiltInScalarFunctionDataType(
functionExpression, expressionTypes.get(NodeRef.of(inputExpressions.get(0)))));
- } else if (functionExpression.isModelInferenceFunction()) {
- return setExpressionType(functionExpression, TSDataType.DOUBLE);
} else {
return setExpressionType(
functionExpression,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IModelFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IModelFetcher.java
deleted file mode 100644
index 1f0221b5940..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IModelFetcher.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.queryengine.plan.analyze;
-
-import org.apache.iotdb.commons.model.ModelInformation;
-
-public interface IModelFetcher {
- /** Get model information by model id from configNode. */
- ModelInformation getModelInformation(String modelId);
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ModelFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ModelFetcher.java
deleted file mode 100644
index d829c51c0d0..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ModelFetcher.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.queryengine.plan.analyze;
-
-import org.apache.iotdb.commons.client.IClientManager;
-import org.apache.iotdb.commons.client.exception.ClientManagerException;
-import org.apache.iotdb.commons.consensus.ConfigRegionId;
-import org.apache.iotdb.commons.model.ModelInformation;
-import org.apache.iotdb.confignode.rpc.thrift.TGetModelInfoReq;
-import org.apache.iotdb.confignode.rpc.thrift.TGetModelInfoResp;
-import org.apache.iotdb.db.exception.sql.StatementAnalyzeException;
-import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
-import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager;
-import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
-import org.apache.iotdb.rpc.TSStatusCode;
-
-import org.apache.thrift.TException;
-
-public class ModelFetcher implements IModelFetcher {
-
- private final IClientManager<ConfigRegionId, ConfigNodeClient> configNodeClientManager =
- ConfigNodeClientManager.getInstance();
-
- private static final class ModelFetcherHolder {
-
- private static final ModelFetcher INSTANCE = new ModelFetcher();
-
- private ModelFetcherHolder() {}
- }
-
- public static ModelFetcher getInstance() {
- return ModelFetcher.ModelFetcherHolder.INSTANCE;
- }
-
- private ModelFetcher() {}
-
- @Override
- public ModelInformation getModelInformation(String modelId) {
- try (ConfigNodeClient client =
- configNodeClientManager.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
- TGetModelInfoResp getModelInfoResp = client.getModelInfo(new TGetModelInfoReq(modelId));
- if (getModelInfoResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- if (getModelInfoResp.modelInfo != null) {
- return ModelInformation.deserialize(getModelInfoResp.modelInfo);
- } else {
- return null;
- }
- } else {
- throw new StatementAnalyzeException(
- "An error occurred when executing getModelInformation():"
- + getModelInfoResp.getStatus().getMessage());
- }
- } catch (ClientManagerException | TException e) {
- throw new StatementAnalyzeException(
- "An error occurred when executing getModelInformation():" + e.getMessage());
- }
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java
index abfcee56cea..798845c76dc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java
@@ -51,10 +51,6 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowTTLTas
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowTriggersTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowVariablesTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.UnSetTTLTask;
-import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.model.CreateModelTask;
-import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.model.DropModelTask;
-import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.model.ShowModelsTask;
-import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.model.ShowTrialsTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.template.AlterSchemaTemplateTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.template.CreateSchemaTemplateTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.template.DeactivateSchemaTemplateTask;
@@ -113,10 +109,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTTLStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTriggersStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowVariablesStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.UnSetTTLStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.CreateModelStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.DropModelStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.ShowModelsStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.ShowTrialsStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.CreatePipePluginStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.CreatePipeStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.DropPipePluginStatement;
@@ -513,29 +505,4 @@ public class ConfigTaskVisitor extends StatementVisitor<IConfigTask, MPPQueryCon
ShowThrottleQuotaStatement showThrottleQuotaStatement, MPPQueryContext context) {
return new ShowThrottleQuotaTask(showThrottleQuotaStatement);
}
-
- /** ML Model Management */
- @Override
- public IConfigTask visitCreateModel(
- CreateModelStatement createModelStatement, MPPQueryContext context) {
- return new CreateModelTask(createModelStatement, context);
- }
-
- @Override
- public IConfigTask visitDropModel(
- DropModelStatement dropModelStatement, MPPQueryContext context) {
- return new DropModelTask(dropModelStatement.getModelId());
- }
-
- @Override
- public IConfigTask visitShowModels(
- ShowModelsStatement showModelsStatement, MPPQueryContext context) {
- return new ShowModelsTask();
- }
-
- @Override
- public IConfigTask visitShowTrials(
- ShowTrialsStatement showTrialsStatement, MPPQueryContext context) {
- return new ShowTrialsTask(showTrialsStatement.getModelId());
- }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
index 66db56300e6..c3803213c99 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
@@ -52,7 +52,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListReq;
import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListResp;
import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreateFunctionReq;
-import org.apache.iotdb.confignode.rpc.thrift.TCreateModelReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreatePipePluginReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq;
import org.apache.iotdb.confignode.rpc.thrift.TCreateTriggerReq;
@@ -64,7 +63,6 @@ import org.apache.iotdb.confignode.rpc.thrift.TDeleteLogicalViewReq;
import org.apache.iotdb.confignode.rpc.thrift.TDeleteTimeSeriesReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropCQReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropFunctionReq;
-import org.apache.iotdb.confignode.rpc.thrift.TDropModelReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropPipePluginReq;
import org.apache.iotdb.confignode.rpc.thrift.TDropTriggerReq;
import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq;
@@ -84,15 +82,11 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowConfigNodesResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowDatabaseResp;
-import org.apache.iotdb.confignode.rpc.thrift.TShowModelReq;
-import org.apache.iotdb.confignode.rpc.thrift.TShowModelResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo;
import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq;
import org.apache.iotdb.confignode.rpc.thrift.TShowRegionReq;
import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowThrottleReq;
-import org.apache.iotdb.confignode.rpc.thrift.TShowTrialReq;
-import org.apache.iotdb.confignode.rpc.thrift.TShowTrialResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp;
import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp;
import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp;
@@ -112,7 +106,6 @@ import org.apache.iotdb.db.queryengine.plan.Coordinator;
import org.apache.iotdb.db.queryengine.plan.analyze.Analysis;
import org.apache.iotdb.db.queryengine.plan.analyze.Analyzer;
import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher;
-import org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer;
import org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher;
import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult;
import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult;
@@ -133,8 +126,6 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowRegion
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowTTLTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowTriggersTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowVariablesTask;
-import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.model.ShowModelsTask;
-import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.model.ShowTrialsTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.template.ShowNodesInSchemaTemplateTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.template.ShowPathSetTemplateTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.template.ShowSchemaTemplateTask;
@@ -143,11 +134,6 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.sys.quota.ShowSpace
import org.apache.iotdb.db.queryengine.plan.execution.config.sys.quota.ShowThrottleQuotaTask;
import org.apache.iotdb.db.queryengine.plan.expression.Expression;
import org.apache.iotdb.db.queryengine.plan.expression.visitor.TransformToViewExpressionVisitor;
-import org.apache.iotdb.db.queryengine.plan.statement.component.FromComponent;
-import org.apache.iotdb.db.queryengine.plan.statement.component.ResultColumn;
-import org.apache.iotdb.db.queryengine.plan.statement.component.SelectComponent;
-import org.apache.iotdb.db.queryengine.plan.statement.component.WhereCondition;
-import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDatabaseStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountTimeSlotListStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.CreateContinuousQueryStatement;
@@ -166,7 +152,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowDataNodesStat
import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowDatabaseStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowRegionStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTTLStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.CreateModelStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.CreatePipePluginStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.CreatePipeStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.DropPipeStatement;
@@ -204,8 +189,6 @@ import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.trigger.api.Trigger;
import org.apache.iotdb.trigger.api.enums.FailureStrategy;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
import org.apache.iotdb.udf.api.UDTF;
@@ -236,14 +219,7 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-import static org.apache.iotdb.commons.model.ForecastModeInformation.INPUT_TYPE_LIST;
-import static org.apache.iotdb.commons.model.ForecastModeInformation.PREDICT_INDEX_LIST;
-import static org.apache.iotdb.commons.schema.SchemaConstant.ROOT;
import static org.apache.iotdb.db.protocol.client.ConfigNodeClient.MSG_RECONNECTION_FAIL;
-import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.and;
-import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.longValue;
-import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.lte;
-import static org.apache.iotdb.db.queryengine.plan.expression.ExpressionFactory.time;
public class ClusterConfigTaskExecutor implements IConfigTaskExecutor {
@@ -2119,148 +2095,6 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor {
return future;
}
- @Override
- public SettableFuture<ConfigTaskResult> createModel(
- CreateModelStatement createModelStatement, MPPQueryContext context) {
- createModelStatement.semanticCheck();
-
- QueryStatement datasetStatement = createModelStatement.getDatasetStatement();
- Analysis analysis = Analyzer.analyze(datasetStatement, context);
-
- SettableFuture<ConfigTaskResult> future = SettableFuture.create();
- if (analysis.getOutputExpressions() == null) {
- future.setException(
- new IoTDBException(
- "The query result of the sql in CREATE MODEL are null.",
- TSStatusCode.ILLEGAL_PARAMETER.getStatusCode()));
- return future;
- }
-
- SelectComponent formattedSelect =
- new SelectComponent(datasetStatement.getSelectComponent().getZoneId());
- List<TSDataType> inputTypeList = new ArrayList<>();
- for (Expression outputExpression :
- analysis.getOutputExpressions().stream().map(Pair::getLeft).collect(Collectors.toList())) {
- ResultColumn.ColumnType columnType =
- ExpressionAnalyzer.identifyOutputColumnType(outputExpression, true);
- formattedSelect.addResultColumn(
- new ResultColumn(ExpressionAnalyzer.removeRootPrefix(outputExpression), columnType));
- inputTypeList.add(analysis.getType(outputExpression));
- }
- datasetStatement.setSelectComponent(formattedSelect);
-
- FromComponent fromRoot = new FromComponent();
- fromRoot.addPrefixPath(new PartialPath(ROOT, false));
- datasetStatement.setFromComponent(fromRoot);
-
- WhereCondition endNowWhereCondition = new WhereCondition();
- Expression endNowExpression = lte(time(), longValue(System.currentTimeMillis()));
- if (!datasetStatement.hasWhere()) {
- endNowWhereCondition.setPredicate(endNowExpression);
- } else {
- endNowWhereCondition.setPredicate(
- and(
- ExpressionAnalyzer.removeRootPrefix(
- datasetStatement.getWhereCondition().getPredicate()),
- endNowExpression));
- }
- datasetStatement.setWhereCondition(endNowWhereCondition);
-
- String datesetFetchSQL = datasetStatement.constructFormattedSQL();
-
- Map<String, String> options = createModelStatement.getOptions();
- options.put(INPUT_TYPE_LIST, Arrays.toString(inputTypeList.toArray()));
- if (!options.containsKey(PREDICT_INDEX_LIST)) {
- List<Integer> predictIndexList = new ArrayList<>();
- for (int i = 0; i < inputTypeList.size(); i++) {
- predictIndexList.add(i);
- }
- options.put(PREDICT_INDEX_LIST, Arrays.toString(predictIndexList.toArray()));
- }
-
- try (ConfigNodeClient client =
- CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
- TCreateModelReq createModelReq =
- new TCreateModelReq(
- createModelStatement.getModelId(),
- createModelStatement.getTaskType(),
- createModelStatement.getOptions(),
- createModelStatement.getHyperparameters(),
- datesetFetchSQL);
-
- final TSStatus executionStatus = client.createModel(createModelReq);
- if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != executionStatus.getCode()) {
- LOGGER.warn(
- "[{}] Failed to create model {}. TSStatus is {}",
- executionStatus,
- createModelStatement.getModelId(),
- executionStatus.message);
- future.setException(new IoTDBException(executionStatus.message, executionStatus.code));
- } else {
- future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS));
- }
- } catch (ClientManagerException | TException e) {
- future.setException(e);
- }
- return future;
- }
-
- @Override
- public SettableFuture<ConfigTaskResult> dropModel(String modelId) {
- SettableFuture<ConfigTaskResult> future = SettableFuture.create();
- try (ConfigNodeClient client =
- CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
- final TSStatus executionStatus = client.dropModel(new TDropModelReq(modelId));
- if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != executionStatus.getCode()) {
- LOGGER.warn("[{}] Failed to drop model {}.", executionStatus, modelId);
- future.setException(new IoTDBException(executionStatus.message, executionStatus.code));
- } else {
- future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS));
- }
- } catch (ClientManagerException | TException e) {
- future.setException(e);
- }
- return future;
- }
-
- @Override
- public SettableFuture<ConfigTaskResult> showModels() {
- SettableFuture<ConfigTaskResult> future = SettableFuture.create();
- try (ConfigNodeClient client =
- CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
- TShowModelResp showModelResp = client.showModel(new TShowModelReq());
- if (showModelResp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- future.setException(
- new IoTDBException(showModelResp.getStatus().message, showModelResp.getStatus().code));
- return future;
- }
- // convert model info list and buildTsBlock
- ShowModelsTask.buildTsBlock(showModelResp.getModelInfoList(), future);
- } catch (ClientManagerException | TException e) {
- future.setException(e);
- }
- return future;
- }
-
- @Override
- public SettableFuture<ConfigTaskResult> showTrials(String modelId) {
- SettableFuture<ConfigTaskResult> future = SettableFuture.create();
- try (ConfigNodeClient client =
- CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
- TShowTrialResp showTrialResp = client.showTrial(new TShowTrialReq(modelId));
- if (showTrialResp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- future.setException(
- new IoTDBException(showTrialResp.getStatus().message, showTrialResp.getStatus().code));
- return future;
- }
- // convert trail info list and buildTsBlock
- ShowTrialsTask.buildTsBlock(showTrialResp.getTrialInfoList(), future);
- } catch (ClientManagerException | TException e) {
- future.setException(e);
- }
- return future;
- }
-
@Override
public SettableFuture<ConfigTaskResult> setSpaceQuota(
SetSpaceQuotaStatement setSpaceQuotaStatement) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java
index a6a3b1eaba5..b3322f93d99 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java
@@ -43,7 +43,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowDataNodesStat
import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowDatabaseStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowRegionStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTTLStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.CreateModelStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.CreatePipePluginStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.CreatePipeStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.DropPipeStatement;
@@ -208,13 +207,4 @@ public interface IConfigTaskExecutor {
ShowThrottleQuotaStatement showThrottleQuotaStatement);
TThrottleQuotaResp getThrottleQuota();
-
- SettableFuture<ConfigTaskResult> createModel(
- CreateModelStatement createModelStatement, MPPQueryContext context);
-
- SettableFuture<ConfigTaskResult> dropModel(String modelId);
-
- SettableFuture<ConfigTaskResult> showModels();
-
- SettableFuture<ConfigTaskResult> showTrials(String modelId);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/model/CreateModelTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/model/CreateModelTask.java
deleted file mode 100644
index 842b558fd8a..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/model/CreateModelTask.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.model;
-
-import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
-import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult;
-import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask;
-import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor;
-import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.CreateModelStatement;
-
-import com.google.common.util.concurrent.ListenableFuture;
-
-public class CreateModelTask implements IConfigTask {
-
- private final CreateModelStatement createModelStatement;
- private final MPPQueryContext context;
-
- public CreateModelTask(CreateModelStatement createModelStatement, MPPQueryContext context) {
- this.createModelStatement = createModelStatement;
- this.context = context;
- }
-
- @Override
- public ListenableFuture<ConfigTaskResult> execute(IConfigTaskExecutor configTaskExecutor)
- throws InterruptedException {
- return configTaskExecutor.createModel(createModelStatement, context);
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/model/DropModelTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/model/DropModelTask.java
deleted file mode 100644
index 990ab694495..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/model/DropModelTask.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.model;
-
-import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult;
-import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask;
-import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor;
-
-import com.google.common.util.concurrent.ListenableFuture;
-
-public class DropModelTask implements IConfigTask {
-
- private final String modelId;
-
- public DropModelTask(String modelId) {
- this.modelId = modelId;
- }
-
- @Override
- public ListenableFuture<ConfigTaskResult> execute(IConfigTaskExecutor configTaskExecutor)
- throws InterruptedException {
- return configTaskExecutor.dropModel(modelId);
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/model/ShowModelsTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/model/ShowModelsTask.java
deleted file mode 100644
index 04e50abbd9f..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/model/ShowModelsTask.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.model;
-
-import org.apache.iotdb.db.queryengine.common.header.ColumnHeader;
-import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant;
-import org.apache.iotdb.db.queryengine.common.header.DatasetHeader;
-import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory;
-import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult;
-import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask;
-import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor;
-import org.apache.iotdb.rpc.TSStatusCode;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
-import org.apache.iotdb.tsfile.utils.Binary;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.SettableFuture;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-public class ShowModelsTask implements IConfigTask {
-
- private static final int MODEL_INFO_COLUMN_NUM = 5;
-
- public ShowModelsTask() {
- // do nothing
- }
-
- @Override
- public ListenableFuture<ConfigTaskResult> execute(IConfigTaskExecutor configTaskExecutor)
- throws InterruptedException {
- return configTaskExecutor.showModels();
- }
-
- public static void buildTsBlock(
- List<ByteBuffer> modelInfoList, SettableFuture<ConfigTaskResult> future) {
- List<TSDataType> outputDataTypes =
- ColumnHeaderConstant.showModelsColumnHeaders.stream()
- .map(ColumnHeader::getColumnType)
- .collect(Collectors.toList());
- TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes);
- for (ByteBuffer modelInfo : modelInfoList) {
- String modelId = ReadWriteIOUtils.readString(modelInfo);
- String modelTask = ReadWriteIOUtils.readString(modelInfo);
- String modelType = ReadWriteIOUtils.readString(modelInfo);
- String queryBody = ReadWriteIOUtils.readString(modelInfo);
- String trainingState = ReadWriteIOUtils.readString(modelInfo);
-
- int listSize = ReadWriteIOUtils.readInt(modelInfo);
- List<String> modelHyperparameter = new ArrayList<>();
- for (int i = 0; i < listSize; i++) {
- modelHyperparameter.add(ReadWriteIOUtils.readString(modelInfo));
- }
-
- builder.getTimeColumnBuilder().writeLong(0L);
- builder.getColumnBuilder(0).writeBinary(Binary.valueOf(modelId));
- builder.getColumnBuilder(1).writeBinary(Binary.valueOf(modelTask));
- builder.getColumnBuilder(2).writeBinary(Binary.valueOf(modelType));
- builder.getColumnBuilder(3).writeBinary(Binary.valueOf(queryBody));
- builder.getColumnBuilder(4).writeBinary(Binary.valueOf(trainingState));
- builder.getColumnBuilder(5).writeBinary(Binary.valueOf(modelHyperparameter.get(0)));
- builder.declarePosition();
-
- for (int i = 1; i < listSize; i++) {
- builder.getTimeColumnBuilder().writeLong(0L);
- for (int columnIndex = 0; columnIndex <= MODEL_INFO_COLUMN_NUM - 1; columnIndex++) {
- builder.getColumnBuilder(columnIndex).writeBinary(Binary.valueOf(""));
- }
- builder
- .getColumnBuilder(MODEL_INFO_COLUMN_NUM)
- .writeBinary(Binary.valueOf(modelHyperparameter.get(i)));
- builder.declarePosition();
- }
- }
- DatasetHeader datasetHeader = DatasetHeaderFactory.getShowModelsHeader();
- future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS, builder.build(), datasetHeader));
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/model/ShowTrialsTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/model/ShowTrialsTask.java
deleted file mode 100644
index 0bc5160acb0..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/model/ShowTrialsTask.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.model;
-
-import org.apache.iotdb.db.queryengine.common.header.ColumnHeader;
-import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant;
-import org.apache.iotdb.db.queryengine.common.header.DatasetHeader;
-import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory;
-import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult;
-import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask;
-import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor;
-import org.apache.iotdb.rpc.TSStatusCode;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
-import org.apache.iotdb.tsfile.utils.Binary;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.SettableFuture;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-public class ShowTrialsTask implements IConfigTask {
-
- private final String modelId;
-
- public ShowTrialsTask(String modelId) {
- this.modelId = modelId;
- }
-
- @Override
- public ListenableFuture<ConfigTaskResult> execute(IConfigTaskExecutor configTaskExecutor)
- throws InterruptedException {
- return configTaskExecutor.showTrials(modelId);
- }
-
- public static void buildTsBlock(
- List<ByteBuffer> trailInfoList, SettableFuture<ConfigTaskResult> future) {
- List<TSDataType> outputDataTypes =
- ColumnHeaderConstant.showTrialsColumnHeaders.stream()
- .map(ColumnHeader::getColumnType)
- .collect(Collectors.toList());
- TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes);
- for (ByteBuffer trailInfo : trailInfoList) {
- String trailId = ReadWriteIOUtils.readString(trailInfo);
- String modelPath = ReadWriteIOUtils.readString(trailInfo);
- int listSize = ReadWriteIOUtils.readInt(trailInfo);
- List<String> modelHyperparameter = new ArrayList<>();
- for (int i = 0; i < listSize; i++) {
- modelHyperparameter.add(ReadWriteIOUtils.readString(trailInfo));
- }
-
- builder.getTimeColumnBuilder().writeLong(0L);
- builder.getColumnBuilder(0).writeBinary(Binary.valueOf(trailId));
- builder.getColumnBuilder(1).writeBinary(Binary.valueOf(modelPath));
- builder.getColumnBuilder(2).writeBinary(Binary.valueOf(modelHyperparameter.get(0)));
- builder.declarePosition();
-
- for (int i = 1; i < listSize; i++) {
- builder.getTimeColumnBuilder().writeLong(0L);
- builder.getColumnBuilder(0).writeBinary(Binary.valueOf(""));
- builder.getColumnBuilder(1).writeBinary(Binary.valueOf(""));
- builder.getColumnBuilder(2).writeBinary(Binary.valueOf(modelHyperparameter.get(i)));
- builder.declarePosition();
- }
- }
- DatasetHeader datasetHeader = DatasetHeaderFactory.getShowTrialsHeader();
- future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS, builder.build(), datasetHeader));
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/FunctionExpression.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/FunctionExpression.java
index 29615c489ae..17b628f1cb5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/FunctionExpression.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/FunctionExpression.java
@@ -23,7 +23,6 @@ import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.udf.builtin.BuiltinAggregationFunction;
import org.apache.iotdb.commons.udf.builtin.BuiltinScalarFunction;
-import org.apache.iotdb.commons.udf.builtin.ModelInferenceFunction;
import org.apache.iotdb.db.queryengine.common.NodeRef;
import org.apache.iotdb.db.queryengine.plan.expression.Expression;
import org.apache.iotdb.db.queryengine.plan.expression.ExpressionType;
@@ -120,8 +119,6 @@ public class FunctionExpression extends Expression {
functionType = FunctionType.AGGREGATION_FUNCTION;
} else if (BuiltinScalarFunction.getNativeFunctionNames().contains(lowerCaseFunctionName)) {
functionType = FunctionType.BUILT_IN_SCALAR_FUNCTION;
- } else if (ModelInferenceFunction.getNativeFunctionNames().contains(functionName)) {
- functionType = FunctionType.MODEL_INFERENCE_FUNCTION;
} else {
functionType = FunctionType.UDF;
}
@@ -142,13 +139,6 @@ public class FunctionExpression extends Expression {
return functionType == FunctionType.BUILT_IN_SCALAR_FUNCTION;
}
- public boolean isModelInferenceFunction() {
- if (functionType == null) {
- initializeFunctionType();
- }
- return functionType == FunctionType.MODEL_INFERENCE_FUNCTION;
- }
-
@Override
public boolean isConstantOperandInternal() {
if (isConstantOperandCache == null) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/FunctionType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/FunctionType.java
index e1d64f29680..336bbcce0bb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/FunctionType.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/expression/multi/FunctionType.java
@@ -23,6 +23,5 @@ package org.apache.iotdb.db.queryengine.plan.expression.multi;
public enum FunctionType {
AGGREGATION_FUNCTION,
BUILT_IN_SCALAR_FUNCTION,
- UDF,
- MODEL_INFERENCE_FUNCTION
+ UDF
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
index b2ef0fd8ea3..63d83858568 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
@@ -153,10 +153,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTimeSeriesSta
import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTriggersStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowVariablesStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.UnSetTTLStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.CreateModelStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.DropModelStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.ShowModelsStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.ShowTrialsStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.CreatePipePluginStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.CreatePipeStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.DropPipePluginStatement;
@@ -1219,69 +1215,6 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
}
}
- // Create Model =====================================================================
- @Override
- public Statement visitCreateModel(IoTDBSqlParser.CreateModelContext ctx) {
- CreateModelStatement createModelStatement = new CreateModelStatement();
- createModelStatement.setModelId(parseIdentifier(ctx.modelId.getText()));
-
- Map<String, String> modelOptions = new HashMap<>();
- for (IoTDBSqlParser.AttributePairContext attribute : ctx.attributePair()) {
- modelOptions.put(
- parseAttributeKey(attribute.key).toLowerCase(), parseAttributeValue(attribute.value));
- }
- createModelStatement.setOptions(modelOptions);
-
- Map<String, String> hyperParameters = new HashMap<>();
- for (IoTDBSqlParser.HparamPairContext attribute : ctx.hparamPair()) {
- hyperParameters.put(
- parseAttributeKey(attribute.hparamKey).toLowerCase(),
- parseHparamValue(attribute.hparamValue()));
- }
- createModelStatement.setHyperparameters(hyperParameters);
-
- createModelStatement.setDatasetStatement(
- (QueryStatement) visitSelectStatement(ctx.selectStatement()));
- return createModelStatement;
- }
-
- private String parseHparamValue(IoTDBSqlParser.HparamValueContext ctx) {
- if (ctx.attributeValue() != null) {
- return parseAttributeValue(ctx.attributeValue());
- } else if (ctx.hparamRange() != null) {
- return parseAttributeValue(ctx.hparamRange().hparamRangeStart)
- + ","
- + parseAttributeValue(ctx.hparamRange().hparamRangeEnd);
- } else if (ctx.hparamCandidates() != null) {
- List<String> candidates = new ArrayList<>();
- for (IoTDBSqlParser.AttributeValueContext candidate :
- ctx.hparamCandidates().attributeValue()) {
- candidates.add(parseAttributeValue(candidate));
- }
- return Arrays.toString(candidates.toArray());
- } else {
- throw new IllegalArgumentException();
- }
- }
-
- // Drop Model =====================================================================
- @Override
- public Statement visitDropModel(IoTDBSqlParser.DropModelContext ctx) {
- return new DropModelStatement(parseIdentifier(ctx.modelId.getText()));
- }
-
- // Show Models =====================================================================
- @Override
- public Statement visitShowModels(IoTDBSqlParser.ShowModelsContext ctx) {
- return new ShowModelsStatement();
- }
-
- // Show Trails =====================================================================
- @Override
- public Statement visitShowTrials(IoTDBSqlParser.ShowTrialsContext ctx) {
- return new ShowTrialsStatement(parseIdentifier(ctx.modelId.getText()));
- }
-
/** Data Manipulation Language (DML). */
// Select Statement ========================================================================
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java
index 6944104b3b3..dfcc59345a0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java
@@ -68,9 +68,6 @@ import org.apache.iotdb.db.schemaengine.template.TemplateQueryType;
import org.apache.iotdb.db.utils.QueryDataSetUtils;
import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
import org.apache.iotdb.db.utils.constant.SqlConstant;
-import org.apache.iotdb.mpp.rpc.thrift.TDeleteModelMetricsReq;
-import org.apache.iotdb.mpp.rpc.thrift.TFetchTimeseriesReq;
-import org.apache.iotdb.mpp.rpc.thrift.TRecordModelMetricsReq;
import org.apache.iotdb.service.rpc.thrift.TSAggregationQueryReq;
import org.apache.iotdb.service.rpc.thrift.TSCreateAlignedTimeseriesReq;
import org.apache.iotdb.service.rpc.thrift.TSCreateMultiTimeseriesReq;
@@ -106,16 +103,12 @@ import org.antlr.v4.runtime.tree.ParseTree;
import java.nio.ByteBuffer;
import java.time.ZoneId;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD;
-import static org.apache.iotdb.db.protocol.thrift.impl.MLNodeRPCServiceImpl.ML_METRICS_PATH_PREFIX;
-
/** Convert SQL and RPC requests to {@link Statement}. */
public class StatementGenerator {
private static final PerformanceOverviewMetrics PERFORMANCE_OVERVIEW_METRICS =
@@ -859,40 +852,4 @@ public class StatementGenerator {
MetaFormatUtils.checkDatabase(database);
return databasePath;
}
-
- public static InsertRowStatement createStatement(TRecordModelMetricsReq recordModelMetricsReq)
- throws IllegalPathException {
- String path =
- ML_METRICS_PATH_PREFIX
- + TsFileConstant.PATH_SEPARATOR
- + recordModelMetricsReq.getModelId()
- + TsFileConstant.PATH_SEPARATOR
- + recordModelMetricsReq.getTrialId();
- InsertRowStatement insertRowStatement = new InsertRowStatement();
- insertRowStatement.setDevicePath(DEVICE_PATH_CACHE.getPartialPath(path));
- insertRowStatement.setTime(recordModelMetricsReq.getTimestamp());
- insertRowStatement.setMeasurements(recordModelMetricsReq.getMetrics().toArray(new String[0]));
- insertRowStatement.setAligned(true);
-
- TSDataType[] dataTypes = new TSDataType[recordModelMetricsReq.getValues().size()];
- Arrays.fill(dataTypes, TSDataType.DOUBLE);
- insertRowStatement.setDataTypes(dataTypes);
- insertRowStatement.setValues(recordModelMetricsReq.getValues().toArray(new Object[0]));
- return insertRowStatement;
- }
-
- public static Statement createStatement(TFetchTimeseriesReq fetchTimeseriesReq, ZoneId zoneId) {
- return invokeParser(fetchTimeseriesReq.getQueryBody(), zoneId);
- }
-
- public static DeleteTimeSeriesStatement createStatement(TDeleteModelMetricsReq req)
- throws IllegalPathException {
- String path =
- ML_METRICS_PATH_PREFIX
- + TsFileConstant.PATH_SEPARATOR
- + req.getModelId()
- + TsFileConstant.PATH_SEPARATOR
- + MULTI_LEVEL_PATH_WILDCARD;
- return new DeleteTimeSeriesStatement(Collections.singletonList(new PartialPath(path)));
- }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java
index 830bd907862..2d84088d4ed 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java
@@ -74,7 +74,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.TopKNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.TransformNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.last.LastQueryNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.last.LastQueryTransformNode;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ml.ForecastNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.AlignedLastQueryScanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.AlignedSeriesAggregationScanNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.AlignedSeriesScanNode;
@@ -92,7 +91,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.GroupByParame
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.GroupByTimeParameter;
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.IntoPathDescriptor;
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.OrderByParameter;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.model.ForecastModelInferenceDescriptor;
import org.apache.iotdb.db.queryengine.plan.statement.component.OrderByKey;
import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering;
import org.apache.iotdb.db.queryengine.plan.statement.component.SortItem;
@@ -1608,12 +1606,4 @@ public class LogicalPlanBuilder {
}
return this;
}
-
- public LogicalPlanBuilder planForecast(
- ForecastModelInferenceDescriptor forecastModelInferenceDescriptor) {
- this.root =
- new ForecastNode(
- context.getQueryId().genPlanNodeId(), root, forecastModelInferenceDescriptor);
- return this;
- }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java
index d7e810b2b5f..a8bf4d38fec 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java
@@ -23,7 +23,6 @@ import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression;
import org.apache.iotdb.commons.udf.builtin.BuiltinAggregationFunction;
-import org.apache.iotdb.commons.udf.builtin.ModelInferenceFunction;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.plan.analyze.Analysis;
import org.apache.iotdb.db.queryengine.plan.analyze.ExpressionAnalyzer;
@@ -54,8 +53,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOf
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.PipeEnrichedInsertNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.AggregationStep;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.model.ForecastModelInferenceDescriptor;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.model.ModelInferenceDescriptor;
import org.apache.iotdb.db.queryengine.plan.statement.StatementNode;
import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor;
import org.apache.iotdb.db.queryengine.plan.statement.crud.DeleteDataStatement;
@@ -102,7 +99,6 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.Set;
import static org.apache.iotdb.db.utils.constant.SqlConstant.COUNT_TIME;
@@ -225,22 +221,6 @@ public class LogicalPlanVisitor extends StatementVisitor<PlanNode, MPPQueryConte
.planOffset(queryStatement.getRowOffset())
.planLimit(queryStatement.getRowLimit());
- if (queryStatement.isModelInferenceQuery()) {
- ModelInferenceDescriptor modelInferenceDescriptor = analysis.getModelInferenceDescriptor();
- if (Objects.requireNonNull(modelInferenceDescriptor.getFunctionType())
- == ModelInferenceFunction.FORECAST) {
- ForecastModelInferenceDescriptor forecastModelInferenceDescriptor =
- (ForecastModelInferenceDescriptor) modelInferenceDescriptor;
- planBuilder
- .planLimit(forecastModelInferenceDescriptor.getModelInputLength())
- .planForecast(forecastModelInferenceDescriptor);
- } else {
- throw new IllegalArgumentException(
- "Unsupported model inference function type: "
- + modelInferenceDescriptor.getFunctionType());
- }
- }
-
// plan select into
if (queryStatement.isAlignByDevice()) {
planBuilder = planBuilder.planDeviceViewInto(analysis.getDeviceViewIntoPathDescriptor());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java
index e0be33d076f..916cf3a6796 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java
@@ -103,7 +103,6 @@ import org.apache.iotdb.db.queryengine.execution.operator.process.last.LastQuery
import org.apache.iotdb.db.queryengine.execution.operator.process.last.LastQueryUtil;
import org.apache.iotdb.db.queryengine.execution.operator.process.last.UpdateLastCacheOperator;
import org.apache.iotdb.db.queryengine.execution.operator.process.last.UpdateViewPathLastCacheOperator;
-import org.apache.iotdb.db.queryengine.execution.operator.process.ml.ForecastOperator;
import org.apache.iotdb.db.queryengine.execution.operator.schema.CountGroupByLevelMergeOperator;
import org.apache.iotdb.db.queryengine.execution.operator.schema.CountGroupByLevelScanOperator;
import org.apache.iotdb.db.queryengine.execution.operator.schema.CountMergeOperator;
@@ -183,7 +182,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.last.LastQ
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.last.LastQueryMergeNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.last.LastQueryNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.last.LastQueryTransformNode;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ml.ForecastNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.sink.IdentitySinkNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.sink.ShuffleSinkNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.AlignedLastQueryScanNode;
@@ -207,7 +205,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.InputLocation
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.IntoPathDescriptor;
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.OutputColumn;
import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.model.ForecastModelInferenceDescriptor;
import org.apache.iotdb.db.queryengine.plan.statement.component.FillPolicy;
import org.apache.iotdb.db.queryengine.plan.statement.component.OrderByKey;
import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering;
@@ -220,8 +217,6 @@ import org.apache.iotdb.db.queryengine.transformation.dag.udf.UDTFContext;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.TimeValuePair;
import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
-import org.apache.iotdb.tsfile.read.common.block.column.DoubleColumn;
-import org.apache.iotdb.tsfile.read.common.block.column.TimeColumn;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.filter.operator.Gt;
import org.apache.iotdb.tsfile.read.filter.operator.GtEq;
@@ -251,7 +246,6 @@ import static com.google.common.base.Preconditions.checkArgument;
import static org.apache.iotdb.db.queryengine.common.DataNodeEndPoints.isSameNode;
import static org.apache.iotdb.db.queryengine.execution.operator.AggregationUtil.calculateMaxAggregationResultSize;
import static org.apache.iotdb.db.queryengine.execution.operator.AggregationUtil.calculateMaxAggregationResultSizeForLastQuery;
-import static org.apache.iotdb.db.queryengine.execution.operator.AggregationUtil.getOutputColumnSizePerLine;
import static org.apache.iotdb.db.queryengine.execution.operator.AggregationUtil.initTimeRangeIterator;
import static org.apache.iotdb.db.queryengine.plan.expression.leaf.TimestampOperand.TIMESTAMP_EXPRESSION_STRING;
import static org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions.updateFilterUsingTTL;
@@ -1682,49 +1676,6 @@ public class OperatorTreeGenerator extends PlanVisitor<Operator, LocalExecutionP
MergeSortComparator.getComparator(sortItemList, sortItemIndexList, sortItemDataTypeList));
}
- @Override
- public Operator visitForecast(ForecastNode node, LocalExecutionPlanContext context) {
- Operator child = node.getChild().accept(this, context);
- OperatorContext operatorContext =
- context
- .getDriverContext()
- .addOperatorContext(
- context.getNextOperatorId(),
- node.getPlanNodeId(),
- ForecastOperator.class.getSimpleName());
-
- ForecastModelInferenceDescriptor forecastModelInferenceDescriptor =
- node.getModelInferenceDescriptor();
-
- List<TSDataType> inputTypeList = forecastModelInferenceDescriptor.getInputTypeList();
- int modelInputLength = forecastModelInferenceDescriptor.getModelInputLength();
- long timeValueColumnsSizePerLine = TimeColumn.SIZE_IN_BYTES_PER_POSITION;
- for (TSDataType dataType : inputTypeList) {
- timeValueColumnsSizePerLine += getOutputColumnSizePerLine(dataType);
- }
- long maxRetainedSize = timeValueColumnsSizePerLine * modelInputLength;
-
- int expectedPredictLength = forecastModelInferenceDescriptor.getExpectedPredictLength();
- int outputColumnNum = forecastModelInferenceDescriptor.getPredictIndexList().size();
- long maxReturnSize =
- (TimeColumn.SIZE_IN_BYTES_PER_POSITION
- + (long) outputColumnNum * DoubleColumn.SIZE_IN_BYTES_PER_POSITION)
- * expectedPredictLength;
-
- context.getTimeSliceAllocator().recordExecutionWeight(operatorContext, 1);
-
- return new ForecastOperator(
- operatorContext,
- child,
- forecastModelInferenceDescriptor.getModelPath(),
- inputTypeList,
- node.getChild().getOutputColumnNames(),
- expectedPredictLength,
- FragmentInstanceManager.getInstance().getModelInferenceExecutor(),
- maxRetainedSize,
- maxReturnSize);
- }
-
@Override
public Operator visitInto(IntoNode node, LocalExecutionPlanContext context) {
Operator child = node.getChild().accept(this, context);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java
index 79165695b7d..5ff7ffef463 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanGraphPrinter.java
@@ -47,7 +47,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.last.LastQ
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.last.LastQueryMergeNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.last.LastQueryNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.last.LastQueryTransformNode;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ml.ForecastNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.sink.IdentitySinkNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.sink.ShuffleSinkNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.AlignedLastQueryScanNode;
@@ -471,17 +470,6 @@ public class PlanGraphPrinter extends PlanVisitor<List<String>, PlanGraphPrinter
return render(node, boxValue, context);
}
- @Override
- public List<String> visitForecast(ForecastNode node, GraphContext context) {
- List<String> boxValue = new ArrayList<>();
- boxValue.add(String.format("Forecast-%s", node.getPlanNodeId().getId()));
- boxValue.add("Output: ");
- for (String outputColumnName : node.getOutputColumnNames()) {
- boxValue.add(String.format(" %s", outputColumnName));
- }
- return render(node, boxValue, context);
- }
-
private String printRegion(TRegionReplicaSet regionReplicaSet) {
return String.format(
"Partition: %s",
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java
index e732129f68c..19e73bfdede 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java
@@ -80,7 +80,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.last.LastQ
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.last.LastQueryMergeNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.last.LastQueryNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.last.LastQueryTransformNode;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ml.ForecastNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.sink.IdentitySinkNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.sink.ShuffleSinkNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.AlignedLastQueryScanNode;
@@ -183,7 +182,9 @@ public enum PlanNodeType {
LOGICAL_VIEW_SCHEMA_SCAN((short) 77),
ALTER_LOGICAL_VIEW((short) 78),
PIPE_ENRICHED_INSERT((short) 79),
- FORECAST((short) 80),
+
+ // NodeId 80 is used by IoTDB-ML which shouldn't be used.
+
LAST_QUERY_TRANSFORM((short) 81),
TOP_K((short) 82);
@@ -390,8 +391,6 @@ public enum PlanNodeType {
return AlterLogicalViewNode.deserialize(buffer);
case 79:
return PipeEnrichedInsertNode.deserialize(buffer);
- case 80:
- return ForecastNode.deserialize(buffer);
case 81:
return LastQueryTransformNode.deserialize(buffer);
case 82:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
index 7589b2fe7f3..0c6ee35add4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java
@@ -80,7 +80,6 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.last.LastQ
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.last.LastQueryMergeNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.last.LastQueryNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.last.LastQueryTransformNode;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ml.ForecastNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.sink.IdentitySinkNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.sink.ShuffleSinkNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.AlignedLastQueryScanNode;
@@ -195,10 +194,6 @@ public abstract class PlanVisitor<R, C> {
return visitSingleChildProcess(node, context);
}
- public R visitForecast(ForecastNode node, C context) {
- return visitSingleChildProcess(node, context);
- }
-
// multi child --------------------------------------------------------------------------------
public R visitMultiChildProcess(MultiChildProcessNode node, C context) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/process/ml/ForecastNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/process/ml/ForecastNode.java
deleted file mode 100644
index 9edccfee589..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/process/ml/ForecastNode.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ml;
-
-import org.apache.iotdb.db.queryengine.plan.expression.Expression;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.SingleChildProcessNode;
-import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.model.ForecastModelInferenceDescriptor;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Objects;
-
-public class ForecastNode extends SingleChildProcessNode {
-
- private final ForecastModelInferenceDescriptor modelInferenceDescriptor;
-
- private List<String> outputColumnNames;
-
- public ForecastNode(
- PlanNodeId id, PlanNode child, ForecastModelInferenceDescriptor modelInferenceDescriptor) {
- super(id, child);
- this.modelInferenceDescriptor = modelInferenceDescriptor;
- }
-
- public ForecastNode(PlanNodeId id, ForecastModelInferenceDescriptor modelInferenceDescriptor) {
- super(id);
- this.modelInferenceDescriptor = modelInferenceDescriptor;
- }
-
- public ForecastModelInferenceDescriptor getModelInferenceDescriptor() {
- return modelInferenceDescriptor;
- }
-
- @Override
- public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
- return visitor.visitForecast(this, context);
- }
-
- @Override
- public PlanNode clone() {
- return new ForecastNode(getPlanNodeId(), child, modelInferenceDescriptor);
- }
-
- @Override
- public List<String> getOutputColumnNames() {
- if (outputColumnNames == null) {
- outputColumnNames = new ArrayList<>();
- for (Expression expression : modelInferenceDescriptor.getModelInferenceOutputExpressions()) {
- outputColumnNames.add(expression.toString());
- }
- }
- return outputColumnNames;
- }
-
- @Override
- protected void serializeAttributes(ByteBuffer byteBuffer) {
- PlanNodeType.FORECAST.serialize(byteBuffer);
- modelInferenceDescriptor.serialize(byteBuffer);
- }
-
- @Override
- protected void serializeAttributes(DataOutputStream stream) throws IOException {
- PlanNodeType.FORECAST.serialize(stream);
- modelInferenceDescriptor.serialize(stream);
- }
-
- public static ForecastNode deserialize(ByteBuffer buffer) {
- ForecastModelInferenceDescriptor modelInferenceDescriptor =
- ForecastModelInferenceDescriptor.deserialize(buffer);
- PlanNodeId planNodeId = PlanNodeId.deserialize(buffer);
- return new ForecastNode(planNodeId, modelInferenceDescriptor);
- }
-
- @Override
- public String toString() {
- return "ForecastNode-" + this.getPlanNodeId();
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- if (!super.equals(o)) {
- return false;
- }
- ForecastNode that = (ForecastNode) o;
- return modelInferenceDescriptor.equals(that.modelInferenceDescriptor);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(super.hashCode(), modelInferenceDescriptor);
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/model/ForecastModelInferenceDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/model/ForecastModelInferenceDescriptor.java
deleted file mode 100644
index f7b075d14c6..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/model/ForecastModelInferenceDescriptor.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.model;
-
-import org.apache.iotdb.commons.model.ForecastModeInformation;
-import org.apache.iotdb.commons.udf.builtin.ModelInferenceFunction;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Objects;
-
-import static org.apache.iotdb.db.utils.constant.SqlConstant.MODEL_ID;
-import static org.apache.iotdb.db.utils.constant.SqlConstant.PREDICT_LENGTH;
-
-public class ForecastModelInferenceDescriptor extends ModelInferenceDescriptor {
-
- private final List<TSDataType> inputTypeList;
- private final List<Integer> predictIndexList;
-
- private final int modelInputLength;
- private final int modelPredictLength;
- private int expectedPredictLength;
-
- private LinkedHashMap<String, String> outputAttributes;
-
- public ForecastModelInferenceDescriptor(
- ModelInferenceFunction functionType, ForecastModeInformation modelInformation) {
- super(functionType, modelInformation);
- this.inputTypeList = modelInformation.getInputTypeList();
- this.predictIndexList = modelInformation.getPredictIndexList();
- this.modelInputLength = modelInformation.getInputLength();
- this.expectedPredictLength = this.modelPredictLength = modelInformation.getPredictLength();
- }
-
- public ForecastModelInferenceDescriptor(ByteBuffer buffer) {
- super(buffer);
- int listSize = ReadWriteIOUtils.readInt(buffer);
- this.inputTypeList = new ArrayList<>(listSize);
- for (int i = 0; i < listSize; i++) {
- this.inputTypeList.add(TSDataType.deserializeFrom(buffer));
- }
- listSize = ReadWriteIOUtils.readInt(buffer);
- this.predictIndexList = new ArrayList<>(listSize);
- for (int i = 0; i < listSize; i++) {
- this.predictIndexList.add(ReadWriteIOUtils.readInt(buffer));
- }
- this.modelInputLength = ReadWriteIOUtils.readInt(buffer);
- this.modelPredictLength = ReadWriteIOUtils.readInt(buffer);
- this.expectedPredictLength = ReadWriteIOUtils.readInt(buffer);
- }
-
- public List<Integer> getPredictIndexList() {
- return predictIndexList;
- }
-
- public List<TSDataType> getInputTypeList() {
- return inputTypeList;
- }
-
- public int getModelInputLength() {
- return modelInputLength;
- }
-
- public int getModelPredictLength() {
- return modelPredictLength;
- }
-
- public int getExpectedPredictLength() {
- return expectedPredictLength;
- }
-
- public void setExpectedPredictLength(int expectedPredictLength) {
- this.expectedPredictLength = expectedPredictLength;
- }
-
- @Override
- public LinkedHashMap<String, String> getOutputAttributes() {
- if (outputAttributes == null) {
- outputAttributes = new LinkedHashMap<>();
- outputAttributes.put(MODEL_ID, modelId);
- if (expectedPredictLength != modelPredictLength) {
- outputAttributes.put(PREDICT_LENGTH, String.valueOf(expectedPredictLength));
- }
- }
- return outputAttributes;
- }
-
- @Override
- public void serialize(ByteBuffer byteBuffer) {
- super.serialize(byteBuffer);
- ReadWriteIOUtils.write(inputTypeList.size(), byteBuffer);
- for (TSDataType dataType : inputTypeList) {
- dataType.serializeTo(byteBuffer);
- }
- ReadWriteIOUtils.write(predictIndexList.size(), byteBuffer);
- for (Integer index : predictIndexList) {
- ReadWriteIOUtils.write(index, byteBuffer);
- }
- ReadWriteIOUtils.write(modelInputLength, byteBuffer);
- ReadWriteIOUtils.write(modelPredictLength, byteBuffer);
- ReadWriteIOUtils.write(expectedPredictLength, byteBuffer);
- }
-
- @Override
- public void serialize(DataOutputStream stream) throws IOException {
- super.serialize(stream);
- ReadWriteIOUtils.write(inputTypeList.size(), stream);
- for (TSDataType dataType : inputTypeList) {
- dataType.serializeTo(stream);
- }
- ReadWriteIOUtils.write(predictIndexList.size(), stream);
- for (Integer index : predictIndexList) {
- ReadWriteIOUtils.write(index, stream);
- }
- ReadWriteIOUtils.write(modelInputLength, stream);
- ReadWriteIOUtils.write(modelPredictLength, stream);
- ReadWriteIOUtils.write(expectedPredictLength, stream);
- }
-
- public static ForecastModelInferenceDescriptor deserialize(ByteBuffer buffer) {
- return new ForecastModelInferenceDescriptor(buffer);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- if (!super.equals(o)) {
- return false;
- }
- ForecastModelInferenceDescriptor that = (ForecastModelInferenceDescriptor) o;
- return modelInputLength == that.modelInputLength
- && modelPredictLength == that.modelPredictLength
- && expectedPredictLength == that.expectedPredictLength
- && inputTypeList.equals(that.inputTypeList)
- && predictIndexList.equals(that.predictIndexList);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(
- super.hashCode(),
- inputTypeList,
- predictIndexList,
- modelInputLength,
- modelPredictLength,
- expectedPredictLength);
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/model/ModelInferenceDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/model/ModelInferenceDescriptor.java
deleted file mode 100644
index dfbb12545e3..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/parameter/model/ModelInferenceDescriptor.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.model;
-
-import org.apache.iotdb.commons.model.ModelInformation;
-import org.apache.iotdb.commons.udf.builtin.ModelInferenceFunction;
-import org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression;
-import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Objects;
-
-public abstract class ModelInferenceDescriptor {
-
- protected final ModelInferenceFunction functionType;
-
- protected final String modelId;
-
- protected final String modelPath;
-
- protected List<FunctionExpression> modelInferenceOutputExpressions;
-
- public ModelInferenceDescriptor(
- ModelInferenceFunction functionType, ModelInformation modelInformation) {
- this.functionType = functionType;
- this.modelId = modelInformation.getModelId();
- this.modelPath = modelInformation.getModelPath();
- }
-
- public ModelInferenceDescriptor(ByteBuffer buffer) {
- this.functionType = ModelInferenceFunction.values()[ReadWriteIOUtils.readInt(buffer)];
- this.modelId = ReadWriteIOUtils.readString(buffer);
- this.modelPath = ReadWriteIOUtils.readString(buffer);
- }
-
- public ModelInferenceFunction getFunctionType() {
- return functionType;
- }
-
- public String getModelId() {
- return modelId;
- }
-
- public String getModelPath() {
- return modelPath;
- }
-
- public List<FunctionExpression> getModelInferenceOutputExpressions() {
- return modelInferenceOutputExpressions;
- }
-
- public void setModelInferenceOutputExpressions(
- List<FunctionExpression> modelInferenceOutputExpressions) {
- this.modelInferenceOutputExpressions = modelInferenceOutputExpressions;
- }
-
- public abstract LinkedHashMap<String, String> getOutputAttributes();
-
- public void serialize(ByteBuffer byteBuffer) {
- ReadWriteIOUtils.write(functionType.ordinal(), byteBuffer);
- ReadWriteIOUtils.write(modelId, byteBuffer);
- ReadWriteIOUtils.write(modelPath, byteBuffer);
- }
-
- public void serialize(DataOutputStream stream) throws IOException {
- ReadWriteIOUtils.write(functionType.ordinal(), stream);
- ReadWriteIOUtils.write(modelId, stream);
- ReadWriteIOUtils.write(modelPath, stream);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- ModelInferenceDescriptor that = (ModelInferenceDescriptor) o;
- return functionType == that.functionType
- && modelId.equals(that.modelId)
- && modelPath.equals(that.modelPath)
- && modelInferenceOutputExpressions.equals(that.modelInferenceOutputExpressions);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(functionType, modelId, modelPath, modelInferenceOutputExpressions);
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java
index ba31db2d843..2d34e5affba 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java
@@ -73,10 +73,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTimeSeriesSta
import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTriggersStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowVariablesStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.UnSetTTLStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.CreateModelStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.DropModelStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.ShowModelsStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.ShowTrialsStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.CreatePipePluginStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.CreatePipeStatement;
import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.DropPipePluginStatement;
@@ -267,23 +263,6 @@ public abstract class StatementVisitor<R, C> {
return visitStatement(alterLogicalViewStatement, context);
}
- // ML Model
- public R visitCreateModel(CreateModelStatement createModelStatement, C context) {
- return visitStatement(createModelStatement, context);
- }
-
- public R visitDropModel(DropModelStatement dropModelStatement, C context) {
- return visitStatement(dropModelStatement, context);
- }
-
- public R visitShowModels(ShowModelsStatement showModelsModelStatement, C context) {
- return visitStatement(showModelsModelStatement, context);
- }
-
- public R visitShowTrials(ShowTrialsStatement showTrialsStatement, C context) {
- return visitStatement(showTrialsStatement, context);
- }
-
/** Data Manipulation Language (DML) */
// Select Statement
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/component/ResultColumn.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/component/ResultColumn.java
index e7299e60e63..f9739133081 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/component/ResultColumn.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/component/ResultColumn.java
@@ -127,7 +127,6 @@ public class ResultColumn extends StatementNode {
public enum ColumnType {
RAW,
AGGREGATION,
- CONSTANT,
- MODEL_INFERENCE
+ CONSTANT
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/component/SelectComponent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/component/SelectComponent.java
index dfe3a6d65cf..aaa803d95bf 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/component/SelectComponent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/component/SelectComponent.java
@@ -36,8 +36,6 @@ public class SelectComponent extends StatementNode {
private boolean hasBuiltInAggregationFunction = false;
- private boolean hasModelInferenceFunction = false;
-
protected List<ResultColumn> resultColumns = new ArrayList<>();
private Map<String, Expression> aliasToColumnMap;
@@ -54,17 +52,11 @@ public class SelectComponent extends StatementNode {
return hasBuiltInAggregationFunction;
}
- public boolean hasModelInferenceFunction() {
- return hasModelInferenceFunction;
- }
-
public void addResultColumn(ResultColumn resultColumn) {
resultColumns.add(resultColumn);
ResultColumn.ColumnType columnType = resultColumn.getColumnType();
if (columnType == ResultColumn.ColumnType.AGGREGATION) {
hasBuiltInAggregationFunction = true;
- } else if (columnType == ResultColumn.ColumnType.MODEL_INFERENCE) {
- hasModelInferenceFunction = true;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/QueryStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/QueryStatement.java
index bdec45762f7..556a4dad374 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/QueryStatement.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/crud/QueryStatement.java
@@ -58,7 +58,6 @@ import java.util.List;
import java.util.Set;
import static org.apache.iotdb.db.utils.constant.SqlConstant.COUNT_TIME;
-import static org.apache.iotdb.db.utils.constant.SqlConstant.MODEL_ID;
/**
* Base class of SELECT statement.
@@ -532,10 +531,6 @@ public class QueryStatement extends AuthorityInformationStatement {
public static final String RAW_AGGREGATION_HYBRID_QUERY_ERROR_MSG =
"Raw data and aggregation hybrid query is not supported.";
- public boolean isModelInferenceQuery() {
- return selectComponent.hasModelInferenceFunction();
- }
-
public static final String COUNT_TIME_NOT_SUPPORT_GROUP_BY_LEVEL =
"Count_time aggregation function using with group by level is not supported.";
@@ -551,51 +546,6 @@ public class QueryStatement extends AuthorityInformationStatement {
@SuppressWarnings({"squid:S3776", "squid:S6541"}) // Suppress high Cognitive Complexity warning
public void semanticCheck() {
- if (isModelInferenceQuery()) {
- if (selectComponent.getResultColumns().size() > 1) {
- throw new SemanticException("Model inference query can't be used with other queries now.");
- }
-
- Expression modelInferenceExpression =
- selectComponent.getResultColumns().get(0).getExpression();
- if (!(modelInferenceExpression instanceof FunctionExpression
- && ((FunctionExpression) modelInferenceExpression).isModelInferenceFunction())) {
- throw new SemanticException("Model inference only supports forecast function now.");
- }
- if (!((FunctionExpression) modelInferenceExpression)
- .getFunctionAttributes()
- .containsKey(MODEL_ID)) {
- throw new SemanticException("Model inference function must indicate the model id.");
- }
- if (!ExpressionAnalyzer.searchAggregationExpressions(modelInferenceExpression).isEmpty()) {
- throw new SemanticException(
- "Expression "
- + modelInferenceExpression.getExpressionString()
- + " doesn't refer to any data.");
- }
-
- if (hasHaving()
- || isGroupBy()
- || isGroupByLevel()
- || isGroupByTag()
- || isAlignByDevice()
- || isLastQuery()
- || seriesLimit > 0
- || seriesOffset > 0
- || isSelectInto()
- || isOrderByDevice()
- || isOrderByTimeseries()) {
- throw new SemanticException("Model inference query can't be used with other queries now.");
- }
-
- if (orderByComponent != null
- && (!orderByComponent.isOrderByTime()
- || orderByComponent.getTimeOrder() != Ordering.ASC)) {
- throw new SemanticException(
- "The order by clause of model inference query must be ORDER BY TIME ASC.");
- }
- }
-
if (isAggregationQuery()) {
if (groupByComponent != null && isGroupByLevel()) {
throw new SemanticException("GROUP BY CLAUSES doesn't support GROUP BY LEVEL now.");
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/model/CreateModelStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/model/CreateModelStatement.java
deleted file mode 100644
index b7e50ee72e2..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/model/CreateModelStatement.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.queryengine.plan.statement.metadata.model;
-
-import org.apache.iotdb.common.rpc.thrift.TaskType;
-import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.db.exception.sql.SemanticException;
-import org.apache.iotdb.db.queryengine.plan.analyze.QueryType;
-import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.Statement;
-import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor;
-import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.iotdb.commons.model.ModelInformation.MODEL_TYPE;
-import static org.apache.iotdb.commons.model.ModelInformation.TASK_TYPE;
-
-public class CreateModelStatement extends Statement implements IConfigStatement {
-
- private String modelId;
-
- private Map<String, String> options;
-
- private Map<String, String> hyperparameters;
-
- private QueryStatement datasetStatement;
-
- private final List<String> requiredOptions = Arrays.asList(TASK_TYPE, MODEL_TYPE);
-
- public CreateModelStatement() {
- // do nothing
- }
-
- public String getModelId() {
- return modelId;
- }
-
- public void setModelId(String modelId) {
- this.modelId = modelId;
- }
-
- public Map<String, String> getOptions() {
- return options;
- }
-
- public void setOptions(Map<String, String> options) {
- this.options = options;
- }
-
- public Map<String, String> getHyperparameters() {
- return hyperparameters;
- }
-
- public void setHyperparameters(Map<String, String> hyperparameters) {
- this.hyperparameters = hyperparameters;
- }
-
- public QueryStatement getDatasetStatement() {
- return datasetStatement;
- }
-
- public void setDatasetStatement(QueryStatement queryBody) {
- this.datasetStatement = queryBody;
- }
-
- public TaskType getTaskType() {
- try {
- return TaskType.valueOf(options.get("task_type").toUpperCase());
- } catch (IllegalArgumentException e) {
- throw new SemanticException("Unknown task type: " + options.get("task_type"));
- }
- }
-
- public void semanticCheck() {
- for (String requiredOption : requiredOptions) {
- if (!options.containsKey(requiredOption)) {
- throw new SemanticException("The option `" + requiredOption + "` must be specified.");
- }
- }
- if (datasetStatement.isAlignByDevice()) {
- throw new SemanticException(
- "Currently the sql statement in CREATE MODEL does not support align by device.");
- }
- if (datasetStatement.isLastQuery()) {
- throw new SemanticException("The sql statement in CREATE MODEL does not support LAST query.");
- }
- if (datasetStatement.isAggregationQuery() && !datasetStatement.isGroupByTime()) {
- throw new SemanticException(
- "The aggregation sql in CREATE MODEL only supports GROUP BY TIME now.");
- }
- }
-
- @Override
- public List<PartialPath> getPaths() {
- return Collections.emptyList();
- }
-
- @Override
- public QueryType getQueryType() {
- return QueryType.WRITE;
- }
-
- @Override
- public <R, C> R accept(StatementVisitor<R, C> visitor, C context) {
- return visitor.visitCreateModel(this, context);
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/model/DropModelStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/model/DropModelStatement.java
deleted file mode 100644
index 34ba98207e4..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/model/DropModelStatement.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.queryengine.plan.statement.metadata.model;
-
-import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.db.queryengine.plan.analyze.QueryType;
-import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.Statement;
-import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor;
-
-import java.util.Collections;
-import java.util.List;
-
-public class DropModelStatement extends Statement implements IConfigStatement {
-
- private final String modelId;
-
- public DropModelStatement(String modelId) {
- this.modelId = modelId;
- }
-
- public String getModelId() {
- return modelId;
- }
-
- @Override
- public List<PartialPath> getPaths() {
- return Collections.emptyList();
- }
-
- @Override
- public QueryType getQueryType() {
- return QueryType.WRITE;
- }
-
- @Override
- public <R, C> R accept(StatementVisitor<R, C> visitor, C context) {
- return visitor.visitDropModel(this, context);
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/model/ShowModelsStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/model/ShowModelsStatement.java
deleted file mode 100644
index 77c3facfed1..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/model/ShowModelsStatement.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.queryengine.plan.statement.metadata.model;
-
-import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.db.queryengine.plan.analyze.QueryType;
-import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.Statement;
-import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor;
-
-import java.util.Collections;
-import java.util.List;
-
-public class ShowModelsStatement extends Statement implements IConfigStatement {
-
- public ShowModelsStatement() {
- // do nothing
- }
-
- @Override
- public List<PartialPath> getPaths() {
- return Collections.emptyList();
- }
-
- @Override
- public QueryType getQueryType() {
- return QueryType.READ;
- }
-
- @Override
- public <R, C> R accept(StatementVisitor<R, C> visitor, C context) {
- return visitor.visitShowModels(this, context);
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/model/ShowTrialsStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/model/ShowTrialsStatement.java
deleted file mode 100644
index 00c60a2fe5f..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/model/ShowTrialsStatement.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.queryengine.plan.statement.metadata.model;
-
-import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.db.queryengine.plan.analyze.QueryType;
-import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement;
-import org.apache.iotdb.db.queryengine.plan.statement.Statement;
-import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor;
-
-import java.util.Collections;
-import java.util.List;
-
-public class ShowTrialsStatement extends Statement implements IConfigStatement {
-
- private final String modelId;
-
- public ShowTrialsStatement(String modelId) {
- this.modelId = modelId;
- }
-
- public String getModelId() {
- return modelId;
- }
-
- @Override
- public List<PartialPath> getPaths() {
- return Collections.emptyList();
- }
-
- @Override
- public QueryType getQueryType() {
- return QueryType.READ;
- }
-
- @Override
- public <R, C> R accept(StatementVisitor<R, C> visitor, C context) {
- return visitor.visitShowTrials(this, context);
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java
index 78b7c0d5795..3f8ee4f3c90 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java
@@ -566,10 +566,6 @@ public class DataNode implements DataNodeMBean {
private void setUpRPCService() throws StartupException {
// Start InternalRPCService to indicate that the current DataNode can accept cluster scheduling
registerManager.register(DataNodeInternalRPCService.getInstance());
- // Start InternalRPCService to indicate that the current DataNode can accept request from MLNode
- if (config.isEnableMLNodeService()) {
- registerManager.register(MLNodeRPCService.getInstance());
- }
// Notice: During the period between starting the internal RPC service
// and starting the client RPC service , some requests may fail because
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/MLNodeRPCService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/MLNodeRPCService.java
deleted file mode 100644
index 7f4300d7a53..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/MLNodeRPCService.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.service;
-
-import org.apache.iotdb.commons.concurrent.ThreadName;
-import org.apache.iotdb.commons.exception.runtime.RPCServiceException;
-import org.apache.iotdb.commons.service.ServiceType;
-import org.apache.iotdb.commons.service.ThriftService;
-import org.apache.iotdb.commons.service.ThriftServiceThread;
-import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.protocol.thrift.handler.MLNodeRPCServiceThriftHandler;
-import org.apache.iotdb.db.protocol.thrift.impl.MLNodeRPCServiceImpl;
-import org.apache.iotdb.mpp.rpc.thrift.IMLNodeInternalRPCService;
-
-public class MLNodeRPCService extends ThriftService implements MLNodeRPCServiceMBean {
-
- private MLNodeRPCServiceImpl impl;
-
- private MLNodeRPCService() {}
-
- @Override
- public ServiceType getID() {
- return ServiceType.MLNode_RPC_SERVICE;
- }
-
- @Override
- public void initTProcessor() {
- impl = new MLNodeRPCServiceImpl();
- initSyncedServiceImpl(null);
- processor = new IMLNodeInternalRPCService.Processor<>(impl);
- }
-
- @Override
- public void initThriftServiceThread()
- throws IllegalAccessException, InstantiationException, ClassNotFoundException {
- try {
- IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
- thriftServiceThread =
- new ThriftServiceThread(
- processor,
- getID().getName(),
- ThreadName.MLNODE_RPC_SERVICE.getName(),
- getBindIP(),
- getBindPort(),
- config.getRpcMaxConcurrentClientNum(),
- config.getThriftServerAwaitTimeForStopService(),
- new MLNodeRPCServiceThriftHandler(impl),
- config.isRpcThriftCompressionEnable());
- } catch (RPCServiceException e) {
- throw new IllegalAccessException(e.getMessage());
- }
- thriftServiceThread.setName(ThreadName.MLNODE_RPC_SERVICE.getName());
- }
-
- @Override
- public String getBindIP() {
- return IoTDBDescriptor.getInstance().getConfig().getRpcAddress();
- }
-
- @Override
- public int getBindPort() {
- return IoTDBDescriptor.getInstance().getConfig().getMLNodePort();
- }
-
- private static class MLNodeRPCServiceHolder {
- private static final MLNodeRPCService INSTANCE = new MLNodeRPCService();
-
- private MLNodeRPCServiceHolder() {}
- }
-
- public static MLNodeRPCService getInstance() {
- return MLNodeRPCService.MLNodeRPCServiceHolder.INSTANCE;
- }
-}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/MLNodeRPCServiceMBean.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/MLNodeRPCServiceMBean.java
deleted file mode 100644
index f4218c79bc4..00000000000
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/MLNodeRPCServiceMBean.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.service;
-
-public interface MLNodeRPCServiceMBean {}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java
index e98b34c7f44..afd47cc8c51 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java
@@ -81,9 +81,6 @@ public class SqlConstant {
public static final String SUBSTRING_IS_STANDARD = "isStandard";
public static final String SUBSTRING_FOR = "FOR";
- public static final String MODEL_ID = "model_id";
- public static final String PREDICT_LENGTH = "predict_length";
-
public static String[] getSingleRootArray() {
return SINGLE_ROOT_ARRAY;
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java
index 268ddae322f..fa6cadc0472 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGeneratorTest.java
@@ -57,9 +57,6 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.UnsetSch
import org.apache.iotdb.db.queryengine.plan.statement.metadata.view.CreateLogicalViewStatement;
import org.apache.iotdb.db.queryengine.plan.statement.sys.AuthorStatement;
import org.apache.iotdb.isession.template.TemplateNode;
-import org.apache.iotdb.mpp.rpc.thrift.TDeleteModelMetricsReq;
-import org.apache.iotdb.mpp.rpc.thrift.TFetchTimeseriesReq;
-import org.apache.iotdb.mpp.rpc.thrift.TRecordModelMetricsReq;
import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.service.rpc.thrift.TSAggregationQueryReq;
import org.apache.iotdb.service.rpc.thrift.TSCreateAlignedTimeseriesReq;
@@ -398,35 +395,6 @@ public class StatementGeneratorTest {
assertEquals(Collections.singletonList(new PartialPath("root.sg.d1")), statement.getPaths());
}
- @Test
- public void testInsertRecordModelMetrics() throws IllegalPathException {
- TRecordModelMetricsReq recordModelMetricsReq =
- new TRecordModelMetricsReq(
- "modelId",
- "trialId",
- Collections.singletonList("metrics"),
- 1L,
- Collections.singletonList(1.0));
- InsertRowStatement statement = StatementGenerator.createStatement(recordModelMetricsReq);
- assertEquals(1L, statement.getTime());
- }
-
- @Test
- public void testFetchTimeseries() {
- TFetchTimeseriesReq req = new TFetchTimeseriesReq("select * from root.sg.d1.s1");
- Statement statement = StatementGenerator.createStatement(req, ZonedDateTime.now().getOffset());
- assertEquals("root.sg.d1.s1.*", statement.getPaths().get(0).getFullPath());
- }
-
- @Test
- public void testDeleteModelMetrics() throws IllegalPathException {
- TDeleteModelMetricsReq req = new TDeleteModelMetricsReq("model");
- DeleteTimeSeriesStatement statement = StatementGenerator.createStatement(req);
- assertEquals(
- Collections.singletonList(new PartialPath("root.__system.ml.exp.model.**")),
- statement.getPaths());
- }
-
private org.apache.iotdb.isession.template.Template getTemplate()
throws StatementExecutionException {
org.apache.iotdb.isession.template.Template sessionTemplate =
diff --git a/iotdb-core/mlnode/.flake8 b/iotdb-core/mlnode/.flake8
deleted file mode 100644
index ce062a0252e..00000000000
--- a/iotdb-core/mlnode/.flake8
+++ /dev/null
@@ -1,33 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-[flake8]
-ignore =
- E203,
- W503
-max-line-length=200
-exclude =
- .git,
- tests/*,
- iotdb/thrift/**/*
-extend-exclude =
- dist,
- build,
- venv
-show-source = True
-statistics = True
-format = %(path)s:%(row)d,%(col)d:%(code)s:%(text)s:https://lintlyci.github.io/Flake8Rules/rules/%(code)s.html
diff --git a/iotdb-core/mlnode/.gitignore b/iotdb-core/mlnode/.gitignore
deleted file mode 100644
index cc99cc4f975..00000000000
--- a/iotdb-core/mlnode/.gitignore
+++ /dev/null
@@ -1,9 +0,0 @@
-# generated by Thrift
-/iotdb/thrift/
-
-# generated by Poetry
-/dist/
-
-# generated by MLNode
-*.pt
-/rebuild.bat
diff --git a/iotdb-core/mlnode/README.md b/iotdb-core/mlnode/README.md
deleted file mode 100644
index 062db10f7bf..00000000000
--- a/iotdb-core/mlnode/README.md
+++ /dev/null
@@ -1,47 +0,0 @@
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-# Apache IoTDB MLNode
-
-For developers, you can start an ML Node through the following steps.
-
-- Step 1: build project
-
-```shell
-mvn clean package -DskipUTs -pl iotdb-core/mlnode -am
-```
-
-```shell
-cd iotdb-core/mlnode
-poetry build
-```
-
-- Step 2: install
-
-```shell
-pip install dist/apache_iotdb_mlnode-1.0.0-py3-none-any.whl --force-reinstall
-```
-
-- Step 3: start node
-
-```shell
-mlnode start
-```
\ No newline at end of file
diff --git a/iotdb-core/mlnode/iotdb/__init__.py b/iotdb-core/mlnode/iotdb/__init__.py
deleted file mode 100644
index 2a1e720805f..00000000000
--- a/iotdb-core/mlnode/iotdb/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
diff --git a/iotdb-core/mlnode/iotdb/mlnode/__init__.py b/iotdb-core/mlnode/iotdb/mlnode/__init__.py
deleted file mode 100644
index 2a1e720805f..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
diff --git a/iotdb-core/mlnode/iotdb/mlnode/algorithm/factory.py b/iotdb-core/mlnode/iotdb/mlnode/algorithm/factory.py
deleted file mode 100644
index fb25a27cac9..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/algorithm/factory.py
+++ /dev/null
@@ -1,66 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-from typing import Dict
-
-import torch.nn as nn
-
-from iotdb.mlnode.algorithm.hyperparameter import HyperparameterName
-from iotdb.mlnode.algorithm.models.forecast.dlinear import (DLinear,
- DLinearIndividual)
-from iotdb.mlnode.algorithm.models.forecast.nbeats import NBeats
-from iotdb.mlnode.constant import ForecastModelType, OptionsKey
-from iotdb.mlnode.exception import BadConfigValueError
-from iotdb.mlnode.parser import ForecastTaskOptions
-
-
-def create_forecast_model(
- model_type: ForecastModelType,
- model_configs: Dict,
-) -> nn.Module:
- """
- Factory method for all support forecasting models
- the given arguments is common configs shared by all forecasting models
- """
- if model_type.value not in ForecastModelType.values():
- raise BadConfigValueError('model_name', f'It should be one of {ForecastModelType.values()}')
-
- if model_type == ForecastModelType.DLINEAR:
- return DLinear(kernel_size=model_configs[HyperparameterName.KERNEL_SIZE.value],
- input_len=model_configs[OptionsKey.INPUT_LENGTH.value],
- pred_len=model_configs[OptionsKey.PREDICT_LENGTH.value],
- input_vars=model_configs[HyperparameterName.INPUT_VARS.value])
- elif model_type == ForecastModelType.DLINEAR_INDIVIDUAL:
- return DLinearIndividual(kernel_size=model_configs[HyperparameterName.KERNEL_SIZE.value],
- input_len=model_configs[OptionsKey.INPUT_LENGTH.value],
- pred_len=model_configs[OptionsKey.PREDICT_LENGTH.value],
- input_vars=model_configs[HyperparameterName.INPUT_VARS.value]
- )
- elif model_type == ForecastModelType.NBEATS:
- return NBeats(
- block_type=model_configs[HyperparameterName.BLOCK_TYPE.value],
- d_model=model_configs[HyperparameterName.D_MODEL.value],
- inner_layers=model_configs[HyperparameterName.INNER_LAYERS.value],
- outer_layers=model_configs[HyperparameterName.OUTER_LAYERS.value],
- input_len=model_configs[OptionsKey.INPUT_LENGTH.value],
- pred_len=model_configs[OptionsKey.PREDICT_LENGTH.value],
- input_vars=model_configs[HyperparameterName.INPUT_VARS.value]
- )
- else:
- raise BadConfigValueError('model_name', model_type,
- f'It should be one of {ForecastModelType.values()}')
diff --git a/iotdb-core/mlnode/iotdb/mlnode/algorithm/hyperparameter.py b/iotdb-core/mlnode/iotdb/mlnode/algorithm/hyperparameter.py
deleted file mode 100644
index 44b8feb9f10..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/algorithm/hyperparameter.py
+++ /dev/null
@@ -1,407 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-from abc import abstractmethod
-from typing import Dict, List, Optional, Tuple
-
-import optuna
-
-from iotdb.mlnode.algorithm.validator import NumberRangeValidator, Validator
-from iotdb.mlnode.constant import HyperparameterName, ForecastModelType
-from iotdb.mlnode.exception import BadConfigValueError, UnsupportedError
-from iotdb.mlnode.parser import ForecastTaskOptions, TaskOptions
-
-
-class Hyperparameter(object):
-
- def __init__(self, name: str):
- """
- Args:
- name: name of the hyperparameter
- """
- self._name = name
-
- @abstractmethod
- def get_default_value(self):
- raise NotImplementedError
-
- @abstractmethod
- def validate_value(self, value):
- raise NotImplementedError
-
- @abstractmethod
- def validate_range(self, min_value, max_value):
- raise NotImplementedError
-
- @abstractmethod
- def suggest_parameter(self, optuna_suggest: optuna.Trial):
- raise NotImplementedError
-
- @abstractmethod
- def parse(self, string_value: str):
- raise NotImplementedError
-
-
-class IntHyperparameter(Hyperparameter):
- def __init__(self, name: str,
- log: bool,
- default_value: int,
- value_validators: List[Validator],
- default_low: int,
- low_validators: List[Validator],
- default_high: int,
- high_validators: List[Validator],
- step: Optional[int] = None,
- tuning: bool = False
- ):
- super(IntHyperparameter, self).__init__(name)
- self.__log = log
- self.__default_value = default_value
- self.__value_validators = value_validators
- self.__default_low = default_low
- self.__low_validators = low_validators
- self.__default_high = default_high
- self.__high_validators = high_validators
- self.__step = step
- self.__tuning = tuning
-
- def get_default_value(self):
- return self.__default_value
-
- def __validate(self, value, validators: List[Validator] = None):
- try:
- for validator in validators:
- validator.validate(int(value))
- except Exception as e:
- raise BadConfigValueError(self._name, value, str(e))
-
- def validate_value(self, value):
- self.__validate(value, self.__value_validators)
-
- def validate_range(self, min_value: int, max_value: int):
- self.__validate(min_value, self.__low_validators)
- self.__validate(max_value, self.__high_validators)
-
- def suggest_parameter(self, optuna_suggest: optuna.Trial):
- if not self.__tuning:
- return self.__default_value
- return optuna_suggest.suggest_int(
- name=self._name,
- low=self.__default_low,
- high=self.__default_high,
- step=self.__step,
- log=self.__log
- )
-
- def parse(self, string_value: str):
- return int(string_value)
-
-
-class FloatHyperparameter(Hyperparameter):
- def __init__(self, name: str,
- log: bool,
- default_value: float,
- value_validators: List[Validator],
- default_low: float,
- low_validators: List[Validator],
- default_high: float,
- high_validators: List[Validator],
- step: Optional[float] = None,
- tuning: bool = False):
- super(FloatHyperparameter, self).__init__(name)
- self.__log = log
- self.__default_value = default_value
- self.__value_validators = value_validators
- self.__default_low = default_low
- self.__low_validators = low_validators
- self.__default_high = default_high
- self.__high_validators = high_validators
- self.__step = step
- self.__tuning = tuning
-
- def get_default_value(self):
- return self.__default_value
-
- def __validate(self, value, validators: List[Validator] = None):
- try:
- for validator in validators:
- validator.validate(float(value))
- except Exception as e:
- raise BadConfigValueError(self._name, value, str(e))
-
- def validate_value(self, value):
- self.__validate(value, self.__value_validators)
-
- def validate_range(self, min_value: float, max_value: float):
- self.__validate(min_value, self.__low_validators)
- self.__validate(max_value, self.__high_validators)
-
- def suggest_parameter(self, optuna_suggest: optuna.Trial):
- if not self.__tuning:
- return self.__default_value
- return optuna_suggest.suggest_float(
- name=self._name,
- low=self.__default_low,
- high=self.__default_high,
- step=self.__step,
- log=self.__log
- )
-
- def parse(self, string_value: str):
- return float(string_value)
-
-
-class StringHyperparameter(Hyperparameter):
- def __init__(self, name: str,
- default_value: str,
- value_validators: List[Validator],
- value_list: List[str],
- tuning: bool = False):
- super(StringHyperparameter, self).__init__(name)
- self.__default_value = default_value
- self.__value_validators = value_validators
- self.__value_list = value_list
- self.__tuning = tuning
-
- def get_default_value(self):
- return self.__default_value
-
- def __validate(self, value, validators: List[Validator] = None):
- try:
- for validator in validators:
- validator.validate(float(value))
- except Exception as e:
- raise BadConfigValueError(self._name, value, str(e))
-
- def validate_value(self, value):
- self.__validate(value, self.__value_validators)
-
- def validate_range(self, min_value: float, max_value: float):
- raise UnsupportedError("validate range in string hyperparameter")
-
- def suggest_parameter(self, optuna_suggest: optuna.Trial):
- if not self.__tuning:
- return self.__default_value
- return optuna_suggest.suggest_categorical(
- name=self._name,
- choices=self.__value_list
- )
-
- def parse(self, string_value: str):
- return string_value
-
-
-class BooleanHyperparameter(Hyperparameter):
- def __init__(self, name: str,
- default_value: bool,
- tuning: bool = False):
- super(BooleanHyperparameter, self).__init__(name)
- self.__default_value = default_value
- self.__tuning = tuning
-
- def get_default_value(self):
- return self.__default_value
-
- def validate_value(self, value):
- # no need to validate value in boolean hyperparameter
- pass
-
- def validate_range(self, min_value: float, max_value: float):
- # no need to validate range in boolean hyperparameter
- pass
-
- def suggest_parameter(self, optuna_suggest: optuna.Trial):
- if not self.__tuning:
- return self.__default_value
- return optuna_suggest.suggest_categorical(
- self._name,
- [True, False]
- )
-
- def parse(self, string_value: str):
- return bool(string_value)
-
-
-training_hyperparameter_map = {
- HyperparameterName.LEARNING_RATE.name(): FloatHyperparameter(
- name=HyperparameterName.LEARNING_RATE.name(),
- log=True,
- default_value=1e-3,
- value_validators=[NumberRangeValidator(1e-8, 10)],
- default_low=1e-5,
- low_validators=[],
- default_high=1e-1,
- high_validators=[],
- tuning=True
- ),
- HyperparameterName.EPOCHS.name(): IntHyperparameter(
- name=HyperparameterName.EPOCHS.name(),
- log=True,
- default_value=10,
- value_validators=[NumberRangeValidator(1, 10000)],
- default_low=1,
- low_validators=[],
- default_high=100,
- high_validators=[],
- tuning=False
- ),
- HyperparameterName.BATCH_SIZE.name(): IntHyperparameter(
- name=HyperparameterName.BATCH_SIZE.name(),
- log=True,
- default_value=32,
- value_validators=[NumberRangeValidator(1, 8196)],
- default_low=1,
- low_validators=[],
- default_high=8196,
- high_validators=[],
- tuning=False
- ),
- HyperparameterName.USE_GPU.name(): BooleanHyperparameter(
- name=HyperparameterName.USE_GPU.name(),
- default_value=False,
- tuning=False
- ),
- HyperparameterName.NUM_WORKERS.name(): IntHyperparameter(
- name=HyperparameterName.NUM_WORKERS.name(),
- log=False,
- default_value=0,
- value_validators=[NumberRangeValidator(0, 8)],
- default_low=0,
- low_validators=[],
- default_high=8,
- high_validators=[],
- tuning=False
- )
-}
-
-dlinear_structure_hyperparameter_map = {
- HyperparameterName.KERNEL_SIZE.value: IntHyperparameter(
- name=HyperparameterName.KERNEL_SIZE.name(),
- log=True,
- default_value=25,
- value_validators=[NumberRangeValidator(1, 1e10)],
- default_low=5,
- low_validators=[],
- default_high=50,
- high_validators=[]
- )
-}
-
-nbeats_structure_hyperparameter_map = {
- HyperparameterName.BLOCK_TYPE.value: StringHyperparameter(
- name=HyperparameterName.BLOCK_TYPE.name(),
- default_value='generic',
- value_validators=[],
- value_list=['generic'],
- tuning=False
- ),
- HyperparameterName.D_MODEL.value: IntHyperparameter(
- name=HyperparameterName.D_MODEL.name(),
- log=True,
- default_value=512,
- value_validators=[NumberRangeValidator(4, 8192)],
- default_low=2,
- low_validators=[],
- default_high=2048,
- high_validators=[]
- ),
- HyperparameterName.INNER_LAYERS.value: IntHyperparameter(
- name=HyperparameterName.INNER_LAYERS.name(),
- log=False,
- default_value=4,
- value_validators=[NumberRangeValidator(1, 128)],
- default_low=1,
- low_validators=[],
- default_high=128,
- high_validators=[]
- ),
- HyperparameterName.OUTER_LAYERS.value: IntHyperparameter(
- name=HyperparameterName.OUTER_LAYERS.name(),
- log=False,
- default_value=4,
- value_validators=[NumberRangeValidator(1, 128)],
- default_low=1,
- low_validators=[],
- default_high=128,
- high_validators=[])
-}
-
-
-def get_structure_hyperparameter_map(model_type: ForecastModelType) -> Dict[str, Hyperparameter]:
- """
- Different model may have different structure hyperparameters.
- This method returns the structure hyperparameter map for a given model type.
- """
- if model_type == ForecastModelType.DLINEAR:
- return dlinear_structure_hyperparameter_map
- elif model_type == ForecastModelType.DLINEAR_INDIVIDUAL:
- return dlinear_structure_hyperparameter_map
- elif model_type == ForecastModelType.NBEATS:
- return nbeats_structure_hyperparameter_map
- else:
- raise NotImplementedError(f"Model type {model_type} is not supported yet")
-
-
-def parse_fixed_hyperparameters(
- task_options: ForecastTaskOptions,
- input_hyperparameters: Dict[str, str]
-) -> Tuple[Dict, Dict]:
- """
- Parse the input hyperparameters into model hyperparameters and task hyperparameters.
- If the input hyperparameters contains hyperparameters that are not defined in the model or task,
- use default value.
- """
- structure_hyperparameter_map = get_structure_hyperparameter_map(task_options.model_type)
- model_hyperparameters = parse_dict(input_hyperparameters, structure_hyperparameter_map)
- task_hyperparameters = parse_dict(input_hyperparameters, training_hyperparameter_map)
- return model_hyperparameters, task_hyperparameters
-
-
-def parse_dict(input_hyperparameters: Dict[str, str], hyperparameter_template_map: Dict[str, Hyperparameter]) -> Dict:
- hyperparameter = {}
- for hyperparameter_name in hyperparameter_template_map.keys():
- hyperparameter_template = hyperparameter_template_map[hyperparameter_name]
-
- # if user defines current hyperparameter
- if hyperparameter_name in input_hyperparameters.keys():
- value = hyperparameter_template.parse(input_hyperparameters[hyperparameter_name])
- hyperparameter_template.validate_value(value)
- hyperparameter[hyperparameter_name] = value
- # use default value
- else:
- hyperparameter[hyperparameter_name] = hyperparameter_template.get_default_value()
- return hyperparameter
-
-
-def generate_hyperparameters(
- optuna_suggest: optuna.Trial,
- task_options: TaskOptions
-) -> Tuple[Dict, Dict]:
- """
- Generate hyperparameters for model and task by optuna in auto-tuning training.
- """
- # TODO : support user to define hyperparameters in auto_tuning training
- structure_hyperparameter_map = get_structure_hyperparameter_map(task_options.model_type)
- model_hyperparameters = {}
- for hyperparameter_name, hyperparameter_template in structure_hyperparameter_map.items():
- model_hyperparameters[hyperparameter_name] = hyperparameter_template.suggest_parameter(optuna_suggest)
-
- task_hyperparameters = {}
- for hyperparameter_name, hyperparameter_template in training_hyperparameter_map.items():
- task_hyperparameters[hyperparameter_name] = hyperparameter_template.suggest_parameter(optuna_suggest)
- return model_hyperparameters, task_hyperparameters
diff --git a/iotdb-core/mlnode/iotdb/mlnode/algorithm/metric.py b/iotdb-core/mlnode/iotdb/mlnode/algorithm/metric.py
deleted file mode 100644
index bcf63d1f68d..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/algorithm/metric.py
+++ /dev/null
@@ -1,80 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-from abc import abstractmethod
-from typing import Dict, List
-
-import numpy as np
-
-all_metrics = ['RSE', 'CORR', 'MAE', 'MSE', 'RMSE', 'MAPE', 'MSPE']
-
-
-class Metric(object):
- def __call__(self, pred, ground_truth):
- return self.calculate(pred, ground_truth)
-
- @abstractmethod
- def calculate(self, pred, ground_truth):
- pass
-
-
-class RSE(Metric):
- def calculate(self, pred, ground_truth):
- return np.sqrt(np.sum((ground_truth - pred) ** 2)) / np.sqrt(np.sum((ground_truth - ground_truth.mean()) ** 2))
-
-
-class CORR(Metric):
- def calculate(self, pred, ground_truth):
- u = ((ground_truth - ground_truth.mean(0)) * (pred - pred.mean(0))).sum(0)
- d = np.sqrt(((ground_truth - ground_truth.mean(0)) ** 2 * (pred - pred.mean(0)) ** 2).sum(0))
- return (u / d).mean(-1)
-
-
-class MAE(Metric):
- def calculate(self, pred, ground_truth):
- return np.mean(np.abs(pred - ground_truth))
-
-
-class MSE(Metric):
- def calculate(self, pred, ground_truth):
- return np.mean((pred - ground_truth) ** 2)
-
-
-class RMSE(Metric):
- def calculate(self, pred, ground_truth):
- mse = MSE()
- return np.sqrt(mse(pred, ground_truth))
-
-
-class MAPE(Metric):
- def calculate(self, pred, ground_truth):
- return np.mean(np.abs((pred - ground_truth) / ground_truth))
-
-
-class MSPE(Metric):
- def calculate(self, pred, ground_truth):
- return np.mean(np.square((pred - ground_truth) / ground_truth))
-
-
-def build_metrics(metric_names: List) -> Dict[str, Metric]:
- metrics_dict = {}
- for metric_name in metric_names:
- metrics_dict[metric_name] = eval(metric_name)()
- return metrics_dict
-
-
-forecast_metric_names = ["MSE"]
diff --git a/iotdb-core/mlnode/iotdb/mlnode/algorithm/models/forecast/__init__.py b/iotdb-core/mlnode/iotdb/mlnode/algorithm/models/forecast/__init__.py
deleted file mode 100644
index 2a1e720805f..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/algorithm/models/forecast/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
diff --git a/iotdb-core/mlnode/iotdb/mlnode/algorithm/models/forecast/dlinear.py b/iotdb-core/mlnode/iotdb/mlnode/algorithm/models/forecast/dlinear.py
deleted file mode 100644
index 679c93d30c9..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/algorithm/models/forecast/dlinear.py
+++ /dev/null
@@ -1,133 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-import math
-from typing import Dict
-
-import torch
-import torch.nn as nn
-
-from iotdb.mlnode.constant import ModelInputName
-
-
-class MovingAverageBlock(nn.Module):
- """ Moving average block to highlight the trend of time series """
-
- def __init__(self, kernel_size, stride):
- super(MovingAverageBlock, self).__init__()
- self.kernel_size = kernel_size
- self.avg = nn.AvgPool1d(kernel_size=kernel_size, stride=stride, padding=0)
-
- def forward(self, x):
- # padding on the both ends of time series
- front = x[:, 0:1, :].repeat(1, self.kernel_size - 1 - math.floor((self.kernel_size - 1) // 2), 1)
- end = x[:, -1:, :].repeat(1, math.floor((self.kernel_size - 1) // 2), 1)
- x = torch.cat([front, x, end], dim=1)
- x = self.avg(x.permute(0, 2, 1))
- x = x.permute(0, 2, 1)
- return x
-
-
-class SeriesDecompositionBlock(nn.Module):
- """ Series decomposition block """
-
- def __init__(self, kernel_size):
- super(SeriesDecompositionBlock, self).__init__()
- self.moving_avg = MovingAverageBlock(kernel_size, stride=1)
-
- def forward(self, x):
- moving_mean = self.moving_avg(x)
- res = x - moving_mean
- return res, moving_mean
-
-
-class DLinear(nn.Module):
- """ Decomposition Linear Model """
-
- def __init__(
- self,
- kernel_size: int,
- input_len: int,
- pred_len: int,
- input_vars: int
- ):
- super(DLinear, self).__init__()
- self.input_len = input_len
- self.pred_len = pred_len
- self.kernel_size = kernel_size
- self.channels = input_vars
-
- # decomposition Kernel Size
- self.decomposition = SeriesDecompositionBlock(kernel_size)
- self.linear_seasonal = nn.Linear(self.input_len, self.pred_len)
- self.linear_trend = nn.Linear(self.input_len, self.pred_len)
-
- def forward(self, input_dict: Dict):
- # x: [Batch, Input length, Channel]
- x = input_dict[ModelInputName.DATA_X.value]
- seasonal_init, trend_init = self.decomposition(x)
- seasonal_init, trend_init = seasonal_init.permute(0, 2, 1), trend_init.permute(0, 2, 1)
-
- seasonal_output = self.linear_seasonal(seasonal_init)
- trend_output = self.linear_trend(trend_init)
-
- x = seasonal_output + trend_output
- return x.permute(0, 2, 1) # to [Batch, Output length, Channel]
-
-
-class DLinearIndividual(nn.Module):
- """ Decomposition Linear Model (individual) """
-
- def __init__(
- self,
- kernel_size=25,
- input_len=96,
- pred_len=96,
- input_vars=1
- ):
- super(DLinearIndividual, self).__init__()
- self.input_len = input_len
- self.pred_len = pred_len
- self.kernel_size = kernel_size
- self.channels = input_vars
-
- self.decomposition = SeriesDecompositionBlock(kernel_size)
- self.Linear_Seasonal = nn.ModuleList(
- [nn.Linear(self.input_len, self.pred_len) for _ in range(self.channels)]
- )
- self.Linear_Trend = nn.ModuleList(
- [nn.Linear(self.input_len, self.pred_len) for _ in range(self.channels)]
- )
-
- def forward(self, input_dict: Dict):
- # x: [Batch, Input length, Channel]
- x = input_dict[ModelInputName.DATA_X.value]
- seasonal_init, trend_init = self.decomposition(x)
- seasonal_init, trend_init = seasonal_init.permute(0, 2, 1), trend_init.permute(0, 2, 1)
-
- seasonal_output = torch.zeros([seasonal_init.size(0), seasonal_init.size(1), self.pred_len],
- dtype=seasonal_init.dtype).to(seasonal_init.device)
- trend_output = torch.zeros([trend_init.size(0), trend_init.size(1), self.pred_len],
- dtype=trend_init.dtype).to(trend_init.device)
- for i, linear_season_layer in enumerate(self.Linear_Seasonal):
- seasonal_output[:, i, :] = linear_season_layer(seasonal_init[:, i, :])
- for i, linear_trend_layer in enumerate(self.Linear_Trend):
- trend_output[:, i, :] = linear_trend_layer(trend_init[:, i, :])
-
- x = seasonal_output + trend_output
- return x.permute(0, 2, 1) # to [Batch, Output length, Channel]
diff --git a/iotdb-core/mlnode/iotdb/mlnode/algorithm/models/forecast/nbeats.py b/iotdb-core/mlnode/iotdb/mlnode/algorithm/models/forecast/nbeats.py
deleted file mode 100644
index bf6d64528d4..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/algorithm/models/forecast/nbeats.py
+++ /dev/null
@@ -1,131 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-from typing import Tuple, Dict
-
-import torch
-import torch.nn as nn
-
-from iotdb.mlnode.constant import ModelInputName
-
-
-class GenericBasis(nn.Module):
- """ Generic basis function """
-
- def __init__(self, backcast_size: int, forecast_size: int):
- super().__init__()
- self.backcast_size = backcast_size
- self.forecast_size = forecast_size
-
- def forward(self, theta: torch.Tensor):
- return theta[:, :self.backcast_size], theta[:, -self.forecast_size:]
-
-
-block_dict = {
- 'generic': GenericBasis,
-}
-
-
-class NBeatsBlock(nn.Module):
- """ N-BEATS block which takes a basis function as an argument """
-
- def __init__(self,
- input_size,
- theta_size: int,
- basis_function: nn.Module,
- layers: int,
- layer_size: int):
- """
- N-BEATS block
-
- Args:
- input_size: input sample size
- theta_size: number of parameters for the basis function
- basis_function: basis function which takes the parameters and produces backcast and forecast
- layers: number of layers
- layer_size: layer size
- """
- super().__init__()
- self.layers = nn.ModuleList([nn.Linear(in_features=input_size, out_features=layer_size)] + [
- nn.Linear(in_features=layer_size, out_features=layer_size) for _ in range(layers - 1)])
- self.basis_parameters = nn.Linear(in_features=layer_size, out_features=theta_size)
- self.basis_function = basis_function
-
- def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
- block_input = x
- for layer in self.layers:
- block_input = torch.relu(layer(block_input))
- basis_parameters = self.basis_parameters(block_input)
- return self.basis_function(basis_parameters)
-
-
-class NBeatsUnivariable(nn.Module):
- """ N-Beats Model (uni-variable) """
-
- def __init__(self, blocks: nn.ModuleList):
- super().__init__()
- self.blocks = blocks
-
- def forward(self, x):
- residuals = x
- forecast = None
- for _, block in enumerate(self.blocks):
- backcast, block_forecast = block(residuals)
- residuals = (residuals - backcast)
- if forecast is None:
- forecast = block_forecast
- else:
- forecast += block_forecast
- return forecast
-
-
-class NBeats(nn.Module):
- """ Neural Basis Expansion Analysis Time Series """
-
- def __init__(
- self,
- block_type='generic',
- d_model=128,
- inner_layers=4,
- outer_layers=4,
- input_len=96,
- pred_len=96,
- input_vars=1
- ):
- super(NBeats, self).__init__()
- self.enc_in = input_vars
- self.block = block_dict[block_type]
- self.model = NBeatsUnivariable(
- torch.nn.ModuleList(
- [NBeatsBlock(input_size=input_len,
- theta_size=input_len + pred_len,
- basis_function=self.block(backcast_size=input_len, forecast_size=pred_len),
- layers=inner_layers,
- layer_size=d_model)
- for _ in range(outer_layers)]
- )
- )
-
- def forward(self, input_dict: Dict):
- # x: [Batch, Input length, Channel]
- x = input_dict[ModelInputName.DATA_X.value]
- res = []
- for i in range(self.enc_in):
- dec_out = self.model(x[:, :, i])
- res.append(dec_out)
- return torch.stack(res, dim=-1) # to [Batch, Output length, Channel]
diff --git a/iotdb-core/mlnode/iotdb/mlnode/algorithm/validator.py b/iotdb-core/mlnode/iotdb/mlnode/algorithm/validator.py
deleted file mode 100644
index 1a671a5464e..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/algorithm/validator.py
+++ /dev/null
@@ -1,45 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-from abc import abstractmethod
-
-
-class Validator(object):
- @abstractmethod
- def validate(self, value):
- """
- Checks whether the given value is valid.
-
- Parameters:
- - value: The value to validate
-
- Returns:
- - True if the value is valid, False otherwise.
- """
- raise NotImplementedError("Subclasses must implement the validate() method.")
-
-
-class NumberRangeValidator(Validator):
- def __init__(self, min_value, max_value):
- self.min_value = min_value
- self.max_value = max_value
-
- def validate(self, value):
- if self.min_value <= value <= self.max_value:
- return True
- raise RuntimeError(
- "Expect value between {0} and {1}, got {2} instead.".format(self.min_value, self.max_value, value))
diff --git a/iotdb-core/mlnode/iotdb/mlnode/client.py b/iotdb-core/mlnode/iotdb/mlnode/client.py
deleted file mode 100644
index 04013b9defa..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/client.py
+++ /dev/null
@@ -1,274 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-import time
-from typing import Dict, List
-
-import pandas as pd
-from thrift.protocol import TBinaryProtocol
-from thrift.Thrift import TException
-from thrift.transport import TSocket, TTransport
-
-from iotdb.mlnode import serde
-from iotdb.mlnode.config import descriptor
-from iotdb.mlnode.constant import TSStatusCode
-from iotdb.mlnode.log import logger
-from iotdb.mlnode.util import verify_success
-from iotdb.thrift.common.ttypes import TEndPoint, TrainingState, TSStatus
-from iotdb.thrift.confignode import IConfigNodeRPCService
-from iotdb.thrift.confignode.ttypes import (TUpdateModelInfoReq,
- TUpdateModelStateReq)
-from iotdb.thrift.datanode import IMLNodeInternalRPCService
-from iotdb.thrift.datanode.ttypes import (TFetchMoreDataReq,
- TFetchTimeseriesReq,
- TRecordModelMetricsReq)
-
-
-class ClientManager(object):
- def __init__(self):
- self.__data_node_endpoint = descriptor.get_config().get_mn_target_data_node()
- self.__config_node_endpoint = descriptor.get_config().get_mn_target_config_node()
-
- def borrow_data_node_client(self):
- return DataNodeClient(host=self.__data_node_endpoint.ip,
- port=self.__data_node_endpoint.port)
-
- def borrow_config_node_client(self):
- return ConfigNodeClient(config_leader=self.__config_node_endpoint)
-
-
-class DataNodeClient(object):
- DEFAULT_FETCH_SIZE = 10000
- DEFAULT_TIMEOUT = 60000
-
- def __init__(self, host, port):
- self.__host = host
- self.__port = port
-
- transport = TTransport.TFramedTransport(
- TSocket.TSocket(self.__host, self.__port)
- )
- if not transport.isOpen():
- try:
- transport.open()
- except TTransport.TTransportException as e:
- logger.exception("TTransportException!", exc_info=e)
- raise e
-
- protocol = TBinaryProtocol.TBinaryProtocol(transport)
- self.__client = IMLNodeInternalRPCService.Client(protocol)
-
- def fetch_timeseries(self,
- query_body: str,
- fetch_size: int = DEFAULT_FETCH_SIZE,
- timeout: int = DEFAULT_TIMEOUT) -> pd.DataFrame:
- req = TFetchTimeseriesReq(
- queryBody=query_body,
- fetchSize=fetch_size,
- timeout=timeout
- )
- try:
- resp = self.__client.fetchTimeseries(req)
- verify_success(resp.status, "An error occurs when calling fetch_timeseries()")
-
- if len(resp.tsDataset) == 0:
- raise RuntimeError(f'No data fetched with sql: {query_body}')
- data = serde.convert_to_df(resp.columnNameList,
- resp.columnTypeList,
- resp.columnNameIndexMap,
- resp.tsDataset)
- if data.empty:
- raise RuntimeError(
- f'Fetched empty data with sql: {query_body}')
- except Exception as e:
- logger.warn(
- f'Fail to fetch data with sql: {query_body}')
- raise e
- query_id = resp.queryId
- column_name_list = resp.columnNameList
- column_type_list = resp.columnTypeList
- column_name_index_map = resp.columnNameIndexMap
- has_more_data = resp.hasMoreData
- while has_more_data:
- req = TFetchMoreDataReq(queryId=query_id, fetchSize=fetch_size)
- try:
- resp = self.__client.fetchMoreData(req)
- verify_success(resp.status, "An error occurs when calling fetch_more_data()")
- data = data.append(serde.convert_to_df(column_name_list,
- column_type_list,
- column_name_index_map,
- resp.tsDataset))
- has_more_data = resp.hasMoreData
- except Exception as e:
- logger.warn(
- f'Fail to fetch more data with query id: {query_id}')
- raise e
- return data
-
- def record_model_metrics(self,
- model_id: str,
- trial_id: str,
- metrics: List[str],
- values: List) -> None:
- req = TRecordModelMetricsReq(
- modelId=model_id,
- trialId=trial_id,
- metrics=metrics,
- timestamp=int(round(time.time() * 1000)),
- values=values
- )
- try:
- status = self.__client.recordModelMetrics(req)
- verify_success(status, "An error occurs when calling record_model_metrics()")
- except TTransport.TException as e:
- logger.exception(e.message)
-
-
-class ConfigNodeClient(object):
- def __init__(self, config_leader: TEndPoint):
- self.__config_leader = config_leader
- self.__config_nodes = []
- self.__cursor = 0
- self.__transport = None
- self.__client = None
-
- self.__MSG_RECONNECTION_FAIL = "Fail to connect to any config node. Please check status of ConfigNodes"
- self.__RETRY_NUM = 5
- self.__RETRY_INTERVAL_MS = 1000
-
- try:
- self.__try_to_connect()
- except TException:
- # can not connect to each config node
- self.__sync_latest_config_node_list()
- self.__try_to_connect()
-
- def __try_to_connect(self) -> None:
- if self.__config_leader is not None:
- try:
- self.__connect(self.__config_leader)
- return
- except TException:
- logger.warn("The current node {} may have been down, try next node", self.__config_leader)
- self.__config_leader = None
-
- if self.__transport is not None:
- self.__transport.close()
-
- try_host_num = 0
- while try_host_num < len(self.__config_nodes):
- self.__cursor = (self.__cursor + 1) % len(self.__config_nodes)
-
- try_endpoint = self.__config_nodes[self.__cursor]
- try:
- self.__connect(try_endpoint)
- return
- except TException:
- logger.warn("The current node {} may have been down, try next node", try_endpoint)
-
- try_host_num = try_host_num + 1
-
- raise TException(self.__MSG_RECONNECTION_FAIL)
-
- def __connect(self, target_config_node: TEndPoint) -> None:
- transport = TTransport.TFramedTransport(
- TSocket.TSocket(target_config_node.ip, target_config_node.port)
- )
- if not transport.isOpen():
- try:
- transport.open()
- except TTransport.TTransportException as e:
- logger.exception("TTransportException!", exc_info=e)
-
- protocol = TBinaryProtocol.TBinaryProtocol(transport)
- self.__client = IConfigNodeRPCService.Client(protocol)
-
- def __wait_and_reconnect(self) -> None:
- # wait to start the next try
- time.sleep(self.__RETRY_INTERVAL_MS)
-
- try:
- self.__try_to_connect()
- except TException:
- # can not connect to each config node
- self.__sync_latest_config_node_list()
- self.__try_to_connect()
-
- def __sync_latest_config_node_list(self) -> None:
- # TODO
- pass
-
- def __update_config_node_leader(self, status: TSStatus) -> bool:
- if status.code == TSStatusCode.REDIRECTION_RECOMMEND:
- if status.redirectNode is not None:
- self.__config_leader = status.redirectNode
- else:
- self.__config_leader = None
- return True
- return False
-
- def update_model_state(self,
- model_id: str,
- training_state: TrainingState,
- best_trail_id: str = None) -> None:
- req = TUpdateModelStateReq(
- modelId=model_id,
- state=training_state,
- bestTrialId=best_trail_id
- )
- for _ in range(0, self.__RETRY_NUM):
- try:
- status = self.__client.updateModelState(req)
- if not self.__update_config_node_leader(status):
- verify_success(status, "An error occurs when calling update_model_state()")
- return
- except TTransport.TException:
- logger.warn("Failed to connect to ConfigNode {} from MLNode when executing update_model_info()",
- self.__config_leader)
- self.__config_leader = None
- self.__wait_and_reconnect()
-
- raise TException(self.__MSG_RECONNECTION_FAIL)
-
- def update_model_info(self,
- model_id: str,
- trial_id: str,
- model_info: Dict) -> None:
- if model_info is None:
- model_info = {}
- req = TUpdateModelInfoReq(
- modelId=model_id,
- trialId=trial_id,
- modelInfo={k: str(v) for k, v in model_info.items()},
- )
-
- for _ in range(0, self.__RETRY_NUM):
- try:
- status = self.__client.updateModelInfo(req)
- if not self.__update_config_node_leader(status):
- verify_success(status, "An error occurs when calling update_model_info()")
- return
- except TTransport.TException:
- logger.warn("Failed to connect to ConfigNode {} from MLNode when executing update_model_info()",
- self.__config_leader)
- self.__config_leader = None
- self.__wait_and_reconnect()
-
- raise TException(self.__MSG_RECONNECTION_FAIL)
-
-
-client_manager = ClientManager()
diff --git a/iotdb-core/mlnode/iotdb/mlnode/config.py b/iotdb-core/mlnode/iotdb/mlnode/config.py
deleted file mode 100644
index c54aa07ff25..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/config.py
+++ /dev/null
@@ -1,168 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-import os
-
-from dynaconf import Dynaconf
-
-from iotdb.mlnode.constant import (MLNODE_CONF_DIRECTORY_NAME,
- MLNODE_CONF_FILE_NAME,
- MLNODE_MODEL_STORAGE_DIRECTORY_NAME)
-from iotdb.mlnode.exception import BadNodeUrlError
-from iotdb.mlnode.log import logger
-from iotdb.mlnode.util import parse_endpoint_url
-from iotdb.thrift.common.ttypes import TEndPoint
-
-
-class MLNodeConfig(object):
- def __init__(self):
- # Used for connection of DataNode/ConfigNode clients
- self.__mn_rpc_address: str = "127.0.0.1"
- self.__mn_rpc_port: int = 10810
-
- # Directory to save models
- self.__mn_model_storage_dir = MLNODE_MODEL_STORAGE_DIRECTORY_NAME
-
- # Cache number of model storage to avoid repeated loading
- self.__mn_model_storage_cache_size = 30
-
- # Maximum number of training model tasks, otherwise the task is pending
- self.__mn_task_pool_size = 10
-
- # Maximum number of trials to be explored in a tuning task
- self.__mn_tuning_trial_num = 20
-
- # Concurrency of trials in a tuning task
- self.__mn_tuning_trial_concurrency = 4
-
- # Target ConfigNode to be connected by MLNode
- self.__mn_target_config_node: TEndPoint = TEndPoint("127.0.0.1", 10710)
-
- # Target DataNode to be connected by MLNode
- self.__mn_target_data_node: TEndPoint = TEndPoint("127.0.0.1", 10780)
-
- def get_mn_rpc_address(self) -> str:
- return self.__mn_rpc_address
-
- def set_mn_rpc_address(self, mn_rpc_address: str) -> None:
- self.__mn_rpc_address = mn_rpc_address
-
- def get_mn_rpc_port(self) -> int:
- return self.__mn_rpc_port
-
- def set_mn_rpc_port(self, mn_rpc_port: int) -> None:
- self.__mn_rpc_port = mn_rpc_port
-
- def get_mn_model_storage_dir(self) -> str:
- return self.__mn_model_storage_dir
-
- def set_mn_model_storage_dir(self, mn_model_storage_dir: str) -> None:
- self.__mn_model_storage_dir = mn_model_storage_dir
-
- def get_mn_model_storage_cache_size(self) -> int:
- return self.__mn_model_storage_cache_size
-
- def set_mn_model_storage_cache_size(self, mn_model_storage_cache_size: int) -> None:
- self.__mn_model_storage_cache_size = mn_model_storage_cache_size
-
- def get_mn_mn_task_pool_size(self) -> int:
- return self.__mn_task_pool_size
-
- def set_mn_task_pool_size(self, mn_task_pool_size: int) -> None:
- self.__mn_task_pool_size = mn_task_pool_size
-
- def get_mn_tuning_trial_num(self) -> int:
- return self.__mn_tuning_trial_num
-
- def set_mn_tuning_trial_num(self, mn_tuning_trial_num: int) -> None:
- self.__mn_tuning_trial_num = mn_tuning_trial_num
-
- def get_mn_tuning_trial_concurrency(self) -> int:
- return self.__mn_tuning_trial_concurrency
-
- def set_mn_tuning_trial_concurrency(self, mn_tuning_trial_concurrency: int) -> None:
- self.__mn_tuning_trial_concurrency = mn_tuning_trial_concurrency
-
- def get_mn_target_config_node(self) -> TEndPoint:
- return self.__mn_target_config_node
-
- def set_mn_target_config_node(self, mn_target_config_node: str) -> None:
- self.__mn_target_config_node = parse_endpoint_url(mn_target_config_node)
-
- def get_mn_target_data_node(self) -> TEndPoint:
- return self.__mn_target_data_node
-
- def set_mn_target_data_node(self, mn_target_data_node: str) -> None:
- self.__mn_target_data_node = parse_endpoint_url(mn_target_data_node)
-
-
-class MLNodeDescriptor(object):
- def __init__(self):
- self.__config = MLNodeConfig()
-
- def load_config_from_file(self) -> None:
- conf_file = os.path.join(os.getcwd(), MLNODE_CONF_DIRECTORY_NAME, MLNODE_CONF_FILE_NAME)
- if not os.path.exists(conf_file):
- logger.info("Cannot find MLNode config file '{}', use default configuration.".format(conf_file))
- return
-
- logger.info("Start to read MLNode config file '{}'...".format(conf_file))
-
- # noinspection PyBroadException
- try:
- file_configs = Dynaconf(
- envvar_prefix="DYNACONF",
- settings_files=[conf_file],
- )
-
- if file_configs.mn_rpc_address is not None:
- self.__config.set_mn_rpc_address(file_configs.mn_rpc_address)
-
- if file_configs.mn_rpc_port is not None:
- self.__config.set_mn_rpc_port(file_configs.mn_rpc_port)
-
- if file_configs.mn_model_storage_dir is not None:
- self.__config.set_mn_model_storage_dir(file_configs.mn_model_storage_dir)
-
- if file_configs.mn_model_storage_cache_size is not None:
- self.__config.set_mn_model_storage_cache_size(file_configs.mn_model_storage_cache_size)
-
- if file_configs.mn_task_pool_size is not None:
- self.__config.set_mn_task_pool_size(file_configs.mn_task_pool_size)
-
- if file_configs.mn_tuning_trial_num is not None:
- self.__config.set_mn_tuning_trial_num(file_configs.mn_tuning_trial_num)
-
- if file_configs.mn_tuning_trial_concurrency is not None:
- self.__config.set_mn_tuning_trial_concurrency(file_configs.mn_tuning_trial_concurrency)
-
- if file_configs.mn_target_config_node is not None:
- self.__config.set_mn_target_config_node(file_configs.mn_target_config_node)
-
- if file_configs.mn_target_data_node is not None:
- self.__config.set_mn_target_data_node(file_configs.mn_target_data_node)
- except BadNodeUrlError:
- logger.warn("Cannot load MLNode conf file, use default configuration.")
- except Exception as e:
- logger.warn("Cannot load MLNode conf file, use default configuration. {}".format(e))
-
- def get_config(self) -> MLNodeConfig:
- return self.__config
-
-
-# initialize a singleton
-descriptor = MLNodeDescriptor()
diff --git a/iotdb-core/mlnode/iotdb/mlnode/constant.py b/iotdb-core/mlnode/iotdb/mlnode/constant.py
deleted file mode 100644
index a3c25a6a7d4..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/constant.py
+++ /dev/null
@@ -1,99 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-from enum import Enum
-from typing import List
-
-MLNODE_CONF_DIRECTORY_NAME = "resources/conf"
-MLNODE_CONF_FILE_NAME = "iotdb-mlnode.toml"
-MLNODE_LOG_CONF_FILE_NAME = "logging_config.ini"
-
-MLNODE_MODEL_STORAGE_DIRECTORY_NAME = "models"
-
-TRIAL_ID_PREFIX = "__trial_"
-DEFAULT_TRIAL_ID = TRIAL_ID_PREFIX + "0"
-
-
-class TSStatusCode(Enum):
- SUCCESS_STATUS = 200
- REDIRECTION_RECOMMEND = 400
- MLNODE_INTERNAL_ERROR = 1510
-
- def get_status_code(self) -> int:
- return self.value
-
-
-class TaskType(Enum):
- FORECAST = "forecast"
-
-
-class OptionsKey(Enum):
- # common
- TASK_TYPE = "task_type"
- MODEL_TYPE = "model_type"
- AUTO_TUNING = "auto_tuning"
- INPUT_VARS = "input_vars"
-
- # forecast
- INPUT_LENGTH = "input_length"
- PREDICT_LENGTH = "predict_length"
- PREDICT_INDEX_LIST = "predict_index_list"
- INPUT_TYPE_LIST = "input_type_list"
-
- def name(self) -> str:
- return self.value
-
-
-class HyperparameterName(Enum):
- # Training hyperparameter
- LEARNING_RATE = "learning_rate"
- EPOCHS = "epochs"
- BATCH_SIZE = "batch_size"
- USE_GPU = "use_gpu"
- NUM_WORKERS = "num_workers"
-
- # Structure hyperparameter
- KERNEL_SIZE = "kernel_size"
- INPUT_VARS = "input_vars"
- BLOCK_TYPE = "block_type"
- D_MODEL = "d_model"
- INNER_LAYERS = "inner_layer"
- OUTER_LAYERS = "outer_layer"
-
- def name(self):
- return self.value
-
-
-class ForecastModelType(Enum):
- DLINEAR = "dlinear"
- DLINEAR_INDIVIDUAL = "dlinear_individual"
- NBEATS = "nbeats"
-
- @classmethod
- def values(cls) -> List[str]:
- values = []
- for item in list(cls):
- values.append(item.value)
- return values
-
-
-class ModelInputName(Enum):
- DATA_X = "data_x"
- TIME_STAMP_X = "time_stamp_x"
- TIME_STAMP_Y = "time_stamp_y"
- DEC_INP = "dec_inp"
-
diff --git a/iotdb-core/mlnode/iotdb/mlnode/dataset/dataset.py b/iotdb-core/mlnode/iotdb/mlnode/dataset/dataset.py
deleted file mode 100644
index ab9a1c1c5b3..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/dataset/dataset.py
+++ /dev/null
@@ -1,74 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-from typing import Tuple
-
-import numpy as np
-from torch.utils.data import Dataset
-
-from iotdb.mlnode.dataset.source import DataSource
-from iotdb.mlnode.dataset.utils.time_features import time_features
-
-
-class TsForecastDataset(Dataset):
- """
- Build Windowed dataset (with each element as multivariable time series
- with a sliding window and corresponding timestamps embedding),
- the sliding step is one unit in give data source
-
- Args:
- data_source: the whole multivariate time series for a while
- time_embed: embedding frequency, see `utils/time_features.py` for more detail
- input_len: input window size (unit) [1, 2, ... I]
- pred_len: output window size (unit) right after the input window [I+1, I+2, ... I+P]
-
- Returns:
- Random accessible dataset
- """
-
- def __init__(self,
- data_source: DataSource,
- input_len: int,
- pred_len: int,
- time_embed: str = 'h'):
- self.input_len = input_len
- self.pred_len = pred_len
- self.time_embed = time_embed
- self.data = data_source.get_data()
- self.data_stamp = time_features(data_source.get_timestamp(), time_embed=self.time_embed).transpose(1, 0)
- self.n_vars = self.data.shape[-1]
- if input_len > self.data.shape[0]:
- raise RuntimeError('input_len should not be larger than the number of time series points')
- if pred_len > self.data.shape[0]:
- raise RuntimeError('pred_len should not be larger than the number of time series points')
-
- def __getitem__(self, index) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
- s_begin = index
- s_end = s_begin + self.input_len
- r_begin = s_end
- r_end = s_end + self.pred_len
- seq_x = self.data[s_begin:s_end]
- seq_y = self.data[r_begin:r_end]
- seq_x_t = self.data_stamp[s_begin:s_end]
- seq_y_t = self.data_stamp[r_begin:r_end]
- return seq_x, seq_y, seq_x_t, seq_y_t
-
- def __len__(self) -> int:
- return len(self.data) - self.input_len - self.pred_len + 1
-
- def get_variable_num(self) -> int:
- return self.n_vars # number of series in data_source
diff --git a/iotdb-core/mlnode/iotdb/mlnode/dataset/factory.py b/iotdb-core/mlnode/iotdb/mlnode/dataset/factory.py
deleted file mode 100644
index de666c52206..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/dataset/factory.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-from typing import cast
-
-from torch.utils.data import Dataset
-
-from iotdb.mlnode.constant import TaskType
-from iotdb.mlnode.dataset.dataset import TsForecastDataset
-from iotdb.mlnode.dataset.source import IoTDBDataSource
-from iotdb.mlnode.exception import UnsupportedError
-from iotdb.mlnode.parser import ForecastTaskOptions, TaskOptions
-
-
-def create_dataset(query_body: str, task_options: TaskOptions) -> Dataset:
- """
- Create a dataset for training according to the task type.
- Currently, only ForcastTask is supported, so only create_forecast_dataset is called.
- """
- task_type = task_options.get_task_type()
- if task_type == TaskType.FORECAST:
- task_options = cast(ForecastTaskOptions, task_options)
- return create_forecast_dataset(query_body, task_options)
- else:
- raise UnsupportedError(f"task type {task_type}")
-
-
-def create_forecast_dataset(query_body: str, task_options: ForecastTaskOptions) -> TsForecastDataset:
- """
- Create a dataset for training for forecasting.
- In forecasting, dataset is created from a IoTDBDataSource, and then wrapped by a TsDataset,
- where the input length and predict length are required.
- """
- datasource = IoTDBDataSource(query_body)
- dataset = TsForecastDataset(datasource, task_options.input_length, task_options.predict_length)
- return dataset
diff --git a/iotdb-core/mlnode/iotdb/mlnode/dataset/source.py b/iotdb-core/mlnode/iotdb/mlnode/dataset/source.py
deleted file mode 100644
index f69e7854374..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/dataset/source.py
+++ /dev/null
@@ -1,80 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-import numpy as np
-import pandas as pd
-
-from iotdb.mlnode.client import client_manager
-
-
-class DataSource(object):
- """
- Pre-fetched in multi-variate time series in memory
-
- Methods:
- get_data: returns time series value (Numpy.2DArray)
- get_timestamp: returns aligned timestamp value
- """
-
- def __init__(self):
- self.data = None
- self.timestamp = None
- self._read_data()
-
- def _read_data(self):
- raise NotImplementedError
-
- def get_data(self) -> np.ndarray:
- return self.data
-
- def get_timestamp(self) -> np.ndarray:
- return self.timestamp
-
-
-class FileDataSource(DataSource):
- def __init__(self, filename: str = None):
- self.filename = filename
- super(FileDataSource, self).__init__()
-
- def _read_data(self) -> None:
- try:
- raw_data = pd.read_csv(self.filename)
- except Exception:
- raise RuntimeError(f'Fail to load data with filename: {self.filename}')
- cols_data = raw_data.columns[1:]
- self.data = raw_data[cols_data].values
- self.timestamp = pd.to_datetime(raw_data[raw_data.columns[0]].values)
-
-
-class IoTDBDataSource(DataSource):
- def __init__(self, query_body: str = None):
- self.query_body = query_body
- super(IoTDBDataSource, self).__init__()
-
- def _read_data(self) -> None:
- try:
- data_client = client_manager.borrow_data_node_client()
- except Exception:
- raise RuntimeError('Fail to establish connection with DataNode')
-
- raw_data = data_client.fetch_timeseries(self.query_body)
-
- cols_data = raw_data.columns[1:]
- self.data = raw_data[cols_data].values
- self.timestamp = pd.to_datetime(raw_data[raw_data.columns[0]].values, unit='ms', utc=True) \
- .tz_convert('Asia/Shanghai')
diff --git a/iotdb-core/mlnode/iotdb/mlnode/dataset/utils/__init__.py b/iotdb-core/mlnode/iotdb/mlnode/dataset/utils/__init__.py
deleted file mode 100644
index 2a1e720805f..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/dataset/utils/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
diff --git a/iotdb-core/mlnode/iotdb/mlnode/dataset/utils/time_features.py b/iotdb-core/mlnode/iotdb/mlnode/dataset/utils/time_features.py
deleted file mode 100644
index 17e5a18267a..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/dataset/utils/time_features.py
+++ /dev/null
@@ -1,149 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-from typing import List
-
-import numpy as np
-import pandas as pd
-from pandas.tseries import offsets
-from pandas.tseries.frequencies import to_offset
-
-
-class TimeFeature:
- def __call__(self, index: pd.DatetimeIndex) -> np.ndarray:
- raise NotImplementedError
-
- def __repr__(self):
- return self.__class__.__name__ + '()'
-
-
-class SecondOfMinute(TimeFeature):
- """Minute of hour encoded as value between [-0.5, 0.5]"""
-
- def __call__(self, index: pd.DatetimeIndex) -> np.ndarray:
- return index.second / 59.0 - 0.5
-
-
-class MinuteOfHour(TimeFeature):
- """Minute of hour encoded as value between [-0.5, 0.5]"""
-
- def __call__(self, index: pd.DatetimeIndex) -> np.ndarray:
- return index.minute / 59.0 - 0.5
-
-
-class HourOfDay(TimeFeature):
- """Hour of day encoded as value between [-0.5, 0.5]"""
-
- def __call__(self, index: pd.DatetimeIndex) -> np.ndarray:
- return index.hour / 23.0 - 0.5
-
-
-class DayOfWeek(TimeFeature):
- """Hour of day encoded as value between [-0.5, 0.5]"""
-
- def __call__(self, index: pd.DatetimeIndex) -> np.ndarray:
- return index.dayofweek / 6.0 - 0.5
-
-
-class DayOfMonth(TimeFeature):
- """Day of month encoded as value between [-0.5, 0.5]"""
-
- def __call__(self, index: pd.DatetimeIndex) -> np.ndarray:
- return (index.day - 1) / 30.0 - 0.5
-
-
-class DayOfYear(TimeFeature):
- """Day of year encoded as value between [-0.5, 0.5]"""
-
- def __call__(self, index: pd.DatetimeIndex) -> np.ndarray:
- return (index.dayofyear - 1) / 365.0 - 0.5
-
-
-class MonthOfYear(TimeFeature):
- """Month of year encoded as value between [-0.5, 0.5]"""
-
- def __call__(self, index: pd.DatetimeIndex) -> np.ndarray:
- return (index.month - 1) / 11.0 - 0.5
-
-
-class WeekOfYear(TimeFeature):
- """Week of year encoded as value between [-0.5, 0.5]"""
-
- def __call__(self, index: pd.DatetimeIndex) -> np.ndarray:
- return (index.isocalendar().week - 1) / 52.0 - 0.5
-
-
-def time_features_from_frequency_str(freq_str: str) -> List[TimeFeature]:
- """
- Embedding timestamp by given frequency string
- Args:
- freq_str: frequency string of the form [multiple][granularity] such as '12H', '5min', '1D' etc.
- Returns:
- a list of time features that will be appropriate for the given frequency string.
- """
-
- features_by_offsets = {
- offsets.YearEnd: [],
- offsets.QuarterEnd: [MonthOfYear],
- offsets.MonthEnd: [MonthOfYear],
- offsets.Week: [DayOfMonth, WeekOfYear],
- offsets.Day: [DayOfWeek, DayOfMonth, DayOfYear],
- offsets.BusinessDay: [DayOfWeek, DayOfMonth, DayOfYear],
- offsets.Hour: [HourOfDay, DayOfWeek, DayOfMonth, DayOfYear],
- offsets.Minute: [
- MinuteOfHour,
- HourOfDay,
- DayOfWeek,
- DayOfMonth,
- DayOfYear,
- ],
- offsets.Second: [
- SecondOfMinute,
- MinuteOfHour,
- HourOfDay,
- DayOfWeek,
- DayOfMonth,
- DayOfYear,
- ],
- }
-
- try:
- offset = to_offset(freq_str)
-
- for offset_type, feature_classes in features_by_offsets.items():
- if isinstance(offset, offset_type):
- return [cls() for cls in feature_classes]
- except ValueError:
- supported_freq_msg = f'''
- Unsupported time embedding frequency ({freq_str})
- The following frequencies are supported (case-insensitive):
- Y - yearly
- alias: A
- M - monthly
- W - weekly
- D - daily
- B - business days
- H - hourly
- T - minutely
- alias: min
- S - secondly
- '''
- raise RuntimeError(supported_freq_msg)
-
-
-def time_features(dates, time_embed='h'):
- return np.vstack([feat(dates) for feat in time_features_from_frequency_str(time_embed)])
diff --git a/iotdb-core/mlnode/iotdb/mlnode/exception.py b/iotdb-core/mlnode/iotdb/mlnode/exception.py
deleted file mode 100644
index 36beab83307..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/exception.py
+++ /dev/null
@@ -1,66 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-class _BaseError(Exception):
- """Base class for exceptions in this module."""
-
- def __init__(self):
- self.message = None
-
- def __str__(self) -> str:
- return self.message
-
-
-class BadNodeUrlError(_BaseError):
- def __init__(self, node_url: str):
- self.message = "Bad node url: {}".format(node_url)
-
-
-class ModelNotExistError(_BaseError):
- def __init__(self, file_path: str):
- self.message = "Model path is not exists: {} ".format(file_path)
-
-
-class BadConfigValueError(_BaseError):
- def __init__(self, config_name: str, config_value, hint: str = ''):
- self.message = "Bad value [{0}] for config {1}. {2}".format(config_value, config_name, hint)
-
-
-class MissingConfigError(_BaseError):
- def __init__(self, config_name: str):
- self.message = "Missing config: {}".format(config_name)
-
-
-class MissingOptionError(_BaseError):
- def __init__(self, config_name: str):
- self.message = "Missing task option: {}".format(config_name)
-
-
-class RedundantOptionError(_BaseError):
- def __init__(self, option_name: str):
- self.message = "Redundant task option: {}".format(option_name)
-
-
-class WrongTypeConfigError(_BaseError):
- def __init__(self, config_name: str, expected_type: str):
- self.message = "Wrong type for config: {0}, expected: {1}".format(config_name, expected_type)
-
-
-class UnsupportedError(_BaseError):
- def __init__(self, msg: str):
- self.message = "{0} is not supported in current version".format(msg)
diff --git a/iotdb-core/mlnode/iotdb/mlnode/handler.py b/iotdb-core/mlnode/iotdb/mlnode/handler.py
deleted file mode 100644
index 0790f58b606..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/handler.py
+++ /dev/null
@@ -1,95 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-from typing import cast
-
-from iotdb.mlnode.config import descriptor
-from iotdb.mlnode.constant import TaskType, TSStatusCode
-from iotdb.mlnode.dataset.dataset import TsForecastDataset
-from iotdb.mlnode.dataset.factory import create_dataset
-from iotdb.mlnode.log import logger
-from iotdb.mlnode.parser import (ForecastTaskOptions, parse_forecast_request,
- parse_task_options)
-from iotdb.mlnode.process.manager import TaskManager
-from iotdb.mlnode.serde import convert_to_binary
-from iotdb.mlnode.storage import model_storage
-from iotdb.mlnode.util import get_status
-from iotdb.thrift.mlnode import IMLNodeRPCService
-from iotdb.thrift.mlnode.ttypes import (TCreateTrainingTaskReq,
- TDeleteModelReq, TForecastReq,
- TForecastResp)
-
-
-class MLNodeRPCServiceHandler(IMLNodeRPCService.Iface):
- def __init__(self):
- self.__task_manager = TaskManager(pool_size=descriptor.get_config().get_mn_mn_task_pool_size())
-
- def deleteModel(self, req: TDeleteModelReq):
- try:
- model_storage.delete_model(req.modelId)
- return get_status(TSStatusCode.SUCCESS_STATUS)
- except Exception as e:
- logger.warn(e)
- return get_status(TSStatusCode.MLNODE_INTERNAL_ERROR, str(e))
-
- def createTrainingTask(self, req: TCreateTrainingTaskReq):
- task = None
- try:
- # parse options
- task_options = parse_task_options(req.options)
- dataset = create_dataset(req.datasetFetchSQL, task_options)
-
- # create task according to task type
- # currently, IoTDB-ML supports forecasting training task only
- if task_options.get_task_type() == TaskType.FORECAST:
- task_options = cast(ForecastTaskOptions, task_options)
- dataset = cast(TsForecastDataset, dataset)
- task = self.__task_manager.create_forecast_training_task(
- model_id=req.modelId,
- task_options=task_options,
- hyperparameters=req.hyperparameters,
- dataset=dataset
- )
- else:
- raise NotImplementedError
-
- return get_status(TSStatusCode.SUCCESS_STATUS)
- except Exception as e:
- logger.warn(e)
- return get_status(TSStatusCode.MLNODE_INTERNAL_ERROR, str(e))
- finally:
- if task is not None:
- # submit task to process pool
- self.__task_manager.submit_training_task(task)
-
- def forecast(self, req: TForecastReq):
- model_path, data, pred_length = parse_forecast_request(req)
- try:
- task = self.__task_manager.create_forecast_task(
- pred_length,
- data,
- model_path
- )
- # submit task stage & check resource and decide pending/start
- res = self.__task_manager.submit_forecast_task(task)
- forecast_result = convert_to_binary(res)
- resp = TForecastResp(get_status(TSStatusCode.SUCCESS_STATUS), forecast_result)
- return resp
- except Exception as e:
- logger.warn(e)
- return get_status(TSStatusCode.MLNODE_INTERNAL_ERROR, str(e))
diff --git a/iotdb-core/mlnode/iotdb/mlnode/log.py b/iotdb-core/mlnode/iotdb/mlnode/log.py
deleted file mode 100644
index c63e4adcc52..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/log.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-import logging
-import os
-from logging.config import fileConfig
-
-from iotdb.mlnode.constant import (MLNODE_CONF_DIRECTORY_NAME,
- MLNODE_LOG_CONF_FILE_NAME)
-
-log_conf_file = os.path.join(os.getcwd(), MLNODE_CONF_DIRECTORY_NAME, MLNODE_LOG_CONF_FILE_NAME)
-if os.path.exists(log_conf_file):
- fileConfig(log_conf_file)
-else:
- logging.basicConfig(
- level=logging.DEBUG,
- format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
- )
-
-logger = logging.getLogger()
diff --git a/iotdb-core/mlnode/iotdb/mlnode/parser.py b/iotdb-core/mlnode/iotdb/mlnode/parser.py
deleted file mode 100644
index 8299ac6f658..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/parser.py
+++ /dev/null
@@ -1,99 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-import argparse
-from abc import abstractmethod
-from typing import Dict
-
-from iotdb.mlnode.constant import OptionsKey, TaskType, ForecastModelType
-from iotdb.mlnode.exception import (MissingOptionError, RedundantOptionError,
- UnsupportedError)
-from iotdb.mlnode.serde import convert_to_df
-from iotdb.thrift.mlnode.ttypes import TForecastReq
-
-
-class TaskOptions(object):
- def __init__(self, options: Dict):
- self._raw_options = options
-
- if OptionsKey.MODEL_TYPE.name() not in self._raw_options:
- raise MissingOptionError(OptionsKey.MODEL_TYPE.name())
- model_name = self._raw_options.pop(OptionsKey.MODEL_TYPE.name())
- self.model_type = getattr(ForecastModelType, model_name.upper(), None)
- if not self.model_type:
- raise UnsupportedError(f"model_type {model_name}")
-
- # training with auto-tuning as default
- self.auto_tuning = str2bool(self._raw_options.pop(OptionsKey.AUTO_TUNING.name(), "false"))
-
- @abstractmethod
- def get_task_type(self) -> TaskType:
- raise NotImplementedError("Subclasses must implement the validate() method.")
-
- def _check_redundant_options(self) -> None:
- if len(self._raw_options):
- raise RedundantOptionError(str(self._raw_options))
-
-
-class ForecastTaskOptions(TaskOptions):
- def __init__(self, options: Dict):
- super().__init__(options)
- self.input_length = self._raw_options.pop(OptionsKey.INPUT_LENGTH.name(), 96)
- self.predict_length = self._raw_options.pop(OptionsKey.PREDICT_LENGTH.name(), 96)
- self.predict_index_list = self._raw_options.pop(OptionsKey.PREDICT_INDEX_LIST.name(), None)
- self.input_type_list = self._raw_options.pop(OptionsKey.INPUT_TYPE_LIST.name(), None)
- super()._check_redundant_options()
-
- def get_task_type(self) -> TaskType:
- return TaskType.FORECAST
-
-
-def parse_task_type(options: Dict) -> TaskType:
- if OptionsKey.TASK_TYPE.name() not in options:
- raise MissingOptionError(OptionsKey.TASK_TYPE.name())
- task_name = options.pop(OptionsKey.TASK_TYPE.name())
- task_type = getattr(TaskType, task_name.upper(), None)
- if not task_type:
- raise UnsupportedError(f"task_type {task_name}")
- return task_type
-
-
-def parse_task_options(options) -> TaskOptions:
- task_type = parse_task_type(options)
- if task_type == TaskType.FORECAST:
- return ForecastTaskOptions(options)
- else:
- raise UnsupportedError(f"task type {task_type}")
-
-
-def parse_forecast_request(req: TForecastReq):
- column_name_list = req.inputColumnNameList
- column_type_list = req.inputTypeList
- ts_dataset = req.inputData
- data = convert_to_df(column_name_list, column_type_list, None, [ts_dataset])
- time_stamp, data = data[data.columns[0:1]], data[data.columns[1:]]
- full_data = (data, time_stamp)
- return req.modelPath, full_data, req.predictLength
-
-
-def str2bool(value):
- if value.lower() in ('yes', 'true', 't', 'y', '1'):
- return True
- elif value.lower() in ('no', 'false', 'f', 'n', '0'):
- return False
- else:
- raise argparse.ArgumentTypeError('Boolean value expected.')
diff --git a/iotdb-core/mlnode/iotdb/mlnode/process/__init__.py b/iotdb-core/mlnode/iotdb/mlnode/process/__init__.py
deleted file mode 100644
index 2a1e720805f..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/process/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
diff --git a/iotdb-core/mlnode/iotdb/mlnode/process/manager.py b/iotdb-core/mlnode/iotdb/mlnode/process/manager.py
deleted file mode 100644
index 83c929f4993..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/process/manager.py
+++ /dev/null
@@ -1,124 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-import multiprocessing as mp
-import signal
-import sys
-from subprocess import call
-from typing import Dict
-
-import pandas as pd
-import psutil
-
-from iotdb.mlnode.dataset.dataset import TsForecastDataset
-from iotdb.mlnode.log import logger
-from iotdb.mlnode.parser import ForecastTaskOptions
-from iotdb.mlnode.process.task import (ForecastAutoTuningTrainingTask,
- ForecastFixedParamTrainingTask,
- ForecastingInferenceTask,
- _BasicTrainingTask)
-
-
-class TaskManager(object):
- def __init__(self, pool_size: int):
- """
- Args:
- pool_size: specify the maximum process number of the process pool
-
- __shared_resource_manager: a manager that manage resources shared between processes
- __pid_info: a map shared between processes, can be used to find the pid with model_id and trial_id
- __training_process_pool: a multiprocessing process pool
- """
- self.__shared_resource_manager = mp.Manager()
- self.__pid_info = self.__shared_resource_manager.dict()
- self.__training_process_pool = mp.Pool(pool_size)
- self.__inference_process_pool = mp.Pool(pool_size)
-
- def create_forecast_training_task(self,
- model_id: str,
- task_options: ForecastTaskOptions,
- hyperparameters: Dict[str, str],
- dataset: TsForecastDataset):
- """
- Create a training task for forecasting, which contains the training process
-
- Args:
- model_id: the unique id of the model
- task_options: the options of the task, contains task_type, model_type, etc.
- dataset: a torch dataset to be used for training
- hyperparameters: a dict of hyperparameters, which is consisted of model_hyperparameters
- and task_hyperparameters
-
- Returns:
- task: a training task for forecasting, which can be submitted to self.__training_process_pool
- """
- if task_options.auto_tuning:
- return ForecastAutoTuningTrainingTask(
- model_id=model_id,
- task_options=task_options,
- hyperparameters=hyperparameters,
- dataset=dataset,
- pid_info=self.__pid_info,
- )
- else:
- return ForecastFixedParamTrainingTask(
- model_id=model_id,
- task_options=task_options,
- hyperparameters=hyperparameters,
- dataset=dataset,
- pid_info=self.__pid_info,
- )
-
- def submit_training_task(self, task: _BasicTrainingTask) -> None:
- self.__training_process_pool.apply_async(task, args=())
- logger.info(f'Task: ({task.model_id}) - Training process submitted successfully')
-
- def create_forecast_task(self,
- predict_length,
- data,
- model_path) -> ForecastingInferenceTask:
- task = ForecastingInferenceTask(
- predict_length,
- self.__pid_info,
- data,
- model_path
- )
- return task
-
- def submit_forecast_task(self, task: ForecastingInferenceTask) -> pd.DataFrame:
- read_pipe, send_pipe = mp.Pipe()
- if task is not None:
- self.__inference_process_pool.apply_async(task, args=(send_pipe,))
- logger.info('Forecasting process submitted successfully')
- return read_pipe.recv()
-
- def kill_task(self, model_id):
- """
- Kill the process by pid, will check whether the pid is training or inference process
- """
- pid = self.__pid_info[model_id]
- if sys.platform == 'win32':
- try:
- process = psutil.Process(pid=pid)
- process.send_signal(signal.CTRL_BREAK_EVENT)
- except psutil.NoSuchProcess:
- print(f'Tried to kill process (pid = {pid}), '
- f'but the process does not exist.')
- else:
- cmds = ['kill', str(pid)]
- call(cmds)
diff --git a/iotdb-core/mlnode/iotdb/mlnode/process/task.py b/iotdb-core/mlnode/iotdb/mlnode/process/task.py
deleted file mode 100644
index dd71a0b3a27..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/process/task.py
+++ /dev/null
@@ -1,265 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-import os
-from abc import abstractmethod
-from multiprocessing.connection import Connection
-from typing import Dict, Tuple
-
-import numpy as np
-import optuna
-import pandas as pd
-import torch
-
-from iotdb.mlnode.algorithm.hyperparameter import (generate_hyperparameters,
- parse_fixed_hyperparameters)
-from iotdb.mlnode.client import client_manager
-from iotdb.mlnode.config import descriptor
-from iotdb.mlnode.constant import DEFAULT_TRIAL_ID, TRIAL_ID_PREFIX, OptionsKey, ModelInputName
-from iotdb.mlnode.dataset.dataset import TsForecastDataset
-from iotdb.mlnode.log import logger
-from iotdb.mlnode.parser import ForecastTaskOptions
-from iotdb.mlnode.process.trial import ForecastingTrainingTrial
-from iotdb.mlnode.storage import model_storage
-from iotdb.mlnode.util import pack_input_dict
-from iotdb.thrift.common.ttypes import TrainingState
-
-
-class _BasicTask(object):
- def __init__(
- self,
- pid_info: Dict
- ):
- self.pid_info = pid_info
-
- @abstractmethod
- def __call__(self):
- raise NotImplementedError
-
-
-# Training Task
-
-class _BasicTrainingTask(_BasicTask):
- def __init__(
- self,
- model_id: str,
- hyperparameters: Dict[str, str],
- pid_info: Dict
- ):
- super().__init__(pid_info)
- self.model_id = model_id
- self.hyperparameters = hyperparameters
- self.configNode_client = client_manager.borrow_config_node_client()
-
- @abstractmethod
- def __call__(self):
- raise NotImplementedError
-
-
-class ForecastFixedParamTrainingTask(_BasicTrainingTask):
- def __init__(
- self,
- model_id: str,
- task_options: ForecastTaskOptions,
- hyperparameters: Dict[str, str],
- dataset: TsForecastDataset,
- pid_info: Dict
- ):
- super().__init__(model_id, hyperparameters, pid_info)
- model_hyperparameters, task_hyperparameters = parse_fixed_hyperparameters(task_options, hyperparameters)
- self.dataset = dataset
- self.trial = ForecastingTrainingTrial(
- trial_id=DEFAULT_TRIAL_ID,
- model_id=model_id,
- task_options=task_options,
- model_hyperparameters=model_hyperparameters,
- task_hyperparameters=task_hyperparameters,
- dataset=dataset)
-
- def __call__(self):
- try:
- self.configNode_client.update_model_state(self.model_id, TrainingState.RUNNING)
- self.pid_info[self.model_id] = os.getpid()
- self.trial.start()
- self.configNode_client.update_model_state(self.model_id, TrainingState.FINISHED, self.trial.trial_id)
- except Exception as e:
- logger.warn(e)
- self.configNode_client.update_model_state(self.model_id, TrainingState.FAILED, self.trial.trial_id)
- raise e
-
-
-class ForestingTrainingObjective:
- """
- A class which serve as a function, should accept trial as args
- and return the optimization objective.
- Optuna will try to minimize the objective.
- Currently, user cannot define the range of hyperparameters.
- """
-
- def __init__(
- self,
- model_id: str,
- task_options: ForecastTaskOptions,
- hyperparameters: Dict[str, str],
- dataset: TsForecastDataset,
- pid_info: Dict
- ):
- self.task_options = task_options
- self.hyperparameters = hyperparameters
- self.dataset = dataset
- self.pid_info = pid_info
- self.model_id = model_id
-
- def __call__(self, optuna_suggest: optuna.Trial):
- model_hyperparameters, task_hyperparameters = generate_hyperparameters(optuna_suggest,
- self.task_options)
- trial = ForecastingTrainingTrial(
- trial_id=TRIAL_ID_PREFIX + str(optuna_suggest._trial_id),
- model_id=self.model_id,
- task_options=self.task_options,
- model_hyperparameters=model_hyperparameters,
- task_hyperparameters=task_hyperparameters,
- dataset=self.dataset)
- loss = trial.start()
- return loss
-
-
-class ForecastAutoTuningTrainingTask(_BasicTrainingTask):
- def __init__(
- self,
- model_id: str,
- task_options: ForecastTaskOptions,
- hyperparameters: Dict[str, str],
- dataset: TsForecastDataset,
- pid_info: Dict
- ):
- super().__init__(model_id, hyperparameters, pid_info)
- self.study = optuna.create_study(direction='minimize')
- self.task_options = task_options
- self.dataset = dataset
-
- def __call__(self):
- self.pid_info[self.model_id] = os.getpid()
- try:
- self.configNode_client.update_model_state(self.model_id, TrainingState.RUNNING)
- self.study.optimize(ForestingTrainingObjective(
- model_id=self.model_id,
- task_options=self.task_options,
- hyperparameters=self.hyperparameters,
- dataset=self.dataset,
- pid_info=self.pid_info),
- n_trials=descriptor.get_config().get_mn_tuning_trial_num(),
- n_jobs=descriptor.get_config().get_mn_tuning_trial_concurrency())
- best_trial_id = TRIAL_ID_PREFIX + str(self.study.best_trial._trial_id)
- self.configNode_client.update_model_state(self.model_id, TrainingState.FINISHED, best_trial_id)
- except Exception as e:
- logger.warn(e)
- self.configNode_client.update_model_state(self.model_id, TrainingState.FAILED)
- raise e
-
-
-# Inference Task
-
-class _BasicInferenceTask(_BasicTask):
- def __init__(
- self,
- pid_info: Dict,
- data: Tuple,
- model_path: str
- ):
- super().__init__(pid_info)
- self.data = data
- self.model_path = model_path
-
- @abstractmethod
- def __call__(self, pipe: Connection = None):
- raise NotImplementedError
-
- @abstractmethod
- def data_align(self, *args):
- raise NotImplementedError
-
-
-class ForecastingInferenceTask(_BasicInferenceTask):
- def __init__(
- self,
- predict_length: int,
- pid_info: Dict,
- data: Tuple,
- model_path: str
- ):
- super().__init__(pid_info, data, model_path)
- self.pred_len = predict_length
-
- def __call__(self, pipe: Connection = None):
- self.model, self.model_configs = model_storage.load_model(self.model_path)
- self.model_pred_len = self.model_configs[OptionsKey.PREDICT_LENGTH.name()]
- self.model_input_len = self.model_configs[OptionsKey.INPUT_LENGTH.name()]
-
- data, time_stamp = self.data
- _, c = data.shape
-
- time_stamp = pd.to_datetime(time_stamp.values[:, 0], unit='ms', utc=True) \
- .tz_convert('Asia/Shanghai') # for iotdb
- data, time_stamp = self.data_align(data, time_stamp)
- full_data, full_data_stamp = data, time_stamp
- current_pred_len = 0
- while current_pred_len < self.pred_len:
- current_data = full_data[:, -self.model_input_len:, :]
- current_data = torch.Tensor(current_data)
-
- input_data = pack_input_dict(batch_x=current_data)
- output_data = self.model(input_data).detach().numpy()
- full_data = np.concatenate([full_data, output_data], axis=1)
- current_pred_len += self.model_pred_len
- full_data_stamp = self.generate_future_mark(full_data_stamp, self.pred_len)
- ret_data = pd.concat(
- [pd.DataFrame(full_data_stamp.astype(np.int64)),
- pd.DataFrame(full_data[0, -self.pred_len:, :]).astype(np.double)], axis=1)
- ret_data.columns = list(np.arange(0, c + 1))
- pipe.send(ret_data)
-
- def data_align(self, data: pd.DataFrame, data_stamp) -> Tuple[np.ndarray, pd.DataFrame]:
- """
- data: L x C, DataFrame, suppose no batch dim
- time_stamp: L x 1, DataFrame
- """
- data_stamp = pd.DataFrame(data_stamp)
- assert len(data.shape) == 2, 'expect inference data to have two dimensions'
- assert len(data_stamp.shape) == 2, 'expect inference timestamps to be shaped as [L, 1]'
- time_deltas = data_stamp.diff().dropna()
- mean_timedelta = time_deltas.mean()[0]
- data = data.values
- if data.shape[0] < self.model_input_len:
- extra_len = self.model_input_len - data.shape[0]
- data = np.concatenate([np.mean(data, axis=0, keepdims=True).repeat(extra_len, axis=0), data], axis=0)
- extrapolated_timestamp = pd.date_range(data_stamp[0][0] - extra_len * mean_timedelta, periods=extra_len,
- freq=mean_timedelta)
- data_stamp = pd.concat([extrapolated_timestamp.to_frame(), data_stamp])
- else:
- data = data[-self.model_input_len:, :]
- data_stamp = data_stamp[-self.model_input_len:]
- data = data[None, :] # add batch dim
- return data, data_stamp
-
- def generate_future_mark(self, time_stamp: pd.DataFrame, future_len: int) -> pd.DatetimeIndex:
- time_deltas = time_stamp.diff().dropna()
- mean_timedelta = time_deltas.mean()[0]
- extrapolated_timestamp = pd.date_range(time_stamp.values[-1][0] + mean_timedelta, periods=future_len,
- freq=mean_timedelta)
- return extrapolated_timestamp[:, None]
diff --git a/iotdb-core/mlnode/iotdb/mlnode/process/trial.py b/iotdb-core/mlnode/iotdb/mlnode/process/trial.py
deleted file mode 100644
index 9a96a526a95..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/process/trial.py
+++ /dev/null
@@ -1,224 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-import time
-from abc import abstractmethod
-from typing import Dict, Tuple
-
-import numpy as np
-import torch
-import torch.nn as nn
-from torch.utils.data import DataLoader, Dataset
-
-from iotdb.mlnode.algorithm.factory import create_forecast_model
-from iotdb.mlnode.algorithm.hyperparameter import HyperparameterName
-from iotdb.mlnode.algorithm.metric import build_metrics, forecast_metric_names
-from iotdb.mlnode.client import client_manager
-from iotdb.mlnode.constant import OptionsKey, ModelInputName
-from iotdb.mlnode.dataset.dataset import TsForecastDataset
-from iotdb.mlnode.log import logger
-from iotdb.mlnode.parser import ForecastTaskOptions
-from iotdb.mlnode.storage import model_storage
-from iotdb.mlnode.util import pack_input_dict
-from iotdb.thrift.common.ttypes import TrainingState
-
-
-class BasicTrial(object):
- def __init__(
- self,
- model_id: str,
- model: nn.Module,
- task_configs: Dict,
- dataset: Dataset
- ):
- self.model_id = model_id
- self.dataset = dataset
- self.trial_configs = task_configs
- self.batch_size = task_configs[HyperparameterName.BATCH_SIZE.name()]
- self.learning_rate = task_configs[HyperparameterName.LEARNING_RATE.name()]
- self.epochs = task_configs[HyperparameterName.EPOCHS.name()]
- self.num_workers = task_configs[HyperparameterName.NUM_WORKERS.name()]
- self.use_gpu = task_configs[HyperparameterName.USE_GPU.name()]
- self.device = self.__acquire_device()
- self.model = model.to(self.device)
-
- def __acquire_device(self):
- if self.use_gpu:
- raise NotImplementedError
- else:
- device = torch.device('cpu')
- return device
-
- def _build_dataloader(self) -> DataLoader:
- """
- Returns:
- training dataloader built with the dataset
- """
- return DataLoader(
- self.dataset,
- shuffle=True,
- drop_last=True,
- batch_size=self.batch_size,
- num_workers=self.num_workers
- )
-
- @abstractmethod
- def start(self):
- raise NotImplementedError
-
-
-class ForecastingTrainingTrial(BasicTrial):
- def __init__(
- self,
- trial_id: str,
- model_id: str,
- task_options: ForecastTaskOptions,
- model_hyperparameters: Dict,
- task_hyperparameters: Dict,
- dataset: TsForecastDataset
- ):
- """
- A training trial, accept all parameters needed and train a single model.
- """
- self.model_configs = model_hyperparameters
- # model_config contains the parameters which are needed to be saved after training.
- self.model_configs[HyperparameterName.INPUT_VARS.value] = dataset.get_variable_num()
- self.model_configs[OptionsKey.INPUT_LENGTH.value] = task_options.input_length
- self.model_configs[OptionsKey.PREDICT_LENGTH.value] = task_options.predict_length
- model = create_forecast_model(task_options.model_type, self.model_configs)
-
- super(ForecastingTrainingTrial, self).__init__(model_id, model, task_hyperparameters, dataset)
- self.trial_id = trial_id
- self.pred_len = task_options.predict_length
- self.dataloader = self._build_dataloader()
- self.datanode_client = client_manager.borrow_data_node_client()
- self.configNode_client = client_manager.borrow_config_node_client()
- self.criterion = torch.nn.MSELoss()
- self.optimizer = torch.optim.Adam(self.model.parameters(), lr=self.learning_rate)
- self.metrics_dict = build_metrics(forecast_metric_names)
-
- def train(self, epoch: int) -> float:
- self.model.train()
- train_loss = []
- epoch_time = time.time()
-
- for i, (batch_x, batch_y, batch_x_mark, batch_y_mark) in enumerate(self.dataloader):
- self.optimizer.zero_grad()
- batch_x = batch_x.float().to(self.device)
- batch_y = batch_y.float().to(self.device)
-
- batch_x_mark = batch_x_mark.float().to(self.device)
- batch_y_mark = batch_y_mark.float().to(self.device)
-
- # decoder input
- dec_inp = torch.zeros_like(batch_y[:, -self.pred_len:, :]).float()
-
- inputs = pack_input_dict(
- batch_x=batch_x,
- batch_x_mark=batch_x_mark,
- dec_inp=dec_inp,
- batch_y_mark=batch_y_mark)
- outputs = self.model(inputs)
-
- outputs = outputs[:, -self.pred_len:]
- batch_y = batch_y[:, -self.pred_len:]
-
- loss = self.criterion(outputs, batch_y)
- train_loss.append(loss.item())
-
- if (i + 1) % 500 == 0:
- logger.info('\titers: {0}, epoch: {1} | loss: {2:.7f}'
- .format(i + 1, epoch + 1, loss.item()))
-
- loss.backward()
- self.optimizer.step()
-
- train_loss = np.average(train_loss)
- logger.info('Epoch: {0} cost time: {1} | Train Loss: {2:.7f}'
- .format(epoch + 1, time.time() - epoch_time, train_loss))
- return train_loss
-
- def vali(self, epoch: int) -> Tuple[float, Dict]:
- self.model.eval()
- val_loss = []
- metrics_value_dict = {name: [] for name in forecast_metric_names}
- for i, (batch_x, batch_y, batch_x_mark, batch_y_mark) in enumerate(self.dataloader):
- batch_x = batch_x.float().to(self.device)
- batch_y = batch_y.float().to(self.device)
-
- batch_x_mark = batch_x_mark.float().to(self.device)
- batch_y_mark = batch_y_mark.float().to(self.device)
-
- # decoder input
- dec_inp = torch.zeros_like(batch_y[:, -self.pred_len:, :]).float()
-
- inputs = pack_input_dict(batch_x, batch_x_mark, dec_inp, batch_y_mark)
- outputs = self.model(inputs)
-
- outputs = outputs[:, -self.pred_len:]
- batch_y = batch_y[:, -self.pred_len:]
-
- loss = self.criterion(outputs, batch_y)
- val_loss.append(loss.item())
-
- for name in forecast_metric_names:
- metric = self.metrics_dict[name]
- value = metric(outputs.detach().cpu().numpy(), batch_y.detach().cpu().numpy())
- metrics_value_dict[name].append(value)
-
- for name, value_list in metrics_value_dict.items():
- metrics_value_dict[name] = np.average(value_list)
-
- self.datanode_client.record_model_metrics(
- model_id=self.model_id,
- trial_id=self.trial_id,
- metrics=list(metrics_value_dict.keys()),
- values=list(metrics_value_dict.values())
- )
- val_loss = np.average(val_loss)
- logger.info('Epoch: {0} Vali Loss: {1:.7f}'.format(epoch + 1, val_loss))
- return val_loss, metrics_value_dict
-
- def start(self) -> float:
- """
- Start training with the specified parameters, save the best model and report metrics to the db.
- """
- try:
- best_loss = np.inf
- best_metrics_dict = None
- model_path = None
- for epoch in range(self.epochs):
- self.train(epoch)
- val_loss, metrics_dict = self.vali(epoch)
- if val_loss < best_loss:
- best_loss = val_loss
- best_metrics_dict = metrics_dict
- model_path = model_storage.save_model(self.model,
- self.model_configs,
- model_id=self.model_id,
- trial_id=self.trial_id)
-
- logger.info(f'Trial: ({self.model_id}_{self.trial_id}) - Finished with best model saved successfully')
- model_info = {}
- model_info.update(best_metrics_dict)
- model_info.update(self.trial_configs)
- model_info['model_path'] = model_path
- self.configNode_client.update_model_info(self.model_id, self.trial_id, model_info)
- return best_loss
- except Exception as e:
- logger.warn(e)
- raise e
diff --git a/iotdb-core/mlnode/iotdb/mlnode/script.py b/iotdb-core/mlnode/iotdb/mlnode/script.py
deleted file mode 100644
index b73dfb78c58..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/script.py
+++ /dev/null
@@ -1,36 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-import sys
-
-from iotdb.mlnode.log import logger
-from iotdb.mlnode.service import MLNode
-
-
-def main():
- arguments = sys.argv
- if len(arguments) == 1:
- logger.info("Command line argument must be specified.")
- return
-
- argument = sys.argv[1]
- # TODO(lmh): support more commands
- if argument == 'start':
- server = MLNode()
- server.start()
- else:
- logger.info("Unknown argument: {}.".format(argument))
diff --git a/iotdb-core/mlnode/iotdb/mlnode/serde.py b/iotdb-core/mlnode/iotdb/mlnode/serde.py
deleted file mode 100644
index 5b6e352866d..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/serde.py
+++ /dev/null
@@ -1,508 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-from enum import Enum
-
-import numpy as np
-import pandas as pd
-
-
-class TSDataType(Enum):
- BOOLEAN = 0
- INT32 = 1
- INT64 = 2
- FLOAT = 3
- DOUBLE = 4
- TEXT = 5
-
- # this method is implemented to avoid the issue reported by:
- # https://bugs.python.org/issue30545
- def __eq__(self, other) -> bool:
- return self.value == other.value
-
- def __hash__(self):
- return self.value
-
- def np_dtype(self):
- return {
- TSDataType.BOOLEAN: np.dtype(">?"),
- TSDataType.FLOAT: np.dtype(">f4"),
- TSDataType.DOUBLE: np.dtype(">f8"),
- TSDataType.INT32: np.dtype(">i4"),
- TSDataType.INT64: np.dtype(">i8"),
- TSDataType.TEXT: np.dtype("str"),
- }[self]
-
-
-TIMESTAMP_STR = "Time"
-START_INDEX = 2
-
-
-# convert dataFrame to tsBlock in binary
-def convert_to_binary(data_frame: pd.DataFrame):
- data_shape = data_frame.shape
- value_column_size = data_shape[1] - 1
- position_count = data_shape[0]
- keys = data_frame.keys()
-
- binary = value_column_size.to_bytes(4, byteorder="big")
-
- # all the tsDataType are double
- for i in range(value_column_size):
- binary += b'\x04'
-
- # position count
- binary += position_count.to_bytes(4, byteorder="big")
-
- # column encoding
- binary += b'\x02'
- for i in range(value_column_size):
- binary += b'\x02'
-
- # write columns, the column in index 0 must be timeColumn
- binary += bool.to_bytes(False, 1, byteorder="big")
- for i in range(position_count):
- value = data_frame[keys[0]][i]
- if value.dtype.byteorder != '>':
- value = value.byteswap()
- binary += value.tobytes()
-
- for i in range(value_column_size):
- # the value can't be null
- binary += bool.to_bytes(False, 1, byteorder="big")
- col = data_frame[keys[i + 1]]
- for j in range(position_count):
- value = col[j]
- if value.dtype.byteorder != '>':
- value = value.byteswap()
- binary += value.tobytes()
-
- return binary
-
-
-# convert tsBlock in binary to dataFrame
-def convert_to_df(name_list, type_list, name_index, binary_list):
- column_name_list = [TIMESTAMP_STR]
- column_type_list = [TSDataType.INT64]
- column_ordinal_dict = {TIMESTAMP_STR: 1}
-
- if name_index is not None:
- column_type_deduplicated_list = [
- None for _ in range(len(name_index))
- ]
- for i in range(len(name_list)):
- name = name_list[i]
- column_name_list.append(name)
- column_type_list.append(TSDataType[type_list[i]])
- if name not in column_ordinal_dict:
- index = name_index[name]
- column_ordinal_dict[name] = index + START_INDEX
- column_type_deduplicated_list[index] = TSDataType[type_list[i]]
- else:
- index = START_INDEX
- column_type_deduplicated_list = []
- for i in range(len(name_list)):
- name = name_list[i]
- column_name_list.append(name)
- column_type_list.append(TSDataType[type_list[i]])
- if name not in column_ordinal_dict:
- column_ordinal_dict[name] = index
- index += 1
- column_type_deduplicated_list.append(
- TSDataType[type_list[i]]
- )
-
- binary_size = len(binary_list)
- binary_index = 0
- result = {}
- for column_name in column_name_list:
- result[column_name] = None
-
- while binary_index < binary_size:
- buffer = binary_list[binary_index]
- binary_index += 1
- time_column_values, column_values, null_indicators, _ = deserialize(buffer)
- time_array = np.frombuffer(
- time_column_values, np.dtype(np.longlong).newbyteorder(">")
- )
- if time_array.dtype.byteorder == ">":
- time_array = time_array.byteswap().newbyteorder("<")
-
- if result[TIMESTAMP_STR] is None:
- result[TIMESTAMP_STR] = time_array
- else:
- result[TIMESTAMP_STR] = np.concatenate(
- (result[TIMESTAMP_STR], time_array), axis=0
- )
- total_length = len(time_array)
-
- for i in range(len(column_values)):
- column_name = column_name_list[i + 1]
-
- location = column_ordinal_dict[column_name] - START_INDEX
- if location < 0:
- continue
-
- data_type = column_type_deduplicated_list[location]
- value_buffer = column_values[location]
- value_buffer_len = len(value_buffer)
-
- if data_type == TSDataType.DOUBLE:
- data_array = np.frombuffer(
- value_buffer, np.dtype(np.double).newbyteorder(">")
- )
- elif data_type == TSDataType.FLOAT:
- data_array = np.frombuffer(
- value_buffer, np.dtype(np.float32).newbyteorder(">")
- )
- elif data_type == TSDataType.BOOLEAN:
- data_array = []
- for index in range(len(value_buffer)):
- data_array.append(value_buffer[index])
- data_array = np.array(data_array).astype("bool")
- elif data_type == TSDataType.INT32:
- data_array = np.frombuffer(
- value_buffer, np.dtype(np.int32).newbyteorder(">")
- )
- elif data_type == TSDataType.INT64:
- data_array = np.frombuffer(
- value_buffer, np.dtype(np.int64).newbyteorder(">")
- )
- elif data_type == TSDataType.TEXT:
- index = 0
- data_array = []
- while index < value_buffer_len:
- value_bytes = value_buffer[index]
- value = value_bytes.decode("utf-8")
- data_array.append(value)
- index += 1
- data_array = np.array(data_array, dtype=object)
- else:
- raise RuntimeError("unsupported data type {}.".format(data_type))
-
- if data_array.dtype.byteorder == ">":
- data_array = data_array.byteswap().newbyteorder("<")
-
- null_indicator = null_indicators[location]
- if len(data_array) < total_length or (data_type == TSDataType.BOOLEAN and null_indicator is not None):
- if data_type == TSDataType.INT32 or data_type == TSDataType.INT64:
- tmp_array = np.full(total_length, np.nan, np.float32)
- elif data_type == TSDataType.FLOAT or data_type == TSDataType.DOUBLE:
- tmp_array = np.full(total_length, np.nan, data_array.dtype)
- elif data_type == TSDataType.BOOLEAN:
- tmp_array = np.full(total_length, np.nan, np.float32)
- elif data_type == TSDataType.TEXT:
- tmp_array = np.full(total_length, np.nan, dtype=data_array.dtype)
- else:
- raise Exception("Unsupported dataType in deserialization")
-
- if null_indicator is not None:
- indexes = [not v for v in null_indicator]
- if data_type == TSDataType.BOOLEAN:
- tmp_array[indexes] = data_array[indexes]
- else:
- tmp_array[indexes] = data_array
-
- if data_type == TSDataType.INT32:
- tmp_array = pd.Series(tmp_array).astype("Int32")
- elif data_type == TSDataType.INT64:
- tmp_array = pd.Series(tmp_array).astype("Int64")
- elif data_type == TSDataType.BOOLEAN:
- tmp_array = pd.Series(tmp_array).astype("boolean")
-
- data_array = tmp_array
-
- if result[column_name] is None:
- result[column_name] = data_array
- else:
- if isinstance(result[column_name], pd.Series):
- if not isinstance(data_array, pd.Series):
- if data_type == TSDataType.INT32:
- data_array = pd.Series(data_array).astype("Int32")
- elif data_type == TSDataType.INT64:
- data_array = pd.Series(data_array).astype("Int64")
- elif data_type == TSDataType.BOOLEAN:
- data_array = pd.Series(data_array).astype("boolean")
- else:
- raise RuntimeError("Series Error")
- result[column_name] = result[column_name].append(data_array)
- else:
- result[column_name] = np.concatenate(
- (result[column_name], data_array), axis=0
- )
- for k, v in result.items():
- if v is None:
- result[k] = []
- df = pd.DataFrame(result)
- df = df.reset_index(drop=True)
- return df
-
-
-# Serialized tsBlock:
-# +-------------+---------------+---------+------------+-----------+----------+
-# | val col cnt | val col types | pos cnt | encodings | time col | val col |
-# +-------------+---------------+---------+------------+-----------+----------+
-# | int32 | list[byte] | int32 | list[byte] | bytes | byte |
-# +-------------+---------------+---------+------------+-----------+----------+
-
-def deserialize(buffer):
- value_column_count, buffer = read_int_from_buffer(buffer)
- data_types, buffer = read_column_types(buffer, value_column_count)
-
- position_count, buffer = read_int_from_buffer(buffer)
- column_encodings, buffer = read_column_encoding(buffer, value_column_count + 1)
-
- time_column_values, buffer = read_time_column(buffer, position_count)
- column_values = [None] * value_column_count
- null_indicators = [None] * value_column_count
- for i in range(value_column_count):
- column_value, null_indicator, buffer = read_column(column_encodings[i + 1], buffer, data_types[i],
- position_count)
- column_values[i] = column_value
- null_indicators[i] = null_indicator
-
- return time_column_values, column_values, null_indicators, position_count
-
-
-# General Methods
-
-def read_int_from_buffer(buffer):
- res, buffer = read_from_buffer(buffer, 4)
- return int.from_bytes(res, "big"), buffer
-
-
-def read_byte_from_buffer(buffer):
- return read_from_buffer(buffer, 1)
-
-
-def read_from_buffer(buffer, size):
- res = buffer[:size]
- buffer = buffer[size:]
- return res, buffer
-
-
-# Read ColumnType
-
-def read_column_types(buffer, value_column_count):
- data_types = []
- for _ in range(value_column_count):
- res, buffer = read_byte_from_buffer(buffer)
- data_types.append(get_data_type(res))
- return data_types, buffer
-
-
-def get_data_type(value):
- if value == b'\x00':
- return TSDataType.BOOLEAN
- elif value == b'\x01':
- return TSDataType.INT32
- elif value == b'\x02':
- return TSDataType.INT64
- elif value == b'\x03':
- return TSDataType.FLOAT
- elif value == b'\x04':
- return TSDataType.DOUBLE
- elif value == b'\x05':
- return TSDataType.TEXT
-
-
-# Read ColumnEncodings
-
-def read_column_encoding(buffer, size):
- encodings = []
- for _ in range(size):
- res, buffer = read_byte_from_buffer(buffer)
- encodings.append(res)
- return encodings, buffer
-
-
-# Read Column
-
-def deserialize_null_indicators(buffer, size):
- may_have_null, buffer = read_byte_from_buffer(buffer)
- if may_have_null != b'\x00':
- return deserialize_from_boolean_array(buffer, size)
- return None, buffer
-
-
-# Serialized data layout:
-# +---------------+-----------------+-------------+
-# | may have null | null indicators | values |
-# +---------------+-----------------+-------------+
-# | byte | list[byte] | list[int64] |
-# +---------------+-----------------+-------------+
-
-def read_time_column(buffer, size):
- null_indicators, buffer = deserialize_null_indicators(buffer, size)
- if null_indicators is None:
- values, buffer = read_from_buffer(
- buffer, size * 8
- )
- else:
- raise Exception("TimeColumn should not contains null value")
- return values, buffer
-
-
-def read_int64_column(buffer, data_type, position_count):
- null_indicators, buffer = deserialize_null_indicators(buffer, position_count)
- if null_indicators is None:
- size = position_count
- else:
- size = null_indicators.count(False)
-
- if TSDataType.INT64 == data_type or TSDataType.DOUBLE == data_type:
- values, buffer = read_from_buffer(buffer, size * 8)
- return values, null_indicators, buffer
- else:
- raise Exception("Invalid data type: " + data_type)
-
-
-# Serialized data layout:
-# +---------------+-----------------+-------------+
-# | may have null | null indicators | values |
-# +---------------+-----------------+-------------+
-# | byte | list[byte] | list[int32] |
-# +---------------+-----------------+-------------+
-
-def read_int32_column(buffer, data_type, position_count):
- null_indicators, buffer = deserialize_null_indicators(buffer, position_count)
- if null_indicators is None:
- size = position_count
- else:
- size = null_indicators.count(False)
-
- if TSDataType.INT32 == data_type or TSDataType.FLOAT == data_type:
- values, buffer = read_from_buffer(buffer, size * 4)
- return values, null_indicators, buffer
- else:
- raise Exception("Invalid data type: " + data_type)
-
-
-# Serialized data layout:
-# +---------------+-----------------+-------------+
-# | may have null | null indicators | values |
-# +---------------+-----------------+-------------+
-# | byte | list[byte] | list[byte] |
-# +---------------+-----------------+-------------+
-
-def read_byte_column(buffer, data_type, position_count):
- if data_type != TSDataType.BOOLEAN:
- raise Exception("Invalid data type: " + data_type)
- null_indicators, buffer = deserialize_null_indicators(buffer, position_count)
- res, buffer = deserialize_from_boolean_array(buffer, position_count)
- return res, null_indicators, buffer
-
-
-def deserialize_from_boolean_array(buffer, size):
- packed_boolean_array, buffer = read_from_buffer(buffer, (size + 7) // 8)
- current_byte = 0
- output = [None] * size
- position = 0
- # read null bits 8 at a time
- while position < (size & ~0b111):
- value = packed_boolean_array[current_byte]
- output[position] = ((value & 0b1000_0000) != 0)
- output[position + 1] = ((value & 0b0100_0000) != 0)
- output[position + 2] = ((value & 0b0010_0000) != 0)
- output[position + 3] = ((value & 0b0001_0000) != 0)
- output[position + 4] = ((value & 0b0000_1000) != 0)
- output[position + 5] = ((value & 0b0000_0100) != 0)
- output[position + 6] = ((value & 0b0000_0010) != 0)
- output[position + 7] = ((value & 0b0000_0001) != 0)
-
- position += 8
- current_byte += 1
- # read last null bits
- if (size & 0b111) > 0:
- value = packed_boolean_array[-1]
- mask = 0b1000_0000
- position = size & ~0b111
- while position < size:
- output[position] = ((value & mask) != 0)
- mask >>= 1
- position += 1
- return output, buffer
-
-
-# Serialized data layout:
-# +---------------+-----------------+-------------+
-# | may have null | null indicators | values |
-# +---------------+-----------------+-------------+
-# | byte | list[byte] | list[entry] |
-# +---------------+-----------------+-------------+
-#
-# Each entry is represented as:
-# +---------------+-------+
-# | value length | value |
-# +---------------+-------+
-# | int32 | bytes |
-# +---------------+-------+
-
-def read_binary_column(buffer, data_type, position_count):
- if data_type != TSDataType.TEXT:
- raise Exception("Invalid data type: " + data_type)
- null_indicators, buffer = deserialize_null_indicators(buffer, position_count)
-
- if null_indicators is None:
- size = position_count
- else:
- size = null_indicators.count(False)
- values = [None] * size
- for i in range(size):
- length, buffer = read_int_from_buffer(buffer)
- res, buffer = read_from_buffer(buffer, length)
- values[i] = res
- return values, null_indicators, buffer
-
-
-def read_column(encoding, buffer, data_type, position_count):
- if encoding == b'\x00':
- return read_byte_column(buffer, data_type, position_count)
- elif encoding == b'\x01':
- return read_int32_column(buffer, data_type, position_count)
- elif encoding == b'\x02':
- return read_int64_column(buffer, data_type, position_count)
- elif encoding == b'\x03':
- return read_binary_column(buffer, data_type, position_count)
- elif encoding == b'\x04':
- return read_run_length_column(buffer, data_type, position_count)
- else:
- raise Exception("Unsupported encoding: " + encoding)
-
-
-# Serialized data layout:
-# +-----------+-------------------------+
-# | encoding | serialized inner column |
-# +-----------+-------------------------+
-# | byte | list[byte] |
-# +-----------+-------------------------+
-
-def read_run_length_column(buffer, data_type, position_count):
- encoding, buffer = read_byte_from_buffer(buffer)
- column, null_indicators, buffer = read_column(encoding, buffer, data_type, 1)
-
- return repeat(column, data_type, position_count), null_indicators * position_count, buffer
-
-
-def repeat(buffer, data_type, position_count):
- if data_type == TSDataType.BOOLEAN or data_type == TSDataType.TEXT:
- return buffer * position_count
- else:
- res = bytes(0)
- for _ in range(position_count):
- res.join(buffer)
- return res
diff --git a/iotdb-core/mlnode/iotdb/mlnode/service.py b/iotdb-core/mlnode/iotdb/mlnode/service.py
deleted file mode 100644
index 30c963aa304..00000000000
--- a/iotdb-core/mlnode/iotdb/mlnode/service.py
+++ /dev/null
@@ -1,58 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
... 2863 lines suppressed ...