You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yc...@apache.org on 2025/03/05 10:54:40 UTC
(iotdb) branch ycy/AINodeTraining updated: adjust training function
This is an automated email from the ASF dual-hosted git repository.
ycycse pushed a commit to branch ycy/AINodeTraining
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/ycy/AINodeTraining by this push:
new e8de8abb174 adjust training function
e8de8abb174 is described below
commit e8de8abb174dc33375d090f6ed57115721c9e899
Author: YangCaiyin <yc...@gmail.com>
AuthorDate: Tue Mar 4 18:27:52 2025 +0800
adjust training function
---
iotdb-core/ainode/.gitignore | 4 +-
.../ainode/ai/ainode/manager/training_manager.py | 191 -----
.../ainode/ai/ainode/model/TimerXL/__init__.py | 0
.../ai/ainode/model/TimerXL/exp/exp_basic.py | 26 -
.../ai/ainode/model/TimerXL/layers/Attn_Bias.py | 78 --
.../ainode/model/TimerXL/layers/Attn_Projection.py | 106 ---
.../ainode/ai/ainode/model/TimerXL/layers/Embed.py | 189 -----
.../model/TimerXL/layers/SelfAttention_Family.py | 142 ----
.../model/TimerXL/layers/Transformer_EncDec.py | 246 ------
.../ai/ainode/model/TimerXL/layers/__init__.py | 0
.../ai/ainode/model/TimerXL/models/__init__.py | 0
.../ai/ainode/model/TimerXL/models/timer_xl.py | 80 --
.../ai/ainode/model/built_in_model_factory.py | 924 ---------------------
iotdb-core/ainode/ai/ainode/model/model_factory.py | 235 ------
iotdb-core/ainode/ai/ainode/model/model_storage.py | 123 ---
iotdb-core/ainode/ai/ainode/util/__init__.py | 17 -
iotdb-core/ainode/ai/ainode/util/masking.py | 35 -
iotdb-core/ainode/ai/ainode/util/metrics.py | 44 -
iotdb-core/ainode/ai/ainode/util/tools.py | 96 ---
iotdb-core/ainode/{ai => ainode}/__init__.py | 0
.../ainode/{ai/ainode => ainode/core}/__init__.py | 0
.../ainode/{ai/ainode => ainode/core}/client.py | 24 +-
.../ainode/{ai/ainode => ainode/core}/config.py | 22 +-
.../ainode/{ai/ainode => ainode/core}/constant.py | 2 +-
.../ainode/{ai/ainode => ainode/core}/exception.py | 2 +-
.../ainode/{ai/ainode => ainode/core}/handler.py | 18 +-
.../ainode/{ai/ainode => ainode/core}/log.py | 8 +-
.../{ai/ainode => ainode/core}/manager/__init__.py | 0
.../core}/manager/cluster_manager.py | 4 +-
.../core}/manager/inference_manager.py | 14 +-
.../core}/manager/model_manager.py | 16 +-
.../core/manager/training_manager.py} | 13 +-
.../ainode/{ai/ainode => ainode/core}/script.py | 16 +-
.../ainode/{ai/ainode => ainode/core}/service.py | 8 +-
.../ainode/model => ainode/core/util}/__init__.py | 0
.../{ai/ainode => ainode/core}/util/decorator.py | 0
.../ainode/{ai/ainode => ainode/core}/util/lock.py | 0
.../{ai/ainode => ainode/core}/util/serde.py | 2 +-
.../{ai/ainode => ainode/core}/util/status.py | 6 +-
iotdb-core/ainode/pom.xml | 8 +-
iotdb-core/ainode/pyproject.toml | 18 +-
.../ainode/resources/conf/iotdb-ainode.properties | 6 +-
iotdb-core/ainode/resources/sbin/remove-ainode.sh | 4 +-
.../iotdb/confignode/manager/ModelManager.java | 8 +-
.../iotdb/confignode/manager/ProcedureManager.java | 5 +-
.../impl/model/CreateTrainingProcedure.java | 11 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 4 +-
.../execution/config/TableConfigTaskVisitor.java | 7 +
.../config/executor/ClusterConfigTaskExecutor.java | 1 +
.../plan/relational/sql/ast/AstVisitor.java | 4 +
.../plan/relational/sql/ast/ShowModels.java | 55 ++
.../plan/relational/sql/parser/AstBuilder.java | 10 +
.../db/relational/grammar/sql/RelationalSql.g4 | 15 +-
.../src/main/thrift/confignode.thrift | 5 +-
54 files changed, 207 insertions(+), 2645 deletions(-)
diff --git a/iotdb-core/ainode/.gitignore b/iotdb-core/ainode/.gitignore
index 0dec9206348..ddf402ed3b3 100644
--- a/iotdb-core/ainode/.gitignore
+++ b/iotdb-core/ainode/.gitignore
@@ -1,8 +1,8 @@
# generated by Thrift
-/ai/thrift/
+/ainode/thrift/
# generated by maven
-/ai/conf/
+/ainode/conf/
# .whl of ainode, generated by Poetry
/dist/
diff --git a/iotdb-core/ainode/ai/ainode/manager/training_manager.py b/iotdb-core/ainode/ai/ainode/manager/training_manager.py
deleted file mode 100644
index c0e3f1966f1..00000000000
--- a/iotdb-core/ainode/ai/ainode/manager/training_manager.py
+++ /dev/null
@@ -1,191 +0,0 @@
-import argparse
-
-import numpy as np
-import torch
-import time
-from iotdb.table_session import TableSession, TableSessionConfig
-from torch.utils.data import Dataset, DataLoader
-
-from ai.thrift.ainode.ttypes import ITableSchema
-from client import ClientManager
-from config import AINodeDescriptor
-from constant import TSStatusCode
-from log import Logger
-from model.TimerXL.exp.exp_forecast import ExpForecast
-from util.status import get_status
-
-config = TableSessionConfig(
- node_urls=["localhost:6667"],
- username="root",
- password="root",
- time_zone="UTC+8",
-)
-
-SELECT_SERIES_FORMAT_SQL = "select distinct item_id from %s.%s"
-COUNT_SERIES_LENGTH_SQL = "select count(value) from %s.%s where item_id = '%s'"
-FETCH_SERIES_SQL = "select value from %s.%s where item_id = '%s' offset %s limit %s"
-SERIES_NAME = "%s.%s.%s"
-
-logger = Logger()
-
-class TrainingParameters:
- def __init__(self):
- self.seed = 2025
-
- # forecasting task
- self.seq_len = 672
- self.input_token_len = 96
- self.output_token_len = 96
-
- # model define
- self.dropout = 0.1
- self.e_layers = 1
- self.d_model = 512
- self.n_heads = 8
- self.d_ff = 2048
- self.activation = 'relu'
- self.covariate = False
- self.node_num = 100
- self.node_list = '23,37,40'
- self.use_norm = False
- self.nonautoregressive = False
- self.output_attention = False
- self.flash_attention = False
-
- # adaptation
- self.adaptation = False
- self.pretrain_model_path = 'pretrain_model.pth'
- self.subset_rand_ratio = 1
-
- # optimization
- self.num_workers = 10
- self.itr = 1
- self.train_epochs = 1
- self.batch_size = 32
- self.patience = 3
- self.learning_rate = 0.0001
- self.des = 'test'
- self.loss = 'MSE'
- self.lradj = 'type1'
- self.cosine = False
- self.tmax = 10
- self.weight_decay = 0
- self.valid_last = False
- self.last_token = False
-
- # GPU
- self.device = 'cpu'
- self.ddp = False
- self.dp = False
- self.devices = '0,1,2,3'
-
- # LLM-based model
- self.gpt_layers = 6
- self.patch_size = 16
- self.kernel_size = 25
- self.stride = 8
-
- self.context_length = self.seq_len + self.output_token_len
-
-class IoTDBDataset(Dataset):
-
- def __init__(self, sorted_series:list, start_index:int, end_index:int, output_tokens:int, input_tokens:int, predict_length:int):
- self.sorted_series = sorted_series
- self.start_index = start_index
- self.end_index = end_index
- self.output_tokens = output_tokens
- self.input_tokens = input_tokens
- self.context_length = predict_length + output_tokens
- self.predict_length = predict_length
- self.iotdb_session = TableSession(config)
-
- def __getitem__(self, index):
- window_index = index + self.start_index
-
- series_index = 0
-
- while self.sorted_series[series_index][2] < window_index:
- series_index += 1
-
- if series_index != 0:
- window_index -= self.sorted_series[series_index - 1][2]
-
- # while self.sorted_series[series_index][1] - real_index < self.context_len:
- # series_index += 1
- if window_index != 0:
- window_index -= 1
- series = self.sorted_series[series_index - 1][0]
- schema = series.split(".")
-
- start_time = time.time()
- result = []
- try:
- with self.iotdb_session.execute_query_statement(FETCH_SERIES_SQL % (schema[0], schema[1], schema[2], window_index, self.context_length)) as query_result:
- while query_result.has_next():
- result.append(query_result.next().get_fields()[0].get_double_value())
- except Exception:
- print("error")
- result = torch.tensor(result)
- x_mark = torch.zeros((result.shape[0], 1))
- y_mark = torch.zeros((result.shape[0], 1))
- logger.info("fetch data from iotdb cost %s", time.time() - start_time)
- return result[0:self.predict_length].unsqueeze(-1), result[-self.output_tokens:].unsqueeze(-1), x_mark, y_mark
-
- def __len__(self):
- return self.end_index - self.start_index - 1
-
-
-def _fetch_schema(table_schema_list: list[ITableSchema], args:TrainingParameters) -> (int, list):
- iotdb_session = TableSession(config)
- series_to_length = {}
- for table_schema in table_schema_list:
- series_list = []
- with iotdb_session.execute_query_statement(SELECT_SERIES_FORMAT_SQL % (table_schema.database, table_schema.tableName)) as show_devices_result:
- while show_devices_result.has_next():
- series_list.append(str(show_devices_result.next().get_fields()[0]))
-
- for series in series_list:
- with iotdb_session.execute_query_statement(COUNT_SERIES_LENGTH_SQL % (table_schema.database, table_schema.tableName, series)) as count_series_result:
- length = int(str(count_series_result.next().get_fields()[0]))
- series_to_length[SERIES_NAME % (table_schema.database, table_schema.tableName, series)] = length
-
- iotdb_session.close()
-
- sorted_series = sorted(series_to_length.items(), key=lambda x: x[1])
- sorted_series_with_prefix_sum = []
- current_sum = 0
- window_count = 0
- for seq_name, seq_length in sorted_series:
- window_count = seq_length - args.context_length + 1
- current_sum += window_count
- sorted_series_with_prefix_sum.append((seq_name, window_count , current_sum))
-
- return window_count, sorted_series_with_prefix_sum
-
-
-class TrainingManager:
- def __init__(self):
- pass
-
- def create_training_task(self, model_id:str, model_type, table_list: list[ITableSchema], parameters: dict[str, str]):
- args = get_args()
- args.model = "timer_xl"
- total_count, sorted_series = _fetch_schema(table_list, args)
-
- training_dataset = IoTDBDataset(sorted_series, 0, total_count, args.input_token_len, args.output_token_len, args.seq_len)
- training_dataloader = DataLoader(training_dataset, batch_size=args.batch_size, shuffle=True, num_workers=1)
- exp = ExpForecast(args)
- exp.train(training_dataloader, model_id)
-
- logger.info(model_id + "finished training")
- ClientManager().borrow_config_node_client().update_model_info(model_id, 3, "finished", [AINodeDescriptor().get_config().get_ainode_id()],args.seq_len,args.output_token_len)
- return get_status(TSStatusCode.SUCCESS_STATUS)
-
-
-def get_args():
- args = TrainingParameters()
- fix_seed = args.seed
- torch.manual_seed(fix_seed)
- np.random.seed(fix_seed)
-
- return args
diff --git a/iotdb-core/ainode/ai/ainode/model/TimerXL/__init__.py b/iotdb-core/ainode/ai/ainode/model/TimerXL/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/iotdb-core/ainode/ai/ainode/model/TimerXL/exp/exp_basic.py b/iotdb-core/ainode/ai/ainode/model/TimerXL/exp/exp_basic.py
deleted file mode 100644
index 3929cf426a9..00000000000
--- a/iotdb-core/ainode/ai/ainode/model/TimerXL/exp/exp_basic.py
+++ /dev/null
@@ -1,26 +0,0 @@
-from ai.ainode.model.TimerXL.models import timer_xl
-
-
-class ExpBasic(object):
- def __init__(self, args):
- self.args = args
- self.model_dict = {
- # 'timer': timer,
- 'timer_xl': timer_xl,
- }
- self.model = self._build_model()
-
- def _build_model(self):
- raise NotImplementedError
-
- def _get_data(self):
- pass
-
- def vali(self):
- pass
-
- def train(self):
- pass
-
- def test(self):
- pass
diff --git a/iotdb-core/ainode/ai/ainode/model/TimerXL/layers/Attn_Bias.py b/iotdb-core/ainode/ai/ainode/model/TimerXL/layers/Attn_Bias.py
deleted file mode 100644
index 4214e4ba7b0..00000000000
--- a/iotdb-core/ainode/ai/ainode/model/TimerXL/layers/Attn_Bias.py
+++ /dev/null
@@ -1,78 +0,0 @@
-import abc
-import math
-import torch
-from einops import rearrange
-from torch import nn
-
-
-class AttentionBias(nn.Module, abc.ABC):
- def __init__(self, dim: int, num_heads: int):
- super().__init__()
- assert num_heads > 0 and dim % num_heads == 0
-
- self.num_heads = num_heads
- self.head_dim = dim // num_heads
-
- @abc.abstractmethod
- def forward(self, query_id, kv_id): ...
-
-
-class BinaryAttentionBias(AttentionBias):
- def __init__(self, dim: int, num_heads: int):
- super().__init__(dim, num_heads)
- self.emb = nn.Embedding(num_embeddings=2, embedding_dim=self.num_heads)
-
- def forward(self, query_id, kv_id):
- ind = torch.eq(query_id.unsqueeze(-1), kv_id.unsqueeze(-2))
- weight = rearrange(
- self.emb.weight, "two num_heads -> two num_heads 1 1")
- bias = ~ind * weight[:1] + ind * weight[1:]
- return bias
-
-
-def _relative_position_bucket(relative_position, bidirectional=True, num_buckets=32, max_distance=128):
- relative_buckets = 0
- if bidirectional:
- num_buckets //= 2
- relative_buckets += (relative_position >
- 0).to(torch.long) * num_buckets
- relative_position = torch.abs(relative_position)
- else:
- relative_position = - \
- torch.min(relative_position, torch.zeros_like(relative_position))
-
- max_exact = num_buckets // 2
- is_small = relative_position < max_exact
- relative_position_if_large = max_exact + (
- torch.log(relative_position.float() / max_exact)
- / math.log(max_distance / max_exact)
- * (num_buckets - max_exact)
- ).to(torch.long)
- relative_position_if_large = torch.min(
- relative_position_if_large, torch.full_like(
- relative_position_if_large, num_buckets - 1)
- )
-
- relative_buckets += torch.where(is_small,
- relative_position, relative_position_if_large)
- return relative_buckets
-
-
-class T5AttentionBias(AttentionBias):
- def __init__(self, dim: int, num_heads: int):
- super().__init__(dim, num_heads)
- self.num_buckets = 32
- self.max_distance = 32
- self.relative_attention_bias = nn.Embedding(self.num_buckets, 1)
-
- def forward(self, n_vars, n_tokens):
- context_position = torch.arange(n_tokens, dtype=torch.long,)[:, None]
- memory_position = torch.arange(n_tokens, dtype=torch.long, )[None, :]
- relative_position = memory_position - context_position
- bucket = _relative_position_bucket(relative_position=relative_position, bidirectional=False,
- num_buckets=self.num_buckets, max_distance=self.max_distance).to(self.relative_attention_bias.weight.device)
- bias = self.relative_attention_bias(bucket).squeeze(-1)
- bias = bias.reshape(1, 1, bias.shape[0], bias.shape[1])
- mask1 = torch.ones((n_vars, n_vars), dtype=torch.bool).to(bias.device)
- final_bias = torch.kron(mask1, bias)
- return final_bias
diff --git a/iotdb-core/ainode/ai/ainode/model/TimerXL/layers/Attn_Projection.py b/iotdb-core/ainode/ai/ainode/model/TimerXL/layers/Attn_Projection.py
deleted file mode 100644
index 0002744c907..00000000000
--- a/iotdb-core/ainode/ai/ainode/model/TimerXL/layers/Attn_Projection.py
+++ /dev/null
@@ -1,106 +0,0 @@
-import abc
-import torch
-from functools import cached_property
-from einops import einsum, rearrange, repeat
-from torch import nn
-
-
-class Projection(nn.Module, abc.ABC):
- def __init__(self, proj_width: int, num_heads: int, **kwargs):
- super().__init__()
- self.proj_width = proj_width
- self.num_heads = num_heads
-
- @abc.abstractmethod
- def forward(self, x, seq_id): ...
-
-
-class RotaryProjection(Projection):
- def __init__(self, *, proj_width: int, num_heads: int, max_len: int = 512, base: int = 10000):
- super().__init__(proj_width, num_heads)
- assert (
- self.proj_width % 2 == 0
- ), f"proj_width must be even, got {self.proj_width}"
- self.register_buffer(
- "theta",
- 1.0
- / torch.pow(
- base,
- torch.arange(0, self.proj_width, 2, dtype=torch.float)
- / self.proj_width,
- ),
- persistent=False,
- )
- self.register_buffer("cos", None, persistent=False)
- self.register_buffer("sin", None, persistent=False)
- self._init_freq(max_len=max_len)
-
- def _init_freq(self, max_len: int):
- if self.cos is None or self.cos.size(-2) < max_len:
- position = torch.arange(
- max_len, device=self.theta.device, dtype=self.theta.dtype
- )
- m_theta = einsum(position, self.theta,
- "length, width -> length width")
- m_theta = repeat(m_theta, "length width -> length (width 2)")
- self.register_buffer("cos", torch.cos(m_theta), persistent=False)
- self.register_buffer("sin", torch.sin(m_theta), persistent=False)
-
- @staticmethod
- def _rotate(x):
- x1, x2 = rearrange(x, "... (dim r) -> r ... dim", r=2)
- return rearrange([-x2, x1], "r ... dim -> ... (dim r)", r=2) # noqa
-
- def forward(self, x, seq_id):
- self._init_freq(max_len=seq_id.max() + 1)
- rot_cos = self.cos[seq_id]
- rot_sin = self.sin[seq_id]
- return rot_cos * x + rot_sin * self._rotate(x)
-
-
-class QueryKeyProjection(nn.Module):
- def __init__(self, dim: int, num_heads: int, proj_layer, kwargs=None, partial_factor=None):
- super().__init__()
- if partial_factor is not None:
- assert (
- 0.0 <= partial_factor[0] < partial_factor[1] <= 1.0
- ), f"got {partial_factor[0]}, {partial_factor[1]}"
- assert num_heads > 0 and dim % num_heads == 0
-
- self.head_dim = dim // num_heads
- self.partial_factor = partial_factor
- self.query_proj = proj_layer(
- proj_width=self.proj_width,
- num_heads=num_heads,
- **(kwargs or {}),
- )
- self.key_proj = self.query_proj
-
- @cached_property
- def proj_width(self) -> int:
- if self.partial_factor is None:
- return self.head_dim
- return int(self.head_dim * (self.partial_factor[1] - self.partial_factor[0]))
-
- @cached_property
- def split_sizes(self):
- if self.partial_factor is None:
- return 0, self.head_dim, 0
- return (
- int(self.partial_factor[0] * self.head_dim),
- self.proj_width,
- int((1.0 - self.partial_factor[1]) * self.head_dim),
- )
-
- def forward(self, query, key, query_id, kv_id):
- if self.partial_factor is not None:
- queries = list(query.split(self.split_sizes, dim=-1))
- keys = list(key.split(self.split_sizes, dim=-1))
- queries[1] = self.query_proj(queries[1], seq_id=query_id)
- keys[1] = self.key_proj(keys[1], seq_id=kv_id)
- query = torch.cat(queries, dim=-1)
- key = torch.cat(keys, dim=-1)
- else:
- query = self.query_proj(query, seq_id=query_id)
- key = self.key_proj(key, seq_id=kv_id)
- return query, key
diff --git a/iotdb-core/ainode/ai/ainode/model/TimerXL/layers/Embed.py b/iotdb-core/ainode/ai/ainode/model/TimerXL/layers/Embed.py
deleted file mode 100644
index 6b476be455f..00000000000
--- a/iotdb-core/ainode/ai/ainode/model/TimerXL/layers/Embed.py
+++ /dev/null
@@ -1,189 +0,0 @@
-import math
-import torch
-import torch.nn as nn
-
-
-class PositionalEmbedding(nn.Module):
- def __init__(self, d_model, max_len=6500):
- super(PositionalEmbedding, self).__init__()
- # Compute the positional encodings once in log space.
- pe = torch.zeros(max_len, d_model).float()
- pe.require_grad = False
-
- position = torch.arange(0, max_len).float().unsqueeze(1)
- div_term = (torch.arange(0, d_model, 2).float()
- * -(math.log(10000.0) / d_model)).exp()
-
- pe[:, 0::2] = torch.sin(position * div_term)
- pe[:, 1::2] = torch.cos(position * div_term)
-
- pe = pe.unsqueeze(0)
- self.register_buffer('pe', pe)
-
- def forward(self, x):
- return self.pe[:, :x.size(1)]
-
-
-class TokenEmbedding(nn.Module):
- def __init__(self, c_in, d_model):
- super(TokenEmbedding, self).__init__()
- padding = 1 if torch.__version__ >= '1.5.0' else 2
- self.tokenConv = nn.Conv1d(in_channels=c_in, out_channels=d_model,
- kernel_size=3, padding=padding, padding_mode='circular', bias=False)
- for m in self.modules():
- if isinstance(m, nn.Conv1d):
- nn.init.kaiming_normal_(
- m.weight, mode='fan_in', nonlinearity='leaky_relu')
-
- def forward(self, x):
- x = self.tokenConv(x.permute(0, 2, 1)).transpose(1, 2)
- return x
-
-
-class FixedEmbedding(nn.Module):
- def __init__(self, c_in, d_model):
- super(FixedEmbedding, self).__init__()
-
- w = torch.zeros(c_in, d_model).float()
- w.require_grad = False
-
- position = torch.arange(0, c_in).float().unsqueeze(1)
- div_term = (torch.arange(0, d_model, 2).float()
- * -(math.log(10000.0) / d_model)).exp()
-
- w[:, 0::2] = torch.sin(position * div_term)
- w[:, 1::2] = torch.cos(position * div_term)
-
- self.emb = nn.Embedding(c_in, d_model)
- self.emb.weight = nn.Parameter(w, requires_grad=False)
-
- def forward(self, x):
- return self.emb(x).detach()
-
-
-class TemporalEmbedding(nn.Module):
- def __init__(self, d_model, embed_type='fixed', freq='h'):
- super(TemporalEmbedding, self).__init__()
-
- minute_size = 4
- hour_size = 24
- weekday_size = 7
- day_size = 32
- month_size = 13
-
- Embed = FixedEmbedding if embed_type == 'fixed' else nn.Embedding
- if freq == 't':
- self.minute_embed = Embed(minute_size, d_model)
- self.hour_embed = Embed(hour_size, d_model)
- self.weekday_embed = Embed(weekday_size, d_model)
- self.day_embed = Embed(day_size, d_model)
- self.month_embed = Embed(month_size, d_model)
-
- def forward(self, x):
- x = x.long()
- minute_x = self.minute_embed(x[:, :, 4]) if hasattr(
- self, 'minute_embed') else 0.
- hour_x = self.hour_embed(x[:, :, 3])
- weekday_x = self.weekday_embed(x[:, :, 2])
- day_x = self.day_embed(x[:, :, 1])
- month_x = self.month_embed(x[:, :, 0])
-
- return hour_x + weekday_x + day_x + month_x + minute_x
-
-
-class TimeFeatureEmbedding(nn.Module):
- def __init__(self, d_model, embed_type='timeF', freq='h'):
- super(TimeFeatureEmbedding, self).__init__()
-
- freq_map = {'h': 4, 't': 5, 's': 6,
- 'm': 1, 'a': 1, 'w': 2, 'd': 3, 'b': 3}
- d_inp = freq_map[freq]
- self.embed = nn.Linear(d_inp, d_model, bias=False)
-
- def forward(self, x):
- return self.embed(x)
-
-
-class DataEmbedding(nn.Module):
- def __init__(self, c_in, d_model, embed_type='fixed', freq='h', dropout=0.1):
- super(DataEmbedding, self).__init__()
-
- self.value_embedding = TokenEmbedding(c_in=c_in, d_model=d_model)
- self.position_embedding = PositionalEmbedding(d_model=d_model)
- self.temporal_embedding = TemporalEmbedding(d_model=d_model, embed_type=embed_type,
- freq=freq) if embed_type != 'timeF' else TimeFeatureEmbedding(
- d_model=d_model, embed_type=embed_type, freq=freq)
- self.dropout = nn.Dropout(p=dropout)
-
- def forward(self, x, x_mark):
- if x_mark is None:
- x = self.value_embedding(x) + self.position_embedding(x)
- else:
- x = self.value_embedding(
- x) + self.temporal_embedding(x_mark) + self.position_embedding(x)
- return self.dropout(x)
-
-
-class DataEmbedding_inverted(nn.Module):
- def __init__(self, c_in, d_model, embed_type='fixed', freq='h', dropout=0.1):
- super(DataEmbedding_inverted, self).__init__()
- self.value_embedding = nn.Linear(c_in, d_model)
- self.dropout = nn.Dropout(p=dropout)
-
- def forward(self, x, x_mark):
- x = x.permute(0, 2, 1)
- # x: [Batch Variate Time]
- if x_mark is None:
- x = self.value_embedding(x)
- else:
- x = self.value_embedding(
- torch.cat([x, x_mark.permute(0, 2, 1)], 1))
- # x: [Batch Variate d_model]
- return self.dropout(x)
-
-
-class DataEmbedding_wo_pos(nn.Module):
- def __init__(self, c_in, d_model, embed_type='fixed', freq='h', dropout=0.1):
- super(DataEmbedding_wo_pos, self).__init__()
-
- self.value_embedding = TokenEmbedding(c_in=c_in, d_model=d_model)
- self.position_embedding = PositionalEmbedding(d_model=d_model)
- self.temporal_embedding = TemporalEmbedding(d_model=d_model, embed_type=embed_type,
- freq=freq) if embed_type != 'timeF' else TimeFeatureEmbedding(
- d_model=d_model, embed_type=embed_type, freq=freq)
- self.dropout = nn.Dropout(p=dropout)
-
- def forward(self, x, x_mark):
- if x_mark is None:
- x = self.value_embedding(x)
- else:
- x = self.value_embedding(x) + self.temporal_embedding(x_mark)
- return self.dropout(x)
-
-
-class PatchEmbedding(nn.Module):
- def __init__(self, d_model, patch_len, stride, padding, dropout):
- super(PatchEmbedding, self).__init__()
- # Patching
- self.patch_len = patch_len
- self.stride = stride
- self.padding_patch_layer = nn.ReplicationPad1d((0, padding))
-
- # Backbone, Input encoding: projection of feature vectors onto a d-dim vector space
- self.value_embedding = nn.Linear(patch_len, d_model, bias=False)
-
- # Positional embedding
- self.position_embedding = PositionalEmbedding(d_model)
-
- # Residual dropout
- self.dropout = nn.Dropout(dropout)
-
- def forward(self, x):
- # do patching
- n_vars = x.shape[1]
- x = self.padding_patch_layer(x)
- x = x.unfold(dimension=-1, size=self.patch_len, step=self.stride)
- x = torch.reshape(x, (x.shape[0] * x.shape[1], x.shape[2], x.shape[3]))
- # Input encoding
- x = self.value_embedding(x) + self.position_embedding(x)
- return self.dropout(x), n_vars
diff --git a/iotdb-core/ainode/ai/ainode/model/TimerXL/layers/SelfAttention_Family.py b/iotdb-core/ainode/ai/ainode/model/TimerXL/layers/SelfAttention_Family.py
deleted file mode 100644
index 43746baf79d..00000000000
--- a/iotdb-core/ainode/ai/ainode/model/TimerXL/layers/SelfAttention_Family.py
+++ /dev/null
@@ -1,142 +0,0 @@
-import torch
-import torch.nn as nn
-import numpy as np
-from math import sqrt
-from einops import repeat
-from ai.ainode.model.TimerXL.layers.Attn_Bias import BinaryAttentionBias
-from ai.ainode.model.TimerXL.layers.Attn_Projection import QueryKeyProjection, RotaryProjection
-from ai.ainode.util.masking import TriangularCausalMask, TimerMultivariateMask, TimerCovariateMask
-
-
-class FullAttention(nn.Module):
- def __init__(self, mask_flag=True, scale=None, attention_dropout=0.1, output_attention=False):
- super(FullAttention, self).__init__()
- self.scale = scale
- self.mask_flag = mask_flag
- self.output_attention = output_attention
- self.dropout = nn.Dropout(attention_dropout)
-
- def forward(self, queries, keys, values, attn_mask, n_vars=None, n_tokens=None, tau=None, delta=None):
- B, L, H, E = queries.shape
- _, S, _, D = values.shape
- scale = self.scale or 1. / sqrt(E)
-
- scores = torch.einsum("blhe,bshe->bhls", queries, keys)
-
- if self.mask_flag:
- if attn_mask is None:
- attn_mask = TriangularCausalMask(B, L, device=queries.device)
-
- scores.masked_fill_(attn_mask.mask, -np.inf)
-
- A = self.dropout(torch.softmax(scale * scores, dim=-1))
- V = torch.einsum("bhls,bshd->blhd", A, values)
-
- if self.output_attention:
- return V.contiguous(), A
- else:
- return V.contiguous(), None
-
-
-class TimeAttention(nn.Module):
- def __init__(self, mask_flag=True, scale=None, attention_dropout=0.1, output_attention=False, d_model=512, num_heads=8, max_len=100, covariate=False, flash_attention=False):
- super(TimeAttention, self).__init__()
- self.scale = scale
- self.mask_flag = mask_flag
- self.output_attention = output_attention
- self.dropout = nn.Dropout(attention_dropout)
- self.covariate = covariate
- self.flash_attention = flash_attention
- self.qk_proj = QueryKeyProjection(dim=d_model, num_heads=num_heads, proj_layer=RotaryProjection, kwargs=dict(max_len=max_len),
- partial_factor=(0.0, 0.5),)
- self.attn_bias = BinaryAttentionBias(dim=d_model, num_heads=num_heads)
-
- def forward(self, queries, keys, values, attn_mask, n_vars, n_tokens, tau=None, delta=None):
- B, L, H, E = queries.shape
- _, S, _, D = values.shape
-
- # [B, H, L, E]
- queries = queries.permute(0, 2, 1, 3)
- keys = keys.permute(0, 2, 1, 3)
- if self.flash_attention:
- values = values.permute(0, 2, 1, 3)
-
- seq_id = torch.arange(n_tokens * n_vars)
- seq_id = repeat(seq_id, 'n -> b h n', b=B, h=H)
-
- queries, keys = self.qk_proj(
- queries, keys, query_id=seq_id, kv_id=seq_id)
-
- scale = self.scale or 1. / sqrt(E)
-
- var_id = repeat(torch.arange(n_vars),
- 'C -> (C n_tokens)', n_tokens=n_tokens)
- var_id = repeat(var_id, 'L -> b h L', b=B, h=1).to(queries.device)
-
- attn_bias = self.attn_bias(var_id, var_id)
-
- if self.mask_flag:
- if attn_mask is None:
- if self.covariate:
- attn_mask = TimerCovariateMask(
- B, n_vars, n_tokens, device=queries.device)
- else:
- attn_mask = TimerMultivariateMask(
- B, n_vars, n_tokens, device=queries.device)
- attn_mask = attn_bias.masked_fill(attn_mask.mask, float("-inf"))
- else:
- attn_mask = attn_bias
-
- if self.flash_attention:
- V = torch.nn.functional.scaled_dot_product_attention(
- queries, keys, values, attn_mask)
- else:
- scores = torch.einsum("bhle,bhse->bhls", queries, keys)
- scores += attn_mask
-
- A = self.dropout(torch.softmax(scale * scores, dim=-1))
- V = torch.einsum("bhls,bshd->blhd", A, values)
-
- if self.output_attention:
- return V.contiguous(), None
- else:
- return V.contiguous(), None
-
-
-class AttentionLayer(nn.Module):
- def __init__(self, attention, d_model, n_heads, d_keys=None, d_values=None):
- super(AttentionLayer, self).__init__()
-
- d_keys = d_keys or (d_model // n_heads)
- d_values = d_values or (d_model // n_heads)
-
- self.inner_attention = attention
- self.query_projection = nn.Linear(d_model, d_keys * n_heads)
- self.key_projection = nn.Linear(d_model, d_keys * n_heads)
- self.value_projection = nn.Linear(d_model, d_values * n_heads)
- self.out_projection = nn.Linear(d_values * n_heads, d_model)
- self.n_heads = n_heads
-
- def forward(self, queries, keys, values, attn_mask, n_vars=None, n_tokens=None, tau=None, delta=None):
- B, L, _ = queries.shape
- _, S, _ = keys.shape
- H = self.n_heads
-
- queries = self.query_projection(queries).view(B, L, H, -1)
- keys = self.key_projection(keys).view(B, S, H, -1)
- values = self.value_projection(values).view(B, S, H, -1)
-
- out, attn = self.inner_attention(
- queries,
- keys,
- values,
- attn_mask,
- n_vars=n_vars,
- n_tokens=n_tokens,
- tau=tau,
- delta=delta
- )
- out = out.view(B, L, -1)
-
- return self.out_projection(out), attn
-
diff --git a/iotdb-core/ainode/ai/ainode/model/TimerXL/layers/Transformer_EncDec.py b/iotdb-core/ainode/ai/ainode/model/TimerXL/layers/Transformer_EncDec.py
deleted file mode 100644
index 963b7984e42..00000000000
--- a/iotdb-core/ainode/ai/ainode/model/TimerXL/layers/Transformer_EncDec.py
+++ /dev/null
@@ -1,246 +0,0 @@
-import torch.nn as nn
-import torch.nn.functional as F
-
-
-class EncoderLayer(nn.Module):
- def __init__(self, attention, d_model, d_ff=None, dropout=0.1, activation="relu"):
- super(EncoderLayer, self).__init__()
- d_ff = d_ff or 4 * d_model
- self.attention = attention
- self.conv1 = nn.Conv1d(in_channels=d_model,
- out_channels=d_ff, kernel_size=1)
- self.conv2 = nn.Conv1d(
- in_channels=d_ff, out_channels=d_model, kernel_size=1)
- self.norm1 = nn.LayerNorm(d_model)
- self.norm2 = nn.LayerNorm(d_model)
- self.dropout = nn.Dropout(dropout)
- self.activation = F.relu if activation == "relu" else F.gelu
-
- def forward(self, x, attn_mask=None, tau=None, delta=None):
- new_x, attn = self.attention(
- x, x, x,
- attn_mask=attn_mask,
- tau=tau, delta=delta
- )
- x = x + self.dropout(new_x)
-
- y = x = self.norm1(x)
- y = self.dropout(self.activation(self.conv1(y.transpose(-1, 1))))
- y = self.dropout(self.conv2(y).transpose(-1, 1))
-
- return self.norm2(x + y), attn
-
-
-class DecoderLayer(nn.Module):
- def __init__(self, self_attention, cross_attention, d_model, d_ff=None,
- dropout=0.1, activation="relu"):
- super(DecoderLayer, self).__init__()
- d_ff = d_ff or 4 * d_model
- self.self_attention = self_attention
- self.cross_attention = cross_attention
- self.conv1 = nn.Conv1d(in_channels=d_model,
- out_channels=d_ff, kernel_size=1)
- self.conv2 = nn.Conv1d(
- in_channels=d_ff, out_channels=d_model, kernel_size=1)
- self.norm1 = nn.LayerNorm(d_model)
- self.norm2 = nn.LayerNorm(d_model)
- self.norm3 = nn.LayerNorm(d_model)
- self.dropout = nn.Dropout(dropout)
- self.activation = F.relu if activation == "relu" else F.gelu
-
- def forward(self, x, cross, x_mask=None, cross_mask=None, tau=None, delta=None):
- x = x + self.dropout(self.self_attention(
- x, x, x,
- attn_mask=x_mask,
- tau=tau, delta=None
- )[0])
- x = self.norm1(x)
-
- x = x + self.dropout(self.cross_attention(
- x, cross, cross,
- attn_mask=cross_mask,
- tau=tau, delta=delta
- )[0])
-
- y = x = self.norm2(x)
- y = self.dropout(self.activation(self.conv1(y.transpose(-1, 1))))
- y = self.dropout(self.conv2(y).transpose(-1, 1))
-
- return self.norm3(x + y)
-
-
-class DecoderOnlyLayer(nn.Module):
- def __init__(self, attention, d_model, d_ff=None, dropout=0.1, activation="relu"):
- super(DecoderOnlyLayer, self).__init__()
- d_ff = d_ff or 4 * d_model
- self.attention = attention
- self.conv1 = nn.Conv1d(in_channels=d_model,
- out_channels=d_ff, kernel_size=1)
- self.conv2 = nn.Conv1d(
- in_channels=d_ff, out_channels=d_model, kernel_size=1)
- self.norm1 = nn.LayerNorm(d_model)
- self.norm2 = nn.LayerNorm(d_model)
- self.dropout = nn.Dropout(dropout)
- self.activation = F.relu if activation == "relu" else F.gelu
-
- def forward(self, x, attn_mask=None, tau=None, delta=None):
- new_x, attn = self.attention(
- x, x, x,
- attn_mask=attn_mask,
- tau=tau, delta=delta
- )
- x = x + self.dropout(new_x)
-
- y = x = self.norm1(x)
- y = self.dropout(self.activation(self.conv1(y.transpose(-1, 1))))
- y = self.dropout(self.conv2(y).transpose(-1, 1))
-
- return self.norm2(x + y), attn
-
-
-class TimerLayer(nn.Module):
- def __init__(self, attention, d_model, d_ff=None, dropout=0.1, activation="relu"):
- super(TimerLayer, self).__init__()
- d_ff = d_ff or 4 * d_model
- self.attention = attention
- self.conv1 = nn.Conv1d(in_channels=d_model,
- out_channels=d_ff, kernel_size=1)
- self.conv2 = nn.Conv1d(
- in_channels=d_ff, out_channels=d_model, kernel_size=1)
- self.norm1 = nn.LayerNorm(d_model)
- self.norm2 = nn.LayerNorm(d_model)
- self.dropout = nn.Dropout(dropout)
- self.activation = F.relu if activation == "relu" else F.gelu
-
- def forward(self, x, n_vars, n_tokens, attn_mask=None, tau=None, delta=None):
- new_x, attn = self.attention(
- x, x, x,
- n_vars=n_vars,
- n_tokens=n_tokens,
- attn_mask=attn_mask,
- tau=tau, delta=delta
- )
- x = x + self.dropout(new_x)
-
- y = x = self.norm1(x)
- y = self.dropout(self.activation(self.conv1(y.transpose(-1, 1))))
- y = self.dropout(self.conv2(y).transpose(-1, 1))
-
- return self.norm2(x + y), attn
-
-
-class Encoder(nn.Module):
- def __init__(self, attn_layers, conv_layers=None, norm_layer=None):
- super(Encoder, self).__init__()
- self.attn_layers = nn.ModuleList(attn_layers)
- self.conv_layers = nn.ModuleList(
- conv_layers) if conv_layers is not None else None
- self.norm = norm_layer
-
- def forward(self, x, attn_mask=None, tau=None, delta=None):
- # x [B, L, D]
- attns = []
- if self.conv_layers is not None:
- for i, (attn_layer, conv_layer) in enumerate(zip(self.attn_layers, self.conv_layers)):
- delta = delta if i == 0 else None
- x, attn = attn_layer(
- x, attn_mask=attn_mask, tau=tau, delta=delta)
- x = conv_layer(x)
- attns.append(attn)
- x, attn = self.attn_layers[-1](x, tau=tau, delta=None)
- attns.append(attn)
- else:
- for attn_layer in self.attn_layers:
- x, attn = attn_layer(
- x, attn_mask=attn_mask, tau=tau, delta=delta)
- attns.append(attn)
-
- if self.norm is not None:
- x = self.norm(x)
-
- return x, attns
-
-
-class Decoder(nn.Module):
- def __init__(self, layers, norm_layer=None, projection=None):
- super(Decoder, self).__init__()
- self.layers = nn.ModuleList(layers)
- self.norm = norm_layer
- self.projection = projection
-
- def forward(self, x, cross, x_mask=None, cross_mask=None, tau=None, delta=None):
- for layer in self.layers:
- x = layer(x, cross, x_mask=x_mask,
- cross_mask=cross_mask, tau=tau, delta=delta)
-
- if self.norm is not None:
- x = self.norm(x)
-
- if self.projection is not None:
- x = self.projection(x)
- return x
-
-
-class DecoderOnly(nn.Module):
- def __init__(self, attn_layers, conv_layers=None, norm_layer=None):
- super(DecoderOnly, self).__init__()
- self.attn_layers = nn.ModuleList(attn_layers)
- self.conv_layers = nn.ModuleList(
- conv_layers) if conv_layers is not None else None
- self.norm = norm_layer
-
- def forward(self, x, attn_mask=None, tau=None, delta=None):
- # x [B, L, D]
- attns = []
- if self.conv_layers is not None:
- for i, (attn_layer, conv_layer) in enumerate(zip(self.attn_layers, self.conv_layers)):
- delta = delta if i == 0 else None
- x, attn = attn_layer(
- x, attn_mask=attn_mask, tau=tau, delta=delta)
- x = conv_layer(x)
- attns.append(attn)
- x, attn = self.attn_layers[-1](x, tau=tau, delta=None)
- attns.append(attn)
- else:
- for attn_layer in self.attn_layers:
- x, attn = attn_layer(
- x, attn_mask=attn_mask, tau=tau, delta=delta)
- attns.append(attn)
-
- if self.norm is not None:
- x = self.norm(x)
-
- return x, attns
-
-
-class TimerBlock(nn.Module):
- def __init__(self, attn_layers, conv_layers=None, norm_layer=None):
- super(TimerBlock, self).__init__()
- self.attn_layers = nn.ModuleList(attn_layers)
- self.conv_layers = nn.ModuleList(
- conv_layers) if conv_layers is not None else None
- self.norm = norm_layer
-
- def forward(self, x, n_vars, n_tokens, attn_mask=None, tau=None, delta=None):
- # x [B, L, D]
- attns = []
- if self.conv_layers is not None:
- for i, (attn_layer, conv_layer) in enumerate(zip(self.attn_layers, self.conv_layers)):
- delta = delta if i == 0 else None
- x, attn = attn_layer(
- x, attn_mask=attn_mask, tau=tau, delta=delta)
- x = conv_layer(x)
- attns.append(attn)
- x, attn = self.attn_layers[-1](x, n_vars,
- n_tokens, tau=tau, delta=None)
- attns.append(attn)
- else:
- for attn_layer in self.attn_layers:
- x, attn = attn_layer(x, n_vars, n_tokens,
- attn_mask=attn_mask, tau=tau, delta=delta)
- attns.append(attn)
-
- if self.norm is not None:
- x = self.norm(x)
-
- return x, attns
diff --git a/iotdb-core/ainode/ai/ainode/model/TimerXL/layers/__init__.py b/iotdb-core/ainode/ai/ainode/model/TimerXL/layers/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/iotdb-core/ainode/ai/ainode/model/TimerXL/models/__init__.py b/iotdb-core/ainode/ai/ainode/model/TimerXL/models/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/iotdb-core/ainode/ai/ainode/model/TimerXL/models/timer_xl.py b/iotdb-core/ainode/ai/ainode/model/TimerXL/models/timer_xl.py
deleted file mode 100644
index 39a1f76b888..00000000000
--- a/iotdb-core/ainode/ai/ainode/model/TimerXL/models/timer_xl.py
+++ /dev/null
@@ -1,80 +0,0 @@
-import torch
-from torch import nn
-from ai.ainode.model.TimerXL.layers.Transformer_EncDec import TimerBlock, TimerLayer
-from ai.ainode.model.TimerXL.layers.SelfAttention_Family import AttentionLayer, TimeAttention
-
-
-class Model(nn.Module):
- """
- Timer-XL: Long-Context Transformers for Unified Time Series Forecasting
-
- Paper: https://arxiv.org/abs/2410.04803
-
- GitHub: https://github.com/thuml/Timer-XL
-
- Citation: @article{liu2024timer,
- title={Timer-XL: Long-Context Transformers for Unified Time Series Forecasting},
- author={Liu, Yong and Qin, Guo and Huang, Xiangdong and Wang, Jianmin and Long, Mingsheng},
- journal={arXiv preprint arXiv:2410.04803},
- year={2024}
- }
- """
- def __init__(self, configs):
- super().__init__()
- self.input_token_len = configs.input_token_len
- self.embedding = nn.Linear(self.input_token_len, configs.d_model)
- self.output_attention = configs.output_attention
- self.blocks = TimerBlock(
- [
- TimerLayer(
- AttentionLayer(
- TimeAttention(True, attention_dropout=configs.dropout,
- output_attention=self.output_attention,
- d_model=configs.d_model, num_heads=configs.n_heads,
- covariate=configs.covariate, flash_attention=configs.flash_attention),
- configs.d_model, configs.n_heads),
- configs.d_model,
- configs.d_ff,
- dropout=configs.dropout,
- activation=configs.activation
- ) for l in range(configs.e_layers)
- ],
- norm_layer=torch.nn.LayerNorm(configs.d_model)
- )
- self.head = nn.Linear(configs.d_model, configs.output_token_len)
- self.use_norm = configs.use_norm
-
- def forecast(self, x, x_mark, y_mark):
- if self.use_norm:
- means = x.mean(1, keepdim=True).detach()
- x = x - means
- stdev = torch.sqrt(
- torch.var(x, dim=1, keepdim=True, unbiased=False) + 1e-5)
- x /= stdev
- B, _, C = x.shape
- # [B, C, L]
- x = x.permute(0, 2, 1)
- # [B, C, N, P]
- x = x.unfold(
- dimension=-1, size=self.input_token_len, step=self.input_token_len)
- N = x.shape[2]
- # [B, C, N, D]
- embed_out = self.embedding(x)
- # [B, C * N, D]
- embed_out = embed_out.reshape(B, C * N, -1)
- embed_out, attns = self.blocks(embed_out, n_vars=C, n_tokens=N)
- # [B, C * N, P]
- dec_out = self.head(embed_out)
- # [B, C, N * P]
- dec_out = dec_out.reshape(B, C, N, -1).reshape(B, C, -1)
- # [B, L, C]
- dec_out = dec_out.permute(0, 2, 1)
-
- if self.use_norm:
- dec_out = dec_out * stdev + means
- if self.output_attention:
- return dec_out, attns
- return dec_out
-
- def forward(self, x, x_mark=None, y_mark=None):
- return self.forecast(x, x_mark, y_mark)
diff --git a/iotdb-core/ainode/ai/ainode/model/built_in_model_factory.py b/iotdb-core/ainode/ai/ainode/model/built_in_model_factory.py
deleted file mode 100644
index 07e46d1932e..00000000000
--- a/iotdb-core/ainode/ai/ainode/model/built_in_model_factory.py
+++ /dev/null
@@ -1,924 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-from abc import abstractmethod
-from typing import List, Dict
-
-import numpy as np
-from sklearn.preprocessing import MinMaxScaler
-from sktime.annotation.hmm_learn import GaussianHMM, GMMHMM
-from sktime.annotation.stray import STRAY
-from sktime.forecasting.arima import ARIMA
-from sktime.forecasting.exp_smoothing import ExponentialSmoothing
-from sktime.forecasting.naive import NaiveForecaster
-from sktime.forecasting.trend import STLForecaster
-
-from ai.ainode.constant import AttributeName, BuiltInModelType
-from ai.ainode.exception import InferenceModelInternalError, AttributeNotSupportError
-from ai.ainode.exception import WrongAttributeTypeError, NumericalRangeException, StringRangeException, \
- ListRangeException, BuiltInModelNotSupportError
-from ai.ainode.log import Logger
-
-logger = Logger()
-
-
-def get_model_attributes(model_id: str):
- if model_id == BuiltInModelType.ARIMA.value:
- attribute_map = arima_attribute_map
- elif model_id == BuiltInModelType.NAIVE_FORECASTER.value:
- attribute_map = naive_forecaster_attribute_map
- elif model_id == BuiltInModelType.EXPONENTIAL_SMOOTHING.value:
- attribute_map = exponential_smoothing_attribute_map
- elif model_id == BuiltInModelType.STL_FORECASTER.value:
- attribute_map = stl_forecaster_attribute_map
- elif model_id == BuiltInModelType.GMM_HMM.value:
- attribute_map = gmmhmm_attribute_map
- elif model_id == BuiltInModelType.GAUSSIAN_HMM.value:
- attribute_map = gaussian_hmm_attribute_map
- elif model_id == BuiltInModelType.STRAY.value:
- attribute_map = stray_attribute_map
- else:
- raise BuiltInModelNotSupportError(model_id)
- return attribute_map
-
-
-def fetch_built_in_model(model_id, inference_attributes):
- """
- Args:
- model_id: the unique id of the model
- inference_attributes: a list of attributes to be inferred, in this function, the attributes will include some
- parameters of the built-in model. Some parameters are optional, and if the parameters are not
- specified, the default value will be used.
- Returns:
- model: the built-in model
- attributes: a dict of attributes, where the key is the attribute name, the value is the parsed value of the
- attribute
- Description:
- the create_built_in_model function will create the built-in model, which does not require user
- registration. This module will parse the inference attributes and create the built-in model.
- """
- attribute_map = get_model_attributes(model_id)
-
- # validate the inference attributes
- for attribute_name in inference_attributes:
- if attribute_name not in attribute_map:
- raise AttributeNotSupportError(model_id, attribute_name)
-
- # parse the inference attributes, attributes is a Dict[str, Any]
- attributes = parse_attribute(inference_attributes, attribute_map)
-
- # build the built-in model
- if model_id == BuiltInModelType.ARIMA.value:
- model = ArimaModel(attributes)
- elif model_id == BuiltInModelType.EXPONENTIAL_SMOOTHING.value:
- model = ExponentialSmoothingModel(attributes)
- elif model_id == BuiltInModelType.NAIVE_FORECASTER.value:
- model = NaiveForecasterModel(attributes)
- elif model_id == BuiltInModelType.STL_FORECASTER.value:
- model = STLForecasterModel(attributes)
- elif model_id == BuiltInModelType.GMM_HMM.value:
- model = GMMHMMModel(attributes)
- elif model_id == BuiltInModelType.GAUSSIAN_HMM.value:
- model = GaussianHmmModel(attributes)
- elif model_id == BuiltInModelType.STRAY.value:
- model = STRAYModel(attributes)
- else:
- raise BuiltInModelNotSupportError(model_id)
-
- return model
-
-
-class Attribute(object):
- def __init__(self, name: str):
- """
- Args:
- name: the name of the attribute
- """
- self._name = name
-
- @abstractmethod
- def get_default_value(self):
- raise NotImplementedError
-
- @abstractmethod
- def validate_value(self, value):
- raise NotImplementedError
-
- @abstractmethod
- def parse(self, string_value: str):
- raise NotImplementedError
-
-
-class IntAttribute(Attribute):
- def __init__(self, name: str,
- default_value: int,
- default_low: int,
- default_high: int,
- ):
- super(IntAttribute, self).__init__(name)
- self.__default_value = default_value
- self.__default_low = default_low
- self.__default_high = default_high
-
- def get_default_value(self):
- return self.__default_value
-
- def validate_value(self, value):
- if self.__default_low <= value <= self.__default_high:
- return True
- raise NumericalRangeException(self._name, value, self.__default_low, self.__default_high)
-
- def parse(self, string_value: str):
- try:
- int_value = int(string_value)
- except:
- raise WrongAttributeTypeError(self._name, "int")
- return int_value
-
-
-class FloatAttribute(Attribute):
- def __init__(self, name: str,
- default_value: float,
- default_low: float,
- default_high: float,
- ):
- super(FloatAttribute, self).__init__(name)
- self.__default_value = default_value
- self.__default_low = default_low
- self.__default_high = default_high
-
- def get_default_value(self):
- return self.__default_value
-
- def validate_value(self, value):
- if self.__default_low <= value <= self.__default_high:
- return True
- raise NumericalRangeException(self._name, value, self.__default_low, self.__default_high)
-
- def parse(self, string_value: str):
- try:
- float_value = float(string_value)
- except:
- raise WrongAttributeTypeError(self._name, "float")
- return float_value
-
-
-class StringAttribute(Attribute):
- def __init__(self, name: str, default_value: str, value_choices: List[str]):
- super(StringAttribute, self).__init__(name)
- self.__default_value = default_value
- self.__value_choices = value_choices
-
- def get_default_value(self):
- return self.__default_value
-
- def validate_value(self, value):
- if value in self.__value_choices:
- return True
- raise StringRangeException(self._name, value, self.__value_choices)
-
- def parse(self, string_value: str):
- return string_value
-
-
-class BooleanAttribute(Attribute):
- def __init__(self, name: str, default_value: bool):
- super(BooleanAttribute, self).__init__(name)
- self.__default_value = default_value
-
- def get_default_value(self):
- return self.__default_value
-
- def validate_value(self, value):
- if isinstance(value, bool):
- return True
- raise WrongAttributeTypeError(self._name, "bool")
-
- def parse(self, string_value: str):
- if string_value.lower() == "true":
- return True
- elif string_value.lower() == "false":
- return False
- else:
- raise WrongAttributeTypeError(self._name, "bool")
-
-
-class ListAttribute(Attribute):
- def __init__(self, name: str, default_value: List, value_type):
- """
- value_type is the type of the elements in the list, e.g. int, float, str
- """
- super(ListAttribute, self).__init__(name)
- self.__default_value = default_value
- self.__value_type = value_type
- self.__type_to_str = {str: "str", int: "int", float: "float"}
-
- def get_default_value(self):
- return self.__default_value
-
- def validate_value(self, value):
- if not isinstance(value, list):
- raise WrongAttributeTypeError(self._name, "list")
- for value_item in value:
- if not isinstance(value_item, self.__value_type):
- raise WrongAttributeTypeError(self._name, self.__value_type)
- return True
-
- def parse(self, string_value: str):
- try:
- list_value = eval(string_value)
- except:
- raise WrongAttributeTypeError(self._name, "list")
- if not isinstance(list_value, list):
- raise WrongAttributeTypeError(self._name, "list")
- for i in range(len(list_value)):
- try:
- list_value[i] = self.__value_type(list_value[i])
- except:
- raise ListRangeException(self._name, list_value, self.__type_to_str[self.__value_type])
- return list_value
-
-
-class TupleAttribute(Attribute):
- def __init__(self, name: str, default_value: tuple, value_type):
- """
- value_type is the type of the elements in the list, e.g. int, float, str
- """
- super(TupleAttribute, self).__init__(name)
- self.__default_value = default_value
- self.__value_type = value_type
- self.__type_to_str = {str: "str", int: "int", float: "float"}
-
- def get_default_value(self):
- return self.__default_value
-
- def validate_value(self, value):
- if not isinstance(value, tuple):
- raise WrongAttributeTypeError(self._name, "tuple")
- for value_item in value:
- if not isinstance(value_item, self.__value_type):
- raise WrongAttributeTypeError(self._name, self.__value_type)
- return True
-
- def parse(self, string_value: str):
- try:
- tuple_value = eval(string_value)
- except:
- raise WrongAttributeTypeError(self._name, "tuple")
- if not isinstance(tuple_value, tuple):
- raise WrongAttributeTypeError(self._name, "tuple")
- list_value = list(tuple_value)
- for i in range(len(list_value)):
- try:
- list_value[i] = self.__value_type(list_value[i])
- except:
- raise ListRangeException(self._name, list_value, self.__type_to_str[self.__value_type])
- tuple_value = tuple(list_value)
- return tuple_value
-
-
-def parse_attribute(input_attributes: Dict[str, str], attribute_map: Dict[str, Attribute]):
- """
- Args:
- input_attributes: a dict of attributes, where the key is the attribute name, the value is the string value of
- the attribute
- attribute_map: a dict of hyperparameters, where the key is the attribute name, the value is the Attribute
- object
- Returns:
- a dict of attributes, where the key is the attribute name, the value is the parsed value of the attribute
- """
- attributes = {}
- for attribute_name in attribute_map:
- # user specified the attribute
- if attribute_name in input_attributes:
- attribute = attribute_map[attribute_name]
- value = attribute.parse(input_attributes[attribute_name])
- attribute.validate_value(value)
- attributes[attribute_name] = value
- # user did not specify the attribute, use the default value
- else:
- try:
- attributes[attribute_name] = attribute_map[attribute_name].get_default_value()
- except NotImplementedError as e:
- logger.error(f"attribute {attribute_name} is not implemented.")
- raise e
- return attributes
-
-
-# built-in sktime model attributes
-# NaiveForecaster
-naive_forecaster_attribute_map = {
- AttributeName.PREDICT_LENGTH.value: IntAttribute(
- name=AttributeName.PREDICT_LENGTH.value,
- default_value=1,
- default_low=1,
- default_high=5000
- ),
- AttributeName.STRATEGY.value: StringAttribute(
- name=AttributeName.STRATEGY.value,
- default_value="last",
- value_choices=["last", "mean"],
- ),
- AttributeName.SP.value: IntAttribute(
- name=AttributeName.SP.value,
- default_value=1,
- default_low=1,
- default_high=5000
- ),
-}
-# ExponentialSmoothing
-exponential_smoothing_attribute_map = {
- AttributeName.PREDICT_LENGTH.value: IntAttribute(
- name=AttributeName.PREDICT_LENGTH.value,
- default_value=1,
- default_low=1,
- default_high=5000
- ),
- AttributeName.DAMPED_TREND.value: BooleanAttribute(
- name=AttributeName.DAMPED_TREND.value,
- default_value=False,
- ),
- AttributeName.INITIALIZATION_METHOD.value: StringAttribute(
- name=AttributeName.INITIALIZATION_METHOD.value,
- default_value="estimated",
- value_choices=["estimated", "heuristic", "legacy-heuristic", "known"],
- ),
- AttributeName.OPTIMIZED.value: BooleanAttribute(
- name=AttributeName.OPTIMIZED.value,
- default_value=True,
- ),
- AttributeName.REMOVE_BIAS.value: BooleanAttribute(
- name=AttributeName.REMOVE_BIAS.value,
- default_value=False,
- ),
- AttributeName.USE_BRUTE.value: BooleanAttribute(
- name=AttributeName.USE_BRUTE.value,
- default_value=False,
- )
-}
-# Arima
-arima_attribute_map = {
- AttributeName.PREDICT_LENGTH.value: IntAttribute(
- name=AttributeName.PREDICT_LENGTH.value,
- default_value=1,
- default_low=1,
- default_high=5000
- ),
- AttributeName.ORDER.value: TupleAttribute(
- name=AttributeName.ORDER.value,
- default_value=(1, 0, 0),
- value_type=int
- ),
- AttributeName.SEASONAL_ORDER.value: TupleAttribute(
- name=AttributeName.SEASONAL_ORDER.value,
- default_value=(0, 0, 0, 0),
- value_type=int
- ),
- AttributeName.METHOD.value: StringAttribute(
- name=AttributeName.METHOD.value,
- default_value="lbfgs",
- value_choices=["lbfgs", "bfgs", "newton", "nm", "cg", "ncg", "powell"],
- ),
- AttributeName.MAXITER.value: IntAttribute(
- name=AttributeName.MAXITER.value,
- default_value=1,
- default_low=1,
- default_high=5000
- ),
- AttributeName.SUPPRESS_WARNINGS.value: BooleanAttribute(
- name=AttributeName.SUPPRESS_WARNINGS.value,
- default_value=True,
- ),
- AttributeName.OUT_OF_SAMPLE_SIZE.value: IntAttribute(
- name=AttributeName.OUT_OF_SAMPLE_SIZE.value,
- default_value=0,
- default_low=0,
- default_high=5000
- ),
- AttributeName.SCORING.value: StringAttribute(
- name=AttributeName.SCORING.value,
- default_value="mse",
- value_choices=["mse", "mae", "rmse", "mape", "smape", "rmsle", "r2"],
- ),
- AttributeName.WITH_INTERCEPT.value: BooleanAttribute(
- name=AttributeName.WITH_INTERCEPT.value,
- default_value=True,
- ),
- AttributeName.TIME_VARYING_REGRESSION.value: BooleanAttribute(
- name=AttributeName.TIME_VARYING_REGRESSION.value,
- default_value=False,
- ),
- AttributeName.ENFORCE_STATIONARITY.value: BooleanAttribute(
- name=AttributeName.ENFORCE_STATIONARITY.value,
- default_value=True,
- ),
- AttributeName.ENFORCE_INVERTIBILITY.value: BooleanAttribute(
- name=AttributeName.ENFORCE_INVERTIBILITY.value,
- default_value=True,
- ),
- AttributeName.SIMPLE_DIFFERENCING.value: BooleanAttribute(
- name=AttributeName.SIMPLE_DIFFERENCING.value,
- default_value=False,
- ),
- AttributeName.MEASUREMENT_ERROR.value: BooleanAttribute(
- name=AttributeName.MEASUREMENT_ERROR.value,
- default_value=False,
- ),
- AttributeName.MLE_REGRESSION.value: BooleanAttribute(
- name=AttributeName.MLE_REGRESSION.value,
- default_value=True,
- ),
- AttributeName.HAMILTON_REPRESENTATION.value: BooleanAttribute(
- name=AttributeName.HAMILTON_REPRESENTATION.value,
- default_value=False,
- ),
- AttributeName.CONCENTRATE_SCALE.value: BooleanAttribute(
- name=AttributeName.CONCENTRATE_SCALE.value,
- default_value=False,
- )
-}
-# STLForecaster
-stl_forecaster_attribute_map = {
- AttributeName.PREDICT_LENGTH.value: IntAttribute(
- name=AttributeName.PREDICT_LENGTH.value,
- default_value=1,
- default_low=1,
- default_high=5000
- ),
- AttributeName.SP.value: IntAttribute(
- name=AttributeName.SP.value,
- default_value=2,
- default_low=1,
- default_high=5000
- ),
- AttributeName.SEASONAL.value: IntAttribute(
- name=AttributeName.SEASONAL.value,
- default_value=7,
- default_low=1,
- default_high=5000
- ),
- AttributeName.SEASONAL_DEG.value: IntAttribute(
- name=AttributeName.SEASONAL_DEG.value,
- default_value=1,
- default_low=0,
- default_high=5000
- ),
- AttributeName.TREND_DEG.value: IntAttribute(
- name=AttributeName.TREND_DEG.value,
- default_value=1,
- default_low=0,
- default_high=5000
- ),
- AttributeName.LOW_PASS_DEG.value: IntAttribute(
- name=AttributeName.LOW_PASS_DEG.value,
- default_value=1,
- default_low=0,
- default_high=5000
- ),
- AttributeName.SEASONAL_JUMP.value: IntAttribute(
- name=AttributeName.SEASONAL_JUMP.value,
- default_value=1,
- default_low=0,
- default_high=5000
- ),
- AttributeName.TREND_JUMP.value: IntAttribute(
- name=AttributeName.TREND_JUMP.value,
- default_value=1,
- default_low=0,
- default_high=5000
- ),
- AttributeName.LOSS_PASS_JUMP.value: IntAttribute(
- name=AttributeName.LOSS_PASS_JUMP.value,
- default_value=1,
- default_low=0,
- default_high=5000
- ),
-}
-
-# GAUSSIAN_HMM
-gaussian_hmm_attribute_map = {
- AttributeName.N_COMPONENTS.value: IntAttribute(
- name=AttributeName.N_COMPONENTS.value,
- default_value=1,
- default_low=1,
- default_high=5000
- ),
- AttributeName.COVARIANCE_TYPE.value: StringAttribute(
- name=AttributeName.COVARIANCE_TYPE.value,
- default_value="diag",
- value_choices=["spherical", "diag", "full", "tied"],
- ),
- AttributeName.MIN_COVAR.value: FloatAttribute(
- name=AttributeName.MIN_COVAR.value,
- default_value=1e-3,
- default_low=-1e10,
- default_high=1e10,
- ),
- AttributeName.STARTPROB_PRIOR.value: FloatAttribute(
- name=AttributeName.STARTPROB_PRIOR.value,
- default_value=1.0,
- default_low=-1e10,
- default_high=1e10,
- ),
- AttributeName.TRANSMAT_PRIOR.value: FloatAttribute(
- name=AttributeName.TRANSMAT_PRIOR.value,
- default_value=1.0,
- default_low=-1e10,
- default_high=1e10,
- ),
- AttributeName.MEANS_PRIOR.value: FloatAttribute(
- name=AttributeName.MEANS_PRIOR.value,
- default_value=0.0,
- default_low=-1e10,
- default_high=1e10,
- ),
- AttributeName.MEANS_WEIGHT.value: FloatAttribute(
- name=AttributeName.MEANS_WEIGHT.value,
- default_value=0.0,
- default_low=-1e10,
- default_high=1e10,
- ),
- AttributeName.COVARS_PRIOR.value: FloatAttribute(
- name=AttributeName.COVARS_PRIOR.value,
- default_value=1e-2,
- default_low=-1e10,
- default_high=1e10,
- ),
- AttributeName.COVARS_WEIGHT.value: FloatAttribute(
- name=AttributeName.COVARS_WEIGHT.value,
- default_value=1.0,
- default_low=-1e10,
- default_high=1e10,
- ),
- AttributeName.ALGORITHM.value: StringAttribute(
- name=AttributeName.ALGORITHM.value,
- default_value="viterbi",
- value_choices=["viterbi", "map"],
- ),
- AttributeName.N_ITER.value: IntAttribute(
- name=AttributeName.N_ITER.value,
- default_value=10,
- default_low=1,
- default_high=5000
- ),
- AttributeName.TOL.value: FloatAttribute(
- name=AttributeName.TOL.value,
- default_value=1e-2,
- default_low=-1e10,
- default_high=1e10,
- ),
- AttributeName.PARAMS.value: StringAttribute(
- name=AttributeName.PARAMS.value,
- default_value="stmc",
- value_choices=["stmc", "stm"],
- ),
- AttributeName.INIT_PARAMS.value: StringAttribute(
- name=AttributeName.INIT_PARAMS.value,
- default_value="stmc",
- value_choices=["stmc", "stm"],
- ),
- AttributeName.IMPLEMENTATION.value: StringAttribute(
- name=AttributeName.IMPLEMENTATION.value,
- default_value="log",
- value_choices=["log", "scaling"],
- )
-}
-
-# GMMHMM
-gmmhmm_attribute_map = {
- AttributeName.N_COMPONENTS.value: IntAttribute(
- name=AttributeName.N_COMPONENTS.value,
- default_value=1,
- default_low=1,
- default_high=5000
- ),
- AttributeName.N_MIX.value: IntAttribute(
- name=AttributeName.N_MIX.value,
- default_value=1,
- default_low=1,
- default_high=5000
- ),
- AttributeName.MIN_COVAR.value: FloatAttribute(
- name=AttributeName.MIN_COVAR.value,
- default_value=1e-3,
- default_low=-1e10,
- default_high=1e10,
- ),
- AttributeName.STARTPROB_PRIOR.value: FloatAttribute(
- name=AttributeName.STARTPROB_PRIOR.value,
- default_value=1.0,
- default_low=-1e10,
- default_high=1e10,
- ),
- AttributeName.TRANSMAT_PRIOR.value: FloatAttribute(
- name=AttributeName.TRANSMAT_PRIOR.value,
- default_value=1.0,
- default_low=-1e10,
- default_high=1e10,
- ),
- AttributeName.WEIGHTS_PRIOR.value: FloatAttribute(
- name=AttributeName.WEIGHTS_PRIOR.value,
- default_value=1.0,
- default_low=-1e10,
- default_high=1e10,
- ),
- AttributeName.MEANS_PRIOR.value: FloatAttribute(
- name=AttributeName.MEANS_PRIOR.value,
- default_value=0.0,
- default_low=-1e10,
- default_high=1e10,
- ),
- AttributeName.MEANS_WEIGHT.value: FloatAttribute(
- name=AttributeName.MEANS_WEIGHT.value,
- default_value=0.0,
- default_low=-1e10,
- default_high=1e10,
- ),
- AttributeName.ALGORITHM.value: StringAttribute(
- name=AttributeName.ALGORITHM.value,
- default_value="viterbi",
- value_choices=["viterbi", "map"],
- ),
- AttributeName.COVARIANCE_TYPE.value: StringAttribute(
- name=AttributeName.COVARIANCE_TYPE.value,
- default_value="diag",
- value_choices=["sperical", "diag", "full", "tied"],
- ),
- AttributeName.N_ITER.value: IntAttribute(
- name=AttributeName.N_ITER.value,
- default_value=10,
- default_low=1,
- default_high=5000
- ),
- AttributeName.TOL.value: FloatAttribute(
- name=AttributeName.TOL.value,
- default_value=1e-2,
- default_low=-1e10,
- default_high=1e10,
- ),
- AttributeName.INIT_PARAMS.value: StringAttribute(
- name=AttributeName.INIT_PARAMS.value,
- default_value="stmcw",
- value_choices=["s", "t", "m", "c", "w", "st", "sm", "sc", "sw", "tm", "tc", "tw", "mc", "mw", "cw", "stm",
- "stc", "stw", "smc", "smw", "scw", "tmc", "tmw", "tcw", "mcw", "stmc", "stmw", "stcw", "smcw",
- "tmcw", "stmcw"]
- ),
- AttributeName.PARAMS.value: StringAttribute(
- name=AttributeName.PARAMS.value,
- default_value="stmcw",
- value_choices=["s", "t", "m", "c", "w", "st", "sm", "sc", "sw", "tm", "tc", "tw", "mc", "mw", "cw", "stm",
- "stc", "stw", "smc", "smw", "scw", "tmc", "tmw", "tcw", "mcw", "stmc", "stmw", "stcw", "smcw",
- "tmcw", "stmcw"]
- ),
- AttributeName.IMPLEMENTATION.value: StringAttribute(
- name=AttributeName.IMPLEMENTATION.value,
- default_value="log",
- value_choices=["log", "scaling"],
- )
-}
-
-# STRAY
-stray_attribute_map = {
- AttributeName.ALPHA.value: FloatAttribute(
- name=AttributeName.ALPHA.value,
- default_value=0.01,
- default_low=-1e10,
- default_high=1e10,
- ),
- AttributeName.K.value: IntAttribute(
- name=AttributeName.K.value,
- default_value=10,
- default_low=1,
- default_high=5000
- ),
- AttributeName.KNN_ALGORITHM.value: StringAttribute(
- name=AttributeName.KNN_ALGORITHM.value,
- default_value="brute",
- value_choices=["brute", "kd_tree", "ball_tree", "auto"],
- ),
- AttributeName.P.value: FloatAttribute(
- name=AttributeName.P.value,
- default_value=0.5,
- default_low=-1e10,
- default_high=1e10,
- ),
- AttributeName.SIZE_THRESHOLD.value: IntAttribute(
- name=AttributeName.SIZE_THRESHOLD.value,
- default_value=50,
- default_low=1,
- default_high=5000
- ),
- AttributeName.OUTLIER_TAIL.value: StringAttribute(
- name=AttributeName.OUTLIER_TAIL.value,
- default_value="max",
- value_choices=["min", "max"],
- ),
-}
-
-
-class BuiltInModel(object):
- def __init__(self, attributes):
- self._attributes = attributes
- self._model = None
-
- @abstractmethod
- def inference(self, data):
- raise NotImplementedError
-
-
-class ArimaModel(BuiltInModel):
- def __init__(self, attributes):
- super(ArimaModel, self).__init__(attributes)
- self._model = ARIMA(
- order=attributes['order'],
- seasonal_order=attributes['seasonal_order'],
- method=attributes['method'],
- suppress_warnings=attributes['suppress_warnings'],
- maxiter=attributes['maxiter'],
- out_of_sample_size=attributes['out_of_sample_size'],
- scoring=attributes['scoring'],
- with_intercept=attributes['with_intercept'],
- time_varying_regression=attributes['time_varying_regression'],
- enforce_stationarity=attributes['enforce_stationarity'],
- enforce_invertibility=attributes['enforce_invertibility'],
- simple_differencing=attributes['simple_differencing'],
- measurement_error=attributes['measurement_error'],
- mle_regression=attributes['mle_regression'],
- hamilton_representation=attributes['hamilton_representation'],
- concentrate_scale=attributes['concentrate_scale']
- )
-
- def inference(self, data):
- try:
- predict_length = self._attributes['predict_length']
- self._model.fit(data)
- output = self._model.predict(fh=range(predict_length))
- output = np.array(output, dtype=np.float64)
- return output
- except Exception as e:
- raise InferenceModelInternalError(str(e))
-
-
-class ExponentialSmoothingModel(BuiltInModel):
- def __init__(self, attributes):
- super(ExponentialSmoothingModel, self).__init__(attributes)
- self._model = ExponentialSmoothing(
- damped_trend=attributes['damped_trend'],
- initialization_method=attributes['initialization_method'],
- optimized=attributes['optimized'],
- remove_bias=attributes['remove_bias'],
- use_brute=attributes['use_brute']
- )
-
- def inference(self, data):
- try:
- predict_length = self._attributes['predict_length']
- self._model.fit(data)
- output = self._model.predict(fh=range(predict_length))
- output = np.array(output, dtype=np.float64)
- return output
- except Exception as e:
- raise InferenceModelInternalError(str(e))
-
-
-class NaiveForecasterModel(BuiltInModel):
- def __init__(self, attributes):
- super(NaiveForecasterModel, self).__init__(attributes)
- self._model = NaiveForecaster(
- strategy=attributes['strategy'],
- sp=attributes['sp']
- )
-
- def inference(self, data):
- try:
- predict_length = self._attributes['predict_length']
- self._model.fit(data)
- output = self._model.predict(fh=range(predict_length))
- output = np.array(output, dtype=np.float64)
- return output
- except Exception as e:
- raise InferenceModelInternalError(str(e))
-
-
-class STLForecasterModel(BuiltInModel):
- def __init__(self, attributes):
- super(STLForecasterModel, self).__init__(attributes)
- self._model = STLForecaster(
- sp=attributes['sp'],
- seasonal=attributes['seasonal'],
- seasonal_deg=attributes['seasonal_deg'],
- trend_deg=attributes['trend_deg'],
- low_pass_deg=attributes['low_pass_deg'],
- seasonal_jump=attributes['seasonal_jump'],
- trend_jump=attributes['trend_jump'],
- low_pass_jump=attributes['low_pass_jump']
- )
-
- def inference(self, data):
- try:
- predict_length = self._attributes['predict_length']
- self._model.fit(data)
- output = self._model.predict(fh=range(predict_length))
- output = np.array(output, dtype=np.float64)
- return output
- except Exception as e:
- raise InferenceModelInternalError(str(e))
-
-
-class GMMHMMModel(BuiltInModel):
- def __init__(self, attributes):
- super(GMMHMMModel, self).__init__(attributes)
- self._model = GMMHMM(
- n_components=attributes['n_components'],
- n_mix=attributes['n_mix'],
- min_covar=attributes['min_covar'],
- startprob_prior=attributes['startprob_prior'],
- transmat_prior=attributes['transmat_prior'],
- means_prior=attributes['means_prior'],
- means_weight=attributes['means_weight'],
- weights_prior=attributes['weights_prior'],
- algorithm=attributes['algorithm'],
- covariance_type=attributes['covariance_type'],
- n_iter=attributes['n_iter'],
- tol=attributes['tol'],
- params=attributes['params'],
- init_params=attributes['init_params'],
- implementation=attributes['implementation']
- )
-
- def inference(self, data):
- try:
- self._model.fit(data)
- output = self._model.predict(data)
- output = np.array(output, dtype=np.int32)
- return output
- except Exception as e:
- raise InferenceModelInternalError(str(e))
-
-
-class GaussianHmmModel(BuiltInModel):
- def __init__(self, attributes):
- super(GaussianHmmModel, self).__init__(attributes)
- self._model = GaussianHMM(
- n_components=attributes['n_components'],
- covariance_type=attributes['covariance_type'],
- min_covar=attributes['min_covar'],
- startprob_prior=attributes['startprob_prior'],
- transmat_prior=attributes['transmat_prior'],
- means_prior=attributes['means_prior'],
- means_weight=attributes['means_weight'],
- covars_prior=attributes['covars_prior'],
- covars_weight=attributes['covars_weight'],
- algorithm=attributes['algorithm'],
- n_iter=attributes['n_iter'],
- tol=attributes['tol'],
- params=attributes['params'],
- init_params=attributes['init_params'],
- implementation=attributes['implementation']
- )
-
- def inference(self, data):
- try:
- self._model.fit(data)
- output = self._model.predict(data)
- output = np.array(output, dtype=np.int32)
- return output
- except Exception as e:
- raise InferenceModelInternalError(str(e))
-
-
-class STRAYModel(BuiltInModel):
- def __init__(self, attributes):
- super(STRAYModel, self).__init__(attributes)
- self._model = STRAY(
- alpha=attributes['alpha'],
- k=attributes['k'],
- knn_algorithm=attributes['knn_algorithm'],
- p=attributes['p'],
- size_threshold=attributes['size_threshold'],
- outlier_tail=attributes['outlier_tail']
- )
-
- def inference(self, data):
- try:
- data = MinMaxScaler().fit_transform(data)
- output = self._model.fit_transform(data)
- # change the output to int
- output = np.array(output, dtype=np.int32)
- return output
- except Exception as e:
- raise InferenceModelInternalError(str(e))
diff --git a/iotdb-core/ainode/ai/ainode/model/model_factory.py b/iotdb-core/ainode/ai/ainode/model/model_factory.py
deleted file mode 100644
index 480676b67a6..00000000000
--- a/iotdb-core/ainode/ai/ainode/model/model_factory.py
+++ /dev/null
@@ -1,235 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-import os
-import shutil
-from urllib.parse import urlparse, urljoin
-
-import yaml
-from requests import Session
-from requests.adapters import HTTPAdapter
-
-from ai.ainode.constant import DEFAULT_RECONNECT_TIMES, DEFAULT_RECONNECT_TIMEOUT, DEFAULT_CHUNK_SIZE, \
- DEFAULT_CONFIG_FILE_NAME, DEFAULT_MODEL_FILE_NAME
-from ai.ainode.exception import InvalidUriError, BadConfigValueError
-from ai.ainode.log import Logger
-from ai.ainode.util.serde import get_data_type_byte_from_str
-from ai.thrift.ainode.ttypes import TConfigs
-
-HTTP_PREFIX = "http://"
-HTTPS_PREFIX = "https://"
-
-logger = Logger()
-
-
-def _parse_uri(uri):
- """
- Args:
- uri (str): uri to parse
- Returns:
- is_network_path (bool): True if the url is a network path, False otherwise
- parsed_uri (str): parsed uri to get related file
- """
-
- parse_result = urlparse(uri)
- is_network_path = parse_result.scheme in ('http', 'https')
- if is_network_path:
- return True, uri
-
- # handle file:// in uri
- if parse_result.scheme == 'file':
- uri = uri[7:]
-
- # handle ~ in uri
- uri = os.path.expanduser(uri)
- return False, uri
-
-
-def _download_file(url: str, storage_path: str) -> None:
- """
- Args:
- url: url of file to download
- storage_path: path to save the file
- Returns:
- None
- """
- logger.debug(f"download file from {url} to {storage_path}")
-
- session = Session()
- adapter = HTTPAdapter(max_retries=DEFAULT_RECONNECT_TIMES)
- session.mount(HTTP_PREFIX, adapter)
- session.mount(HTTPS_PREFIX, adapter)
-
- response = session.get(url, timeout=DEFAULT_RECONNECT_TIMEOUT, stream=True)
- response.raise_for_status()
-
- with open(storage_path, 'wb') as file:
- for chunk in response.iter_content(chunk_size=DEFAULT_CHUNK_SIZE):
- if chunk:
- file.write(chunk)
-
- logger.debug(f"download file from {url} to {storage_path} success")
-
-
-def _register_model_from_network(uri: str, model_storage_path: str,
- config_storage_path: str) -> [TConfigs, str]:
- """
- Args:
- uri: network dir path of model to register, where model.pt and config.yaml are required,
- e.g. https://huggingface.co/user/modelname/resolve/main/
- model_storage_path: path to save model.pt
- config_storage_path: path to save config.yaml
- Returns:
- configs: TConfigs
- attributes: str
- """
- # concat uri to get complete url
- uri = uri if uri.endswith("/") else uri + "/"
- target_model_path = urljoin(uri, DEFAULT_MODEL_FILE_NAME)
- target_config_path = urljoin(uri, DEFAULT_CONFIG_FILE_NAME)
-
- # download config file
- _download_file(target_config_path, config_storage_path)
-
- # read and parse config dict from config.yaml
- with open(config_storage_path, 'r', encoding='utf-8') as file:
- config_dict = yaml.safe_load(file)
- configs, attributes = _parse_inference_config(config_dict)
-
- # if config.yaml is correct, download model file
- _download_file(target_model_path, model_storage_path)
- return configs, attributes
-
-
-def _register_model_from_local(uri: str, model_storage_path: str,
- config_storage_path: str) -> [TConfigs, str]:
- """
- Args:
- uri: local dir path of model to register, where model.pt and config.yaml are required,
- e.g. /Users/admin/Desktop/model
- model_storage_path: path to save model.pt
- config_storage_path: path to save config.yaml
- Returns:
- configs: TConfigs
- attributes: str
- """
- # concat uri to get complete path
- target_model_path = os.path.join(uri, DEFAULT_MODEL_FILE_NAME)
- target_config_path = os.path.join(uri, DEFAULT_CONFIG_FILE_NAME)
-
- # check if file exist
- exist_model_file = os.path.exists(target_model_path)
- exist_config_file = os.path.exists(target_config_path)
-
- configs = None
- attributes = None
- if exist_model_file and exist_config_file:
- # copy config.yaml
- logger.debug(f"copy file from {target_config_path} to {config_storage_path}")
- shutil.copy(target_config_path, config_storage_path)
- logger.debug(f"copy file from {target_config_path} to {config_storage_path} success")
-
- # read and parse config dict from config.yaml
- with open(config_storage_path, 'r', encoding='utf-8') as file:
- config_dict = yaml.safe_load(file)
- configs, attributes = _parse_inference_config(config_dict)
-
- # if config.yaml is correct, copy model file
- logger.debug(f"copy file from {target_model_path} to {model_storage_path}")
- shutil.copy(target_model_path, model_storage_path)
- logger.debug(f"copy file from {target_model_path} to {model_storage_path} success")
-
- elif not exist_model_file or not exist_config_file:
- raise InvalidUriError(uri)
-
- return configs, attributes
-
-
-def _parse_inference_config(config_dict):
- """
- Args:
- config_dict: dict
- - configs: dict
- - input_shape (list<i32>): input shape of the model and needs to be two-dimensional array like [96, 2]
- - output_shape (list<i32>): output shape of the model and needs to be two-dimensional array like [96, 2]
- - input_type (list<str>): input type of the model and each element needs to be in ['bool', 'int32', 'int64', 'float32', 'float64', 'text'], default float64
- - output_type (list<str>): output type of the model and each element needs to be in ['bool', 'int32', 'int64', 'float32', 'float64', 'text'], default float64
- - attributes: dict
- Returns:
- configs: TConfigs
- attributes: str
- """
- configs = config_dict['configs']
-
- # check if input_shape and output_shape are two-dimensional array
- if not (isinstance(configs['input_shape'], list) and len(configs['input_shape']) == 2):
- raise BadConfigValueError('input_shape', configs['input_shape'],
- 'input_shape should be a two-dimensional array.')
- if not (isinstance(configs['output_shape'], list) and len(configs['output_shape']) == 2):
- raise BadConfigValueError('output_shape', configs['output_shape'],
- 'output_shape should be a two-dimensional array.')
-
- # check if input_shape and output_shape are positive integer
- input_shape_is_positive_number = isinstance(configs['input_shape'][0], int) and isinstance(
- configs['input_shape'][1], int) and configs['input_shape'][0] > 0 and configs['input_shape'][1] > 0
- if not input_shape_is_positive_number:
- raise BadConfigValueError('input_shape', configs['input_shape'],
- 'element in input_shape should be positive integer.')
-
- output_shape_is_positive_number = isinstance(configs['output_shape'][0], int) and isinstance(
- configs['output_shape'][1], int) and configs['output_shape'][0] > 0 and configs['output_shape'][1] > 0
- if not output_shape_is_positive_number:
- raise BadConfigValueError('output_shape', configs['output_shape'],
- 'element in output_shape should be positive integer.')
-
- # check if input_type and output_type are one-dimensional array with right length
- if 'input_type' in configs and not (
- isinstance(configs['input_type'], list) and len(configs['input_type']) == configs['input_shape'][1]):
- raise BadConfigValueError('input_type', configs['input_type'],
- 'input_type should be a one-dimensional array and length of it should be equal to input_shape[1].')
-
- if 'output_type' in configs and not (
- isinstance(configs['output_type'], list) and len(configs['output_type']) == configs['output_shape'][1]):
- raise BadConfigValueError('output_type', configs['output_type'],
- 'output_type should be a one-dimensional array and length of it should be equal to output_shape[1].')
-
- # parse input_type and output_type to byte
- if 'input_type' in configs:
- input_type = [get_data_type_byte_from_str(x) for x in configs['input_type']]
- else:
- input_type = [get_data_type_byte_from_str('float32')] * configs['input_shape'][1]
-
- if 'output_type' in configs:
- output_type = [get_data_type_byte_from_str(x) for x in configs['output_type']]
- else:
- output_type = [get_data_type_byte_from_str('float32')] * configs['output_shape'][1]
-
- # parse attributes
- attributes = ""
- if 'attributes' in config_dict:
- attributes = str(config_dict['attributes'])
-
- return TConfigs(configs['input_shape'], configs['output_shape'], input_type, output_type), attributes
-
-
-def fetch_model_by_uri(uri: str, model_storage_path: str, config_storage_path: str):
- is_network_path, uri = _parse_uri(uri)
-
- if is_network_path:
- return _register_model_from_network(uri, model_storage_path, config_storage_path)
- else:
- return _register_model_from_local(uri, model_storage_path, config_storage_path)
diff --git a/iotdb-core/ainode/ai/ainode/model/model_storage.py b/iotdb-core/ainode/ai/ainode/model/model_storage.py
deleted file mode 100644
index de5227d0d83..00000000000
--- a/iotdb-core/ainode/ai/ainode/model/model_storage.py
+++ /dev/null
@@ -1,123 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-import os
-import shutil
-from collections.abc import Callable
-
-import torch
-import torch._dynamo
-from pylru import lrucache
-
-from ai.ainode.config import AINodeDescriptor
-from ai.ainode.constant import (DEFAULT_MODEL_FILE_NAME,
- DEFAULT_CONFIG_FILE_NAME)
-from ai.ainode.exception import ModelNotExistError
-from ai.ainode.log import Logger
-from ai.ainode.model.model_factory import fetch_model_by_uri
-from ai.ainode.util.lock import ModelLockPool
-from manager.training_manager import get_args
-from model.TimerXL.models.timer_xl import Model
-
-logger = Logger()
-
-
-class ModelStorage(object):
- def __init__(self):
- self._model_dir = os.path.join(os.getcwd(), AINodeDescriptor().get_config().get_ain_models_dir())
- if not os.path.exists(self._model_dir):
- try:
- os.makedirs(self._model_dir)
- except PermissionError as e:
- logger.error(e)
- raise e
- self._lock_pool = ModelLockPool()
- self._model_cache = lrucache(AINodeDescriptor().get_config().get_ain_model_storage_cache_size())
-
- def register_model(self, model_id: str, uri: str):
- """
- Args:
- model_id: id of model to register
- uri: network dir path or local dir path of model to register, where model.pt and config.yaml are required,
- e.g. https://huggingface.co/user/modelname/resolve/main/ or /Users/admin/Desktop/model
- Returns:
- configs: TConfigs
- attributes: str
- """
- storage_path = os.path.join(self._model_dir, f'{model_id}')
- # create storage dir if not exist
- if not os.path.exists(storage_path):
- os.makedirs(storage_path)
- model_storage_path = os.path.join(storage_path, DEFAULT_MODEL_FILE_NAME)
- config_storage_path = os.path.join(storage_path, DEFAULT_CONFIG_FILE_NAME)
- return fetch_model_by_uri(uri, model_storage_path, config_storage_path)
-
- def load_model(self, model_id: str, acceleration: bool) -> Callable:
- """
- Returns:
- model: a ScriptModule contains model architecture and parameters, which can be deployed cross-platform
- """
- ain_models_dir = os.path.join(self._model_dir, f'{model_id}')
- model_path = os.path.join(ain_models_dir, DEFAULT_MODEL_FILE_NAME)
- with self._lock_pool.get_lock(model_id).read_lock():
- if model_path in self._model_cache:
- model = self._model_cache[model_path]
- if isinstance(model, torch._dynamo.eval_frame.OptimizedModule) or not acceleration:
- return model
- else:
- model = torch.compile(model)
- self._model_cache[model_path] = model
- return model
- else:
- # todo: use modelType instead
- if 'timer' in model_id:
- model_file_path = os.path.join(ain_models_dir, 'checkpoint.pth')
- state_dict = torch.load(model_file_path)
- model = Model(get_args())
- model.load_state_dict(state_dict)
- self._model_cache[model_path] = model
- return model
- elif not os.path.exists(model_path):
- raise ModelNotExistError(model_path)
- else:
- model = torch.jit.load(model_path)
- if acceleration:
- try:
- model = torch.compile(model)
- except Exception as e:
- logger.warning(f"acceleration failed, fallback to normal mode: {str(e)}")
- self._model_cache[model_path] = model
- return model
-
- def delete_model(self, model_id: str) -> None:
- """
- Args:
- model_id: id of model to delete
- Returns:
- None
- """
- storage_path = os.path.join(self._model_dir, f'{model_id}')
- with self._lock_pool.get_lock(model_id).write_lock():
- if os.path.exists(storage_path):
- for file_name in os.listdir(storage_path):
- self._remove_from_cache(os.path.join(storage_path, file_name))
- shutil.rmtree(storage_path)
-
- def _remove_from_cache(self, file_path: str) -> None:
- if file_path in self._model_cache:
- del self._model_cache[file_path]
diff --git a/iotdb-core/ainode/ai/ainode/util/__init__.py b/iotdb-core/ainode/ai/ainode/util/__init__.py
deleted file mode 100644
index 2a1e720805f..00000000000
--- a/iotdb-core/ainode/ai/ainode/util/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
diff --git a/iotdb-core/ainode/ai/ainode/util/masking.py b/iotdb-core/ainode/ai/ainode/util/masking.py
deleted file mode 100644
index 63a0bf5f280..00000000000
--- a/iotdb-core/ainode/ai/ainode/util/masking.py
+++ /dev/null
@@ -1,35 +0,0 @@
-import torch
-
-class TriangularCausalMask():
- def __init__(self, B, L, device="cpu"):
- mask_shape = [B, 1, L, L]
- with torch.no_grad():
- self._mask = torch.triu(torch.ones(mask_shape, dtype=torch.bool), diagonal=1).to(device)
-
- @property
- def mask(self):
- return self._mask
-
-class TimerMultivariateMask():
- def __init__(self, B, n_vars, n_tokens, device="cpu"):
- mask_shape = [B, 1, n_tokens, n_tokens]
- with torch.no_grad():
- self._mask1 = torch.ones((n_vars, n_vars), dtype=torch.bool).to(device)
- self._mask2 = torch.triu(torch.ones(mask_shape, dtype=torch.bool), diagonal=1).to(device)
- self._mask = torch.kron(self._mask1, self._mask2)
- @property
- def mask(self):
- return self._mask
-
-class TimerCovariateMask():
- def __init__(self, B, n_vars, n_tokens, device="cpu"):
- mask_shape = [B, 1, n_tokens, n_tokens]
- with torch.no_grad():
- self._mask1 = torch.eye(n_vars, dtype=torch.bool).to(device)
- self._mask2 = torch.tril(torch.ones(mask_shape, dtype=torch.bool)).to(device)
- self._mask = ~torch.kron(self._mask1, self._mask2)
- self._mask[:, :, -n_tokens:, :-n_tokens] = False
-
- @property
- def mask(self):
- return self._mask
\ No newline at end of file
diff --git a/iotdb-core/ainode/ai/ainode/util/metrics.py b/iotdb-core/ainode/ai/ainode/util/metrics.py
deleted file mode 100644
index e39dcd4285c..00000000000
--- a/iotdb-core/ainode/ai/ainode/util/metrics.py
+++ /dev/null
@@ -1,44 +0,0 @@
-import numpy as np
-
-
-def RSE(pred, true):
- return np.sqrt(np.sum((true - pred) ** 2)) / np.sqrt(np.sum((true - true.mean()) ** 2))
-
-
-def CORR(pred, true):
- u = ((true - true.mean(0)) * (pred - pred.mean(0))).sum(0)
- d = np.sqrt(((true - true.mean(0)) ** 2 *
- (pred - pred.mean(0)) ** 2).sum(0))
- return (u / d).mean(-1)
-
-
-def MAE(pred, true):
- return np.mean(np.abs(pred - true))
-
-
-def MSE(pred, true):
- return np.mean((pred - true) ** 2)
-
-
-def RMSE(pred, true):
- return np.sqrt(MSE(pred, true))
-
-
-def MAPE(pred, true):
- return np.mean(np.abs((pred - true) / true))
-
-
-def MSPE(pred, true):
- return np.mean(np.square((pred - true) / true))
-
-def SMAPE(pred, true):
- return np.mean(200 * np.abs(pred - true) / (np.abs(pred) + np.abs(true) + 1e-8))
-
-def metric(pred, true):
- mae = MAE(pred, true)
- mse = MSE(pred, true)
- rmse = RMSE(pred, true)
- mape = MAPE(pred, true)
- mspe = MSPE(pred, true)
- smape = SMAPE(pred, true)
- return mae, mse, rmse, mape, mspe, smape
diff --git a/iotdb-core/ainode/ai/ainode/util/tools.py b/iotdb-core/ainode/ai/ainode/util/tools.py
deleted file mode 100644
index 6e0dd6595d5..00000000000
--- a/iotdb-core/ainode/ai/ainode/util/tools.py
+++ /dev/null
@@ -1,96 +0,0 @@
-import numpy as np
-import torch
-import matplotlib.pyplot as plt
-import torch.distributed as dist
-plt.switch_backend('agg')
-
-
-def adjust_learning_rate(optimizer, epoch, args):
- if args.lradj == 'type1':
- lr_adjust = {epoch: args.learning_rate * (0.1 ** epoch)}
- elif args.lradj == 'type2':
- lr_adjust = {epoch: args.learning_rate * (0.5 ** epoch)}
- elif args.lradj == 'type3':
- lr_adjust = {epoch: args.learning_rate * (0.9 ** epoch)}
-
-
- if epoch in lr_adjust.keys():
- lr = lr_adjust[epoch]
- for param_group in optimizer.param_groups:
- param_group['lr'] = lr
- print('Updating learning rate to {}'.format(lr))
-
-
-class EarlyStopping:
- def __init__(self, args, verbose=False, delta=0):
- self.patience = args.patience
- self.verbose = verbose
- self.counter = 0
- self.best_score = None
- self.early_stop = False
- self.val_loss_min = np.inf
- self.delta = delta
- self.dp = args.dp
- self.ddp = args.ddp
- if self.ddp:
- self.local_rank = args.local_rank
- else:
- self.local_rank = None
-
- def __call__(self, val_loss, model, path):
- score = -val_loss
- if self.best_score is None:
- self.best_score = score
- if self.verbose:
- print(
- f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).')
- self.val_loss_min = val_loss
- if self.ddp:
- if self.local_rank == 0:
- self.save_checkpoint(val_loss, model, path)
- dist.barrier()
- else:
- self.save_checkpoint(val_loss, model, path)
- elif score < self.best_score + self.delta:
- self.counter += 1
- print(
- f'EarlyStopping counter: {self.counter} out of {self.patience}')
- if self.counter >= self.patience:
- self.early_stop = True
- else:
- self.best_score = score
- if self.ddp:
- if self.local_rank == 0:
- self.save_checkpoint(val_loss, model, path)
- dist.barrier()
- else:
- self.save_checkpoint(val_loss, model, path)
- if self.verbose:
- print(
- f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).')
- self.val_loss_min = val_loss
- self.counter = 0
-
- def save_checkpoint(self, val_loss, model, path):
- if self.dp:
- model = model.module
- param_grad_dic = {
- k: v.requires_grad for (k, v) in model.named_parameters()
- }
- state_dict = model.state_dict()
- for k in list(state_dict.keys()):
- if k in param_grad_dic.keys() and not param_grad_dic[k]:
- # delete parameters that do not require gradient
- del state_dict[k]
- torch.save(state_dict, path + '/' + f'checkpoint.pth')
-
-def visual(true, preds=None, name='./pic/test.pdf'):
- """
- Results visualization
- """
- plt.figure()
- plt.plot(true, label='GroundTruth', linewidth=2)
- if preds is not None:
- plt.plot(preds, label='Prediction', linewidth=2)
- plt.legend()
- plt.savefig(name, bbox_inches='tight')
\ No newline at end of file
diff --git a/iotdb-core/ainode/ai/__init__.py b/iotdb-core/ainode/ainode/__init__.py
similarity index 100%
rename from iotdb-core/ainode/ai/__init__.py
rename to iotdb-core/ainode/ainode/__init__.py
diff --git a/iotdb-core/ainode/ai/ainode/__init__.py b/iotdb-core/ainode/ainode/core/__init__.py
similarity index 100%
rename from iotdb-core/ainode/ai/ainode/__init__.py
rename to iotdb-core/ainode/ainode/core/__init__.py
diff --git a/iotdb-core/ainode/ai/ainode/client.py b/iotdb-core/ainode/ainode/core/client.py
similarity index 92%
rename from iotdb-core/ainode/ai/ainode/client.py
rename to iotdb-core/ainode/ainode/core/client.py
index d7408917724..2796d46ab93 100644
--- a/iotdb-core/ainode/ai/ainode/client.py
+++ b/iotdb-core/ainode/ainode/core/client.py
@@ -21,16 +21,16 @@ from thrift.Thrift import TException
from thrift.protocol import TCompactProtocol, TBinaryProtocol
from thrift.transport import TSocket, TTransport
-from ai.ainode.config import AINodeDescriptor
-from ai.ainode.constant import TSStatusCode
-from ai.ainode.log import Logger
-from ai.ainode.util.decorator import singleton
-from ai.ainode.util.status import verify_success
-from ai.thrift.common.ttypes import TEndPoint, TSStatus, TAINodeLocation, TAINodeConfiguration
-from ai.thrift.confignode import IConfigNodeRPCService
-from ai.thrift.confignode.ttypes import (TAINodeRemoveReq, TNodeVersionInfo,
- TAINodeRegisterReq, TAINodeRestartReq)
-from ai.thrift.confignode.ttypes import TUpdateModelInfoReq
+from ainode.core.config import AINodeDescriptor
+from ainode.core.constant import TSStatusCode
+from ainode.core.log import Logger
+from ainode.core.util.decorator import singleton
+from ainode.core.util.status import verify_success
+from ainode.thrift.common.ttypes import TEndPoint, TSStatus, TAINodeLocation, TAINodeConfiguration
+from ainode.thrift.confignode import IConfigNodeRPCService
+from ainode.thrift.confignode.ttypes import (TAINodeRemoveReq, TNodeVersionInfo,
+ TAINodeRegisterReq, TAINodeRestartReq)
+from ainode.thrift.confignode.ttypes import TUpdateModelInfoReq
logger = Logger()
@@ -214,8 +214,8 @@ class ConfigNodeClient(object):
)
if ainode_id is not None:
req.aiNodeIds = ainode_id
- req.inputShapes = [input_length]
- req.outputShapes = [output_length]
+ req.inputLength = input_length
+ req.outputLength = output_length
status = self._client.updateModelInfo(req)
if not self._update_config_node_leader(status):
verify_success(status, "An error occurs when calling update model info")
diff --git a/iotdb-core/ainode/ai/ainode/config.py b/iotdb-core/ainode/ainode/core/config.py
similarity index 91%
rename from iotdb-core/ainode/ai/ainode/config.py
rename to iotdb-core/ainode/ainode/core/config.py
index a68b978d43e..036dc20b9c2 100644
--- a/iotdb-core/ainode/ai/ainode/config.py
+++ b/iotdb-core/ainode/ainode/core/config.py
@@ -17,17 +17,17 @@
#
import os
-from ai.ainode.constant import (AINODE_CONF_DIRECTORY_NAME,
- AINODE_CONF_FILE_NAME,
- AINODE_MODELS_DIR, AINODE_LOG_DIR, AINODE_SYSTEM_DIR, AINODE_INFERENCE_RPC_ADDRESS,
- AINODE_INFERENCE_RPC_PORT, AINODE_THRIFT_COMPRESSION_ENABLED,
- AINODE_SYSTEM_FILE_NAME, AINODE_CLUSTER_NAME, AINODE_VERSION_INFO, AINODE_BUILD_INFO,
- AINODE_CONF_GIT_FILE_NAME, AINODE_CONF_POM_FILE_NAME, AINODE_ROOT_DIR,
- AINODE_ROOT_CONF_DIRECTORY_NAME)
-from ai.ainode.exception import BadNodeUrlError
-from ai.ainode.log import Logger
-from ai.ainode.util.decorator import singleton
-from ai.thrift.common.ttypes import TEndPoint
+from ainode.core.constant import (AINODE_CONF_DIRECTORY_NAME,
+ AINODE_CONF_FILE_NAME,
+ AINODE_MODELS_DIR, AINODE_LOG_DIR, AINODE_SYSTEM_DIR, AINODE_INFERENCE_RPC_ADDRESS,
+ AINODE_INFERENCE_RPC_PORT, AINODE_THRIFT_COMPRESSION_ENABLED,
+ AINODE_SYSTEM_FILE_NAME, AINODE_CLUSTER_NAME, AINODE_VERSION_INFO, AINODE_BUILD_INFO,
+ AINODE_CONF_GIT_FILE_NAME, AINODE_CONF_POM_FILE_NAME, AINODE_ROOT_DIR,
+ AINODE_ROOT_CONF_DIRECTORY_NAME)
+from ainode.core.exception import BadNodeUrlError
+from ainode.core.log import Logger
+from ainode.core.util.decorator import singleton
+from ainode.thrift.common.ttypes import TEndPoint
logger = Logger()
diff --git a/iotdb-core/ainode/ai/ainode/constant.py b/iotdb-core/ainode/ainode/core/constant.py
similarity index 99%
rename from iotdb-core/ainode/ai/ainode/constant.py
rename to iotdb-core/ainode/ainode/core/constant.py
index a50ac382e65..596cde0b224 100644
--- a/iotdb-core/ainode/ai/ainode/constant.py
+++ b/iotdb-core/ainode/ainode/core/constant.py
@@ -23,7 +23,7 @@ from typing import List
AINODE_CONF_DIRECTORY_NAME = "conf"
AINODE_ROOT_CONF_DIRECTORY_NAME = "conf"
-AINODE_CONF_FILE_NAME = "ai-ainode.properties"
+AINODE_CONF_FILE_NAME = "iotdb-ainode.properties"
AINODE_CONF_GIT_FILE_NAME = "git.properties"
AINODE_CONF_POM_FILE_NAME = "pom.properties"
AINODE_SYSTEM_FILE_NAME = "system.properties"
diff --git a/iotdb-core/ainode/ai/ainode/exception.py b/iotdb-core/ainode/ainode/core/exception.py
similarity index 98%
rename from iotdb-core/ainode/ai/ainode/exception.py
rename to iotdb-core/ainode/ainode/core/exception.py
index cb8e2f0b1ba..a9b8c496d65 100644
--- a/iotdb-core/ainode/ai/ainode/exception.py
+++ b/iotdb-core/ainode/ainode/core/exception.py
@@ -17,7 +17,7 @@
#
import re
-from ai.ainode.constant import DEFAULT_MODEL_FILE_NAME, DEFAULT_CONFIG_FILE_NAME
+from ainode.core.constant import DEFAULT_MODEL_FILE_NAME, DEFAULT_CONFIG_FILE_NAME
class _BaseError(Exception):
diff --git a/iotdb-core/ainode/ai/ainode/handler.py b/iotdb-core/ainode/ainode/core/handler.py
similarity index 73%
rename from iotdb-core/ainode/ai/ainode/handler.py
rename to iotdb-core/ainode/ainode/core/handler.py
index 3b7e47cd505..292fd2d4b5f 100644
--- a/iotdb-core/ainode/ai/ainode/handler.py
+++ b/iotdb-core/ainode/ainode/core/handler.py
@@ -16,15 +16,15 @@
# under the License.
#
-from ai.ainode.manager.cluster_manager import ClusterManager
-from ai.ainode.manager.inference_manager import InferenceManager
-from ai.ainode.manager.model_manager import ModelManager
-from ai.thrift.ainode import IAINodeRPCService
-from ai.thrift.ainode.ttypes import (TDeleteModelReq, TRegisterModelReq,
- TAIHeartbeatReq, TInferenceReq, TRegisterModelResp, TInferenceResp,
- TAIHeartbeatResp, TTrainingReq)
-from ai.thrift.common.ttypes import TSStatus
-from manager.training_manager import TrainingManager
+from ainode.core.manager.cluster_manager import ClusterManager
+from ainode.core.manager.inference_manager import InferenceManager
+from ainode.core.manager.model_manager import ModelManager
+from ainode.thrift.ainode import IAINodeRPCService
+from ainode.thrift.ainode.ttypes import (TDeleteModelReq, TRegisterModelReq,
+ TAIHeartbeatReq, TInferenceReq, TRegisterModelResp, TInferenceResp,
+ TAIHeartbeatResp, TTrainingReq)
+from ainode.thrift.common.ttypes import TSStatus
+from ainode.core.manager.training_manager import TrainingManager
class AINodeRPCServiceHandler(IAINodeRPCService.Iface):
diff --git a/iotdb-core/ainode/ai/ainode/log.py b/iotdb-core/ainode/ainode/core/log.py
similarity index 94%
rename from iotdb-core/ainode/ai/ainode/log.py
rename to iotdb-core/ainode/ainode/core/log.py
index 04e5c2f1679..34a24d370c4 100644
--- a/iotdb-core/ainode/ai/ainode/log.py
+++ b/iotdb-core/ainode/ainode/core/log.py
@@ -23,8 +23,8 @@ import random
import sys
import threading
-from ai.ainode.constant import STD_LEVEL, AINODE_LOG_FILE_NAMES, AINODE_LOG_FILE_LEVELS
-from ai.ainode.util.decorator import singleton
+from ainode.core.constant import STD_LEVEL, AINODE_LOG_FILE_NAMES, AINODE_LOG_FILE_LEVELS
+from ainode.core.util.decorator import singleton
class LoggerFilter(logging.Filter):
@@ -43,10 +43,10 @@ class LoggerFilter(logging.Filter):
stack_info = ""
frame_info = stack_trace[7]
file_name = frame_info.filename
- # if file_name is not in current working directory, find the first "ai" in the path
+ # if file_name is not in current working directory, find the first "core" in the path
for l in range(len(file_name)):
i = len(file_name) - l - 1
- if file_name[i:].startswith("ai/") or file_name[i:].startswith("ai\\"):
+ if file_name[i:].startswith("core/") or file_name[i:].startswith("core\\"):
file_name = file_name[i:]
break
diff --git a/iotdb-core/ainode/ai/ainode/manager/__init__.py b/iotdb-core/ainode/ainode/core/manager/__init__.py
similarity index 100%
rename from iotdb-core/ainode/ai/ainode/manager/__init__.py
rename to iotdb-core/ainode/ainode/core/manager/__init__.py
diff --git a/iotdb-core/ainode/ai/ainode/manager/cluster_manager.py b/iotdb-core/ainode/ainode/core/manager/cluster_manager.py
similarity index 93%
rename from iotdb-core/ainode/ai/ainode/manager/cluster_manager.py
rename to iotdb-core/ainode/ainode/core/manager/cluster_manager.py
index 7948f95b125..da7008b7762 100644
--- a/iotdb-core/ainode/ai/ainode/manager/cluster_manager.py
+++ b/iotdb-core/ainode/ainode/core/manager/cluster_manager.py
@@ -17,8 +17,8 @@
#
import psutil
-from ai.thrift.ainode.ttypes import TAIHeartbeatResp, TAIHeartbeatReq
-from ai.thrift.common.ttypes import TLoadSample
+from ainode.thrift.ainode.ttypes import TAIHeartbeatResp, TAIHeartbeatReq
+from ainode.thrift.common.ttypes import TLoadSample
class ClusterManager:
diff --git a/iotdb-core/ainode/ai/ainode/manager/inference_manager.py b/iotdb-core/ainode/ainode/core/manager/inference_manager.py
similarity index 95%
rename from iotdb-core/ainode/ai/ainode/manager/inference_manager.py
rename to iotdb-core/ainode/ainode/core/manager/inference_manager.py
index c5480575adb..ebfc6d41c9e 100644
--- a/iotdb-core/ainode/ai/ainode/manager/inference_manager.py
+++ b/iotdb-core/ainode/ainode/core/manager/inference_manager.py
@@ -18,13 +18,13 @@
import pandas as pd
from torch import tensor
-from ai.ainode.constant import TSStatusCode
-from ai.ainode.exception import InvalidWindowArgumentError, InferenceModelInternalError, runtime_error_extractor
-from ai.ainode.log import Logger
-from ai.ainode.manager.model_manager import ModelManager
-from ai.ainode.util.serde import convert_to_binary, convert_to_df
-from ai.ainode.util.status import get_status
-from ai.thrift.ainode.ttypes import TInferenceReq, TInferenceResp
+from ainode.core.constant import TSStatusCode
+from ainode.core.exception import InvalidWindowArgumentError, InferenceModelInternalError, runtime_error_extractor
+from ainode.core.log import Logger
+from ainode.core.manager.model_manager import ModelManager
+from ainode.core.util.serde import convert_to_binary, convert_to_df
+from ainode.core.util.status import get_status
+from ainode.thrift.ainode.ttypes import TInferenceReq, TInferenceResp
logger = Logger()
diff --git a/iotdb-core/ainode/ai/ainode/manager/model_manager.py b/iotdb-core/ainode/ainode/core/manager/model_manager.py
similarity index 86%
rename from iotdb-core/ainode/ai/ainode/manager/model_manager.py
rename to iotdb-core/ainode/ainode/core/manager/model_manager.py
index c4119d16d8b..ead833a5839 100644
--- a/iotdb-core/ainode/ai/ainode/manager/model_manager.py
+++ b/iotdb-core/ainode/ainode/core/manager/model_manager.py
@@ -19,14 +19,14 @@ from typing import Callable
from yaml import YAMLError
-from ai.ainode.constant import TSStatusCode, BuiltInModelType
-from ai.ainode.exception import InvalidUriError, BadConfigValueError, BuiltInModelNotSupportError
-from ai.ainode.log import Logger
-from ai.ainode.model.built_in_model_factory import fetch_built_in_model
-from ai.ainode.model.model_storage import ModelStorage
-from ai.ainode.util.status import get_status
-from ai.thrift.ainode.ttypes import TRegisterModelReq, TRegisterModelResp, TDeleteModelReq
-from ai.thrift.common.ttypes import TSStatus
+from ainode.core.constant import TSStatusCode, BuiltInModelType
+from ainode.core.exception import InvalidUriError, BadConfigValueError, BuiltInModelNotSupportError
+from ainode.core.log import Logger
+from ainode.core.model.built_in_model_factory import fetch_built_in_model
+from ainode.core.model.model_storage import ModelStorage
+from ainode.core.util.status import get_status
+from ainode.thrift.ainode.ttypes import TRegisterModelReq, TRegisterModelResp, TDeleteModelReq
+from ainode.thrift.common.ttypes import TSStatus
logger = Logger()
diff --git a/iotdb-core/ainode/ai/ainode/util/decorator.py b/iotdb-core/ainode/ainode/core/manager/training_manager.py
similarity index 78%
copy from iotdb-core/ainode/ai/ainode/util/decorator.py
copy to iotdb-core/ainode/ainode/core/manager/training_manager.py
index 33b9f4835ac..0233d270469 100644
--- a/iotdb-core/ainode/ai/ainode/util/decorator.py
+++ b/iotdb-core/ainode/ainode/core/manager/training_manager.py
@@ -15,12 +15,9 @@
# specific language governing permissions and limitations
# under the License.
#
-def singleton(cls):
- instances = {}
+class TrainingManager:
+ def __init__(self):
+ pass
- def get_instance(*args, **kwargs):
- if cls not in instances:
- instances[cls] = cls(*args, **kwargs)
- return instances[cls]
-
- return get_instance
+ def create_training_task(self, model_id:str, model_type, table_list: list[ITableSchema], parameters: dict[str, str]):
+ raise NotImplementedError
diff --git a/iotdb-core/ainode/ai/ainode/script.py b/iotdb-core/ainode/ainode/core/script.py
similarity index 93%
rename from iotdb-core/ainode/ai/ainode/script.py
rename to iotdb-core/ainode/ainode/core/script.py
index 6c74fc5f26f..b27bb6ab61b 100644
--- a/iotdb-core/ainode/ai/ainode/script.py
+++ b/iotdb-core/ainode/ainode/core/script.py
@@ -22,14 +22,14 @@ from datetime import datetime
import psutil
-from ai.ainode.client import ClientManager
-from ai.ainode.config import AINodeDescriptor
-from ai.ainode.constant import TSStatusCode, AINODE_SYSTEM_FILE_NAME
-from ai.ainode.exception import MissingConfigError
-from ai.ainode.log import Logger
-from ai.ainode.service import RPCService
-from ai.thrift.common.ttypes import TAINodeLocation, TEndPoint, TAINodeConfiguration, TNodeResource
-from ai.thrift.confignode.ttypes import TNodeVersionInfo
+from ainode.core.client import ClientManager
+from ainode.core.config import AINodeDescriptor
+from ainode.core.constant import TSStatusCode, AINODE_SYSTEM_FILE_NAME
+from ainode.core.exception import MissingConfigError
+from ainode.core.log import Logger
+from ainode.core.service import RPCService
+from ainode.thrift.common.ttypes import TAINodeLocation, TEndPoint, TAINodeConfiguration, TNodeResource
+from ainode.thrift.confignode.ttypes import TNodeVersionInfo
logger = Logger()
diff --git a/iotdb-core/ainode/ai/ainode/service.py b/iotdb-core/ainode/ainode/core/service.py
similarity index 91%
rename from iotdb-core/ainode/ai/ainode/service.py
rename to iotdb-core/ainode/ainode/core/service.py
index 5c21ce5cd7c..9532093d1da 100644
--- a/iotdb-core/ainode/ai/ainode/service.py
+++ b/iotdb-core/ainode/ainode/core/service.py
@@ -21,10 +21,10 @@ from thrift.protocol import TCompactProtocol, TBinaryProtocol
from thrift.server import TServer
from thrift.transport import TSocket, TTransport
-from ai.ainode.config import AINodeDescriptor
-from ai.ainode.handler import AINodeRPCServiceHandler
-from ai.ainode.log import Logger
-from ai.thrift.ainode import IAINodeRPCService
+from ainode.core.config import AINodeDescriptor
+from ainode.core.handler import AINodeRPCServiceHandler
+from ainode.core.log import Logger
+from ainode.thrift.ainode import IAINodeRPCService
logger = Logger()
diff --git a/iotdb-core/ainode/ai/ainode/model/__init__.py b/iotdb-core/ainode/ainode/core/util/__init__.py
similarity index 100%
rename from iotdb-core/ainode/ai/ainode/model/__init__.py
rename to iotdb-core/ainode/ainode/core/util/__init__.py
diff --git a/iotdb-core/ainode/ai/ainode/util/decorator.py b/iotdb-core/ainode/ainode/core/util/decorator.py
similarity index 100%
rename from iotdb-core/ainode/ai/ainode/util/decorator.py
rename to iotdb-core/ainode/ainode/core/util/decorator.py
diff --git a/iotdb-core/ainode/ai/ainode/util/lock.py b/iotdb-core/ainode/ainode/core/util/lock.py
similarity index 100%
rename from iotdb-core/ainode/ai/ainode/util/lock.py
rename to iotdb-core/ainode/ainode/core/util/lock.py
diff --git a/iotdb-core/ainode/ai/ainode/util/serde.py b/iotdb-core/ainode/ainode/core/util/serde.py
similarity index 99%
rename from iotdb-core/ainode/ai/ainode/util/serde.py
rename to iotdb-core/ainode/ainode/core/util/serde.py
index c6b677896c8..b9edccfd03e 100644
--- a/iotdb-core/ainode/ai/ainode/util/serde.py
+++ b/iotdb-core/ainode/ainode/core/util/serde.py
@@ -21,7 +21,7 @@ from enum import Enum
import numpy as np
import pandas as pd
-from ai.ainode.exception import BadConfigValueError
+from ainode.core.exception import BadConfigValueError
class TSDataType(Enum):
diff --git a/iotdb-core/ainode/ai/ainode/util/status.py b/iotdb-core/ainode/ainode/core/util/status.py
similarity index 90%
rename from iotdb-core/ainode/ai/ainode/util/status.py
rename to iotdb-core/ainode/ainode/core/util/status.py
index 8cb983b5b44..37368b0068b 100644
--- a/iotdb-core/ainode/ai/ainode/util/status.py
+++ b/iotdb-core/ainode/ainode/core/util/status.py
@@ -16,9 +16,9 @@
# under the License.
#
-from ai.ainode.constant import TSStatusCode
-from ai.ainode.log import Logger
-from ai.thrift.common.ttypes import TSStatus
+from ainode.core.constant import TSStatusCode
+from ainode.core.log import Logger
+from ainode.thrift.common.ttypes import TSStatus
def get_status(status_code: TSStatusCode, message: str = None) -> TSStatus:
diff --git a/iotdb-core/ainode/pom.xml b/iotdb-core/ainode/pom.xml
index ad6679c7701..ee3137cf664 100644
--- a/iotdb-core/ainode/pom.xml
+++ b/iotdb-core/ainode/pom.xml
@@ -67,7 +67,7 @@
<directory>dist</directory>
</fileset>
<fileset>
- <directory>iotdb</directory>
+ <directory>ainode</directory>
<includes>
<include>conf/</include>
<include>thrift/</include>
@@ -138,7 +138,7 @@
<goal>copy-resources</goal>
</goals>
<configuration>
- <outputDirectory>${basedir}/iotdb/thrift/</outputDirectory>
+ <outputDirectory>${basedir}/ainode/thrift/</outputDirectory>
<resources>
<resource>
<directory>${basedir}/../../iotdb-protocol/thrift-commons/target/generated-sources-python/iotdb/thrift/</directory>
@@ -167,7 +167,7 @@
<goal>copy-resources</goal>
</goals>
<configuration>
- <outputDirectory>${basedir}/iotdb/conf/</outputDirectory>
+ <outputDirectory>${basedir}/ainode/conf/</outputDirectory>
<resources>
<resource>
<directory>${basedir}/resources/</directory>
@@ -192,7 +192,7 @@
</goals>
<configuration>
<generateGitPropertiesFile>true</generateGitPropertiesFile>
- <generateGitPropertiesFilename>${project.basedir}/iotdb/conf/git.properties</generateGitPropertiesFilename>
+ <generateGitPropertiesFilename>${project.basedir}/ainode/conf/git.properties</generateGitPropertiesFilename>
<includeOnlyProperties>
<includeOnlyProperty>^git.commit.id.abbrev$</includeOnlyProperty>
<includeOnlyProperty>^git.dirty$</includeOnlyProperty>
diff --git a/iotdb-core/ainode/pyproject.toml b/iotdb-core/ainode/pyproject.toml
index 161757d21c7..4f80ca6b03b 100644
--- a/iotdb-core/ainode/pyproject.toml
+++ b/iotdb-core/ainode/pyproject.toml
@@ -34,15 +34,15 @@ classifiers = [
"Topic :: Software Development :: Libraries :: Python Modules",
]
include = [
- {path = "iotdb/thrift/*", format = "wheel"},
- {path = "iotdb/thrift/common/*", format = "wheel"},
- {path = "iotdb/thrift/confignode/*", format = "wheel"},
- {path = "iotdb/thrift/datanode/*", format = "wheel"},
- {path = "iotdb/thrift/ainode/*", format = "wheel"},
- {path = "iotdb/conf/*", format = "wheel"},
+ {path = "ainode/thrift/*", format = "wheel"},
+ {path = "ainode/thrift/common/*", format = "wheel"},
+ {path = "ainode/thrift/confignode/*", format = "wheel"},
+ {path = "ainode/thrift/datanode/*", format = "wheel"},
+ {path = "ainode/thrift/ainode/*", format = "wheel"},
+ {path = "ainode/conf/*", format = "wheel"},
]
packages = [
- { include = "iotdb" }
+ { include = "ainode" }
]
[tool.poetry.dependencies]
@@ -53,7 +53,7 @@ pandas = "^1.3.5"
torch = ">=2.2.0"
pylru = "^1.2.1"
-thrift = "^0.13.0"
+thrift = ">=0.14.0"
dynaconf = "^3.1.11"
requests = "^2.31.0"
optuna = "^3.2.0"
@@ -63,4 +63,4 @@ pmdarima = "^2.0.4"
hmmlearn = "^0.3.0"
[tool.poetry.scripts]
-ainode = "iotdb.ainode.script:main"
\ No newline at end of file
+ainode = "ainode.core.script:main"
\ No newline at end of file
diff --git a/iotdb-core/ainode/resources/conf/iotdb-ainode.properties b/iotdb-core/ainode/resources/conf/iotdb-ainode.properties
index 2b208e72125..207bfcf9bef 100644
--- a/iotdb-core/ainode/resources/conf/iotdb-ainode.properties
+++ b/iotdb-core/ainode/resources/conf/iotdb-ainode.properties
@@ -40,19 +40,19 @@
# The starting directory of the relative path is related to the operating system.
# It is recommended to use an absolute path.
# Datatype: String
-# ain_system_dir=data/ainode/system
+# ain_system_dir=data/core/system
# The path where AINode stores model files
# The starting directory of the relative path is related to the operating system.
# It is recommended to use an absolute path.
# Datatype: String
-# ain_models_dir=data/ainode/models
+# ain_models_dir=data/core/models
# The path where AINode stores logs
# The starting directory of the relative path is related to the operating system.
# It is recommended to use an absolute path.
# Datatype: String
-# ain_logs_dir=logs/ainode
+# ain_logs_dir=logs/core
# Whether to use compression in Thrift
# Please use 0 or 1
diff --git a/iotdb-core/ainode/resources/sbin/remove-ainode.sh b/iotdb-core/ainode/resources/sbin/remove-ainode.sh
index 2a27661460d..24074f7064a 100755
--- a/iotdb-core/ainode/resources/sbin/remove-ainode.sh
+++ b/iotdb-core/ainode/resources/sbin/remove-ainode.sh
@@ -74,7 +74,7 @@ else
ain_interpreter_dir="$p_ain_interpreter_dir"
fi
-# If ain_system_dir is empty, set default value to ../data/ainode/system
+# If ain_system_dir is empty, set default value to ../data/core/system
if [ -z "$ain_system_dir" ]
then
ain_system_dir="$SCRIPT_DIR/../data/ainode/system"
@@ -89,7 +89,7 @@ fi
# Change the working directory to the parent directory
cd "$SCRIPT_DIR/.."
-ain_ainode_dir=$(dirname "$ain_interpreter_dir")/ainode
+ain_ainode_dir=$(dirname "$ain_interpreter_dir")/core
if [ -z "$p_ain_remove_target" ]; then
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ModelManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ModelManager.java
index ad19be00041..8a98d473a86 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ModelManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ModelManager.java
@@ -153,11 +153,11 @@ public class ModelManager {
modelInformation.updateStatus(ModelStatus.values()[req.getModelStatus()]);
modelInformation.setAttribute(req.getAttributes());
modelInformation.setInputColumnSize(1);
- if (req.getOutputShapesSize() > 0) {
- modelInformation.setOutputLength(req.getOutputShapes().get(0));
+ if (req.isSetOutputLength()) {
+ modelInformation.setOutputLength(req.getOutputLength());
}
- if (req.getInputShapesSize() > 0) {
- modelInformation.setInputLength(req.getInputShapes().get(0));
+ if (req.isSetInputLength()) {
+ modelInformation.setInputLength(req.getInputLength());
}
UpdateModelInfoPlan updateModelInfoPlan =
new UpdateModelInfoPlan(req.getModelId(), modelInformation);
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
index 9cdb6af607b..71ca36612c2 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
@@ -1292,8 +1292,9 @@ public class ProcedureManager {
new CreateTrainingProcedure(
req.modelId,
req.curDatabase,
- req.targetTables == null ? Collections.emptyList() : req.targetTables,
- req.targetDbs == null ? Collections.emptyList() : req.targetDbs,
+ req.isSetTargetTables() ? req.getTargetTables() : Collections.emptyList(),
+ req.isSetTargetDbs() ? req.targetDbs : Collections.emptyList(),
+ req.isSetParameters() ? req.getParameters() : Collections.emptyMap(),
req.useAllData));
LOGGER.info("CreateModelProcedure was submitted, procedureId: {}.", procedureId);
return RpcUtils.SUCCESS_STATUS;
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/model/CreateTrainingProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/model/CreateTrainingProcedure.java
index 5257a86ffe5..dbd84ddf1b7 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/model/CreateTrainingProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/model/CreateTrainingProcedure.java
@@ -34,6 +34,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
public class CreateTrainingProcedure extends AbstractNodeProcedure<CreateTrainingState> {
@@ -44,6 +45,7 @@ public class CreateTrainingProcedure extends AbstractNodeProcedure<CreateTrainin
private List<String> targetTables;
private List<String> targetDbs;
private boolean useAllData;
+ private Map<String, String> parameters;
private static final IClientManager<ConfigRegionId, ConfigNodeClient> CONFIG_NODE_CLIENT_MANAGER =
ConfigNodeClientManager.getInstance();
@@ -53,12 +55,14 @@ public class CreateTrainingProcedure extends AbstractNodeProcedure<CreateTrainin
String curDatabase,
List<String> targetTables,
List<String> targetDbs,
+ Map<String, String> parameters,
boolean useAllData) {
this.modelId = modelId;
this.curDatabase = curDatabase;
this.targetTables = targetTables;
this.targetDbs = targetDbs;
this.useAllData = useAllData;
+ this.parameters = parameters;
}
public CreateTrainingProcedure() {
@@ -77,9 +81,9 @@ public class CreateTrainingProcedure extends AbstractNodeProcedure<CreateTrainin
// trainingReq.setExistingModelId(createTraining.getExistingModelId());
// }
- // if (createTraining.getParameters() != null) {
- // trainingReq.setParameters(createTraining.getParameters());
- // }
+ if (!parameters.isEmpty()) {
+ trainingReq.setParameters(parameters);
+ }
List<ITableSchema> tableSchemaList = new ArrayList<>();
@@ -111,7 +115,6 @@ public class CreateTrainingProcedure extends AbstractNodeProcedure<CreateTrainin
tableSchemaList.add(new ITableSchema(curDatabase, tableName));
}
trainingReq.setTargetTables(tableSchemaList);
-
try (AINodeClient client =
AINodeClientManager.getInstance().borrowClient(AINodeInfo.endPoint)) {
status = client.createTrainingTask(trainingReq);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java
index cbabf929375..d8e5e23f6dd 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java
@@ -63,8 +63,8 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterDB;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ClearCache;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateDB;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateFunction;
-import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateTraining;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateTable;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateTraining;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DeleteDevice;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DescribeTable;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropColumn;
@@ -97,6 +97,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCurrentUser;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDB;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDataNodes;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowFunctions;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowModels;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowRegions;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowTables;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowVariables;
@@ -441,6 +442,7 @@ public class Coordinator {
|| statement instanceof ReconstructRegion
|| statement instanceof ExtendRegion
|| statement instanceof CreateTraining
+ || statement instanceof ShowModels
|| statement instanceof RemoveRegion) {
return new ConfigExecution(
queryContext,
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java
index cb0c2a53166..655bb77ad16 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java
@@ -46,6 +46,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowPipePl
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowRegionTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowVariablesTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ai.CreateTrainingTask;
+import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ai.ShowModelsTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.ExtendRegionTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.MigrateRegionTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.region.ReconstructRegionTask;
@@ -158,6 +159,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCurrentUser;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDB;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDataNodes;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowFunctions;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowModels;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowPipePlugins;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowPipes;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowRegions;
@@ -1095,4 +1097,9 @@ public class TableConfigTaskVisitor extends AstVisitor<IConfigTask, MPPQueryCont
node.setCurDatabase(curDatabase);
return new CreateTrainingTask(node);
}
+
+ @Override
+ protected IConfigTask visitShowModels(ShowModels node, MPPQueryContext context) {
+ return new ShowModelsTask(node.getModelId());
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
index 42010e3f33c..4583ee9c525 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
@@ -3147,6 +3147,7 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor {
CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) {
final TCreateTrainingReq req =
new TCreateTrainingReq(createTraining.getModelId(), createTraining.getCurDatabase());
+ req.setParameters(createTraining.getParameters());
req.setTargetDbs(createTraining.getTargetDbs());
req.setUseAllData(createTraining.isUseAllData());
req.setTargetTables(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java
index 3f3e2562fca..03456444001 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java
@@ -672,4 +672,8 @@ public abstract class AstVisitor<R, C> {
protected R visitCreateTraining(CreateTraining node, C context) {
return visitStatement(node, context);
}
+
+ protected R visitShowModels(ShowModels node, C context) {
+ return visitStatement(node, context);
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowModels.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowModels.java
new file mode 100644
index 00000000000..a3b803636e0
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowModels.java
@@ -0,0 +1,55 @@
+package org.apache.iotdb.db.queryengine.plan.relational.sql.ast;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+import static com.google.common.base.MoreObjects.toStringHelper;
+
+public class ShowModels extends Statement {
+
+ private String modelId = null;
+
+ public ShowModels() {
+ super(null);
+ }
+
+ @Override
+ public List<? extends Node> getChildren() {
+ return ImmutableList.of();
+ }
+
+ public void setModelId(String modelId) {
+ this.modelId = modelId;
+ }
+
+ public String getModelId() {
+ return modelId;
+ }
+
+ @Override
+ public <R, C> R accept(AstVisitor<R, C> visitor, C context) {
+ return visitor.visitShowModels(this, context);
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof ShowModels)) {
+ return false;
+ }
+ if (modelId == null) {
+ return ((ShowModels) obj).getModelId() == null;
+ }
+ return modelId.equals(((ShowModels) obj).getModelId());
+ }
+
+ @Override
+ public String toString() {
+ return toStringHelper(this).toString();
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java
index 820c54c3bdf..408a1192104 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java
@@ -158,6 +158,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDataNodes;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDevice;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowFunctions;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowIndex;
+import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowModels;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowPipePlugins;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowPipes;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowRegions;
@@ -2809,6 +2810,15 @@ public class AstBuilder extends RelationalSqlBaseVisitor<Node> {
return createTraining;
}
+ @Override
+ public Node visitShowModelsStatement(RelationalSqlParser.ShowModelsStatementContext ctx) {
+ ShowModels showModels = new ShowModels();
+ if (ctx.modelId != null) {
+ showModels.setModelId(ctx.modelId.getText());
+ }
+ return showModels;
+ }
+
// ***************** arguments *****************
@Override
public Node visitGenericType(RelationalSqlParser.GenericTypeContext ctx) {
diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4
index 91a8456f925..272f8cd303b 100644
--- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4
+++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4
@@ -149,6 +149,8 @@ statement
// AI
| createModelStatement
+ | showModelsStatement
+ | callInferenceStatement
// View, Trigger, pipe, CQ, Quota are not supported yet
;
@@ -734,6 +736,15 @@ hparamPair
: hparamKey=identifier '=' hyparamValue=primaryExpression
;
+showModelsStatement
+ : SHOW MODELS
+ | SHOW MODELS modelId=identifier
+ ;
+
+callInferenceStatement
+ : CALL INFERENCE '(' modelId=identifier COMMA inputSql=string (COMMA hparamPair)* ')'
+ ;
+
// ------------------------------------------- Query Statement ---------------------------------------------------------
queryStatement
: query #statementDefault
@@ -1135,7 +1146,7 @@ nonReserved
| JSON
| KEEP | KEY | KEYS | KILL
| LANGUAGE | LAST | LATERAL | LEADING | LEAVE | LEVEL | LIMIT | LINEAR | LOAD | LOCAL | LOGICAL | LOOP
- | MAP | MATCH | MATCHED | MATCHES | MATCH_RECOGNIZE | MATERIALIZED | MEASURES | METHOD | MERGE | MICROSECOND | MIGRATE | MILLISECOND | MINUTE | MODEL | MODIFY | MONTH
+ | MAP | MATCH | MATCHED | MATCHES | MATCH_RECOGNIZE | MATERIALIZED | MEASURES | METHOD | MERGE | MICROSECOND | MIGRATE | MILLISECOND | MINUTE | MODEL | MODELS | MODIFY | MONTH
| NANOSECOND | NESTED | NEXT | NFC | NFD | NFKC | NFKD | NO | NODEID | NONE | NULLIF | NULLS
| OBJECT | OF | OFFSET | OMIT | ONE | ONLY | OPTION | ORDINALITY | OUTPUT | OVER | OVERFLOW
| PARTITION | PARTITIONS | PASSING | PAST | PATH | PATTERN | PER | PERIOD | PERMUTE | PIPE | PIPEPLUGIN | PIPEPLUGINS | PIPES | PLAN | POSITION | PRECEDING | PRECISION | PRIVILEGES | PREVIOUS | PROCESSLIST | PROCESSOR | PROPERTIES | PRUNE
@@ -1293,6 +1304,7 @@ IGNORE: 'IGNORE';
IMMEDIATE: 'IMMEDIATE';
IN: 'IN';
INCLUDING: 'INCLUDING';
+INFERENCE: 'INFERENCE';
INITIAL: 'INITIAL';
INNER: 'INNER';
INPUT: 'INPUT';
@@ -1352,6 +1364,7 @@ MIGRATE: 'MIGRATE';
MILLISECOND: 'MS';
MINUTE: 'MINUTE' | 'M';
MODEL: 'MODEL';
+MODELS: 'MODELS';
MODIFY: 'MODIFY';
MONTH: 'MONTH' | 'MO';
NANOSECOND: 'NS';
diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
index 9b7e225fb84..8eca91b428d 100644
--- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
+++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
@@ -1040,8 +1040,8 @@ struct TUpdateModelInfoReq {
2: required i32 modelStatus
3: optional string attributes
4: optional list<i32> aiNodeIds
- 5: optional list<i32> inputShapes
- 6: optional list<i32> outputShapes
+ 5: optional i32 inputLength
+ 6: optional i32 outputLength
}
struct TCreateTrainingReq {
@@ -1050,6 +1050,7 @@ struct TCreateTrainingReq {
3: optional list<string> targetTables
4: optional list<string> targetDbs
5: optional bool useAllData
+ 6: optional map<string, string> parameters
}
// ====================================================