You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yo...@apache.org on 2025/08/04 12:40:29 UTC
(iotdb) branch master updated: [AINode] Support dynamic adjustment for pool size (#16079)
This is an automated email from the ASF dual-hosted git repository.
yongzao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 738470121dd [AINode] Support dynamic adjustment for pool size (#16079)
738470121dd is described below
commit 738470121ddf10c18229e69bf4eebafc1384f745
Author: Leo <16...@users.noreply.github.com>
AuthorDate: Mon Aug 4 20:40:24 2025 +0800
[AINode] Support dynamic adjustment for pool size (#16079)
---
iotdb-core/ainode/ainode/core/ai_node.py | 2 +
iotdb-core/ainode/ainode/core/config.py | 52 +++++++-
iotdb-core/ainode/ainode/core/constant.py | 9 ++
.../core/inference/inference_request_pool.py | 39 +++---
.../core/inference/scheduler/basic_scheduler.py | 2 +-
.../strategy/timerxl_inference_pipeline.py | 51 ++++++++
.../ainode/core/manager/inference_manager.py | 134 ++++++++++++++++-----
iotdb-core/ainode/ainode/core/manager/utils.py | 109 +++++++++++++++++
iotdb-core/ainode/ainode/core/rpc/service.py | 5 +-
.../ainode/resources/conf/iotdb-ainode.properties | 8 ++
10 files changed, 367 insertions(+), 44 deletions(-)
diff --git a/iotdb-core/ainode/ainode/core/ai_node.py b/iotdb-core/ainode/ainode/core/ai_node.py
index fc03e67921f..71536344391 100644
--- a/iotdb-core/ainode/ainode/core/ai_node.py
+++ b/iotdb-core/ainode/ainode/core/ai_node.py
@@ -162,4 +162,6 @@ class AINode:
if self._rpc_service:
self._rpc_service.stop()
self._rpc_service.join(1)
+ if self._rpc_service.is_alive():
+ logger.warning("RPC service thread failed to stop in time.")
logger.info("IoTDB-AINode has successfully stopped.")
diff --git a/iotdb-core/ainode/ainode/core/config.py b/iotdb-core/ainode/ainode/core/config.py
index 5126d0e53e3..d0244b3784e 100644
--- a/iotdb-core/ainode/ainode/core/config.py
+++ b/iotdb-core/ainode/ainode/core/config.py
@@ -31,7 +31,10 @@ from ainode.core.constant import (
AINODE_CONF_GIT_FILE_NAME,
AINODE_CONF_POM_FILE_NAME,
AINODE_INFERENCE_BATCH_INTERVAL_IN_MS,
+ AINODE_INFERENCE_EXTRA_MEMORY_RATIO,
AINODE_INFERENCE_MAX_PREDICT_LENGTH,
+ AINODE_INFERENCE_MEMORY_USAGE_RATIO,
+ AINODE_INFERENCE_MODEL_MEM_USAGE_MAP,
AINODE_LOG_DIR,
AINODE_MODELS_DIR,
AINODE_ROOT_CONF_DIRECTORY_NAME,
@@ -76,7 +79,15 @@ class AINodeConfig(object):
self._ain_inference_max_predict_length: int = (
AINODE_INFERENCE_MAX_PREDICT_LENGTH
)
-
+ self._ain_inference_model_mem_usage_map: dict[str, int] = (
+ AINODE_INFERENCE_MODEL_MEM_USAGE_MAP
+ )
+ self._ain_inference_memory_usage_ratio: float = (
+ AINODE_INFERENCE_MEMORY_USAGE_RATIO
+ )
+ self._ain_inference_extra_memory_ratio: float = (
+ AINODE_INFERENCE_EXTRA_MEMORY_RATIO
+ )
# log directory
self._ain_logs_dir: str = AINODE_LOG_DIR
@@ -152,6 +163,30 @@ class AINodeConfig(object):
) -> None:
self._ain_inference_max_predict_length = ain_inference_max_predict_length
+ def get_ain_inference_model_mem_usage_map(self) -> dict[str, int]:
+ return self._ain_inference_model_mem_usage_map
+
+ def set_ain_inference_model_mem_usage_map(
+ self, ain_inference_model_mem_usage_map: dict[str, int]
+ ) -> None:
+ self._ain_inference_model_mem_usage_map = ain_inference_model_mem_usage_map
+
+ def get_ain_inference_memory_usage_ratio(self) -> float:
+ return self._ain_inference_memory_usage_ratio
+
+ def set_ain_inference_memory_usage_ratio(
+ self, ain_inference_memory_usage_ratio: float
+ ) -> None:
+ self._ain_inference_memory_usage_ratio = ain_inference_memory_usage_ratio
+
+ def get_ain_inference_extra_memory_ratio(self) -> float:
+ return self._ain_inference_extra_memory_ratio
+
+ def set_ain_inference_extra_memory_ratio(
+ self, ain_inference_extra_memory_ratio: float
+ ) -> None:
+ self._ain_inference_extra_memory_ratio = ain_inference_extra_memory_ratio
+
def get_ain_logs_dir(self) -> str:
return self._ain_logs_dir
@@ -294,6 +329,21 @@ class AINodeDescriptor(object):
int(file_configs["ain_inference_batch_interval_in_ms"])
)
+ if "ain_inference_model_mem_usage_map" in config_keys:
+ self._config.set_ain_inference_model_mem_usage_map(
+ eval(file_configs["ain_inference_model_mem_usage_map"])
+ )
+
+ if "ain_inference_memory_usage_ratio" in config_keys:
+ self._config.set_ain_inference_memory_usage_ratio(
+ float(file_configs["ain_inference_memory_usage_ratio"])
+ )
+
+ if "ain_inference_extra_memory_ratio" in config_keys:
+ self._config.set_ain_inference_extra_memory_ratio(
+ float(file_configs["ain_inference_extra_memory_ratio"])
+ )
+
if "ain_models_dir" in config_keys:
self._config.set_ain_models_dir(file_configs["ain_models_dir"])
diff --git a/iotdb-core/ainode/ainode/core/constant.py b/iotdb-core/ainode/ainode/core/constant.py
index c04985d8287..0665caf090b 100644
--- a/iotdb-core/ainode/ainode/core/constant.py
+++ b/iotdb-core/ainode/ainode/core/constant.py
@@ -21,6 +21,7 @@ import os
from enum import Enum
from typing import List
+from ainode.core.model.model_info import BuiltInModelType
from ainode.thrift.common.ttypes import TEndPoint
AINODE_VERSION_INFO = "UNKNOWN"
@@ -51,6 +52,14 @@ DEFAULT_RECONNECT_TIMES = 3
# AINode inference configuration
AINODE_INFERENCE_BATCH_INTERVAL_IN_MS = 15
AINODE_INFERENCE_MAX_PREDICT_LENGTH = 2880
+AINODE_INFERENCE_MODEL_MEM_USAGE_MAP = {
+ BuiltInModelType.SUNDIAL.value: 1036 * 1024**2, # 1036 MiB
+ BuiltInModelType.TIMER_XL.value: 856 * 1024**2, # 856 MiB
+} # the memory usage of each model in bytes
+AINODE_INFERENCE_MEMORY_USAGE_RATIO = 0.4 # the device space allocated for inference
+AINODE_INFERENCE_EXTRA_MEMORY_RATIO = (
+ 1.2 # the overhead ratio for inference, used to estimate the pool size
+)
# AINode folder structure
AINODE_ROOT_DIR = os.path.dirname(
diff --git a/iotdb-core/ainode/ainode/core/inference/inference_request_pool.py b/iotdb-core/ainode/ainode/core/inference/inference_request_pool.py
index f16fc93a511..4bf594860f7 100644
--- a/iotdb-core/ainode/ainode/core/inference/inference_request_pool.py
+++ b/iotdb-core/ainode/ainode/core/inference/inference_request_pool.py
@@ -50,6 +50,7 @@ class InferenceRequestPool(mp.Process):
config: PretrainedConfig,
request_queue: mp.Queue,
result_queue: mp.Queue,
+ ready_event,
**pool_kwargs,
):
super().__init__()
@@ -59,11 +60,8 @@ class InferenceRequestPool(mp.Process):
self.pool_kwargs = pool_kwargs
self.model = None
self._model_manager = None
- # TODO: Assign device immediately when the pool is created
self.device = None
- self.logger = Logger(
- INFERENCE_LOG_FILE_NAME_PREFIX_TEMPLATE.format(self.device)
- )
+ self.ready_event = ready_event
self._threads = []
self._waiting_queue = request_queue # Requests that are waiting to be processed
@@ -128,15 +126,25 @@ class InferenceRequestPool(mp.Process):
requests = self._scheduler.schedule_step()
# TODO: We need a batcher to accelerate the concurrent inference
for request in requests:
- request.inputs = request.inputs.to(self.device)
- output = self.model.generate(
- request.inputs,
- max_new_tokens=request.max_new_tokens,
- num_samples=10,
- revin=True,
- )
- request.output_tensor = request.output_tensor.to(self.device)
- request.write_step_output(output[0].mean(dim=0))
+ if self.model_id == "sundial":
+ request.inputs = request.inputs.to(self.device)
+ output = self.model.generate(
+ request.inputs,
+ max_new_tokens=request.max_new_tokens,
+ num_samples=10,
+ revin=True,
+ )
+ request.output_tensor = request.output_tensor.to(self.device)
+ request.write_step_output(output[0].mean(dim=0))
+ elif self.model_id == "timer_xl":
+ request.inputs = request.inputs.to(self.device)
+ output = self.model.generate(
+ request.inputs,
+ max_new_tokens=request.max_new_tokens,
+ revin=True,
+ )
+ request.output_tensor = request.output_tensor.to(self.device)
+ request.write_step_output(output[0])
request.inference_pipeline.post_decode()
if request.is_finished():
request.inference_pipeline.post_inference()
@@ -160,8 +168,12 @@ class InferenceRequestPool(mp.Process):
def run(self):
self._model_manager = ModelManager()
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+ self.logger = Logger(
+ INFERENCE_LOG_FILE_NAME_PREFIX_TEMPLATE.format(self.device)
+ )
self._scheduler.device = self.device
self.model = self._model_manager.load_model(self.model_id, {}).to(self.device)
+ self.ready_event.set()
# self._warm_up_and_estimate_memory()
@@ -183,4 +195,3 @@ class InferenceRequestPool(mp.Process):
def stop(self):
self._stop_event.set()
- self.logger.info(f"[Inference][Pool-{self.pool_id}] stop() called")
diff --git a/iotdb-core/ainode/ainode/core/inference/scheduler/basic_scheduler.py b/iotdb-core/ainode/ainode/core/inference/scheduler/basic_scheduler.py
index 26d4b76ac30..65dee81ca1c 100644
--- a/iotdb-core/ainode/ainode/core/inference/scheduler/basic_scheduler.py
+++ b/iotdb-core/ainode/ainode/core/inference/scheduler/basic_scheduler.py
@@ -66,7 +66,7 @@ class BasicScheduler(AbstractScheduler):
)
logger.debug(
f"[Inference][Device-{self.device}][Pool-{self.pool_id}] "
- f"Memory used: {used} bytes, Max memory: {self.max_memory_bytes} bytes"
+ f"Memory used: {used/1024**2:.2f} MB, Max memory: {self.max_memory_bytes/1024**2:.2f} MB"
)
return used < self.max_memory_bytes
diff --git a/iotdb-core/ainode/ainode/core/inference/strategy/timerxl_inference_pipeline.py b/iotdb-core/ainode/ainode/core/inference/strategy/timerxl_inference_pipeline.py
new file mode 100644
index 00000000000..4add928cb38
--- /dev/null
+++ b/iotdb-core/ainode/ainode/core/inference/strategy/timerxl_inference_pipeline.py
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import torch
+
+from ainode.core.exception import InferenceModelInternalError
+from ainode.core.inference.strategy.abstract_inference_pipeline import (
+ AbstractInferencePipeline,
+)
+from ainode.core.model.timerxl.configuration_timer import TimerConfig
+
+
+class TimerXLInferencePipeline(AbstractInferencePipeline):
+ """
+ Strategy for Timer-XL model inference.
+ """
+
+ def __init__(self, model_config: TimerConfig, **infer_kwargs):
+ super().__init__(model_config, infer_kwargs=infer_kwargs)
+
+ def preprocess_inputs(self, inputs: torch.Tensor):
+ super().preprocess_inputs(inputs)
+ if len(inputs.shape) != 2:
+ raise InferenceModelInternalError(
+ f"[Inference] Input shape must be: [batch_size, seq_len], but receives {inputs.shape}"
+ )
+ # TODO: Disassemble and adapt with TimerXL's ts_generation_mixin.py
+ return inputs
+
+ def post_decode(self):
+ # TODO: Disassemble and adapt with TimerXL's ts_generation_mixin.py
+ pass
+
+ def post_inference(self):
+ # TODO: Disassemble and adapt with TimerXL's ts_generation_mixin.py
+ pass
diff --git a/iotdb-core/ainode/ainode/core/manager/inference_manager.py b/iotdb-core/ainode/ainode/core/manager/inference_manager.py
index 3cac5315b6f..19187778bac 100644
--- a/iotdb-core/ainode/ainode/core/manager/inference_manager.py
+++ b/iotdb-core/ainode/ainode/core/manager/inference_manager.py
@@ -15,12 +15,14 @@
# specific language governing permissions and limitations
# under the License.
#
+import gc
import threading
import time
from abc import ABC, abstractmethod
from typing import Dict, List
import pandas as pd
+import psutil
import torch
import torch.multiprocessing as mp
from iotdb.tsfile.utils.tsblock_serde import deserialize
@@ -41,11 +43,19 @@ from ainode.core.inference.inference_request_pool import InferenceRequestPool
from ainode.core.inference.strategy.timer_sundial_inference_pipeline import (
TimerSundialInferencePipeline,
)
+from ainode.core.inference.strategy.timerxl_inference_pipeline import (
+ TimerXLInferencePipeline,
+)
from ainode.core.inference.utils import generate_req_id
from ainode.core.log import Logger
from ainode.core.manager.model_manager import ModelManager
+from ainode.core.manager.utils import (
+ _estimate_pool_size,
+ _measure_model_memory,
+)
from ainode.core.model.sundial.configuration_sundial import SundialConfig
from ainode.core.model.sundial.modeling_sundial import SundialForPrediction
+from ainode.core.model.timerxl.configuration_timer import TimerConfig
from ainode.core.model.timerxl.modeling_timer import TimerForPrediction
from ainode.core.rpc.status import get_status
from ainode.core.util.serde import convert_to_binary
@@ -135,12 +145,9 @@ class RegisteredStrategy(InferenceStrategy):
class InferenceManager:
- ACCELERATE_MODEL_ID = "sundial"
- # DEFAULT_DEVICE = "cpu"
- DEFAULT_DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
- DEFAULT_POOL_SIZE = (
- 0 # TODO: Remove these parameter by sampling model inference consumption
- )
+ ACCELERATE_MODEL_ID = ["sundial", "timer_xl"]
+ DEFAULT_DEVICE = "cpu"
+ # DEFAULT_DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
WAITING_INTERVAL_IN_MS = (
AINodeDescriptor().get_config().get_ain_inference_batch_interval_in_ms()
) # How often to check for requests in the result queue
@@ -153,33 +160,81 @@ class InferenceManager:
# structure: {model_id: [(InferenceRequestPool, request_queue), ...]}
self._request_pool_map: Dict[str, List[(InferenceRequestPool, mp.Queue)]] = {}
self._stop_event = mp.Event()
- self._init_inference_request_pool()
self._result_handler_thread = threading.Thread(
target=self._handle_results, daemon=True
)
self._result_handler_thread.start()
+ self._model_mem_usage_map: Dict[str, int] = (
+ {}
+ ) # store model memory usage for each model
+ # self._preload_model_benchmarks()
+
+ def _preload_model_benchmarks(self):
+ if "cuda" in str(self.DEFAULT_DEVICE):
+ for model_id in self.ACCELERATE_MODEL_ID:
+ mem_usage = _measure_model_memory(self.DEFAULT_DEVICE, model_id)
+ self._model_mem_usage_map[model_id] = mem_usage
+ logger.info(
+ f"[Inference] Preloaded benchmark for {model_id}, mem_usage={mem_usage/1024**2:.2f} MB"
+ )
+ else:
+ logger.warning(
+ f"[Inference] Skipped preloading benchmarks for {self.DEFAULT_DEVICE}, only supports CUDA currently"
+ )
- def _init_inference_request_pool(self):
- """
- Initialize the inference request pool for each model.
- TODO: This is a temporary solution, we need a automatic algorithm to adjust the pool size for different models
- """
- self._request_pool_map[self.ACCELERATE_MODEL_ID] = []
- for idx in range(self.DEFAULT_POOL_SIZE):
- sundial_config = SundialConfig()
- request_queue = mp.Queue()
- request_pool = InferenceRequestPool(
- pool_id=idx,
- model_id=self.ACCELERATE_MODEL_ID,
- config=sundial_config,
- request_queue=request_queue,
- result_queue=self._result_queue,
+ def _first_pool_init(self, model_id: str):
+ if model_id == "sundial":
+ config = SundialConfig()
+ elif model_id == "timer_xl":
+ config = TimerConfig()
+ first_queue = mp.Queue()
+ ready_event = mp.Event()
+ first_pool = InferenceRequestPool(
+ pool_id=0,
+ model_id=model_id,
+ config=config,
+ request_queue=first_queue,
+ result_queue=self._result_queue,
+ ready_event=ready_event,
+ )
+ first_pool.start()
+ if not ready_event.wait(timeout=30):
+ logger.error(
+ f"[Inference][Device-{self.DEFAULT_DEVICE}][Pool-0] First pool failed to be ready in time"
)
- request_pool.start()
- self._request_pool_map[self.ACCELERATE_MODEL_ID].append(
- (request_pool, request_queue)
+ else:
+ self._request_pool_map[model_id] = [(first_pool, first_queue)]
+ logger.info(
+ f"[Inference][Device-{self.DEFAULT_DEVICE}][Pool-0] Initialized inference request pool for model {model_id}"
)
+ def _expand_pools(self, model_id, start_idx, count):
+ for idx in range(count):
+ queue = mp.Queue()
+ if model_id == "sundial":
+ config = SundialConfig()
+ elif model_id == "timer_xl":
+ config = TimerConfig()
+ pool = InferenceRequestPool(
+ pool_id=start_idx + idx,
+ model_id=model_id,
+ config=config,
+ request_queue=queue,
+ result_queue=self._result_queue,
+ ready_event=mp.Event(),
+ )
+ pool.start()
+ if not pool.ready_event.wait(timeout=30):
+ logger.error(
+ f"[Inference][Device-{self.DEFAULT_DEVICE}][Pool-{start_idx + idx}] Pool failed to be ready in time"
+ )
+ continue
+ else:
+ self._request_pool_map[model_id].append((pool, queue))
+ logger.info(
+ f"[Inference][Device-{self.DEFAULT_DEVICE}][Pool-{pool.pool_id}] New inference request pool started for model {model_id}"
+ )
+
def _handle_results(self):
while not self._stop_event.is_set():
if self._result_queue.empty():
@@ -229,7 +284,25 @@ class InferenceManager:
predict_length,
)
- if model_id == self.ACCELERATE_MODEL_ID and self.DEFAULT_POOL_SIZE > 0:
+ if model_id in self.ACCELERATE_MODEL_ID and "cuda" in str(
+ self.DEFAULT_DEVICE
+ ):
+ # lazy initialization for first request
+ if model_id not in self._request_pool_map:
+ pool_num = _estimate_pool_size(self.DEFAULT_DEVICE, model_id)
+ if pool_num <= 0:
+ raise InferenceModelInternalError(
+ f"Not enough memory to run model {model_id}."
+ )
+ # initialize the first pool
+ self._first_pool_init(model_id)
+ # start a background thread to expand pools
+ expand_thread = threading.Thread(
+ target=self._expand_pools,
+ args=(model_id, 1, pool_num - 1),
+ daemon=True,
+ )
+ expand_thread.start()
# TODO: Logic in this branch shall handle all LTSM inferences
# TODO: TSBlock -> Tensor codes should be unified
data = full_data[1][0]
@@ -237,10 +310,14 @@ class InferenceManager:
data = data.byteswap().newbyteorder()
# the inputs should be on CPU before passing to the inference request
inputs = torch.tensor(data).unsqueeze(0).float().to("cpu")
+ if model_id == "sundial":
+ inference_pipeline = TimerSundialInferencePipeline(SundialConfig())
+ elif model_id == "timer_xl":
+ inference_pipeline = TimerXLInferencePipeline(TimerConfig())
infer_req = InferenceRequest(
req_id=generate_req_id(),
inputs=inputs,
- inference_pipeline=TimerSundialInferencePipeline(SundialConfig()),
+ inference_pipeline=inference_pipeline,
max_new_tokens=predict_length,
)
infer_proxy = InferenceRequestProxy(infer_req.req_id)
@@ -250,6 +327,9 @@ class InferenceManager:
self._request_pool_map[model_id]
)
self._request_pool_map[model_id][pool_idx][1].put(infer_req)
+ logger.debug(
+ f"[Inference][Device-{self.DEFAULT_DEVICE}][Pool-{pool_idx}][ID-{infer_req.req_id}] Request is queued for inference"
+ )
outputs = infer_proxy.wait_for_completion()
outputs = convert_to_binary(pd.DataFrame(outputs[0]))
with self._result_wrapper_lock:
diff --git a/iotdb-core/ainode/ainode/core/manager/utils.py b/iotdb-core/ainode/ainode/core/manager/utils.py
new file mode 100644
index 00000000000..5fbd444c38c
--- /dev/null
+++ b/iotdb-core/ainode/ainode/core/manager/utils.py
@@ -0,0 +1,109 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import gc
+
+import psutil
+import torch
+
+from ainode.core.config import AINodeDescriptor
+from ainode.core.log import Logger
+from ainode.core.manager.model_manager import ModelManager
+from ainode.core.model.model_info import BUILT_IN_LTSM_MAP
+
+logger = Logger()
+
+MODEL_MEM_USAGE_MAP = (
+ AINodeDescriptor().get_config().get_ain_inference_model_mem_usage_map()
+) # the memory usage of each model in bytes
+INFERENCE_MEMORY_USAGE_RATIO = (
+ AINodeDescriptor().get_config().get_ain_inference_memory_usage_ratio()
+) # the device space allocated for inference
+INFERENCE_EXTRA_MEMORY_RATIO = (
+ AINodeDescriptor().get_config().get_ain_inference_extra_memory_ratio()
+) # the overhead ratio for inference, used to estimate the pool size
+
+
+def _measure_model_memory(device: torch.device, model_id: str) -> int:
+ # TODO: support CPU in the future
+ # TODO: we can estimate the memory usage by running a dummy inference
+ torch.cuda.empty_cache()
+ torch.cuda.synchronize(device)
+ start = torch.cuda.memory_reserved(device)
+
+ model = ModelManager().load_model(model_id, {}).to(device)
+ torch.cuda.synchronize(device)
+ end = torch.cuda.memory_reserved(device)
+ usage = end - start
+
+ # delete model to free memory
+ del model
+ torch.cuda.empty_cache()
+ gc.collect()
+
+ # add inference factor and cuda context overhead
+ overhead = 500 * 1024**2 # 500 MiB
+ final = int(max(usage, 1) * INFERENCE_EXTRA_MEMORY_RATIO + overhead)
+ return final
+
+
+def _evaluate_system_resources(device: torch.device) -> dict:
+ if torch.cuda.is_available():
+ free_mem, total_mem = torch.cuda.mem_get_info()
+ logger.info(
+ f"[Inference][Device-{device}] CUDA device memory: free={free_mem/1024**2:.2f} MB, total={total_mem/1024**2:.2f} MB"
+ )
+ return {"device": "cuda", "free_mem": free_mem, "total_mem": total_mem}
+ else:
+ # TODO: test cpu in the future
+ free_mem = psutil.virtual_memory().available
+ total_mem = psutil.virtual_memory().total
+ logger.info(
+ f"[Inference][Device-{device}] CPU memory: free={free_mem/1024**2:.2f} MB, total={total_mem/1024**2:.2f} MB"
+ )
+ return {"device": "cpu", "free_mem": free_mem, "total_mem": total_mem}
+
+
+def _estimate_pool_size(device: torch.device, model_id: str) -> int:
+ model_info = BUILT_IN_LTSM_MAP.get(model_id, None)
+ if model_info is None:
+ logger.error(f"[Inference][Device-{device}] Model {model_id} not found")
+ return 0
+
+ model_type = model_info.model_type
+ if model_type not in MODEL_MEM_USAGE_MAP:
+ logger.error(f"[Inference][Device-{device}] Model {model_id} not supported now")
+ return 0
+
+ system_res = _evaluate_system_resources(device)
+ free_mem = system_res["free_mem"]
+
+ mem_usage = MODEL_MEM_USAGE_MAP[model_type] * INFERENCE_EXTRA_MEMORY_RATIO
+ size = int((free_mem * INFERENCE_MEMORY_USAGE_RATIO) // mem_usage)
+ if size <= 0:
+ logger.error(
+ f"[Inference][Device-{device}] Not enough memory to run model {model_id}. free={free_mem/1024**2:.2f} MB, need>={mem_usage/1024**2:.2f} MB"
+ )
+ return 0
+
+ logger.info(
+ f"[Inference][Device-{device}] "
+ f"model={model_id}, mem_usage={mem_usage/1024**2:.2f} MB, "
+ f"pool_num={size}"
+ )
+ return size
diff --git a/iotdb-core/ainode/ainode/core/rpc/service.py b/iotdb-core/ainode/ainode/core/rpc/service.py
index 946fb53abda..b97abeee65b 100644
--- a/iotdb-core/ainode/ainode/core/rpc/service.py
+++ b/iotdb-core/ainode/ainode/core/rpc/service.py
@@ -49,12 +49,15 @@ class AINodeThreadPoolServer(TServer.TThreadPoolServer):
self.serverTransport.listen()
while not self._stop_event.is_set():
try:
- client = self.serverTransport.accept()
+ client = self.serverTransport.accept() # TODO: Fix the block problem
if not client:
continue
self.clients.put(client)
except Exception as x:
logger.error(x)
+ logger.info(
+ "The RPC service thread pool of IoTDB-AINode has successfully stopped."
+ )
def stop(self) -> None:
if not self._stop_event.is_set():
diff --git a/iotdb-core/ainode/resources/conf/iotdb-ainode.properties b/iotdb-core/ainode/resources/conf/iotdb-ainode.properties
index b31079ce00d..6fc9d852715 100644
--- a/iotdb-core/ainode/resources/conf/iotdb-ainode.properties
+++ b/iotdb-core/ainode/resources/conf/iotdb-ainode.properties
@@ -56,6 +56,14 @@ ain_cluster_ingress_password=root
# Datatype: String
ain_cluster_ingress_time_zone=UTC+8
+# The device space allocated for inference
+# Datatype: Float
+ain_inference_memory_usage_ratio=0.4
+
+# The overhead ratio for inference, used to estimate the pool size
+# Datatype: Float
+ain_inference_extra_memory_ratio=1.2
+
# The AINode metadata storage path.
# The starting directory of the relative path is related to the operating system.
# It is recommended to use an absolute path.