You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yo...@apache.org on 2025/08/04 12:40:29 UTC

(iotdb) branch master updated: [AINode] Support dynamic adjustment for pool size (#16079)

This is an automated email from the ASF dual-hosted git repository.

yongzao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new 738470121dd [AINode] Support dynamic adjustment for pool size (#16079)
738470121dd is described below

commit 738470121ddf10c18229e69bf4eebafc1384f745
Author: Leo <16...@users.noreply.github.com>
AuthorDate: Mon Aug 4 20:40:24 2025 +0800

    [AINode] Support dynamic adjustment for pool size (#16079)
---
 iotdb-core/ainode/ainode/core/ai_node.py           |   2 +
 iotdb-core/ainode/ainode/core/config.py            |  52 +++++++-
 iotdb-core/ainode/ainode/core/constant.py          |   9 ++
 .../core/inference/inference_request_pool.py       |  39 +++---
 .../core/inference/scheduler/basic_scheduler.py    |   2 +-
 .../strategy/timerxl_inference_pipeline.py         |  51 ++++++++
 .../ainode/core/manager/inference_manager.py       | 134 ++++++++++++++++-----
 iotdb-core/ainode/ainode/core/manager/utils.py     | 109 +++++++++++++++++
 iotdb-core/ainode/ainode/core/rpc/service.py       |   5 +-
 .../ainode/resources/conf/iotdb-ainode.properties  |   8 ++
 10 files changed, 367 insertions(+), 44 deletions(-)

diff --git a/iotdb-core/ainode/ainode/core/ai_node.py b/iotdb-core/ainode/ainode/core/ai_node.py
index fc03e67921f..71536344391 100644
--- a/iotdb-core/ainode/ainode/core/ai_node.py
+++ b/iotdb-core/ainode/ainode/core/ai_node.py
@@ -162,4 +162,6 @@ class AINode:
             if self._rpc_service:
                 self._rpc_service.stop()
                 self._rpc_service.join(1)
+                if self._rpc_service.is_alive():
+                    logger.warning("RPC service thread failed to stop in time.")
             logger.info("IoTDB-AINode has successfully stopped.")
diff --git a/iotdb-core/ainode/ainode/core/config.py b/iotdb-core/ainode/ainode/core/config.py
index 5126d0e53e3..d0244b3784e 100644
--- a/iotdb-core/ainode/ainode/core/config.py
+++ b/iotdb-core/ainode/ainode/core/config.py
@@ -31,7 +31,10 @@ from ainode.core.constant import (
     AINODE_CONF_GIT_FILE_NAME,
     AINODE_CONF_POM_FILE_NAME,
     AINODE_INFERENCE_BATCH_INTERVAL_IN_MS,
+    AINODE_INFERENCE_EXTRA_MEMORY_RATIO,
     AINODE_INFERENCE_MAX_PREDICT_LENGTH,
+    AINODE_INFERENCE_MEMORY_USAGE_RATIO,
+    AINODE_INFERENCE_MODEL_MEM_USAGE_MAP,
     AINODE_LOG_DIR,
     AINODE_MODELS_DIR,
     AINODE_ROOT_CONF_DIRECTORY_NAME,
@@ -76,7 +79,15 @@ class AINodeConfig(object):
         self._ain_inference_max_predict_length: int = (
             AINODE_INFERENCE_MAX_PREDICT_LENGTH
         )
-
+        self._ain_inference_model_mem_usage_map: dict[str, int] = (
+            AINODE_INFERENCE_MODEL_MEM_USAGE_MAP
+        )
+        self._ain_inference_memory_usage_ratio: float = (
+            AINODE_INFERENCE_MEMORY_USAGE_RATIO
+        )
+        self._ain_inference_extra_memory_ratio: float = (
+            AINODE_INFERENCE_EXTRA_MEMORY_RATIO
+        )
         # log directory
         self._ain_logs_dir: str = AINODE_LOG_DIR
 
@@ -152,6 +163,30 @@ class AINodeConfig(object):
     ) -> None:
         self._ain_inference_max_predict_length = ain_inference_max_predict_length
 
+    def get_ain_inference_model_mem_usage_map(self) -> dict[str, int]:
+        return self._ain_inference_model_mem_usage_map
+
+    def set_ain_inference_model_mem_usage_map(
+        self, ain_inference_model_mem_usage_map: dict[str, int]
+    ) -> None:
+        self._ain_inference_model_mem_usage_map = ain_inference_model_mem_usage_map
+
+    def get_ain_inference_memory_usage_ratio(self) -> float:
+        return self._ain_inference_memory_usage_ratio
+
+    def set_ain_inference_memory_usage_ratio(
+        self, ain_inference_memory_usage_ratio: float
+    ) -> None:
+        self._ain_inference_memory_usage_ratio = ain_inference_memory_usage_ratio
+
+    def get_ain_inference_extra_memory_ratio(self) -> float:
+        return self._ain_inference_extra_memory_ratio
+
+    def set_ain_inference_extra_memory_ratio(
+        self, ain_inference_extra_memory_ratio: float
+    ) -> None:
+        self._ain_inference_extra_memory_ratio = ain_inference_extra_memory_ratio
+
     def get_ain_logs_dir(self) -> str:
         return self._ain_logs_dir
 
@@ -294,6 +329,21 @@ class AINodeDescriptor(object):
                     int(file_configs["ain_inference_batch_interval_in_ms"])
                 )
 
+            if "ain_inference_model_mem_usage_map" in config_keys:
+                self._config.set_ain_inference_model_mem_usage_map(
+                    eval(file_configs["ain_inference_model_mem_usage_map"])
+                )
+
+            if "ain_inference_memory_usage_ratio" in config_keys:
+                self._config.set_ain_inference_memory_usage_ratio(
+                    float(file_configs["ain_inference_memory_usage_ratio"])
+                )
+
+            if "ain_inference_extra_memory_ratio" in config_keys:
+                self._config.set_ain_inference_extra_memory_ratio(
+                    float(file_configs["ain_inference_extra_memory_ratio"])
+                )
+
             if "ain_models_dir" in config_keys:
                 self._config.set_ain_models_dir(file_configs["ain_models_dir"])
 
diff --git a/iotdb-core/ainode/ainode/core/constant.py b/iotdb-core/ainode/ainode/core/constant.py
index c04985d8287..0665caf090b 100644
--- a/iotdb-core/ainode/ainode/core/constant.py
+++ b/iotdb-core/ainode/ainode/core/constant.py
@@ -21,6 +21,7 @@ import os
 from enum import Enum
 from typing import List
 
+from ainode.core.model.model_info import BuiltInModelType
 from ainode.thrift.common.ttypes import TEndPoint
 
 AINODE_VERSION_INFO = "UNKNOWN"
@@ -51,6 +52,14 @@ DEFAULT_RECONNECT_TIMES = 3
 # AINode inference configuration
 AINODE_INFERENCE_BATCH_INTERVAL_IN_MS = 15
 AINODE_INFERENCE_MAX_PREDICT_LENGTH = 2880
+AINODE_INFERENCE_MODEL_MEM_USAGE_MAP = {
+    BuiltInModelType.SUNDIAL.value: 1036 * 1024**2,  # 1036 MiB
+    BuiltInModelType.TIMER_XL.value: 856 * 1024**2,  # 856 MiB
+}  # the memory usage of each model in bytes
+AINODE_INFERENCE_MEMORY_USAGE_RATIO = 0.4  # the device space allocated for inference
+AINODE_INFERENCE_EXTRA_MEMORY_RATIO = (
+    1.2  # the overhead ratio for inference, used to estimate the pool size
+)
 
 # AINode folder structure
 AINODE_ROOT_DIR = os.path.dirname(
diff --git a/iotdb-core/ainode/ainode/core/inference/inference_request_pool.py b/iotdb-core/ainode/ainode/core/inference/inference_request_pool.py
index f16fc93a511..4bf594860f7 100644
--- a/iotdb-core/ainode/ainode/core/inference/inference_request_pool.py
+++ b/iotdb-core/ainode/ainode/core/inference/inference_request_pool.py
@@ -50,6 +50,7 @@ class InferenceRequestPool(mp.Process):
         config: PretrainedConfig,
         request_queue: mp.Queue,
         result_queue: mp.Queue,
+        ready_event,
         **pool_kwargs,
     ):
         super().__init__()
@@ -59,11 +60,8 @@ class InferenceRequestPool(mp.Process):
         self.pool_kwargs = pool_kwargs
         self.model = None
         self._model_manager = None
-        # TODO: Assign device immediately when the pool is created
         self.device = None
-        self.logger = Logger(
-            INFERENCE_LOG_FILE_NAME_PREFIX_TEMPLATE.format(self.device)
-        )
+        self.ready_event = ready_event
 
         self._threads = []
         self._waiting_queue = request_queue  # Requests that are waiting to be processed
@@ -128,15 +126,25 @@ class InferenceRequestPool(mp.Process):
         requests = self._scheduler.schedule_step()
         # TODO: We need a batcher to accelerate the concurrent inference
         for request in requests:
-            request.inputs = request.inputs.to(self.device)
-            output = self.model.generate(
-                request.inputs,
-                max_new_tokens=request.max_new_tokens,
-                num_samples=10,
-                revin=True,
-            )
-            request.output_tensor = request.output_tensor.to(self.device)
-            request.write_step_output(output[0].mean(dim=0))
+            if self.model_id == "sundial":
+                request.inputs = request.inputs.to(self.device)
+                output = self.model.generate(
+                    request.inputs,
+                    max_new_tokens=request.max_new_tokens,
+                    num_samples=10,
+                    revin=True,
+                )
+                request.output_tensor = request.output_tensor.to(self.device)
+                request.write_step_output(output[0].mean(dim=0))
+            elif self.model_id == "timer_xl":
+                request.inputs = request.inputs.to(self.device)
+                output = self.model.generate(
+                    request.inputs,
+                    max_new_tokens=request.max_new_tokens,
+                    revin=True,
+                )
+                request.output_tensor = request.output_tensor.to(self.device)
+                request.write_step_output(output[0])
             request.inference_pipeline.post_decode()
             if request.is_finished():
                 request.inference_pipeline.post_inference()
@@ -160,8 +168,12 @@ class InferenceRequestPool(mp.Process):
     def run(self):
         self._model_manager = ModelManager()
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.logger = Logger(
+            INFERENCE_LOG_FILE_NAME_PREFIX_TEMPLATE.format(self.device)
+        )
         self._scheduler.device = self.device
         self.model = self._model_manager.load_model(self.model_id, {}).to(self.device)
+        self.ready_event.set()
 
         # self._warm_up_and_estimate_memory()
 
@@ -183,4 +195,3 @@ class InferenceRequestPool(mp.Process):
 
     def stop(self):
         self._stop_event.set()
-        self.logger.info(f"[Inference][Pool-{self.pool_id}] stop() called")
diff --git a/iotdb-core/ainode/ainode/core/inference/scheduler/basic_scheduler.py b/iotdb-core/ainode/ainode/core/inference/scheduler/basic_scheduler.py
index 26d4b76ac30..65dee81ca1c 100644
--- a/iotdb-core/ainode/ainode/core/inference/scheduler/basic_scheduler.py
+++ b/iotdb-core/ainode/ainode/core/inference/scheduler/basic_scheduler.py
@@ -66,7 +66,7 @@ class BasicScheduler(AbstractScheduler):
             )
         logger.debug(
             f"[Inference][Device-{self.device}][Pool-{self.pool_id}] "
-            f"Memory used: {used} bytes, Max memory: {self.max_memory_bytes} bytes"
+            f"Memory used: {used/1024**2:.2f} MB, Max memory: {self.max_memory_bytes/1024**2:.2f} MB"
         )
         return used < self.max_memory_bytes
 
diff --git a/iotdb-core/ainode/ainode/core/inference/strategy/timerxl_inference_pipeline.py b/iotdb-core/ainode/ainode/core/inference/strategy/timerxl_inference_pipeline.py
new file mode 100644
index 00000000000..4add928cb38
--- /dev/null
+++ b/iotdb-core/ainode/ainode/core/inference/strategy/timerxl_inference_pipeline.py
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import torch
+
+from ainode.core.exception import InferenceModelInternalError
+from ainode.core.inference.strategy.abstract_inference_pipeline import (
+    AbstractInferencePipeline,
+)
+from ainode.core.model.timerxl.configuration_timer import TimerConfig
+
+
+class TimerXLInferencePipeline(AbstractInferencePipeline):
+    """
+    Strategy for Timer-XL model inference.
+    """
+
+    def __init__(self, model_config: TimerConfig, **infer_kwargs):
+        super().__init__(model_config, infer_kwargs=infer_kwargs)
+
+    def preprocess_inputs(self, inputs: torch.Tensor):
+        super().preprocess_inputs(inputs)
+        if len(inputs.shape) != 2:
+            raise InferenceModelInternalError(
+                f"[Inference] Input shape must be: [batch_size, seq_len], but receives {inputs.shape}"
+            )
+        # TODO: Disassemble and adapt with TimerXL's ts_generation_mixin.py
+        return inputs
+
+    def post_decode(self):
+        # TODO: Disassemble and adapt with TimerXL's ts_generation_mixin.py
+        pass
+
+    def post_inference(self):
+        # TODO: Disassemble and adapt with TimerXL's ts_generation_mixin.py
+        pass
diff --git a/iotdb-core/ainode/ainode/core/manager/inference_manager.py b/iotdb-core/ainode/ainode/core/manager/inference_manager.py
index 3cac5315b6f..19187778bac 100644
--- a/iotdb-core/ainode/ainode/core/manager/inference_manager.py
+++ b/iotdb-core/ainode/ainode/core/manager/inference_manager.py
@@ -15,12 +15,14 @@
 # specific language governing permissions and limitations
 # under the License.
 #
+import gc
 import threading
 import time
 from abc import ABC, abstractmethod
 from typing import Dict, List
 
 import pandas as pd
+import psutil
 import torch
 import torch.multiprocessing as mp
 from iotdb.tsfile.utils.tsblock_serde import deserialize
@@ -41,11 +43,19 @@ from ainode.core.inference.inference_request_pool import InferenceRequestPool
 from ainode.core.inference.strategy.timer_sundial_inference_pipeline import (
     TimerSundialInferencePipeline,
 )
+from ainode.core.inference.strategy.timerxl_inference_pipeline import (
+    TimerXLInferencePipeline,
+)
 from ainode.core.inference.utils import generate_req_id
 from ainode.core.log import Logger
 from ainode.core.manager.model_manager import ModelManager
+from ainode.core.manager.utils import (
+    _estimate_pool_size,
+    _measure_model_memory,
+)
 from ainode.core.model.sundial.configuration_sundial import SundialConfig
 from ainode.core.model.sundial.modeling_sundial import SundialForPrediction
+from ainode.core.model.timerxl.configuration_timer import TimerConfig
 from ainode.core.model.timerxl.modeling_timer import TimerForPrediction
 from ainode.core.rpc.status import get_status
 from ainode.core.util.serde import convert_to_binary
@@ -135,12 +145,9 @@ class RegisteredStrategy(InferenceStrategy):
 
 
 class InferenceManager:
-    ACCELERATE_MODEL_ID = "sundial"
-    # DEFAULT_DEVICE = "cpu"
-    DEFAULT_DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-    DEFAULT_POOL_SIZE = (
-        0  # TODO: Remove these parameter by sampling model inference consumption
-    )
+    ACCELERATE_MODEL_ID = ["sundial", "timer_xl"]
+    DEFAULT_DEVICE = "cpu"
+    # DEFAULT_DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
     WAITING_INTERVAL_IN_MS = (
         AINodeDescriptor().get_config().get_ain_inference_batch_interval_in_ms()
     )  # How often to check for requests in the result queue
@@ -153,33 +160,81 @@ class InferenceManager:
         # structure: {model_id: [(InferenceRequestPool, request_queue), ...]}
         self._request_pool_map: Dict[str, List[(InferenceRequestPool, mp.Queue)]] = {}
         self._stop_event = mp.Event()
-        self._init_inference_request_pool()
         self._result_handler_thread = threading.Thread(
             target=self._handle_results, daemon=True
         )
         self._result_handler_thread.start()
+        self._model_mem_usage_map: Dict[str, int] = (
+            {}
+        )  # store model memory usage for each model
+        # self._preload_model_benchmarks()
+
+    def _preload_model_benchmarks(self):
+        if "cuda" in str(self.DEFAULT_DEVICE):
+            for model_id in self.ACCELERATE_MODEL_ID:
+                mem_usage = _measure_model_memory(self.DEFAULT_DEVICE, model_id)
+                self._model_mem_usage_map[model_id] = mem_usage
+                logger.info(
+                    f"[Inference] Preloaded benchmark for {model_id}, mem_usage={mem_usage/1024**2:.2f} MB"
+                )
+        else:
+            logger.warning(
+                f"[Inference] Skipped preloading benchmarks for {self.DEFAULT_DEVICE}, only supports CUDA currently"
+            )
 
-    def _init_inference_request_pool(self):
-        """
-        Initialize the inference request pool for each model.
-        TODO: This is a temporary solution, we need a automatic algorithm to adjust the pool size for different models
-        """
-        self._request_pool_map[self.ACCELERATE_MODEL_ID] = []
-        for idx in range(self.DEFAULT_POOL_SIZE):
-            sundial_config = SundialConfig()
-            request_queue = mp.Queue()
-            request_pool = InferenceRequestPool(
-                pool_id=idx,
-                model_id=self.ACCELERATE_MODEL_ID,
-                config=sundial_config,
-                request_queue=request_queue,
-                result_queue=self._result_queue,
+    def _first_pool_init(self, model_id: str):
+        if model_id == "sundial":
+            config = SundialConfig()
+        elif model_id == "timer_xl":
+            config = TimerConfig()
+        first_queue = mp.Queue()
+        ready_event = mp.Event()
+        first_pool = InferenceRequestPool(
+            pool_id=0,
+            model_id=model_id,
+            config=config,
+            request_queue=first_queue,
+            result_queue=self._result_queue,
+            ready_event=ready_event,
+        )
+        first_pool.start()
+        if not ready_event.wait(timeout=30):
+            logger.error(
+                f"[Inference][Device-{self.DEFAULT_DEVICE}][Pool-0] First pool failed to be ready in time"
             )
-            request_pool.start()
-            self._request_pool_map[self.ACCELERATE_MODEL_ID].append(
-                (request_pool, request_queue)
+        else:
+            self._request_pool_map[model_id] = [(first_pool, first_queue)]
+            logger.info(
+                f"[Inference][Device-{self.DEFAULT_DEVICE}][Pool-0] Initialized inference request pool for model {model_id}"
             )
 
+    def _expand_pools(self, model_id, start_idx, count):
+        for idx in range(count):
+            queue = mp.Queue()
+            if model_id == "sundial":
+                config = SundialConfig()
+            elif model_id == "timer_xl":
+                config = TimerConfig()
+            pool = InferenceRequestPool(
+                pool_id=start_idx + idx,
+                model_id=model_id,
+                config=config,
+                request_queue=queue,
+                result_queue=self._result_queue,
+                ready_event=mp.Event(),
+            )
+            pool.start()
+            if not pool.ready_event.wait(timeout=30):
+                logger.error(
+                    f"[Inference][Device-{self.DEFAULT_DEVICE}][Pool-{start_idx + idx}] Pool failed to be ready in time"
+                )
+                continue
+            else:
+                self._request_pool_map[model_id].append((pool, queue))
+                logger.info(
+                    f"[Inference][Device-{self.DEFAULT_DEVICE}][Pool-{pool.pool_id}] New inference request pool started for model {model_id}"
+                )
+
     def _handle_results(self):
         while not self._stop_event.is_set():
             if self._result_queue.empty():
@@ -229,7 +284,25 @@ class InferenceManager:
                     predict_length,
                 )
 
-            if model_id == self.ACCELERATE_MODEL_ID and self.DEFAULT_POOL_SIZE > 0:
+            if model_id in self.ACCELERATE_MODEL_ID and "cuda" in str(
+                self.DEFAULT_DEVICE
+            ):
+                # lazy initialization for first request
+                if model_id not in self._request_pool_map:
+                    pool_num = _estimate_pool_size(self.DEFAULT_DEVICE, model_id)
+                    if pool_num <= 0:
+                        raise InferenceModelInternalError(
+                            f"Not enough memory to run model {model_id}."
+                        )
+                    # initialize the first pool
+                    self._first_pool_init(model_id)
+                    # start a background thread to expand pools
+                    expand_thread = threading.Thread(
+                        target=self._expand_pools,
+                        args=(model_id, 1, pool_num - 1),
+                        daemon=True,
+                    )
+                    expand_thread.start()
                 # TODO: Logic in this branch shall handle all LTSM inferences
                 # TODO: TSBlock -> Tensor codes should be unified
                 data = full_data[1][0]
@@ -237,10 +310,14 @@ class InferenceManager:
                     data = data.byteswap().newbyteorder()
                 # the inputs should be on CPU before passing to the inference request
                 inputs = torch.tensor(data).unsqueeze(0).float().to("cpu")
+                if model_id == "sundial":
+                    inference_pipeline = TimerSundialInferencePipeline(SundialConfig())
+                elif model_id == "timer_xl":
+                    inference_pipeline = TimerXLInferencePipeline(TimerConfig())
                 infer_req = InferenceRequest(
                     req_id=generate_req_id(),
                     inputs=inputs,
-                    inference_pipeline=TimerSundialInferencePipeline(SundialConfig()),
+                    inference_pipeline=inference_pipeline,
                     max_new_tokens=predict_length,
                 )
                 infer_proxy = InferenceRequestProxy(infer_req.req_id)
@@ -250,6 +327,9 @@ class InferenceManager:
                     self._request_pool_map[model_id]
                 )
                 self._request_pool_map[model_id][pool_idx][1].put(infer_req)
+                logger.debug(
+                    f"[Inference][Device-{self.DEFAULT_DEVICE}][Pool-{pool_idx}][ID-{infer_req.req_id}] Request is queued for inference"
+                )
                 outputs = infer_proxy.wait_for_completion()
                 outputs = convert_to_binary(pd.DataFrame(outputs[0]))
                 with self._result_wrapper_lock:
diff --git a/iotdb-core/ainode/ainode/core/manager/utils.py b/iotdb-core/ainode/ainode/core/manager/utils.py
new file mode 100644
index 00000000000..5fbd444c38c
--- /dev/null
+++ b/iotdb-core/ainode/ainode/core/manager/utils.py
@@ -0,0 +1,109 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import gc
+
+import psutil
+import torch
+
+from ainode.core.config import AINodeDescriptor
+from ainode.core.log import Logger
+from ainode.core.manager.model_manager import ModelManager
+from ainode.core.model.model_info import BUILT_IN_LTSM_MAP
+
+logger = Logger()
+
+MODEL_MEM_USAGE_MAP = (
+    AINodeDescriptor().get_config().get_ain_inference_model_mem_usage_map()
+)  # the memory usage of each model in bytes
+INFERENCE_MEMORY_USAGE_RATIO = (
+    AINodeDescriptor().get_config().get_ain_inference_memory_usage_ratio()
+)  # the device space allocated for inference
+INFERENCE_EXTRA_MEMORY_RATIO = (
+    AINodeDescriptor().get_config().get_ain_inference_extra_memory_ratio()
+)  # the overhead ratio for inference, used to estimate the pool size
+
+
+def _measure_model_memory(device: torch.device, model_id: str) -> int:
+    # TODO: support CPU in the future
+    # TODO: we can estimate the memory usage by running a dummy inference
+    torch.cuda.empty_cache()
+    torch.cuda.synchronize(device)
+    start = torch.cuda.memory_reserved(device)
+
+    model = ModelManager().load_model(model_id, {}).to(device)
+    torch.cuda.synchronize(device)
+    end = torch.cuda.memory_reserved(device)
+    usage = end - start
+
+    # delete model to free memory
+    del model
+    torch.cuda.empty_cache()
+    gc.collect()
+
+    # add inference factor and cuda context overhead
+    overhead = 500 * 1024**2  # 500 MiB
+    final = int(max(usage, 1) * INFERENCE_EXTRA_MEMORY_RATIO + overhead)
+    return final
+
+
+def _evaluate_system_resources(device: torch.device) -> dict:
+    if torch.cuda.is_available():
+        free_mem, total_mem = torch.cuda.mem_get_info()
+        logger.info(
+            f"[Inference][Device-{device}] CUDA device memory: free={free_mem/1024**2:.2f} MB, total={total_mem/1024**2:.2f} MB"
+        )
+        return {"device": "cuda", "free_mem": free_mem, "total_mem": total_mem}
+    else:
+        # TODO: test cpu in the future
+        free_mem = psutil.virtual_memory().available
+        total_mem = psutil.virtual_memory().total
+        logger.info(
+            f"[Inference][Device-{device}] CPU memory: free={free_mem/1024**2:.2f} MB, total={total_mem/1024**2:.2f} MB"
+        )
+        return {"device": "cpu", "free_mem": free_mem, "total_mem": total_mem}
+
+
+def _estimate_pool_size(device: torch.device, model_id: str) -> int:
+    model_info = BUILT_IN_LTSM_MAP.get(model_id, None)
+    if model_info is None:
+        logger.error(f"[Inference][Device-{device}] Model {model_id} not found")
+        return 0
+
+    model_type = model_info.model_type
+    if model_type not in MODEL_MEM_USAGE_MAP:
+        logger.error(f"[Inference][Device-{device}] Model {model_id} not supported now")
+        return 0
+
+    system_res = _evaluate_system_resources(device)
+    free_mem = system_res["free_mem"]
+
+    mem_usage = MODEL_MEM_USAGE_MAP[model_type] * INFERENCE_EXTRA_MEMORY_RATIO
+    size = int((free_mem * INFERENCE_MEMORY_USAGE_RATIO) // mem_usage)
+    if size <= 0:
+        logger.error(
+            f"[Inference][Device-{device}] Not enough memory to run model {model_id}. free={free_mem/1024**2:.2f} MB, need>={mem_usage/1024**2:.2f} MB"
+        )
+        return 0
+
+    logger.info(
+        f"[Inference][Device-{device}] "
+        f"model={model_id}, mem_usage={mem_usage/1024**2:.2f} MB, "
+        f"pool_num={size}"
+    )
+    return size
diff --git a/iotdb-core/ainode/ainode/core/rpc/service.py b/iotdb-core/ainode/ainode/core/rpc/service.py
index 946fb53abda..b97abeee65b 100644
--- a/iotdb-core/ainode/ainode/core/rpc/service.py
+++ b/iotdb-core/ainode/ainode/core/rpc/service.py
@@ -49,12 +49,15 @@ class AINodeThreadPoolServer(TServer.TThreadPoolServer):
         self.serverTransport.listen()
         while not self._stop_event.is_set():
             try:
-                client = self.serverTransport.accept()
+                client = self.serverTransport.accept()  # TODO: Fix the block problem
                 if not client:
                     continue
                 self.clients.put(client)
             except Exception as x:
                 logger.error(x)
+        logger.info(
+            "The RPC service thread pool of IoTDB-AINode has successfully stopped."
+        )
 
     def stop(self) -> None:
         if not self._stop_event.is_set():
diff --git a/iotdb-core/ainode/resources/conf/iotdb-ainode.properties b/iotdb-core/ainode/resources/conf/iotdb-ainode.properties
index b31079ce00d..6fc9d852715 100644
--- a/iotdb-core/ainode/resources/conf/iotdb-ainode.properties
+++ b/iotdb-core/ainode/resources/conf/iotdb-ainode.properties
@@ -56,6 +56,14 @@ ain_cluster_ingress_password=root
 # Datatype: String
 ain_cluster_ingress_time_zone=UTC+8
 
+# The device space allocated for inference
+# Datatype: Float
+ain_inference_memory_usage_ratio=0.4
+
+# The overhead ratio for inference, used to estimate the pool size
+# Datatype: Float
+ain_inference_extra_memory_ratio=1.2
+
 # The AINode metadata storage path.
 # The starting directory of the relative path is related to the operating system.
 # It is recommended to use an absolute path.