You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2026/05/06 06:45:48 UTC
(iotdb) 01/03: Add thrift max frame size calculate logic
This is an automated email from the ASF dual-hosted git repository.
HTHou pushed a commit to branch codex/cherry-pick-16724-dev-1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ba266caf644a4152b7efeeca8d32143bd6195e9a
Author: HTHou <ha...@apache.org>
AuthorDate: Mon Nov 10 14:42:35 2025 +0800
Add thrift max frame size calculate logic
---
.../rpc/TCompressedElasticFramedTransport.java | 8 +--
.../apache/iotdb/rpc/TElasticFramedTransport.java | 80 +++++++++++++++++-----
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 9 ++-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 4 --
.../conf/iotdb-system.properties.template | 4 +-
.../apache/iotdb/commons/conf/IoTDBConstant.java | 1 -
6 files changed, 71 insertions(+), 35 deletions(-)
diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TCompressedElasticFramedTransport.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TCompressedElasticFramedTransport.java
index 8579f46a3cb..07768dd98bd 100644
--- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TCompressedElasticFramedTransport.java
+++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TCompressedElasticFramedTransport.java
@@ -44,13 +44,7 @@ public abstract class TCompressedElasticFramedTransport extends TElasticFramedTr
protected void readFrame() throws TTransportException {
underlying.readAll(i32buf, 0, 4);
int size = TFramedTransport.decodeFrameSize(i32buf);
-
- if (size < 0) {
- close();
- throw new TTransportException(
- TTransportException.CORRUPTED_DATA, "Read a negative frame size (" + size + ")!");
- }
-
+ checkFrameSize(size);
readBuffer.fill(underlying, size);
RpcStat.readCompressedBytes.addAndGet(size);
try {
diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TElasticFramedTransport.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TElasticFramedTransport.java
index d98914ce934..2524e8ae4cd 100644
--- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TElasticFramedTransport.java
+++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TElasticFramedTransport.java
@@ -174,29 +174,71 @@ public class TElasticFramedTransport extends TTransport {
protected void readFrame() throws TTransportException {
underlying.readAll(i32buf, 0, 4);
int size = TFramedTransport.decodeFrameSize(i32buf);
+ checkFrameSize(size);
+ readBuffer.fill(underlying, size);
+ }
- if (size < 0) {
- close();
- throw new TTransportException(
- TTransportException.CORRUPTED_DATA, "Read a negative frame size (" + size + ")!");
+ protected void checkFrameSize(int size) throws TTransportException {
+ final int HTTP_GET_SIGNATURE = 0x47455420; // "GET "
+ final int HTTP_POST_SIGNATURE = 0x504F5354; // "POST"
+ final int TLS_MIN_VERSION = 0x160300;
+ final int TLS_MAX_VERSION = 0x160303;
+ final int TLS_LENGTH_HIGH_MAX = 0x02;
+
+ FrameError error = null;
+ if (size == HTTP_GET_SIGNATURE || size == HTTP_POST_SIGNATURE) {
+ error = FrameError.HTTP_REQUEST;
+ } else {
+ int high24 = size >>> 8;
+ if (high24 >= TLS_MIN_VERSION
+ && high24 <= TLS_MAX_VERSION
+ && (i32buf[3] & 0xFF) <= TLS_LENGTH_HIGH_MAX) {
+ error = FrameError.TLS_REQUEST;
+ } else if (size < 0) {
+ error = FrameError.NEGATIVE_FRAME_SIZE;
+ } else if (size > thriftMaxFrameSize) {
+ error = FrameError.FRAME_SIZE_EXCEEDED;
+ }
}
- if (size > thriftMaxFrameSize) {
- close();
- if (size == 1195725856L || size == 1347375956L) {
- // if someone sends HTTP GET/POST to this port, the size will be read as the following
- throw new TTransportException(
- TTransportException.CORRUPTED_DATA,
- "Singular frame size ("
- + size
- + ") detected, you may be sending HTTP GET/POST requests to the Thrift-RPC port, please confirm that you are using the right port");
- } else {
- throw new TTransportException(
- TTransportException.CORRUPTED_DATA,
- "Frame size (" + size + ") larger than protect max size (" + thriftMaxFrameSize + ")!");
- }
+ if (error == null) {
+ return;
+ }
+
+ SocketAddress remoteAddress = null;
+ if (underlying instanceof TSocket) {
+ remoteAddress = ((TSocket) underlying).getSocket().getRemoteSocketAddress();
+ }
+ String remoteInfo = (remoteAddress == null) ? "" : " from " + remoteAddress;
+ close();
+
+ error.throwException(size, remoteInfo, thriftMaxFrameSize);
+ }
+
+ private enum FrameError {
+ HTTP_REQUEST(
+ "Singular frame size (%d) detected, you may be sending HTTP GET/POST%s "
+ + "requests to the Thrift-RPC port, please confirm that you are using the right port"),
+ TLS_REQUEST(
+ "Singular frame size (%d) detected, you may be sending TLS ClientHello "
+ + "requests%s to the Non-SSL Thrift-RPC port, please confirm that you are using "
+ + "the right configuration"),
+ NEGATIVE_FRAME_SIZE("Read a negative frame size (%d)%s!"),
+ FRAME_SIZE_EXCEEDED("Frame size (%d) larger than protect max size (%d)%s!");
+
+ private final String messageFormat;
+
+ FrameError(String messageFormat) {
+ this.messageFormat = messageFormat;
+ }
+
+ void throwException(int size, String remoteInfo, int maxSize) throws TTransportException {
+ String message =
+ (this == FRAME_SIZE_EXCEEDED)
+ ? String.format(messageFormat, size, maxSize, remoteInfo)
+ : String.format(messageFormat, size, remoteInfo);
+ throw new TTransportException(TTransportException.CORRUPTED_DATA, message);
}
- readBuffer.fill(underlying, size);
}
protected void checkWriteFrameSize(int size) throws TTransportException {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 989dc757464..af9e40ffa73 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -868,7 +868,7 @@ public class IoTDBConfig {
private float udfCollectorMemoryBudgetInMB = (float) (1.0 / 3 * udfMemoryBudgetInMB);
/** Unit: byte */
- private int thriftMaxFrameSize = 536870912;
+ private int thriftMaxFrameSize = 0;
private int thriftDefaultBufferSize = RpcUtils.THRIFT_DEFAULT_BUF_CAPACITY;
@@ -2784,7 +2784,12 @@ public class IoTDBConfig {
}
public void setThriftMaxFrameSize(int thriftMaxFrameSize) {
- this.thriftMaxFrameSize = thriftMaxFrameSize;
+ this.thriftMaxFrameSize =
+ thriftMaxFrameSize <= 0
+ ? Math.min(
+ 64 * 1024 * 1024,
+ (int) Math.min(Runtime.getRuntime().maxMemory() / 64, Integer.MAX_VALUE))
+ : thriftMaxFrameSize;
BaseRpcTransportFactory.setThriftMaxFrameSize(this.thriftMaxFrameSize);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index c755381e88d..e965611016d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -861,10 +861,6 @@ public class IoTDBDescriptor {
properties.getProperty(
"dn_thrift_max_frame_size", String.valueOf(conf.getThriftMaxFrameSize()))));
- if (conf.getThriftMaxFrameSize() < IoTDBConstant.LEFT_SIZE_IN_REQUEST * 2) {
- conf.setThriftMaxFrameSize(IoTDBConstant.LEFT_SIZE_IN_REQUEST * 2);
- }
-
conf.setThriftDefaultBufferSize(
Integer.parseInt(
properties.getProperty(
diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
index 018551d889a..e3b1e4da1bb 100644
--- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
+++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -509,10 +509,10 @@ dn_rpc_min_concurrent_client_num=1
# Datatype: int
dn_rpc_max_concurrent_client_num=1000
-# thrift max frame size, 512MB by default
+# thrift max frame size in bytes, When = 0, use min(64MB, datanode heap memory / 64)
# effectiveMode: restart
# Datatype: int
-dn_thrift_max_frame_size=536870912
+dn_thrift_max_frame_size=0
# thrift init buffer size
# effectiveMode: restart
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
index 18ac41409ba..f9f69f486e6 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
@@ -268,7 +268,6 @@ public class IoTDBConstant {
public static final String MQTT_MAX_MESSAGE_SIZE = "mqtt_max_message_size";
// thrift
- public static final int LEFT_SIZE_IN_REQUEST = 4 * 1024 * 1024;
public static final int DEFAULT_FETCH_SIZE = 5000;
public static final int DEFAULT_CONNECTION_TIMEOUT_MS = 0;