You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by ho...@apache.org on 2026/09/20 03:47:44 UTC
(paimon) 04/20: [common] Reject out-of-range range-bitmap chunk-size option (#9765)
This is an automated email from the ASF dual-hosted git repository.
XiaoHongbo-Hope pushed a commit to branch release-2.1
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 2e937fa45b9b4f1cf5085eb46bae597f63236d74
Author: YangJie <ya...@baidu.com>
AuthorDate: Sun Sep 13 22:37:57 2026 -0400
[common] Reject out-of-range range-bitmap chunk-size option (#9765)
---
.../rangebitmap/RangeBitmapFileIndex.java | 13 ++++++++--
.../rangebitmap/RangeBitmapFileIndexTest.java | 29 ++++++++++++++++++++++
2 files changed, 40 insertions(+), 2 deletions(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java b/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java
index 8b3ef92f22..0cfa5fa56f 100644
--- a/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java
+++ b/paimon-common/src/main/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndex.java
@@ -72,9 +72,18 @@ public class RangeBitmapFileIndex implements FileIndexer {
public Writer(DataType dataType, Options options) {
KeyFactory factory = KeyFactory.create(dataType);
String chunkSize = options.getString(CHUNK_SIZE, factory.defaultChunkSize());
+ long bytes = MemorySize.parse(chunkSize).getBytes();
+ // the chunk size becomes an eagerly allocated per-chunk buffer, so it has to fit an
+ // int. Narrowing it silently substitutes a different size: "2g" becomes negative and
+ // fails only once the writer allocates, "4g" becomes 0 and "5g" becomes 1g.
+ if (bytes > Integer.MAX_VALUE) {
+ throw new IllegalArgumentException(
+ String.format(
+ "The '%s' option must not exceed 2147483647 bytes, but was '%s'.",
+ CHUNK_SIZE, chunkSize));
+ }
this.converter = factory.createConverter();
- this.appender =
- new RangeBitmap.Appender(factory, (int) MemorySize.parse(chunkSize).getBytes());
+ this.appender = new RangeBitmap.Appender(factory, (int) bytes);
}
@Override
diff --git a/paimon-common/src/test/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndexTest.java b/paimon-common/src/test/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndexTest.java
index 8e94eba873..5edac83812 100644
--- a/paimon-common/src/test/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndexTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/fileindex/rangebitmap/RangeBitmapFileIndexTest.java
@@ -50,6 +50,7 @@ import static org.apache.paimon.predicate.SortValue.NullOrdering.NULLS_LAST;
import static org.apache.paimon.predicate.SortValue.SortDirection.ASCENDING;
import static org.apache.paimon.predicate.SortValue.SortDirection.DESCENDING;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** test for {@link RangeBitmapFileIndex}. */
public class RangeBitmapFileIndexTest {
@@ -57,6 +58,34 @@ public class RangeBitmapFileIndexTest {
private static final int ROW_COUNT = 10000;
private static final int BOUND = 1000000;
+ @Test
+ public void testChunkSizeBeyondIntRangeRejected() {
+ VarCharType varCharType = new VarCharType();
+
+ // the boundary is the whole guard: one byte past int range is rejected, int range itself
+ // is accepted. "2g" and "4g" would both only re-test the same comparison
+ Options justPastIntRange = new Options();
+ justPastIntRange.setString(RangeBitmapFileIndex.CHUNK_SIZE, "2147483648 bytes");
+ assertThatThrownBy(
+ () ->
+ new RangeBitmapFileIndex(varCharType, justPastIntRange)
+ .createWriter())
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("chunk-size");
+
+ Options atIntRange = new Options();
+ atIntRange.setString(RangeBitmapFileIndex.CHUNK_SIZE, "2147483647 bytes");
+ assertThat(new RangeBitmapFileIndex(varCharType, atIntRange).createWriter()).isNotNull();
+
+ // a large but in-range chunk size still writes and serializes
+ Options valid = new Options();
+ valid.setString(RangeBitmapFileIndex.CHUNK_SIZE, "16mb");
+ FileIndexWriter writer = new RangeBitmapFileIndex(varCharType, valid).createWriter();
+ writer.write(BinaryString.fromString("a"));
+ writer.write(BinaryString.fromString("b"));
+ assertThat(writer.serializedBytes()).isNotEmpty();
+ }
+
@RepeatedTest(10)
public void test() {
String prefix = "hello-";