You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2026/03/25 06:46:35 UTC
(paimon) branch master updated: [python] Fix BINARY(N) type mapping to use variable-length binary (#7518)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 098ef8e8d2 [python] Fix BINARY(N) type mapping to use variable-length binary (#7518)
098ef8e8d2 is described below
commit 098ef8e8d2be98b345bd35264f1cfb46f2488287
Author: XiaoHongbo <13...@qq.com>
AuthorDate: Wed Mar 25 14:46:30 2026 +0800
[python] Fix BINARY(N) type mapping to use variable-length binary (#7518)
---
.../test/java/org/apache/paimon/JavaPyE2ETest.java | 63 +++++++++++++++++-----
paimon-python/pypaimon/schema/data_types.py | 10 +---
.../pypaimon/tests/e2e/java_py_read_write_test.py | 11 +++-
paimon-python/pypaimon/tests/reader_base_test.py | 2 +-
paimon-python/pypaimon/write/table_write.py | 30 +++++++++--
5 files changed, 86 insertions(+), 30 deletions(-)
diff --git a/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java b/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
index fad9507067..ca640aa68d 100644
--- a/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
@@ -155,6 +155,7 @@ public class JavaPyE2ETest {
.column("ts", DataTypes.TIMESTAMP())
.column("ts_ltz", DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE())
.column("t", DataTypes.TIME())
+ .column("bin_data", DataTypes.BINARY(20))
.column(
"metadata",
DataTypes.ROW(
@@ -186,8 +187,18 @@ public class JavaPyE2ETest {
write.write(
createRow7Cols(
- 1, "Apple", "Fruit", 1.5, 1000000L, 2000000L, 1000, "store1", 1001L,
- "Beijing", "China"));
+ 1,
+ "Apple",
+ "Fruit",
+ 1.5,
+ 1000000L,
+ 2000000L,
+ 1000,
+ "apple_bin_data".getBytes(),
+ "store1",
+ 1001L,
+ "Beijing",
+ "China"));
write.write(
createRow7Cols(
2,
@@ -197,6 +208,7 @@ public class JavaPyE2ETest {
1000001L,
2000001L,
2000,
+ "banana_bin".getBytes(),
"store1",
1002L,
"Shanghai",
@@ -210,6 +222,7 @@ public class JavaPyE2ETest {
1000002L,
2000002L,
3000,
+ "carrot".getBytes(),
"store2",
1003L,
"Tokyo",
@@ -223,18 +236,39 @@ public class JavaPyE2ETest {
1000003L,
2000003L,
4000,
+ "broccoli_binary_data".getBytes(),
"store2",
1004L,
"Seoul",
"Korea"));
write.write(
createRow7Cols(
- 5, "Chicken", "Meat", 5.0, 1000004L, 2000004L, 5000, "store3",
- 1005L, "NewYork", "USA"));
+ 5,
+ "Chicken",
+ "Meat",
+ 5.0,
+ 1000004L,
+ 2000004L,
+ 5000,
+ "chicken".getBytes(),
+ "store3",
+ 1005L,
+ "NewYork",
+ "USA"));
write.write(
createRow7Cols(
- 6, "Beef", "Meat", 8.0, 1000005L, 2000005L, 6000, "store3", 1006L,
- "London", "UK"));
+ 6,
+ "Beef",
+ "Meat",
+ 8.0,
+ 1000005L,
+ 2000005L,
+ 6000,
+ "beef_data".getBytes(),
+ "store3",
+ 1006L,
+ "London",
+ "UK"));
// Row with null partition value -> __DEFAULT_PARTITION__
write.write(
GenericRow.of(
@@ -245,6 +279,7 @@ public class JavaPyE2ETest {
org.apache.paimon.data.Timestamp.fromEpochMillis(1000006L),
org.apache.paimon.data.Timestamp.fromEpochMillis(2000006L),
7000,
+ "tofu".getBytes(),
GenericRow.of(
BinaryString.fromString("store4"),
1007L,
@@ -262,13 +297,13 @@ public class JavaPyE2ETest {
getResult(read, splits, row -> rowToStringWithStruct(row, table.rowType()));
assertThat(res)
.containsExactlyInAnyOrder(
- "+I[1, Apple, Fruit, 1.5, 1970-01-01T00:16:40, 1970-01-01T00:33:20, 1000, (store1, 1001, (Beijing, China))]",
- "+I[2, Banana, Fruit, 0.8, 1970-01-01T00:16:40.001, 1970-01-01T00:33:20.001, 2000, (store1, 1002, (Shanghai, China))]",
- "+I[3, Carrot, Vegetable, 0.6, 1970-01-01T00:16:40.002, 1970-01-01T00:33:20.002, 3000, (store2, 1003, (Tokyo, Japan))]",
- "+I[4, Broccoli, Vegetable, 1.2, 1970-01-01T00:16:40.003, 1970-01-01T00:33:20.003, 4000, (store2, 1004, (Seoul, Korea))]",
- "+I[5, Chicken, Meat, 5.0, 1970-01-01T00:16:40.004, 1970-01-01T00:33:20.004, 5000, (store3, 1005, (NewYork, USA))]",
- "+I[6, Beef, Meat, 8.0, 1970-01-01T00:16:40.005, 1970-01-01T00:33:20.005, 6000, (store3, 1006, (London, UK))]",
- "+I[7, Tofu, NULL, 3.0, 1970-01-01T00:16:40.006, 1970-01-01T00:33:20.006, 7000, (store4, 1007, (Paris, France))]");
+ "+I[1, Apple, Fruit, 1.5, 1970-01-01T00:16:40, 1970-01-01T00:33:20, 1000, [97, 112, 112, 108, 101, 95, 98, 105, 110, 95, 100, 97, 116, 97], (store1, 1001, (Beijing, China))]",
+ "+I[2, Banana, Fruit, 0.8, 1970-01-01T00:16:40.001, 1970-01-01T00:33:20.001, 2000, [98, 97, 110, 97, 110, 97, 95, 98, 105, 110], (store1, 1002, (Shanghai, China))]",
+ "+I[3, Carrot, Vegetable, 0.6, 1970-01-01T00:16:40.002, 1970-01-01T00:33:20.002, 3000, [99, 97, 114, 114, 111, 116], (store2, 1003, (Tokyo, Japan))]",
+ "+I[4, Broccoli, Vegetable, 1.2, 1970-01-01T00:16:40.003, 1970-01-01T00:33:20.003, 4000, [98, 114, 111, 99, 99, 111, 108, 105, 95, 98, 105, 110, 97, 114, 121, 95, 100, 97, 116, 97], (store2, 1004, (Seoul, Korea))]",
+ "+I[5, Chicken, Meat, 5.0, 1970-01-01T00:16:40.004, 1970-01-01T00:33:20.004, 5000, [99, 104, 105, 99, 107, 101, 110], (store3, 1005, (NewYork, USA))]",
+ "+I[6, Beef, Meat, 8.0, 1970-01-01T00:16:40.005, 1970-01-01T00:33:20.005, 6000, [98, 101, 101, 102, 95, 100, 97, 116, 97], (store3, 1006, (London, UK))]",
+ "+I[7, Tofu, NULL, 3.0, 1970-01-01T00:16:40.006, 1970-01-01T00:33:20.006, 7000, [116, 111, 102, 117], (store4, 1007, (Paris, France))]");
}
}
@@ -792,6 +827,7 @@ public class JavaPyE2ETest {
long ts,
long tsLtz,
int timeMillis,
+ byte[] binData,
String metadataSource,
long metadataCreatedAt,
String city,
@@ -809,6 +845,7 @@ public class JavaPyE2ETest {
org.apache.paimon.data.Timestamp.fromEpochMillis(ts),
org.apache.paimon.data.Timestamp.fromEpochMillis(tsLtz),
timeMillis,
+ binData,
metadataRow);
}
diff --git a/paimon-python/pypaimon/schema/data_types.py b/paimon-python/pypaimon/schema/data_types.py
index 0d95e7b95c..ebb5612c43 100755
--- a/paimon-python/pypaimon/schema/data_types.py
+++ b/paimon-python/pypaimon/schema/data_types.py
@@ -477,18 +477,10 @@ class PyarrowFieldParser:
return pyarrow.bool_()
elif type_name == 'STRING' or type_name.startswith('CHAR') or type_name.startswith('VARCHAR'):
return pyarrow.string()
- elif type_name == 'BYTES' or type_name.startswith('VARBINARY'):
+ elif type_name == 'BYTES' or type_name.startswith('VARBINARY') or type_name.startswith('BINARY'):
return pyarrow.binary()
elif type_name == 'BLOB':
return pyarrow.large_binary()
- elif type_name.startswith('BINARY'):
- if type_name == 'BINARY':
- return pyarrow.binary(1)
- match = re.fullmatch(r'BINARY\((\d+)\)', type_name)
- if match:
- length = int(match.group(1))
- if length > 0:
- return pyarrow.binary(length)
elif type_name.startswith('DECIMAL'):
if type_name == 'DECIMAL':
return pyarrow.decimal128(10, 0) # default to 10, 0
diff --git a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
index f53d244e54..0732925bae 100644
--- a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
+++ b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
@@ -240,9 +240,10 @@ class JavaPyReadWriteTest(unittest.TestCase):
self.assertEqual(table.fields[4].type.type, "TIMESTAMP(6)")
self.assertEqual(table.fields[5].type.type, "TIMESTAMP(6) WITH LOCAL TIME ZONE")
self.assertEqual(table.fields[6].type.type, "TIME(0)")
+ self.assertEqual(table.fields[7].type.type, "BINARY(20)")
from pypaimon.schema.data_types import RowType
- self.assertIsInstance(table.fields[7].type, RowType)
- metadata_fields = table.fields[7].type.fields
+ self.assertIsInstance(table.fields[8].type, RowType)
+ metadata_fields = table.fields[8].type.fields
self.assertEqual(len(metadata_fields), 3)
self.assertEqual(metadata_fields[0].name, 'source')
self.assertEqual(metadata_fields[1].name, 'created_at')
@@ -259,6 +260,12 @@ class JavaPyReadWriteTest(unittest.TestCase):
self.assertEqual(len(tofu_row), 1)
self.assertTrue(pd.isna(tofu_row['category'].iloc[0]))
+ if file_format != "lance" and 'bin_data' in res.columns:
+ apple_row = res[res['name'] == 'Apple']
+ self.assertEqual(apple_row['bin_data'].iloc[0], b'apple_bin_data')
+ carrot_row = res[res['name'] == 'Carrot']
+ self.assertEqual(carrot_row['bin_data'].iloc[0], b'carrot')
+
# Verify metadata column can be read and contains nested structures
if 'metadata' in res.columns:
self.assertFalse(res['metadata'].isnull().all())
diff --git a/paimon-python/pypaimon/tests/reader_base_test.py b/paimon-python/pypaimon/tests/reader_base_test.py
index 201811228f..657678f9ea 100644
--- a/paimon-python/pypaimon/tests/reader_base_test.py
+++ b/paimon-python/pypaimon/tests/reader_base_test.py
@@ -645,7 +645,7 @@ class ReaderBasicTest(unittest.TestCase):
DataField(5, "f5", AtomicType('DOUBLE'), 'desc'),
DataField(6, "f6", AtomicType('BOOLEAN'), 'desc'),
DataField(7, "f7", AtomicType('STRING'), 'desc'),
- DataField(8, "f8", AtomicType('BINARY(12)'), 'desc'),
+ DataField(8, "f8", AtomicType('BYTES'), 'desc'),
DataField(9, "f9", AtomicType('DECIMAL(10, 6)'), 'desc'),
DataField(10, "f10", AtomicType('BYTES'), 'desc'),
DataField(11, "f11", AtomicType('DATE'), 'desc'),
diff --git a/paimon-python/pypaimon/write/table_write.py b/paimon-python/pypaimon/write/table_write.py
index a2c9a9a4c3..d14d9ca049 100644
--- a/paimon-python/pypaimon/write/table_write.py
+++ b/paimon-python/pypaimon/write/table_write.py
@@ -102,11 +102,31 @@ class TableWrite:
self.file_store_write.close()
def _validate_pyarrow_schema(self, data_schema: pa.Schema):
- if data_schema != self.table_pyarrow_schema and data_schema.names != self.file_store_write.write_cols:
- raise ValueError(f"Input schema isn't consistent with table schema and write cols. "
- f"Input schema is: {data_schema} "
- f"Table schema is: {self.table_pyarrow_schema} "
- f"Write cols is: {self.file_store_write.write_cols}")
+ if data_schema == self.table_pyarrow_schema:
+ return
+ if data_schema.names == self.file_store_write.write_cols:
+ return
+ # Allow compatible binary types: binary, fixed_size_binary[N] are interchangeable
+ if data_schema.names == self.table_pyarrow_schema.names:
+ compatible = True
+ for i in range(len(data_schema)):
+ input_type = data_schema.field(i).type
+ table_type = self.table_pyarrow_schema.field(i).type
+ if input_type != table_type:
+ if self._is_binary_family(input_type) and self._is_binary_family(table_type):
+ continue
+ compatible = False
+ break
+ if compatible:
+ return
+ raise ValueError(f"Input schema isn't consistent with table schema and write cols. "
+ f"Input schema is: {data_schema} "
+ f"Table schema is: {self.table_pyarrow_schema} "
+ f"Write cols is: {self.file_store_write.write_cols}")
+
+ @staticmethod
+ def _is_binary_family(arrow_type) -> bool:
+ return pa.types.is_binary(arrow_type) or pa.types.is_fixed_size_binary(arrow_type)
class BatchTableWrite(TableWrite):