You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2026/03/08 11:12:37 UTC
(paimon) branch master updated: [python] Support dynamic bucket write (#7363)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new d707b1c743 [python] Support dynamic bucket write (#7363)
d707b1c743 is described below
commit d707b1c74344d98ad42c8e91a8c0c01e2f1e34da
Author: XiaoHongbo <13...@qq.com>
AuthorDate: Sun Mar 8 19:12:32 2026 +0800
[python] Support dynamic bucket write (#7363)
---
.../pypaimon/common/options/core_options.py | 25 ++++
.../pypaimon/tests/rest/rest_read_write_test.py | 2 -
.../pypaimon/tests/rest/rest_simple_test.py | 19 ++-
.../write/simple_hash_bucket_assigner_test.py | 60 +++++++++
.../pypaimon/tests/write/table_write_test.py | 30 +++++
paimon-python/pypaimon/write/row_key_extractor.py | 137 ++++++++++++++++++++-
6 files changed, 265 insertions(+), 8 deletions(-)
diff --git a/paimon-python/pypaimon/common/options/core_options.py b/paimon-python/pypaimon/common/options/core_options.py
index 33384316cf..85cf965f91 100644
--- a/paimon-python/pypaimon/common/options/core_options.py
+++ b/paimon-python/pypaimon/common/options/core_options.py
@@ -107,6 +107,25 @@ class CoreOptions:
)
)
+ DYNAMIC_BUCKET_TARGET_ROW_NUM: ConfigOption[int] = (
+ ConfigOptions.key("dynamic-bucket.target-row-num")
+ .int_type()
+ .default_value(2000000)
+ .with_description(
+ "In dynamic bucket mode (bucket=-1), target row number per bucket; "
+ "when exceeded, a new bucket is created (aligned with Java SimpleHashBucketAssigner)."
+ )
+ )
+
+ DYNAMIC_BUCKET_MAX_BUCKETS: ConfigOption[int] = (
+ ConfigOptions.key("dynamic-bucket.max-buckets")
+ .int_type()
+ .default_value(-1)
+ .with_description(
+ "In dynamic bucket mode, max buckets per partition. -1 means unlimited."
+ )
+ )
+
SCAN_MANIFEST_PARALLELISM: ConfigOption[int] = (
ConfigOptions.key("scan.manifest.parallelism")
.int_type()
@@ -389,6 +408,12 @@ class CoreOptions:
def bucket_key(self, default=None):
return self.options.get(CoreOptions.BUCKET_KEY, default)
+ def dynamic_bucket_target_row_num(self, default=None):
+ return self.options.get(CoreOptions.DYNAMIC_BUCKET_TARGET_ROW_NUM, default)
+
+ def dynamic_bucket_max_buckets(self, default=None):
+ return self.options.get(CoreOptions.DYNAMIC_BUCKET_MAX_BUCKETS, default)
+
def scan_manifest_parallelism(self, default=None):
return self.options.get(CoreOptions.SCAN_MANIFEST_PARALLELISM, default)
diff --git a/paimon-python/pypaimon/tests/rest/rest_read_write_test.py b/paimon-python/pypaimon/tests/rest/rest_read_write_test.py
index f2ed1342e5..726ef134d3 100644
--- a/paimon-python/pypaimon/tests/rest/rest_read_write_test.py
+++ b/paimon-python/pypaimon/tests/rest/rest_read_write_test.py
@@ -20,7 +20,6 @@ import logging
import pandas as pd
import pyarrow as pa
-import unittest
from pypaimon.common.options import Options
from pypaimon.catalog.catalog_context import CatalogContext
@@ -363,7 +362,6 @@ class RESTTableReadWriteTest(RESTBaseTest):
])
self.assertEqual(actual, expected)
- @unittest.skip("does not support dynamic bucket in dummy rest server")
def test_pk_lance_reader_no_bucket(self):
"""Test Lance format with PrimaryKey table without specifying bucket."""
schema = Schema.from_pyarrow_schema(self.pa_schema,
diff --git a/paimon-python/pypaimon/tests/rest/rest_simple_test.py b/paimon-python/pypaimon/tests/rest/rest_simple_test.py
index b55e42f841..7fde2955c5 100644
--- a/paimon-python/pypaimon/tests/rest/rest_simple_test.py
+++ b/paimon-python/pypaimon/tests/rest/rest_simple_test.py
@@ -600,14 +600,25 @@ class RESTSimpleTest(RESTBaseTest):
write_builder = table.new_batch_write_builder()
table_write = write_builder.new_write()
+ table_commit = write_builder.new_commit()
self.assertIsInstance(table_write.row_key_extractor, DynamicBucketRowKeyExtractor)
pa_table = pa.Table.from_pydict(self.data, schema=self.pa_schema)
+ table_write.write_arrow(pa_table)
+ table_commit.commit(table_write.prepare_commit())
+ table_write.close()
+ table_commit.close()
- with self.assertRaises(ValueError) as context:
- table_write.write_arrow(pa_table)
-
- self.assertEqual(str(context.exception), "Can't extract bucket from row in dynamic bucket mode")
+ splits = []
+ read_builder = table.new_read_builder()
+ splits.extend(read_builder.new_scan().with_shard(0, 3).plan().splits())
+ splits.extend(read_builder.new_scan().with_shard(1, 3).plan().splits())
+ splits.extend(read_builder.new_scan().with_shard(2, 3).plan().splits())
+ table_read = read_builder.new_read()
+ actual = table_read.to_arrow(splits)
+ expected_sorted = table_sort_by(self.expected, 'user_id')
+ actual_sorted = table_sort_by(actual, 'user_id')
+ self.assertEqual(actual_sorted, expected_sorted)
def test_with_shard_pk_fixed_bucket(self):
schema = Schema.from_pyarrow_schema(self.pa_schema, partition_keys=['user_id'], primary_keys=['user_id', 'dt'],
diff --git a/paimon-python/pypaimon/tests/write/simple_hash_bucket_assigner_test.py b/paimon-python/pypaimon/tests/write/simple_hash_bucket_assigner_test.py
new file mode 100644
index 0000000000..5644cf0d25
--- /dev/null
+++ b/paimon-python/pypaimon/tests/write/simple_hash_bucket_assigner_test.py
@@ -0,0 +1,60 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+import unittest
+
+from pypaimon.write.row_key_extractor import SimpleHashBucketAssigner
+
+
+class SimpleHashBucketAssignerTest(unittest.TestCase):
+
+ def test_assign(self):
+ assigner = SimpleHashBucketAssigner(2, 0, 100, -1)
+ partition = ()
+ buckets = [assigner.assign(partition, h) for h in range(301)]
+ for b in buckets[:100]:
+ self.assertEqual(b, 0)
+ for b in buckets[100:200]:
+ self.assertEqual(b, 2)
+ self.assertEqual(buckets[200], 4)
+
+ def test_assign_with_upper_bound(self):
+ assigner = SimpleHashBucketAssigner(1, 0, 100, 3)
+ partition = ()
+ buckets = [assigner.assign(partition, h) for h in range(400)]
+ for b in buckets[:100]:
+ self.assertEqual(b, 0)
+ for b in buckets[100:200]:
+ self.assertEqual(b, 1)
+ for b in buckets[200:300]:
+ self.assertEqual(b, 2)
+ for b in buckets[300:]:
+ self.assertIn(b, [0, 1, 2])
+
+ def test_assign_with_same_hash(self):
+ for max_buckets in [-1, 1, 2]:
+ with self.subTest(max_buckets=max_buckets):
+ assigner = SimpleHashBucketAssigner(1, 0, 100, max_buckets)
+ partition = ()
+ hashes = list(range(100)) + list(range(100))
+ buckets = [assigner.assign(partition, h) for h in hashes]
+ for b in buckets[100:]:
+ self.assertEqual(b, 0)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/paimon-python/pypaimon/tests/write/table_write_test.py b/paimon-python/pypaimon/tests/write/table_write_test.py
index a73dc631b1..34a892a725 100644
--- a/paimon-python/pypaimon/tests/write/table_write_test.py
+++ b/paimon-python/pypaimon/tests/write/table_write_test.py
@@ -354,3 +354,33 @@ class TableWriteTest(unittest.TestCase):
for file_name in data_files:
self.assertRegex(file_name, expected_pattern,
f"File name '{file_name}' does not match expected prefix format")
+
+ def test_dynamic_bucket_write(self):
+ schema = Schema.from_pyarrow_schema(
+ self.pa_schema,
+ partition_keys=['dt'],
+ primary_keys=['user_id', 'dt'],
+ options={'bucket': '-1'}
+ )
+ self.catalog.create_table(
+ 'default.test_dynamic_bucket', schema, False)
+ table = self.catalog.get_table(
+ 'default.test_dynamic_bucket')
+ write_builder = table.new_batch_write_builder()
+
+ table_write = write_builder.new_write()
+ table_commit = write_builder.new_commit()
+ table_write.write_arrow(self.expected)
+ table_commit.commit(table_write.prepare_commit())
+ table_write.close()
+ table_commit.close()
+
+ read_builder = table.new_read_builder()
+ table_read = read_builder.new_read()
+ splits = read_builder.new_scan().plan().splits()
+ actual = table_read.to_arrow(splits)
+ sort_keys = [('user_id', 'ascending'), ('dt', 'ascending')]
+ self.assertEqual(
+ self.expected.sort_by(sort_keys),
+ actual.sort_by(sort_keys),
+ )
diff --git a/paimon-python/pypaimon/write/row_key_extractor.py b/paimon-python/pypaimon/write/row_key_extractor.py
index 9ff23404c3..2f09e6577b 100644
--- a/paimon-python/pypaimon/write/row_key_extractor.py
+++ b/paimon-python/pypaimon/write/row_key_extractor.py
@@ -17,9 +17,10 @@
################################################################################
import math
+import random
import struct
from abc import ABC, abstractmethod
-from typing import List, Tuple
+from typing import Dict, List, Tuple
import pyarrow as pa
@@ -167,6 +168,102 @@ class UnawareBucketRowKeyExtractor(RowKeyExtractor):
return [0] * data.num_rows
+_SHORT_MAX_VALUE = 32767
+
+
+def _is_my_bucket(bucket: int, num_assigners: int, assign_id: int) -> bool:
+ return bucket % num_assigners == assign_id % num_assigners
+
+
+def _pick_randomly(bucket_list: List[int]) -> int:
+ return random.choice(bucket_list)
+
+
+class _SimplePartitionIndex:
+ def __init__(self, num_assigners: int, assign_id: int, max_buckets_num: int) -> None:
+ self.hash2bucket: Dict[int, int] = {}
+ self.bucket_information: Dict[int, int] = {}
+ self.bucket_list: List[int] = []
+ self.current_bucket: int = 0
+ self._load_new_bucket(max_buckets_num, num_assigners, assign_id)
+
+ def assign(
+ self,
+ hash_value: int,
+ max_bucket_id: int,
+ target_bucket_row_number: int,
+ max_buckets_num: int,
+ num_assigners: int,
+ assign_id: int,
+ ) -> Tuple[int, int]:
+ if hash_value in self.hash2bucket:
+ assigned = self.hash2bucket[hash_value]
+ return assigned, max(max_bucket_id, assigned)
+
+ if self.current_bucket not in self.bucket_information:
+ self.bucket_list.append(self.current_bucket)
+ self.bucket_information[self.current_bucket] = 0
+ num = self.bucket_information[self.current_bucket]
+
+ if num >= target_bucket_row_number:
+ if (
+ max_buckets_num == -1
+ or not self.bucket_information
+ or max_bucket_id < max_buckets_num - 1
+ ):
+ self._load_new_bucket(max_buckets_num, num_assigners, assign_id)
+ else:
+ self.current_bucket = _pick_randomly(self.bucket_list)
+
+ self.bucket_information[self.current_bucket] = (
+ self.bucket_information.get(self.current_bucket, 0) + 1
+ )
+ self.hash2bucket[hash_value] = self.current_bucket
+ new_max = max(max_bucket_id, self.current_bucket)
+ return self.current_bucket, new_max
+
+ def _load_new_bucket(
+ self, max_buckets_num: int, num_assigners: int, assign_id: int
+ ) -> None:
+ for i in range(_SHORT_MAX_VALUE):
+ if _is_my_bucket(i, num_assigners, assign_id) and (
+ i not in self.bucket_information
+ ):
+ if max_buckets_num == -1 or i <= max_buckets_num - 1:
+ self.current_bucket = i
+ return
+ return
+ raise RuntimeError(
+ "Can't find a suitable bucket to assign, all the bucket are assigned?"
+ )
+
+
+class SimpleHashBucketAssigner:
+ def __init__(self, num_assigners, assign_id, target_bucket_row_number, max_buckets_num):
+ self.num_assigners = num_assigners
+ self.assign_id = assign_id
+ self.target_bucket_row_number = target_bucket_row_number
+ self.max_buckets_num = max_buckets_num
+ self.max_bucket_id = 0
+ self._partition_index: Dict[Tuple, _SimplePartitionIndex] = {}
+
+ def assign(self, partition: Tuple, hash_value: int) -> int:
+ if partition not in self._partition_index:
+ self._partition_index[partition] = _SimplePartitionIndex(
+ self.num_assigners, self.assign_id, self.max_buckets_num)
+ index = self._partition_index[partition]
+
+ assigned, self.max_bucket_id = index.assign(
+ hash_value,
+ self.max_bucket_id,
+ self.target_bucket_row_number,
+ self.max_buckets_num,
+ self.num_assigners,
+ self.assign_id,
+ )
+ return assigned
+
+
class DynamicBucketRowKeyExtractor(RowKeyExtractor):
"""
Row key extractor for dynamic bucket mode
@@ -181,9 +278,45 @@ class DynamicBucketRowKeyExtractor(RowKeyExtractor):
raise ValueError(
f"Only 'bucket' = '-1' is allowed for 'DynamicBucketRowKeyExtractor', but found: {num_buckets}"
)
+ opts = CoreOptions.from_dict(table_schema.options)
+ self._assigner = SimpleHashBucketAssigner(
+ num_assigners=1,
+ assign_id=0,
+ target_bucket_row_number=opts.dynamic_bucket_target_row_num(),
+ max_buckets_num=opts.dynamic_bucket_max_buckets(),
+ )
+ # TODO: extract bucket key init logic to base class (shared with FixedBucketRowKeyExtractor)
+ bucket_key_option = opts.bucket_key()
+ if bucket_key_option and bucket_key_option.strip():
+ self.bucket_keys = [k.strip() for k in bucket_key_option.split(',')]
+ else:
+ self.bucket_keys = [
+ pk for pk in table_schema.primary_keys
+ if pk not in table_schema.partition_keys
+ ]
+ self.bucket_key_indices = self._get_field_indices(self.bucket_keys)
+ field_map = {f.name: f for f in table_schema.fields}
+ self._bucket_key_fields = [
+ field_map[name] for name in self.bucket_keys if name in field_map
+ ]
def _extract_buckets_batch(self, data: pa.RecordBatch) -> List[int]:
- raise ValueError("Can't extract bucket from row in dynamic bucket mode")
+ partitions = self._extract_partitions_batch(data)
+ columns = [data.column(i) for i in self.bucket_key_indices]
+ buckets = []
+ for row_idx in range(data.num_rows):
+ key_hash = _hash_bytes_by_words(
+ GenericRowSerializer.to_bytes(
+ GenericRow(
+ [columns[j][row_idx].as_py() for j in range(len(columns))],
+ self._bucket_key_fields,
+ RowKind.INSERT,
+ )
+ )[4:]
+ )
+ buckets.append(
+ self._assigner.assign(partitions[row_idx], key_hash))
+ return buckets
class PostponeBucketRowKeyExtractor(RowKeyExtractor):