You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2026/03/08 11:12:37 UTC

(paimon) branch master updated: [python] Support dynamic bucket write (#7363)

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new d707b1c743 [python] Support dynamic bucket write (#7363)
d707b1c743 is described below

commit d707b1c74344d98ad42c8e91a8c0c01e2f1e34da
Author: XiaoHongbo <13...@qq.com>
AuthorDate: Sun Mar 8 19:12:32 2026 +0800

    [python] Support dynamic bucket write (#7363)
---
 .../pypaimon/common/options/core_options.py        |  25 ++++
 .../pypaimon/tests/rest/rest_read_write_test.py    |   2 -
 .../pypaimon/tests/rest/rest_simple_test.py        |  19 ++-
 .../write/simple_hash_bucket_assigner_test.py      |  60 +++++++++
 .../pypaimon/tests/write/table_write_test.py       |  30 +++++
 paimon-python/pypaimon/write/row_key_extractor.py  | 137 ++++++++++++++++++++-
 6 files changed, 265 insertions(+), 8 deletions(-)

diff --git a/paimon-python/pypaimon/common/options/core_options.py b/paimon-python/pypaimon/common/options/core_options.py
index 33384316cf..85cf965f91 100644
--- a/paimon-python/pypaimon/common/options/core_options.py
+++ b/paimon-python/pypaimon/common/options/core_options.py
@@ -107,6 +107,25 @@ class CoreOptions:
         )
     )
 
+    DYNAMIC_BUCKET_TARGET_ROW_NUM: ConfigOption[int] = (
+        ConfigOptions.key("dynamic-bucket.target-row-num")
+        .int_type()
+        .default_value(2000000)
+        .with_description(
+            "In dynamic bucket mode (bucket=-1), target row number per bucket; "
+            "when exceeded, a new bucket is created (aligned with Java SimpleHashBucketAssigner)."
+        )
+    )
+
+    DYNAMIC_BUCKET_MAX_BUCKETS: ConfigOption[int] = (
+        ConfigOptions.key("dynamic-bucket.max-buckets")
+        .int_type()
+        .default_value(-1)
+        .with_description(
+            "In dynamic bucket mode, max buckets per partition. -1 means unlimited."
+        )
+    )
+
     SCAN_MANIFEST_PARALLELISM: ConfigOption[int] = (
         ConfigOptions.key("scan.manifest.parallelism")
         .int_type()
@@ -389,6 +408,12 @@ class CoreOptions:
     def bucket_key(self, default=None):
         return self.options.get(CoreOptions.BUCKET_KEY, default)
 
+    def dynamic_bucket_target_row_num(self, default=None):
+        return self.options.get(CoreOptions.DYNAMIC_BUCKET_TARGET_ROW_NUM, default)
+
+    def dynamic_bucket_max_buckets(self, default=None):
+        return self.options.get(CoreOptions.DYNAMIC_BUCKET_MAX_BUCKETS, default)
+
     def scan_manifest_parallelism(self, default=None):
         return self.options.get(CoreOptions.SCAN_MANIFEST_PARALLELISM, default)
 
diff --git a/paimon-python/pypaimon/tests/rest/rest_read_write_test.py b/paimon-python/pypaimon/tests/rest/rest_read_write_test.py
index f2ed1342e5..726ef134d3 100644
--- a/paimon-python/pypaimon/tests/rest/rest_read_write_test.py
+++ b/paimon-python/pypaimon/tests/rest/rest_read_write_test.py
@@ -20,7 +20,6 @@ import logging
 
 import pandas as pd
 import pyarrow as pa
-import unittest
 
 from pypaimon.common.options import Options
 from pypaimon.catalog.catalog_context import CatalogContext
@@ -363,7 +362,6 @@ class RESTTableReadWriteTest(RESTBaseTest):
         ])
         self.assertEqual(actual, expected)
 
-    @unittest.skip("does not support dynamic bucket in dummy rest server")
     def test_pk_lance_reader_no_bucket(self):
         """Test Lance format with PrimaryKey table without specifying bucket."""
         schema = Schema.from_pyarrow_schema(self.pa_schema,
diff --git a/paimon-python/pypaimon/tests/rest/rest_simple_test.py b/paimon-python/pypaimon/tests/rest/rest_simple_test.py
index b55e42f841..7fde2955c5 100644
--- a/paimon-python/pypaimon/tests/rest/rest_simple_test.py
+++ b/paimon-python/pypaimon/tests/rest/rest_simple_test.py
@@ -600,14 +600,25 @@ class RESTSimpleTest(RESTBaseTest):
 
         write_builder = table.new_batch_write_builder()
         table_write = write_builder.new_write()
+        table_commit = write_builder.new_commit()
         self.assertIsInstance(table_write.row_key_extractor, DynamicBucketRowKeyExtractor)
 
         pa_table = pa.Table.from_pydict(self.data, schema=self.pa_schema)
+        table_write.write_arrow(pa_table)
+        table_commit.commit(table_write.prepare_commit())
+        table_write.close()
+        table_commit.close()
 
-        with self.assertRaises(ValueError) as context:
-            table_write.write_arrow(pa_table)
-
-        self.assertEqual(str(context.exception), "Can't extract bucket from row in dynamic bucket mode")
+        splits = []
+        read_builder = table.new_read_builder()
+        splits.extend(read_builder.new_scan().with_shard(0, 3).plan().splits())
+        splits.extend(read_builder.new_scan().with_shard(1, 3).plan().splits())
+        splits.extend(read_builder.new_scan().with_shard(2, 3).plan().splits())
+        table_read = read_builder.new_read()
+        actual = table_read.to_arrow(splits)
+        expected_sorted = table_sort_by(self.expected, 'user_id')
+        actual_sorted = table_sort_by(actual, 'user_id')
+        self.assertEqual(actual_sorted, expected_sorted)
 
     def test_with_shard_pk_fixed_bucket(self):
         schema = Schema.from_pyarrow_schema(self.pa_schema, partition_keys=['user_id'], primary_keys=['user_id', 'dt'],
diff --git a/paimon-python/pypaimon/tests/write/simple_hash_bucket_assigner_test.py b/paimon-python/pypaimon/tests/write/simple_hash_bucket_assigner_test.py
new file mode 100644
index 0000000000..5644cf0d25
--- /dev/null
+++ b/paimon-python/pypaimon/tests/write/simple_hash_bucket_assigner_test.py
@@ -0,0 +1,60 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+import unittest
+
+from pypaimon.write.row_key_extractor import SimpleHashBucketAssigner
+
+
+class SimpleHashBucketAssignerTest(unittest.TestCase):
+
+    def test_assign(self):
+        assigner = SimpleHashBucketAssigner(2, 0, 100, -1)
+        partition = ()
+        buckets = [assigner.assign(partition, h) for h in range(301)]
+        for b in buckets[:100]:
+            self.assertEqual(b, 0)
+        for b in buckets[100:200]:
+            self.assertEqual(b, 2)
+        self.assertEqual(buckets[200], 4)
+
+    def test_assign_with_upper_bound(self):
+        assigner = SimpleHashBucketAssigner(1, 0, 100, 3)
+        partition = ()
+        buckets = [assigner.assign(partition, h) for h in range(400)]
+        for b in buckets[:100]:
+            self.assertEqual(b, 0)
+        for b in buckets[100:200]:
+            self.assertEqual(b, 1)
+        for b in buckets[200:300]:
+            self.assertEqual(b, 2)
+        for b in buckets[300:]:
+            self.assertIn(b, [0, 1, 2])
+
+    def test_assign_with_same_hash(self):
+        for max_buckets in [-1, 1, 2]:
+            with self.subTest(max_buckets=max_buckets):
+                assigner = SimpleHashBucketAssigner(1, 0, 100, max_buckets)
+                partition = ()
+                hashes = list(range(100)) + list(range(100))
+                buckets = [assigner.assign(partition, h) for h in hashes]
+                for b in buckets[100:]:
+                    self.assertEqual(b, 0)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/paimon-python/pypaimon/tests/write/table_write_test.py b/paimon-python/pypaimon/tests/write/table_write_test.py
index a73dc631b1..34a892a725 100644
--- a/paimon-python/pypaimon/tests/write/table_write_test.py
+++ b/paimon-python/pypaimon/tests/write/table_write_test.py
@@ -354,3 +354,33 @@ class TableWriteTest(unittest.TestCase):
         for file_name in data_files:
             self.assertRegex(file_name, expected_pattern,
                              f"File name '{file_name}' does not match expected prefix format")
+
+    def test_dynamic_bucket_write(self):
+        schema = Schema.from_pyarrow_schema(
+            self.pa_schema,
+            partition_keys=['dt'],
+            primary_keys=['user_id', 'dt'],
+            options={'bucket': '-1'}
+        )
+        self.catalog.create_table(
+            'default.test_dynamic_bucket', schema, False)
+        table = self.catalog.get_table(
+            'default.test_dynamic_bucket')
+        write_builder = table.new_batch_write_builder()
+
+        table_write = write_builder.new_write()
+        table_commit = write_builder.new_commit()
+        table_write.write_arrow(self.expected)
+        table_commit.commit(table_write.prepare_commit())
+        table_write.close()
+        table_commit.close()
+
+        read_builder = table.new_read_builder()
+        table_read = read_builder.new_read()
+        splits = read_builder.new_scan().plan().splits()
+        actual = table_read.to_arrow(splits)
+        sort_keys = [('user_id', 'ascending'), ('dt', 'ascending')]
+        self.assertEqual(
+            self.expected.sort_by(sort_keys),
+            actual.sort_by(sort_keys),
+        )
diff --git a/paimon-python/pypaimon/write/row_key_extractor.py b/paimon-python/pypaimon/write/row_key_extractor.py
index 9ff23404c3..2f09e6577b 100644
--- a/paimon-python/pypaimon/write/row_key_extractor.py
+++ b/paimon-python/pypaimon/write/row_key_extractor.py
@@ -17,9 +17,10 @@
 ################################################################################
 
 import math
+import random
 import struct
 from abc import ABC, abstractmethod
-from typing import List, Tuple
+from typing import Dict, List, Tuple
 
 import pyarrow as pa
 
@@ -167,6 +168,102 @@ class UnawareBucketRowKeyExtractor(RowKeyExtractor):
         return [0] * data.num_rows
 
 
+_SHORT_MAX_VALUE = 32767
+
+
+def _is_my_bucket(bucket: int, num_assigners: int, assign_id: int) -> bool:
+    return bucket % num_assigners == assign_id % num_assigners
+
+
+def _pick_randomly(bucket_list: List[int]) -> int:
+    return random.choice(bucket_list)
+
+
+class _SimplePartitionIndex:
+    def __init__(self, num_assigners: int, assign_id: int, max_buckets_num: int) -> None:
+        self.hash2bucket: Dict[int, int] = {}
+        self.bucket_information: Dict[int, int] = {}
+        self.bucket_list: List[int] = []
+        self.current_bucket: int = 0
+        self._load_new_bucket(max_buckets_num, num_assigners, assign_id)
+
+    def assign(
+        self,
+        hash_value: int,
+        max_bucket_id: int,
+        target_bucket_row_number: int,
+        max_buckets_num: int,
+        num_assigners: int,
+        assign_id: int,
+    ) -> Tuple[int, int]:
+        if hash_value in self.hash2bucket:
+            assigned = self.hash2bucket[hash_value]
+            return assigned, max(max_bucket_id, assigned)
+
+        if self.current_bucket not in self.bucket_information:
+            self.bucket_list.append(self.current_bucket)
+            self.bucket_information[self.current_bucket] = 0
+        num = self.bucket_information[self.current_bucket]
+
+        if num >= target_bucket_row_number:
+            if (
+                max_buckets_num == -1
+                or not self.bucket_information
+                or max_bucket_id < max_buckets_num - 1
+            ):
+                self._load_new_bucket(max_buckets_num, num_assigners, assign_id)
+            else:
+                self.current_bucket = _pick_randomly(self.bucket_list)
+
+        self.bucket_information[self.current_bucket] = (
+            self.bucket_information.get(self.current_bucket, 0) + 1
+        )
+        self.hash2bucket[hash_value] = self.current_bucket
+        new_max = max(max_bucket_id, self.current_bucket)
+        return self.current_bucket, new_max
+
+    def _load_new_bucket(
+        self, max_buckets_num: int, num_assigners: int, assign_id: int
+    ) -> None:
+        for i in range(_SHORT_MAX_VALUE):
+            if _is_my_bucket(i, num_assigners, assign_id) and (
+                i not in self.bucket_information
+            ):
+                if max_buckets_num == -1 or i <= max_buckets_num - 1:
+                    self.current_bucket = i
+                    return
+                return
+        raise RuntimeError(
+            "Can't find a suitable bucket to assign, all the bucket are assigned?"
+        )
+
+
+class SimpleHashBucketAssigner:
+    def __init__(self, num_assigners, assign_id, target_bucket_row_number, max_buckets_num):
+        self.num_assigners = num_assigners
+        self.assign_id = assign_id
+        self.target_bucket_row_number = target_bucket_row_number
+        self.max_buckets_num = max_buckets_num
+        self.max_bucket_id = 0
+        self._partition_index: Dict[Tuple, _SimplePartitionIndex] = {}
+
+    def assign(self, partition: Tuple, hash_value: int) -> int:
+        if partition not in self._partition_index:
+            self._partition_index[partition] = _SimplePartitionIndex(
+                self.num_assigners, self.assign_id, self.max_buckets_num)
+        index = self._partition_index[partition]
+
+        assigned, self.max_bucket_id = index.assign(
+            hash_value,
+            self.max_bucket_id,
+            self.target_bucket_row_number,
+            self.max_buckets_num,
+            self.num_assigners,
+            self.assign_id,
+        )
+        return assigned
+
+
 class DynamicBucketRowKeyExtractor(RowKeyExtractor):
     """
     Row key extractor for dynamic bucket mode
@@ -181,9 +278,45 @@ class DynamicBucketRowKeyExtractor(RowKeyExtractor):
             raise ValueError(
                 f"Only 'bucket' = '-1' is allowed for 'DynamicBucketRowKeyExtractor', but found: {num_buckets}"
             )
+        opts = CoreOptions.from_dict(table_schema.options)
+        self._assigner = SimpleHashBucketAssigner(
+            num_assigners=1,
+            assign_id=0,
+            target_bucket_row_number=opts.dynamic_bucket_target_row_num(),
+            max_buckets_num=opts.dynamic_bucket_max_buckets(),
+        )
+        # TODO: extract bucket key init logic to base class (shared with FixedBucketRowKeyExtractor)
+        bucket_key_option = opts.bucket_key()
+        if bucket_key_option and bucket_key_option.strip():
+            self.bucket_keys = [k.strip() for k in bucket_key_option.split(',')]
+        else:
+            self.bucket_keys = [
+                pk for pk in table_schema.primary_keys
+                if pk not in table_schema.partition_keys
+            ]
+        self.bucket_key_indices = self._get_field_indices(self.bucket_keys)
+        field_map = {f.name: f for f in table_schema.fields}
+        self._bucket_key_fields = [
+            field_map[name] for name in self.bucket_keys if name in field_map
+        ]
 
     def _extract_buckets_batch(self, data: pa.RecordBatch) -> List[int]:
-        raise ValueError("Can't extract bucket from row in dynamic bucket mode")
+        partitions = self._extract_partitions_batch(data)
+        columns = [data.column(i) for i in self.bucket_key_indices]
+        buckets = []
+        for row_idx in range(data.num_rows):
+            key_hash = _hash_bytes_by_words(
+                GenericRowSerializer.to_bytes(
+                    GenericRow(
+                        [columns[j][row_idx].as_py() for j in range(len(columns))],
+                        self._bucket_key_fields,
+                        RowKind.INSERT,
+                    )
+                )[4:]
+            )
+            buckets.append(
+                self._assigner.assign(partitions[row_idx], key_hash))
+        return buckets
 
 
 class PostponeBucketRowKeyExtractor(RowKeyExtractor):