You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by yu...@apache.org on 2026/04/12 10:53:59 UTC
(paimon) branch release-1.4 updated (f509a8fe15 -> 1fe4cf22ae)
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a change to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
from f509a8fe15 [spark] Fix view resolution for CTE and ORDER BY ordinal (#7552)
new f2c41f14d7 [flink] Support dropping primary keys for empty tables in Flink (#7566)
new 861252a702 [python] Fix REST signature mismatch by encoding query param values (#7568)
new 97813f617c [core] Fix that latestSnapshotOfUser might list snapshot directory to find the earliest (#7572)
new 500c9c9219 [python] Fix token cache isolation by adding user identity to cache key (#7571)
new 2aecba09e6 [arrow] Reuse length array to avoid allocate array memory repeatedly (#7573)
new c1974391d2 [python] Fix update_by_row_id conflict when file rolling splits (#7574)
new 7ce5d01bce [core] Extract Parquet stats from in-memory footer instead of re-reading file (#7489)
new de76b56763 [api] Fix locale-sensitive date formatting in DLFOpenApiSigner (#7577)
new e87c30c4e1 [python] Fix rolling stack overflow by replacing recursion with iteration (#7578)
new 07eac94c18 [spark] Fix view query failure with VARCHAR/CHAR column types (#7585)
new 227d631200 [core] Fix partial-update sequence-group delete mismatch under projected reads (#7586)
new daa619b019 [hotfix] Fix incorrect integer comparison prevent misjudgment in chain table. (#7591)
new da948fa1cd [python] Fix BlockHandle varlen decoding in SstFileIterator.read_batch (#7597)
new c25ddfa415 [core] Fix TagAutoCreation.forceCreatingSnapshot to use SINK_PROCESS_TIME_ZONE (#7600)
new 06f3a6d172 [python] Fix stats evolution index mapping to use field ID instead of field name (#7593)
new 32b886a324 [docs] Add Spark SQL example for fast_forward in manage-branches (#7604)
new 8fb5b3e3bd [flink] fix de-merge-into runtimeContext may throw NoSuchMethod error (#7607)
new b60d1bac4d [python] Add file_source assertion in row-tracking commit to fail fast on missing file_source (#7610)
new 14b0816b6c [spark] Fix RowTrackingTable reading by add _ROW_ID existing check before adding _ROW_ID (#7606)
new 3579ba1663 [core] Add hashCode to AbstractFileStoreTable for consistency with equals (#7616)
new d3589fa11b [flink] fix compatibility issues of StreamExecutionEnv (#7615)
new a352fcfc2c [core] When scan.primary-branch is set, allow primary branch to be an append only table (#7614)
new c258aea8e9 [core] Fix blob read failure after alter table and compaction (#7618)
new f27a002af4 [python] Fix duplicate _ROW_ID when file exceeds read batch size (#7626)
new 1fe4cf22ae [python] Add conflict detection in shard update (#7630)
The 25 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/content/maintenance/manage-branches.md | 11 +
.../apache/paimon/rest/auth/DLFOpenApiSigner.java | 3 +-
.../org/apache/paimon/schema/SchemaChange.java | 27 +++
.../paimon/rest/auth/DLFRequestSignerTest.java | 41 ++++
.../paimon/arrow/writer/ArrowFieldWriters.java | 39 ++--
.../org/apache/paimon/format/FormatWriter.java | 17 ++
.../apache/paimon/format/SimpleStatsExtractor.java | 20 ++
.../org/apache/paimon/io/SimpleStatsProducer.java | 18 ++
.../org/apache/paimon/io/SingleFileWriter.java | 11 +
.../paimon/io/StatsCollectingSingleFileWriter.java | 2 +-
.../compact/PartialUpdateMergeFunction.java | 9 +-
.../paimon/operation/DataEvolutionSplitRead.java | 8 +-
.../org/apache/paimon/schema/SchemaManager.java | 26 ++-
.../paimon/table/AbstractFileStoreTable.java | 5 +
.../apache/paimon/table/ChainGroupReadTable.java | 2 +-
.../paimon/table/FallbackReadFileStoreTable.java | 24 --
.../org/apache/paimon/tag/TagAutoCreation.java | 19 +-
.../org/apache/paimon/utils/SnapshotManager.java | 28 +--
.../test/java/org/apache/paimon/JavaPyE2ETest.java | 149 ++++++++++++
.../org/apache/paimon/append/BlobTableTest.java | 239 +++++++++++++++++++
.../compact/PartialUpdateMergeFunctionTest.java | 40 ++++
.../paimon/operation/DataEvolutionReadTest.java | 44 ++++
.../apache/paimon/schema/SchemaManagerTest.java | 45 ++++
.../paimon/table/AppendOnlySimpleTableTest.java | 27 +++
.../paimon/table/PrimaryKeySimpleTableTest.java | 27 +++
.../org/apache/paimon/tag/TagAutoManagerTest.java | 45 ++++
.../apache/paimon/utils/SnapshotManagerTest.java | 4 +-
.../java/org/apache/paimon/flink/FlinkCatalog.java | 38 ++-
.../flink/action/RemoveUnexistingFilesAction.java | 4 +-
.../paimon/flink/btree/BTreeIndexTopoBuilder.java | 6 +-
.../dataevolution/MergeIntoUpdateChecker.java | 3 +-
.../flink/globalindex/GenericIndexTopoBuilder.java | 7 +-
.../org/apache/paimon/flink/BranchSqlITCase.java | 42 ++++
.../apache/paimon/flink/SchemaChangeITCase.java | 24 ++
.../utils/StreamExecutionEnvironmentUtils.java} | 27 ++-
.../utils/StreamExecutionEnvironmentUtils.java} | 26 ++-
.../parquet/ParquetSimpleStatsExtractor.java | 26 +++
.../apache/paimon/format/parquet/ParquetUtil.java | 45 ++--
.../format/parquet/writer/ParquetBulkWriter.java | 13 ++
.../format/parquet/ParquetInMemoryStatsTest.java | 213 +++++++++++++++++
paimon-python/dev/run_mixed_tests.sh | 81 ++++++-
paimon-python/pypaimon/api/typedef.py | 17 +-
.../pypaimon/catalog/rest/rest_token_file_io.py | 76 +++++-
.../pypaimon/globalindex/btree/sst_file_reader.py | 33 ++-
.../pypaimon/manifest/simple_stats_evolutions.py | 10 +-
paimon-python/pypaimon/read/plan.py | 5 +-
.../pypaimon/read/reader/data_file_batch_reader.py | 1 +
paimon-python/pypaimon/read/reader/field_bunch.py | 9 +-
.../pypaimon/read/scanner/file_scanner.py | 23 +-
.../pypaimon/read/streaming_table_scan.py | 2 +-
paimon-python/pypaimon/read/table_scan.py | 12 +-
paimon-python/pypaimon/schema/schema_manager.py | 13 ++
paimon-python/pypaimon/tests/binary_row_test.py | 8 +-
paimon-python/pypaimon/tests/blob_table_test.py | 25 ++
.../pypaimon/tests/data_evolution_test.py | 41 ++++
.../pypaimon/tests/e2e/java_py_read_write_test.py | 75 ++++++
.../tests/manifest/simple_stats_evolutions_test.py | 156 +++++++++++++
.../pypaimon/tests/reader_append_only_test.py | 45 ++++
.../pypaimon/tests/rest/dlf_signer_test.py | 7 +
.../pypaimon/tests/rest/rest_token_file_io_test.py | 254 ++++++++++++++-------
.../pypaimon/tests/sst_file_iterator_test.py | 221 ++++++++++++++++++
paimon-python/pypaimon/tests/table_update_test.py | 67 ++++++
.../pypaimon/tests/write/table_write_test.py | 35 +++
.../pypaimon/write/commit/commit_scanner.py | 4 +-
paimon-python/pypaimon/write/file_store_commit.py | 5 +-
paimon-python/pypaimon/write/file_store_write.py | 5 +
paimon-python/pypaimon/write/table_update.py | 6 +-
.../pypaimon/write/table_update_by_row_id.py | 8 +-
paimon-python/pypaimon/write/writer/data_writer.py | 22 +-
.../paimon/spark/PaimonBaseScanBuilder.scala | 5 +-
.../catalyst/analysis/PaimonViewResolver.scala | 9 +-
.../org/apache/paimon/spark/read/BaseScan.scala | 5 +-
.../paimon/spark/sql/PaimonViewTestBase.scala | 48 ++++
.../paimon/spark/sql/RowTrackingTestBase.scala | 17 ++
74 files changed, 2457 insertions(+), 297 deletions(-)
copy paimon-flink/{paimon-flink-common/src/test/java/org/apache/paimon/flink/utils/JavaTypeInfoTest.java => paimon-flink1-common/src/main/java/org/apache/paimon/flink/utils/StreamExecutionEnvironmentUtils.java} (50%)
copy paimon-flink/{paimon-flink-common/src/test/java/org/apache/paimon/flink/utils/JavaTypeInfoTest.java => paimon-flink2-common/src/main/java/org/apache/paimon/flink/utils/StreamExecutionEnvironmentUtils.java} (52%)
create mode 100644 paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetInMemoryStatsTest.java
create mode 100644 paimon-python/pypaimon/tests/manifest/simple_stats_evolutions_test.py
create mode 100644 paimon-python/pypaimon/tests/sst_file_iterator_test.py
(paimon) 06/25: [python] Fix update_by_row_id conflict when file rolling splits (#7574)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit c1974391d25755e3771da8c574df24bd550b1c1e
Author: XiaoHongbo <13...@qq.com>
AuthorDate: Wed Apr 1 17:29:51 2026 +0800
[python] Fix update_by_row_id conflict when file rolling splits (#7574)
When calling `update_by_arrow_with_row_id` or `upsert_by_arrow_with_key`
on large files, there is `RuntimeError: For Data Evolution table,
multiple 'MERGE INTO' and 'COMPACT' operations have encountered
conflicts`. This PR fixes the above issue by disabling rolling for one
file when `update_by_arrow_with_row_id`.
---
paimon-python/pypaimon/tests/table_update_test.py | 67 ++++++++++++++++++++++
paimon-python/pypaimon/write/file_store_write.py | 5 ++
.../pypaimon/write/table_update_by_row_id.py | 3 +-
3 files changed, 74 insertions(+), 1 deletion(-)
diff --git a/paimon-python/pypaimon/tests/table_update_test.py b/paimon-python/pypaimon/tests/table_update_test.py
index bf46f7bbac..204e31ff0f 100644
--- a/paimon-python/pypaimon/tests/table_update_test.py
+++ b/paimon-python/pypaimon/tests/table_update_test.py
@@ -1052,6 +1052,73 @@ class TableUpdateTest(unittest.TestCase):
self.assertEqual(40, ages[3], "Row 3 should remain unchanged")
self.assertEqual(45, ages[4], "Row 4 should remain unchanged")
+ def test_update_with_large_file(self):
+ import uuid
+ import random
+ import string
+
+ table_name = f'test_row_id_split_{uuid.uuid4().hex[:8]}'
+ schema = Schema.from_pyarrow_schema(
+ pa.schema([('id', pa.int64()), ('name', pa.string())]),
+ options={
+ 'row-tracking.enabled': 'true',
+ 'data-evolution.enabled': 'true',
+ 'write-only': 'true',
+ }
+ )
+ self.catalog.create_table(
+ f'default.{table_name}', schema, False)
+ table = self.catalog.get_table(f'default.{table_name}')
+
+ N = 5000
+ data = pa.table({
+ 'id': list(range(N)),
+ 'name': [
+ ''.join(random.choices(
+ string.ascii_letters, k=200))
+ for _ in range(N)],
+ })
+ wb = table.new_batch_write_builder()
+ tw = wb.new_write()
+ tc = wb.new_commit()
+ tw.write_arrow(data)
+ tc.commit(tw.prepare_commit())
+ tw.close()
+ tc.close()
+
+ from pypaimon.schema.schema_change import SetOption
+ self.catalog.alter_table(
+ f'default.{table_name}',
+ [SetOption('target-file-size', '10kb')])
+ table = self.catalog.get_table(f'default.{table_name}')
+
+ wb = table.new_batch_write_builder()
+ updator = wb.new_update()
+ updator.with_update_type(['name'])
+ update_data = pa.table({
+ '_ROW_ID': pa.array(
+ list(range(N)), type=pa.int64()),
+ 'name': [
+ ''.join(random.choices(
+ string.ascii_letters, k=200))
+ for _ in range(N)],
+ })
+ msgs = updator.update_by_arrow_with_row_id(update_data)
+
+ all_files = []
+ for msg in msgs:
+ all_files.extend(msg.new_files)
+
+ self.assertEqual(
+ len(all_files), 1,
+ "Update should produce exactly one file per group")
+ self.assertEqual(all_files[0].first_row_id, 0)
+ self.assertEqual(all_files[0].row_count, N)
+
+ tc = wb.new_commit()
+ tc.commit(msgs)
+ tc.close()
+
if __name__ == '__main__':
unittest.main()
diff --git a/paimon-python/pypaimon/write/file_store_write.py b/paimon-python/pypaimon/write/file_store_write.py
index 95113a0689..75b1d3a7d7 100644
--- a/paimon-python/pypaimon/write/file_store_write.py
+++ b/paimon-python/pypaimon/write/file_store_write.py
@@ -46,6 +46,11 @@ class FileStoreWrite:
(f"{self.options.data_file_prefix()}-u-{commit_user}"
f"-s-{random.randint(0, 2 ** 31 - 2)}-w-"))
+ def disable_rolling(self):
+ """Disable file rolling by setting target_file_size to max."""
+ self.options.set(
+ CoreOptions.TARGET_FILE_SIZE, str(2 ** 63 - 1))
+
def write(self, partition: Tuple, bucket: int, data: pa.RecordBatch):
key = (partition, bucket)
if key not in self.data_writers:
diff --git a/paimon-python/pypaimon/write/table_update_by_row_id.py b/paimon-python/pypaimon/write/table_update_by_row_id.py
index 48f61751ec..089fde047c 100644
--- a/paimon-python/pypaimon/write/table_update_by_row_id.py
+++ b/paimon-python/pypaimon/write/table_update_by_row_id.py
@@ -293,7 +293,9 @@ class TableUpdateByRowId:
merged_data = self._merge_update_with_original(original_data, data, column_names, first_row_id)
# Create a file store write for this partition
+ # Disable rolling to ensure one output file per first_row_id group,
file_store_write = FileStoreWrite(self.table, self.commit_user)
+ file_store_write.disable_rolling()
# Set write columns to only update specific columns
write_cols = column_names
@@ -313,7 +315,6 @@ class TableUpdateByRowId:
for msg in commit_messages:
msg.check_from_snapshot = self.snapshot_id
for file in msg.new_files:
- # Assign the same first_row_id as the original file
file.first_row_id = first_row_id
file.write_cols = write_cols
(paimon) 13/25: [python] Fix BlockHandle varlen decoding in SstFileIterator.read_batch (#7597)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit da948fa1cd6d65e531417408aba702b9aa9344a1
Author: Jiajia Li <pl...@alibaba-inc.com>
AuthorDate: Tue Apr 7 09:58:38 2026 +0800
[python] Fix BlockHandle varlen decoding in SstFileIterator.read_batch (#7597)
read_batch() incorrectly used fixed-width struct.unpack to decode
BlockHandle, while the SST format uses variable-length encoding. Extract
shared _parse_block_handle() to ensure both seek_to() and read_batch()
use consistent varlen decoding.
---
.../pypaimon/globalindex/btree/sst_file_reader.py | 33 ++-
.../pypaimon/tests/e2e/java_py_read_write_test.py | 7 +
.../pypaimon/tests/sst_file_iterator_test.py | 221 +++++++++++++++++++++
3 files changed, 242 insertions(+), 19 deletions(-)
diff --git a/paimon-python/pypaimon/globalindex/btree/sst_file_reader.py b/paimon-python/pypaimon/globalindex/btree/sst_file_reader.py
index 9d7f2eafc2..c7a28850ca 100644
--- a/paimon-python/pypaimon/globalindex/btree/sst_file_reader.py
+++ b/paimon-python/pypaimon/globalindex/btree/sst_file_reader.py
@@ -30,7 +30,7 @@ import zlib
from typing import Optional, Callable
from typing import BinaryIO
-from pypaimon.globalindex.btree.btree_file_footer import BlockHandle
+from pypaimon.globalindex.btree.block_handle import BlockHandle
from pypaimon.globalindex.btree.block_entry import BlockEntry
from pypaimon.globalindex.btree.block_reader import BlockReader, BlockIterator
from pypaimon.globalindex.btree.memory_slice_input import MemorySliceInput
@@ -48,27 +48,28 @@ class SstFileIterator:
self.index_iterator = index_block_iterator
self.sought_data_block: Optional[BlockIterator] = None
+ @staticmethod
+ def _parse_block_handle(block_handle_bytes: bytes) -> BlockHandle:
+ handle_input = MemorySliceInput(block_handle_bytes)
+ return BlockHandle(
+ handle_input.read_var_len_long(),
+ handle_input.read_var_len_int()
+ )
+
def seek_to(self, key: bytes) -> None:
"""
Seek to the position of the record whose key is exactly equal to or
greater than the specified key.
-
+
Args:
key: The key to seek to
"""
self.index_iterator.seek_to(key)
-
+
if self.index_iterator.has_next():
index_entry: BlockEntry = self.index_iterator.__next__()
- block_handle_bytes = index_entry.__getattribute__("value")
- handle_input = MemorySliceInput(block_handle_bytes)
-
- # Parse block handle
- block_handle = BlockHandle(
- handle_input.read_var_len_long(),
- handle_input.read_var_len_int()
- )
-
+ block_handle = self._parse_block_handle(index_entry.value)
+
# Create data block reader and seek
data_block_reader = self.read_block(block_handle)
self.sought_data_block = data_block_reader.iterator()
@@ -93,13 +94,7 @@ class SstFileIterator:
return None
index_entry = self.index_iterator.__next__()
- block_handle_bytes = index_entry.value
-
- # Parse block handle
- block_handle = BlockHandle(
- struct.unpack('<Q', block_handle_bytes[0:8])[0],
- struct.unpack('<I', block_handle_bytes[8:12])[0]
- )
+ block_handle = self._parse_block_handle(index_entry.value)
# Create data block reader
data_block_reader = self.read_block(block_handle)
diff --git a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
index 4d888c3cb4..1f906c656d 100644
--- a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
+++ b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
@@ -452,6 +452,13 @@ class JavaPyReadWriteTest(unittest.TestCase):
})
self.assertEqual(expected, actual)
+ # read is_not_null index (full scan across all data blocks)
+ read_builder.with_filter(predicate_builder.is_not_null('k'))
+ table_read = read_builder.new_read()
+ splits = read_builder.new_scan().plan().splits()
+ actual = table_read.to_arrow(splits)
+ self.assertEqual(len(actual), 2000)
+
def _test_read_btree_index_null(self):
table = self.catalog.get_table('default.test_btree_index_null')
diff --git a/paimon-python/pypaimon/tests/sst_file_iterator_test.py b/paimon-python/pypaimon/tests/sst_file_iterator_test.py
new file mode 100644
index 0000000000..c7c0a9b3a9
--- /dev/null
+++ b/paimon-python/pypaimon/tests/sst_file_iterator_test.py
@@ -0,0 +1,221 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+"""Tests for SstFileIterator BlockHandle varlen decoding."""
+
+import unittest
+from unittest.mock import MagicMock
+
+from pypaimon.globalindex.btree.block_handle import BlockHandle
+from pypaimon.globalindex.btree.block_entry import BlockEntry
+from pypaimon.globalindex.btree.memory_slice_input import MemorySliceInput
+from pypaimon.globalindex.btree.sst_file_reader import SstFileIterator
+
+
+def _encode_var_len(value):
+ result = bytearray()
+ while value > 0x7F:
+ result.append((value & 0x7F) | 0x80)
+ value >>= 7
+ result.append(value & 0x7F)
+ return bytes(result)
+
+
+def _encode_block_handle(offset, size):
+ return _encode_var_len(offset) + _encode_var_len(size)
+
+
+def _mock_block_iterator(entries):
+ """Mock a BlockIterator with has_next/next/seek_to over a list of BlockEntry."""
+ state = {'pos': 0}
+ entry_list = list(entries)
+
+ mock = MagicMock()
+ mock.has_next = lambda: state['pos'] < len(entry_list)
+
+ def next_entry(_self=None):
+ if state['pos'] >= len(entry_list):
+ raise StopIteration
+ entry = entry_list[state['pos']]
+ state['pos'] += 1
+ return entry
+ mock.__next__ = next_entry
+ mock.__iter__ = lambda _self=None: mock
+
+ def seek_to(target_key):
+ for i, entry in enumerate(entry_list):
+ if entry.key >= target_key:
+ state['pos'] = i
+ return entry.key == target_key
+ state['pos'] = len(entry_list)
+ return False
+ mock.seek_to = seek_to
+
+ return mock
+
+
+class SstFileIteratorTest(unittest.TestCase):
+
+ def _make_iterator(self, index_entries, data_blocks):
+ mock_index_entries = []
+ for key, handle in index_entries:
+ value = _encode_block_handle(handle.offset, handle.size)
+ mock_index_entries.append(BlockEntry(key, value))
+
+ index_iter = _mock_block_iterator(mock_index_entries)
+
+ def read_block(block_handle):
+ entries = data_blocks.get((block_handle.offset, block_handle.size))
+ if entries is None:
+ raise ValueError(
+ "Unexpected BlockHandle(offset={}, size={})".format(
+ block_handle.offset, block_handle.size))
+ reader = MagicMock()
+ reader.iterator = lambda e=entries: _mock_block_iterator(e)
+ return reader
+
+ return SstFileIterator(read_block, index_iter)
+
+ def test_read_batch_varlen_small_values(self):
+ handle = BlockHandle(100, 50)
+ data = [BlockEntry(b"k1", b"v1"), BlockEntry(b"k2", b"v2")]
+
+ it = self._make_iterator(
+ [(b"k2", handle)],
+ {(100, 50): data}
+ )
+
+ batch = it.read_batch()
+ self.assertIsNotNone(batch)
+ entries = [batch.__next__() for _ in range(2)]
+ self.assertEqual(len(entries), 2)
+ self.assertEqual(entries[0].key, b"k1")
+ self.assertEqual(entries[1].key, b"k2")
+ self.assertIsNone(it.read_batch())
+
+ def test_read_batch_varlen_large_offset(self):
+ handle = BlockHandle(300, 200)
+ data = [BlockEntry(b"a", b"1")]
+
+ it = self._make_iterator(
+ [(b"a", handle)],
+ {(300, 200): data}
+ )
+
+ batch = it.read_batch()
+ self.assertIsNotNone(batch)
+ entry = batch.__next__()
+ self.assertEqual(entry.key, b"a")
+
+ def test_read_batch_varlen_very_large_offset(self):
+ handle = BlockHandle(1000000, 65535)
+ data = [BlockEntry(b"big", b"val")]
+
+ it = self._make_iterator(
+ [(b"big", handle)],
+ {(1000000, 65535): data}
+ )
+
+ batch = it.read_batch()
+ self.assertIsNotNone(batch)
+ entry = batch.__next__()
+ self.assertEqual(entry.key, b"big")
+
+ def test_read_batch_multiple_blocks(self):
+ h1 = BlockHandle(0, 100)
+ h2 = BlockHandle(200, 150)
+ h3 = BlockHandle(500, 80)
+
+ it = self._make_iterator(
+ [(b"b", h1), (b"d", h2), (b"f", h3)],
+ {
+ (0, 100): [BlockEntry(b"a", b"1"), BlockEntry(b"b", b"2")],
+ (200, 150): [BlockEntry(b"c", b"3"), BlockEntry(b"d", b"4")],
+ (500, 80): [BlockEntry(b"e", b"5"), BlockEntry(b"f", b"6")],
+ }
+ )
+
+ all_entries = []
+ while True:
+ batch = it.read_batch()
+ if batch is None:
+ break
+ while batch.has_next():
+ all_entries.append(batch.__next__())
+
+ self.assertEqual(len(all_entries), 6)
+ keys = [e.key for e in all_entries]
+ self.assertEqual(keys, [b"a", b"b", b"c", b"d", b"e", b"f"])
+
+ def test_seek_then_read_batch_crosses_blocks(self):
+ h1 = BlockHandle(0, 100)
+ h2 = BlockHandle(256, 128)
+
+ it = self._make_iterator(
+ [(b"b", h1), (b"d", h2)],
+ {
+ (0, 100): [BlockEntry(b"a", b"1"), BlockEntry(b"b", b"2")],
+ (256, 128): [BlockEntry(b"c", b"3"), BlockEntry(b"d", b"4")],
+ }
+ )
+
+ it.seek_to(b"a")
+ self.assertIsNotNone(it.sought_data_block)
+
+ batch1 = it.read_batch()
+ self.assertIsNotNone(batch1)
+ self.assertEqual(batch1.__next__().key, b"a")
+
+ batch2 = it.read_batch()
+ self.assertIsNotNone(batch2)
+ entries2 = []
+ while batch2.has_next():
+ entries2.append(batch2.__next__())
+ self.assertEqual(len(entries2), 2)
+ self.assertEqual(entries2[0].key, b"c")
+ self.assertEqual(entries2[1].key, b"d")
+
+ self.assertIsNone(it.read_batch())
+
+ def test_read_batch_empty_index(self):
+ it = self._make_iterator([], {})
+ self.assertIsNone(it.read_batch())
+
+ def test_varlen_encoding_roundtrip(self):
+ test_cases = [
+ (0, 0),
+ (127, 127),
+ (128, 128),
+ (300, 200),
+ (16384, 255),
+ (1000000, 65535),
+ (2**31 - 1, 2**31 - 1),
+ ]
+ for offset, size in test_cases:
+ encoded = _encode_block_handle(offset, size)
+ inp = MemorySliceInput(encoded)
+ decoded_offset = inp.read_var_len_long()
+ decoded_size = inp.read_var_len_int()
+ self.assertEqual(decoded_offset, offset,
+ "offset mismatch for ({}, {})".format(offset, size))
+ self.assertEqual(decoded_size, size,
+ "size mismatch for ({}, {})".format(offset, size))
+
+
+if __name__ == '__main__':
+ unittest.main()
(paimon) 09/25: [python] Fix rolling stack overflow by replacing recursion with iteration (#7578)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit e87c30c4e12656eace5329b11cc5e91a97819d02
Author: XiaoHongbo <13...@qq.com>
AuthorDate: Thu Apr 2 14:28:44 2026 +0800
[python] Fix rolling stack overflow by replacing recursion with iteration (#7578)
Fix potential stack overflow in `_check_and_roll_if_needed` by replacing
recursion with a while loop. This can occur when `target_file_size` is
set to a small value, causing excessive recursive splits.
---
.../pypaimon/tests/write/table_write_test.py | 35 ++++++++++++++++++++++
paimon-python/pypaimon/write/writer/data_writer.py | 22 +++++++-------
2 files changed, 45 insertions(+), 12 deletions(-)
diff --git a/paimon-python/pypaimon/tests/write/table_write_test.py b/paimon-python/pypaimon/tests/write/table_write_test.py
index 60ddcdc0bd..2c685f5830 100644
--- a/paimon-python/pypaimon/tests/write/table_write_test.py
+++ b/paimon-python/pypaimon/tests/write/table_write_test.py
@@ -28,6 +28,8 @@ import pyarrow as pa
from parameterized import parameterized
from pypaimon.common.json_util import JSON
+from pypaimon.common.options.core_options import CoreOptions
+from pypaimon.write.writer.append_only_data_writer import AppendOnlyDataWriter
class TableWriteTest(unittest.TestCase):
@@ -436,3 +438,36 @@ class TableWriteTest(unittest.TestCase):
splits = read_builder.new_scan().plan().splits()
actual = table_read.to_arrow(splits)
self.assertEqual(expected, actual)
+
+ def test_rolling(self):
+ pa_schema = pa.schema([('name', pa.string())])
+ schema = Schema.from_pyarrow_schema(pa_schema, partition_keys=[])
+ self.catalog.create_table('default.test_rolling_recursion', schema, True)
+ table = self.catalog.get_table('default.test_rolling_recursion')
+
+ row_value = 'x' * 100
+ sample = pa.Table.from_batches([
+ pa.RecordBatch.from_pydict({'name': pa.array([row_value], type=pa.string())})
+ ])
+ # Set target just above single chunk nbytes so best_split=1 every time
+ target = sample.nbytes + 1
+
+ options = CoreOptions.copy(table.options)
+ options.set(CoreOptions.TARGET_FILE_SIZE, str(target))
+ writer = AppendOnlyDataWriter(
+ table=table, partition=(), bucket=0,
+ max_seq_number=0, options=options,
+ )
+
+ num_rows = 1500
+ big_batch = pa.RecordBatch.from_pydict(
+ {'name': pa.array([row_value] * num_rows, type=pa.string())}
+ )
+ writer.write(big_batch)
+
+ pending_rows = writer.pending_data.num_rows if writer.pending_data is not None else 0
+ committed_rows = sum(f.row_count for f in writer.committed_files)
+ self.assertEqual(committed_rows + pending_rows, num_rows)
+ self.assertGreater(len(writer.committed_files), 0)
+ if writer.pending_data is not None:
+ self.assertLessEqual(writer.pending_data.nbytes, target)
diff --git a/paimon-python/pypaimon/write/writer/data_writer.py b/paimon-python/pypaimon/write/writer/data_writer.py
index e0cbe607ca..4d2513a28c 100644
--- a/paimon-python/pypaimon/write/writer/data_writer.py
+++ b/paimon-python/pypaimon/write/writer/data_writer.py
@@ -142,19 +142,17 @@ class DataWriter(ABC):
"""Merge existing data with new data. Must be implemented by subclasses."""
def _check_and_roll_if_needed(self):
- if self.pending_data is None:
- return
-
- current_size = self.pending_data.nbytes
- if current_size > self.target_file_size:
+ while self.pending_data is not None:
+ current_size = self.pending_data.nbytes
+ if current_size <= self.target_file_size:
+ break
split_row = self._find_optimal_split_point(self.pending_data, self.target_file_size)
- if split_row > 0:
- data_to_write = self.pending_data.slice(0, split_row)
- remaining_data = self.pending_data.slice(split_row)
-
- self._write_data_to_file(data_to_write)
- self.pending_data = remaining_data
- self._check_and_roll_if_needed()
+ if split_row <= 0:
+ break
+ data_to_write = self.pending_data.slice(0, split_row)
+ remaining_data = self.pending_data.slice(split_row)
+ self._write_data_to_file(data_to_write)
+ self.pending_data = remaining_data
def _write_data_to_file(self, data: pa.Table):
if data.num_rows == 0:
(paimon) 05/25: [arrow] Reuse length array to avoid allocate array memory repeatedly (#7573)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 2aecba09e60a6396ff4f64f2f79a02dcad99c5c0
Author: xuzifu666 <12...@qq.com>
AuthorDate: Wed Apr 1 17:16:54 2026 +0800
[arrow] Reuse length array to avoid allocate array memory repeatedly (#7573)
Note that there are TODO points in ArrowFieldWriters here. The array
memory was indeed allocated repeatedly. Maybe we can improve here by
reusing it.
---
.../paimon/arrow/writer/ArrowFieldWriters.java | 39 ++++++++++++++--------
1 file changed, 26 insertions(+), 13 deletions(-)
diff --git a/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriters.java b/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriters.java
index 7cb64de7fd..2999acdaf6 100644
--- a/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriters.java
+++ b/paimon-arrow/src/main/java/org/apache/paimon/arrow/writer/ArrowFieldWriters.java
@@ -638,6 +638,8 @@ public class ArrowFieldWriters {
private int offset;
+ private int[] reusableLengths;
+
public ArrayWriter(
FieldVector fieldVector, ArrowFieldWriter elementWriter, boolean isNullable) {
super(fieldVector, isNullable);
@@ -667,8 +669,12 @@ public class ArrowFieldWriters {
}
// length for arrays in [0, startIndex + batchRows)
- // TODO: reuse this
- int[] lengths = new int[lenSize];
+ int[] lengths;
+ if (reusableLengths == null || reusableLengths.length < lenSize) {
+ reusableLengths = new int[lenSize];
+ }
+ lengths = reusableLengths;
+ // Only use the first lenSize elements, reset if needed
for (int i = 0; i < lenSize; i++) {
if (arrayColumnVector.isNullAt(i)) {
// null values don't occupy space
@@ -680,7 +686,7 @@ public class ArrowFieldWriters {
}
ArrayChildWriteInfo arrayChildWriteInfo =
- getArrayChildWriteInfo(pickedInColumn, startIndex, lengths);
+ getArrayChildWriteInfo(pickedInColumn, startIndex, lengths, lenSize);
elementWriter.write(
arrayColumnVector.getColumnVector(),
arrayChildWriteInfo.pickedInColumn,
@@ -809,6 +815,8 @@ public class ArrowFieldWriters {
private int offset;
+ private int[] reusableLengths;
+
public MapWriter(
FieldVector fieldVector,
ArrowFieldWriter keyWriter,
@@ -843,8 +851,10 @@ public class ArrowFieldWriters {
}
// length for arrays in [0, startIndex + batchRows)
- // TODO: reuse this
- int[] lengths = new int[lenSize];
+ if (reusableLengths == null || reusableLengths.length < lenSize) {
+ reusableLengths = new int[lenSize];
+ }
+ int[] lengths = reusableLengths;
for (int i = 0; i < lenSize; i++) {
if (mapColumnVector.isNullAt(i)) {
// null values don't occupy space
@@ -856,7 +866,7 @@ public class ArrowFieldWriters {
}
ArrayChildWriteInfo arrayChildWriteInfo =
- getArrayChildWriteInfo(pickedInColumn, startIndex, lengths);
+ getArrayChildWriteInfo(pickedInColumn, startIndex, lengths, lenSize);
keyWriter.write(
mapColumnVector.getChildren()[0],
arrayChildWriteInfo.pickedInColumn,
@@ -920,20 +930,23 @@ public class ArrowFieldWriters {
}
private static ArrayChildWriteInfo getArrayChildWriteInfo(
- @Nullable int[] pickedInParentColumn, int parentStartIndex, int[] parentLengths) {
+ @Nullable int[] pickedInParentColumn,
+ int parentStartIndex,
+ int[] parentLengths,
+ int lenSize) {
return pickedInParentColumn == null
- ? getArrayChildWriteInfoWithoutDelete(parentStartIndex, parentLengths)
+ ? getArrayChildWriteInfoWithoutDelete(parentStartIndex, parentLengths, lenSize)
: getArrayChildWriteInfoWithDelete(
- pickedInParentColumn, parentStartIndex, parentLengths);
+ pickedInParentColumn, parentStartIndex, parentLengths, lenSize);
}
private static ArrayChildWriteInfo getArrayChildWriteInfoWithoutDelete(
- int parentStartIndex, int[] parentLengths) {
+ int parentStartIndex, int[] parentLengths, int lenSize) {
// the first element index which is to be written
int firstElementIndex = 0;
// batchRows of child column vector
int childBatchRows = 0;
- for (int i = 0; i < parentLengths.length; i++) {
+ for (int i = 0; i < lenSize; i++) {
if (i < parentStartIndex) {
firstElementIndex += parentLengths[i];
} else {
@@ -944,14 +957,14 @@ public class ArrowFieldWriters {
}
private static ArrayChildWriteInfo getArrayChildWriteInfoWithDelete(
- int[] pickedInParentColumn, int parentStartIndex, int[] parentLengths) {
+ int[] pickedInParentColumn, int parentStartIndex, int[] parentLengths, int lenSize) {
// the first element index which is to be written
int firstElementIndex = 0;
// objects to calculate child pickedInColumn
IntArrayList childPicked = new IntArrayList(1024);
int offset = 0;
int currentParentPickedIndex = parentStartIndex;
- for (int i = 0; i < parentLengths.length; i++) {
+ for (int i = 0; i < lenSize; i++) {
if (i < pickedInParentColumn[parentStartIndex]) {
firstElementIndex += parentLengths[i];
offset = firstElementIndex;
(paimon) 14/25: [core] Fix TagAutoCreation.forceCreatingSnapshot to use SINK_PROCESS_TIME_ZONE (#7600)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit c25ddfa41561977bb1d9b63a4c2af288a1138749
Author: yuzelin <33...@users.noreply.github.com>
AuthorDate: Tue Apr 7 11:43:43 2026 +0800
[core] Fix TagAutoCreation.forceCreatingSnapshot to use SINK_PROCESS_TIME_ZONE (#7600)
TagAutoCreation.forceCreatingSnapshot in the ProcessTimeExtractor branch
uses LocalDateTime.now() (machine timezone) to determine whether to
force creating a snapshot. When sink.process-time-zone is configured
differently from the machine timezone (e.g. UTC on an Asia/Shanghai
machine), the tag creation time is incorrect — it triggers at the
machine's midnight instead of the configured timezone's midnight.
---
.../org/apache/paimon/tag/TagAutoCreation.java | 19 ++++++---
.../org/apache/paimon/tag/TagAutoManagerTest.java | 45 ++++++++++++++++++++++
2 files changed, 59 insertions(+), 5 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/tag/TagAutoCreation.java b/paimon-core/src/main/java/org/apache/paimon/tag/TagAutoCreation.java
index 08dfe25378..5f72763c70 100644
--- a/paimon-core/src/main/java/org/apache/paimon/tag/TagAutoCreation.java
+++ b/paimon-core/src/main/java/org/apache/paimon/tag/TagAutoCreation.java
@@ -20,6 +20,7 @@ package org.apache.paimon.tag;
import org.apache.paimon.CoreOptions;
import org.apache.paimon.Snapshot;
+import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.operation.TagDeletion;
import org.apache.paimon.table.sink.TagCallback;
import org.apache.paimon.tag.TagTimeExtractor.ProcessTimeExtractor;
@@ -60,6 +61,7 @@ public class TagAutoCreation {
private final List<TagCallback> callbacks;
private final Duration idlenessTimeout;
private final boolean automaticCompletion;
+ private final ZoneId sinkProcessTimeZone;
private LocalDateTime nextTag;
private long nextSnapshot;
@@ -75,7 +77,8 @@ public class TagAutoCreation {
@Nullable Duration defaultTimeRetained,
Duration idlenessTimeout,
boolean automaticCompletion,
- List<TagCallback> callbacks) {
+ List<TagCallback> callbacks,
+ ZoneId sinkProcessTimeZone) {
this.snapshotManager = snapshotManager;
this.tagManager = tagManager;
this.tagDeletion = tagDeletion;
@@ -87,6 +90,7 @@ public class TagAutoCreation {
this.callbacks = callbacks;
this.idlenessTimeout = idlenessTimeout;
this.automaticCompletion = automaticCompletion;
+ this.sinkProcessTimeZone = sinkProcessTimeZone;
this.periodHandler.validateDelay(delay);
@@ -123,13 +127,17 @@ public class TagAutoCreation {
return isAfterOrEqual(LocalDateTime.now().minus(idlenessTimeout), snapshotTime);
} else if (timeExtractor instanceof ProcessTimeExtractor) {
- return nextTag == null
- || isAfterOrEqual(
- LocalDateTime.now().minus(delay), periodHandler.nextTagTime(nextTag));
+ return forceCreatingSnapshotProcessTime(LocalDateTime.now(sinkProcessTimeZone));
}
return false;
}
+ @VisibleForTesting
+ boolean forceCreatingSnapshotProcessTime(LocalDateTime now) {
+ return nextTag == null
+ || isAfterOrEqual(now.minus(delay), periodHandler.nextTagTime(nextTag));
+ }
+
public void run() {
while (true) {
if (snapshotManager.snapshotExists(nextSnapshot)) {
@@ -230,6 +238,7 @@ public class TagAutoCreation {
options.tagDefaultTimeRetained(),
options.snapshotWatermarkIdleTimeout(),
options.tagAutomaticCompletion(),
- callbacks);
+ callbacks,
+ options.sinkProcessTimeZone());
}
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/tag/TagAutoManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/tag/TagAutoManagerTest.java
index c8954f7b31..ef0e1627ee 100644
--- a/paimon-core/src/test/java/org/apache/paimon/tag/TagAutoManagerTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/tag/TagAutoManagerTest.java
@@ -36,6 +36,7 @@ import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.Collections;
+import static org.apache.paimon.CoreOptions.SINK_PROCESS_TIME_ZONE;
import static org.apache.paimon.CoreOptions.SINK_WATERMARK_TIME_ZONE;
import static org.apache.paimon.CoreOptions.SNAPSHOT_NUM_RETAINED_MAX;
import static org.apache.paimon.CoreOptions.SNAPSHOT_NUM_RETAINED_MIN;
@@ -519,6 +520,50 @@ public class TagAutoManagerTest extends PrimaryKeyTableTestBase {
assertThat(tagManager.allTagNames()).containsOnly("20230718");
}
+ @Test
+ public void testForceCreatingSnapshotProcessTime() throws Exception {
+ // sink.process-time-zone=UTC, machine timezone=Asia/Shanghai.
+ // Daily tag should trigger at UTC 00:00 (Shanghai 08:00), not Shanghai 00:00.
+
+ Options options = new Options();
+ options.set(TAG_AUTOMATIC_CREATION, TagCreationMode.PROCESS_TIME);
+ options.set(TAG_CREATION_PERIOD, TagCreationPeriod.DAILY);
+ options.set(SINK_PROCESS_TIME_ZONE, "UTC");
+
+ FileStoreTable table = this.table.copy(options.toMap());
+
+ // Commit a snapshot to set nextTag
+ TableCommitImpl commit = table.newCommit(commitUser).ignoreEmptyCommit(false);
+ commit.commit(new ManifestCommittable(0));
+ commit.close();
+
+ TagAutoCreation tagAutoCreation =
+ TagAutoCreation.create(
+ table.coreOptions(),
+ table.snapshotManager(),
+ table.store().newTagManager(),
+ table.store().newTagDeletion(),
+ Collections.emptyList());
+
+ // threshold = tagTime + 2 days (nextTag + 1 period)
+ TagManager tagManager = table.store().newTagManager();
+ String createdTag = tagManager.allTagNames().get(0);
+ LocalDateTime tagTime = LocalDateTime.parse(createdTag + "T00:00:00");
+ LocalDateTime thresholdUtc = tagTime.plusDays(2);
+
+ // Shanghai 00:00 = UTC 16:00 previous day, before threshold -> false
+ LocalDateTime shanghaiMidnightAsUtc = thresholdUtc.minusHours(8);
+ assertThat(tagAutoCreation.forceCreatingSnapshotProcessTime(shanghaiMidnightAsUtc))
+ .isFalse();
+
+ // UTC 00:00 = threshold -> true
+ assertThat(tagAutoCreation.forceCreatingSnapshotProcessTime(thresholdUtc)).isTrue();
+
+ // After threshold -> true
+ assertThat(tagAutoCreation.forceCreatingSnapshotProcessTime(thresholdUtc.plusHours(1)))
+ .isTrue();
+ }
+
private long localZoneMills(String timestamp) {
return LocalDateTime.parse(timestamp)
.atZone(ZoneId.systemDefault())
(paimon) 21/25: [flink] fix compatibility issues of StreamExecutionEnv (#7615)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit d3589fa11b18d73b88fa4f01b5f40b21dc647713
Author: Faiz <wx...@antgroup.com>
AuthorDate: Thu Apr 9 22:24:24 2026 +0800
[flink] fix compatibility issues of StreamExecutionEnv (#7615)
---
.../flink/action/RemoveUnexistingFilesAction.java | 4 ++-
.../paimon/flink/btree/BTreeIndexTopoBuilder.java | 6 +++-
.../flink/globalindex/GenericIndexTopoBuilder.java | 7 ++--
.../utils/StreamExecutionEnvironmentUtils.java | 41 ++++++++++++++++++++++
.../utils/StreamExecutionEnvironmentUtils.java | 40 +++++++++++++++++++++
5 files changed, 94 insertions(+), 4 deletions(-)
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/RemoveUnexistingFilesAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/RemoveUnexistingFilesAction.java
index c67ddd7ea3..5639da0871 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/RemoveUnexistingFilesAction.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/RemoveUnexistingFilesAction.java
@@ -22,6 +22,7 @@ import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.flink.sink.Committable;
import org.apache.paimon.flink.sink.CommittableTypeInfo;
import org.apache.paimon.flink.utils.BoundedOneInputOperator;
+import org.apache.paimon.flink.utils.StreamExecutionEnvironmentUtils;
import org.apache.paimon.io.CompactIncrement;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.io.DataIncrement;
@@ -108,7 +109,8 @@ public class RemoveUnexistingFilesAction extends TableActionBase {
List<BinaryRow> binaryPartitions = fileStoreTable.newScan().listPartitions();
SingleOutputStreamOperator<byte[]> source =
- env.fromData(
+ StreamExecutionEnvironmentUtils.fromData(
+ env,
binaryPartitions.stream()
.map(BinaryRow::toBytes)
.collect(Collectors.toList()),
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java
index 35830faac1..dbdfdc8fe0 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/btree/BTreeIndexTopoBuilder.java
@@ -34,6 +34,7 @@ import org.apache.paimon.flink.sorter.TableSortInfo;
import org.apache.paimon.flink.sorter.TableSorter;
import org.apache.paimon.flink.utils.BoundedOneInputOperator;
import org.apache.paimon.flink.utils.JavaTypeInfo;
+import org.apache.paimon.flink.utils.StreamExecutionEnvironmentUtils;
import org.apache.paimon.globalindex.GlobalIndexParallelWriter;
import org.apache.paimon.globalindex.btree.BTreeGlobalIndexBuilder;
import org.apache.paimon.globalindex.btree.BTreeIndexOptions;
@@ -211,7 +212,10 @@ public class BTreeIndexTopoBuilder {
parallelism = Math.min(parallelism, maxParallelism);
DataStream<Split> sourceStream =
- env.fromData(new JavaTypeInfo<>(Split.class), rangeSplits.toArray(new Split[0]))
+ StreamExecutionEnvironmentUtils.fromData(
+ env,
+ new JavaTypeInfo<>(Split.class),
+ rangeSplits.toArray(new Split[0]))
.name("Global Index Source " + " range=" + range)
.setParallelism(1);
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java
index 865280763c..bcbabac793 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/globalindex/GenericIndexTopoBuilder.java
@@ -28,6 +28,7 @@ import org.apache.paimon.flink.sink.NoopCommittableStateManager;
import org.apache.paimon.flink.sink.StoreCommitter;
import org.apache.paimon.flink.utils.BoundedOneInputOperator;
import org.apache.paimon.flink.utils.JavaTypeInfo;
+import org.apache.paimon.flink.utils.StreamExecutionEnvironmentUtils;
import org.apache.paimon.globalindex.GlobalIndexSingletonWriter;
import org.apache.paimon.globalindex.ResultEntry;
import org.apache.paimon.index.IndexFileMeta;
@@ -179,7 +180,8 @@ public class GenericIndexTopoBuilder {
ReadBuilder readBuilder = table.newReadBuilder().withReadType(projectedRowType);
DataStream<ShardTask> source =
- env.fromData(
+ StreamExecutionEnvironmentUtils.fromData(
+ env,
new JavaTypeInfo<>(ShardTask.class),
shardTasks.toArray(new ShardTask[0]))
.name("Generic Index Source")
@@ -201,7 +203,8 @@ public class GenericIndexTopoBuilder {
if (!deletedIndexEntries.isEmpty()) {
List<Committable> deleteCommittables = createDeleteCommittables(deletedIndexEntries);
DataStream<Committable> deletes =
- env.fromData(
+ StreamExecutionEnvironmentUtils.fromData(
+ env,
new CommittableTypeInfo(),
deleteCommittables.toArray(new Committable[0]))
.name("Index Delete Source")
diff --git a/paimon-flink/paimon-flink1-common/src/main/java/org/apache/paimon/flink/utils/StreamExecutionEnvironmentUtils.java b/paimon-flink/paimon-flink1-common/src/main/java/org/apache/paimon/flink/utils/StreamExecutionEnvironmentUtils.java
new file mode 100644
index 0000000000..c602bd3162
--- /dev/null
+++ b/paimon-flink/paimon-flink1-common/src/main/java/org/apache/paimon/flink/utils/StreamExecutionEnvironmentUtils.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.utils;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+/** Utility methods about {@link StreamExecutionEnvironment} to resolve compatibility issues. */
+public class StreamExecutionEnvironmentUtils {
+
+ @SafeVarargs
+ public static <T> DataStreamSource<T> fromData(
+ StreamExecutionEnvironment env, TypeInformation<T> typeInfo, T... data) {
+ return env.fromCollection(Arrays.asList(data), typeInfo);
+ }
+
+ public static <T> DataStreamSource<T> fromData(
+ StreamExecutionEnvironment env, Collection<T> data, TypeInformation<T> typeInfo) {
+ return env.fromCollection(data, typeInfo);
+ }
+}
diff --git a/paimon-flink/paimon-flink2-common/src/main/java/org/apache/paimon/flink/utils/StreamExecutionEnvironmentUtils.java b/paimon-flink/paimon-flink2-common/src/main/java/org/apache/paimon/flink/utils/StreamExecutionEnvironmentUtils.java
new file mode 100644
index 0000000000..1fb5edf496
--- /dev/null
+++ b/paimon-flink/paimon-flink2-common/src/main/java/org/apache/paimon/flink/utils/StreamExecutionEnvironmentUtils.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.utils;
+
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.streaming.api.datastream.DataStreamSource;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+
+import java.util.Collection;
+
+/** Utility methods about {@link StreamExecutionEnvironment} to resolve compatibility issues. */
+public class StreamExecutionEnvironmentUtils {
+
+ @SafeVarargs
+ public static <T> DataStreamSource<T> fromData(
+ StreamExecutionEnvironment env, TypeInformation<T> typeInfo, T... data) {
+ return env.fromData(typeInfo, data);
+ }
+
+ public static <T> DataStreamSource<T> fromData(
+ StreamExecutionEnvironment env, Collection<T> data, TypeInformation<T> typeInfo) {
+ return env.fromData(data, typeInfo);
+ }
+}
(paimon) 01/25: [flink] Support dropping primary keys for empty tables in Flink (#7566)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit f2c41f14d75844588a52d64b6fcceb4ac64673d3
Author: tsreaper <ts...@gmail.com>
AuthorDate: Wed Apr 1 10:41:43 2026 +0800
[flink] Support dropping primary keys for empty tables in Flink (#7566)
Paimon has supported adding primary keys for empty tables before. So in
this PR, we support the symmetrical operation: dropping primary keys for
empty tables.
---
.../org/apache/paimon/schema/SchemaChange.java | 27 +++++++++++++
.../org/apache/paimon/schema/SchemaManager.java | 11 +++++-
.../apache/paimon/schema/SchemaManagerTest.java | 45 ++++++++++++++++++++++
.../java/org/apache/paimon/flink/FlinkCatalog.java | 38 ++++++++++++++----
.../apache/paimon/flink/SchemaChangeITCase.java | 24 ++++++++++++
5 files changed, 136 insertions(+), 9 deletions(-)
diff --git a/paimon-api/src/main/java/org/apache/paimon/schema/SchemaChange.java b/paimon-api/src/main/java/org/apache/paimon/schema/SchemaChange.java
index 4b68ad105a..9b381a3490 100644
--- a/paimon-api/src/main/java/org/apache/paimon/schema/SchemaChange.java
+++ b/paimon-api/src/main/java/org/apache/paimon/schema/SchemaChange.java
@@ -78,6 +78,9 @@ import java.util.Objects;
@JsonSubTypes.Type(
value = SchemaChange.UpdateColumnPosition.class,
name = SchemaChange.Actions.UPDATE_COLUMN_POSITION_ACTION),
+ @JsonSubTypes.Type(
+ value = SchemaChange.DropPrimaryKey.class,
+ name = SchemaChange.Actions.DROP_PRIMARY_KEY_ACTION),
})
public interface SchemaChange extends Serializable {
@@ -164,6 +167,10 @@ public interface SchemaChange extends Serializable {
return new UpdateColumnPosition(move);
}
+ static SchemaChange dropPrimaryKey() {
+ return new DropPrimaryKey();
+ }
+
/** A SchemaChange to set a table option. */
final class SetOption implements SchemaChange {
@@ -810,6 +817,25 @@ public interface SchemaChange extends Serializable {
}
}
+ /** A SchemaChange to drop primary key. */
+ final class DropPrimaryKey implements SchemaChange {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ return o != null && getClass() == o.getClass();
+ }
+
+ @Override
+ public int hashCode() {
+ return getClass().hashCode();
+ }
+ }
+
/** Actions for schema changes: identify for schema change. */
class Actions {
public static final String FIELD_ACTION = "action";
@@ -824,6 +850,7 @@ public interface SchemaChange extends Serializable {
public static final String UPDATE_COLUMN_COMMENT_ACTION = "updateColumnComment";
public static final String UPDATE_COLUMN_DEFAULT_VALUE_ACTION = "updateColumnDefaultValue";
public static final String UPDATE_COLUMN_POSITION_ACTION = "updateColumnPosition";
+ public static final String DROP_PRIMARY_KEY_ACTION = "dropPrimaryKey";
private Actions() {}
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
index 13da6677f7..8667f2271d 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
@@ -68,6 +68,7 @@ import java.io.Serializable;
import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -311,6 +312,7 @@ public class SchemaManager implements Serializable {
List<DataField> newFields = new ArrayList<>(oldTableSchema.fields());
AtomicInteger highestFieldId = new AtomicInteger(oldTableSchema.highestFieldId());
String newComment = oldTableSchema.comment();
+ List<String> newPrimaryKeys = oldTableSchema.primaryKeys();
for (SchemaChange change : changes) {
if (change instanceof SetOption) {
SetOption setOption = (SetOption) change;
@@ -549,6 +551,12 @@ public class SchemaManager implements Serializable {
update.newDefaultValue());
},
lazyIdentifier);
+ } else if (change instanceof SchemaChange.DropPrimaryKey) {
+ if (hasSnapshots.get()) {
+ throw new UnsupportedOperationException(
+ "Cannot drop primary keys on a non-empty table.");
+ }
+ newPrimaryKeys = Collections.emptyList();
} else {
throw new UnsupportedOperationException("Unsupported change: " + change.getClass());
}
@@ -561,8 +569,7 @@ public class SchemaManager implements Serializable {
newFields,
oldTableSchema.partitionKeys(),
applyNotNestedColumnRename(
- oldTableSchema.primaryKeys(),
- Iterables.filter(changes, RenameColumn.class)),
+ newPrimaryKeys, Iterables.filter(changes, RenameColumn.class)),
applyRenameColumnsToOptions(newOptions, changes),
newComment);
diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java
index 31496f8bb8..9a15cab361 100644
--- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java
@@ -460,6 +460,51 @@ public class SchemaManagerTest {
.hasMessage("Change 'merge-engine' is not supported yet.");
}
+ @Test
+ public void testDropPrimaryKeyOnEmptyTable() throws Exception {
+ Path tableRoot = new Path(tempDir.toString(), "table");
+ SchemaManager manager = new SchemaManager(LocalFileIO.create(), tableRoot);
+ manager.createTable(schema);
+
+ // drop primary keys on empty table should succeed
+ manager.commitChanges(SchemaChange.dropPrimaryKey());
+
+ FileStoreTable table = FileStoreTableFactory.create(LocalFileIO.create(), tableRoot);
+ assertThat(table.schema().primaryKeys()).isEmpty();
+ }
+
+ @Test
+ public void testDropPrimaryKeyOnNonEmptyTable() throws Exception {
+ Map<String, String> tableOptions = new HashMap<>(options);
+ tableOptions.put("bucket", "1");
+ Schema pkSchema =
+ new Schema(
+ rowType.getFields(),
+ Collections.emptyList(),
+ primaryKeys,
+ tableOptions,
+ "");
+ Path tableRoot = new Path(tempDir.toString(), "table");
+ SchemaManager manager = new SchemaManager(LocalFileIO.create(), tableRoot);
+ manager.createTable(pkSchema);
+
+ // write data to create a snapshot
+ FileStoreTable table = FileStoreTableFactory.create(LocalFileIO.create(), tableRoot);
+ String commitUser = UUID.randomUUID().toString();
+ TableWriteImpl<?> write =
+ table.newWrite(commitUser).withIOManager(IOManager.create(tempDir + "/io"));
+ TableCommitImpl commit = table.newCommit(commitUser);
+ write.write(GenericRow.of(1, 10L, BinaryString.fromString("apple")));
+ commit.commit(1, write.prepareCommit(false, 1));
+ write.close();
+ commit.close();
+
+ // drop primary keys on non-empty table should fail
+ assertThatThrownBy(() -> manager.commitChanges(SchemaChange.dropPrimaryKey()))
+ .isInstanceOf(UnsupportedOperationException.class)
+ .hasMessage("Cannot drop primary keys on a non-empty table.");
+ }
+
@Test
public void testAddAndDropNestedColumns() throws Exception {
RowType innerType =
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java
index e05106eef6..5f59063668 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkCatalog.java
@@ -510,7 +510,9 @@ public class FlinkCatalog extends AbstractCatalog {
}
private List<SchemaChange> toSchemaChange(
- TableChange change, Map<String, Integer> oldTableNonPhysicalColumnIndex) {
+ TableChange change,
+ Map<String, Integer> oldTableNonPhysicalColumnIndex,
+ @Nullable String primaryKeyConstraintName) {
List<SchemaChange> schemaChanges = new ArrayList<>();
if (change instanceof AddColumn) {
if (((AddColumn) change).getColumn().isPhysical()) {
@@ -627,6 +629,15 @@ public class FlinkCatalog extends AbstractCatalog {
} else if (change instanceof MaterializedTableChange
&& handleMaterializedTableChange(change, schemaChanges)) {
return schemaChanges;
+ } else if (change instanceof TableChange.DropConstraint) {
+ TableChange.DropConstraint dropConstraint = (TableChange.DropConstraint) change;
+ if (primaryKeyConstraintName == null
+ || !primaryKeyConstraintName.equals(dropConstraint.getConstraintName())) {
+ throw new UnsupportedOperationException(
+ "Only dropping primary key constraint is supported.");
+ }
+ schemaChanges.add(SchemaChange.dropPrimaryKey());
+ return schemaChanges;
}
throw new UnsupportedOperationException("Change is not supported: " + change.getClass());
}
@@ -740,10 +751,17 @@ public class FlinkCatalog extends AbstractCatalog {
checkArgument(
table instanceof FileStoreTable,
"Only support alter data table, but is: " + table.getClass());
- validateAlterTable(toCatalogTable(table), newTable);
+ CatalogBaseTable oldCatalogTable = toCatalogTable(table);
+ validateAlterTable(oldCatalogTable, newTable);
Map<String, Integer> oldTableNonPhysicalColumnIndex =
FlinkCatalogPropertiesUtil.nonPhysicalColumns(
table.options(), table.rowType().getFieldNames());
+ String primaryKeyConstraintName =
+ oldCatalogTable
+ .getUnresolvedSchema()
+ .getPrimaryKey()
+ .map(pk -> pk.getConstraintName())
+ .orElse(null);
List<SchemaChange> changes = new ArrayList<>();
@@ -773,7 +791,9 @@ public class FlinkCatalog extends AbstractCatalog {
.flatMap(
tableChange ->
toSchemaChange(
- tableChange, oldTableNonPhysicalColumnIndex)
+ tableChange,
+ oldTableNonPhysicalColumnIndex,
+ primaryKeyConstraintName)
.stream())
.collect(Collectors.toList());
changes.addAll(schemaChanges);
@@ -848,10 +868,10 @@ public class FlinkCatalog extends AbstractCatalog {
if (!table1IsMaterialized) {
org.apache.flink.table.api.Schema ts1 = ct1.getUnresolvedSchema();
org.apache.flink.table.api.Schema ts2 = ct2.getUnresolvedSchema();
- boolean pkEquality = false;
+ boolean allowAlterPk = false;
if (ts1.getPrimaryKey().isPresent() && ts2.getPrimaryKey().isPresent()) {
- pkEquality =
+ allowAlterPk =
Objects.equals(
ts1.getPrimaryKey().get().getConstraintName(),
ts2.getPrimaryKey().get().getConstraintName())
@@ -859,10 +879,14 @@ public class FlinkCatalog extends AbstractCatalog {
ts1.getPrimaryKey().get().getColumnNames(),
ts2.getPrimaryKey().get().getColumnNames());
} else if (!ts1.getPrimaryKey().isPresent() && !ts2.getPrimaryKey().isPresent()) {
- pkEquality = true;
+ allowAlterPk = true;
+ } else if (ts1.getPrimaryKey().isPresent() && !ts2.getPrimaryKey().isPresent()) {
+ // dropping primary key is allowed on empty tables,
+ // SchemaManager will validate this
+ allowAlterPk = true;
}
- if (!pkEquality) {
+ if (!allowAlterPk) {
throw new UnsupportedOperationException(
"Altering primary key is not supported yet.");
}
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java
index 840dce7942..cd7805b9b3 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java
@@ -1776,4 +1776,28 @@ public class SchemaChangeITCase extends CatalogITCaseBase {
Row.of(1L, 100L, "buy", null, null, null, "2024-01-01", "10"),
Row.of(2L, 200L, "sell", 99.5, 10, 3.14, "2024-01-02", "11"));
}
+
+ @Test
+ public void testDropPrimaryKeyOnEmptyTable() {
+ sql("CREATE TABLE T (a INT, b INT, c STRING, PRIMARY KEY (a) NOT ENFORCED)");
+
+ // drop primary key on empty table should succeed
+ sql("ALTER TABLE T DROP PRIMARY KEY");
+
+ List<Row> result = sql("SHOW CREATE TABLE T");
+ assertThat(result.get(0).toString()).doesNotContain("PRIMARY KEY");
+ }
+
+ @Test
+ public void testDropPrimaryKeyOnNonEmptyTable() {
+ sql("CREATE TABLE T (a INT, b INT, c STRING, PRIMARY KEY (a) NOT ENFORCED)");
+ sql("INSERT INTO T VALUES (1, 2, 'hello')");
+
+ // drop primary key on non-empty table should fail
+ assertThatThrownBy(() -> sql("ALTER TABLE T DROP PRIMARY KEY"))
+ .satisfies(
+ anyCauseMatches(
+ UnsupportedOperationException.class,
+ "Cannot drop primary keys on a non-empty table."));
+ }
}
(paimon) 10/25: [spark] Fix view query failure with VARCHAR/CHAR column types (#7585)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 07eac94c18e5efe833ef187e299449380a491e65
Author: Zouxxyy <zo...@alibaba-inc.com>
AuthorDate: Thu Apr 2 21:52:18 2026 +0800
[spark] Fix view query failure with VARCHAR/CHAR column types (#7585)
Fix querying Paimon View fails with "Cannot up cast from STRING to
VARCHAR(n)" when the underlying table has VARCHAR or CHAR columns.
Spark replaces CharType/VarcharType with StringType during V2 table
resolution, but the view schema still preserves the original types. This
causes UpCast(StringType → VarcharType(n)) to fail. The fix uses
CharVarcharUtils.replaceCharVarcharWithStringInSchema to normalize the
view schema before applying UpCast.
---
.../catalyst/analysis/PaimonViewResolver.scala | 9 +++-
.../paimon/spark/sql/PaimonViewTestBase.scala | 48 ++++++++++++++++++++++
2 files changed, 56 insertions(+), 1 deletion(-)
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonViewResolver.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonViewResolver.scala
index b45463e916..95b8ab6e18 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonViewResolver.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/catalyst/analysis/PaimonViewResolver.scala
@@ -30,6 +30,7 @@ import org.apache.spark.sql.catalyst.parser.ParseException
import org.apache.spark.sql.catalyst.parser.extensions.{CurrentOrigin, Origin}
import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Project, SubqueryAlias}
import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.util.CharVarcharUtils
import org.apache.spark.sql.connector.catalog.{Identifier, PaimonLookupCatalog}
case class PaimonViewResolver(spark: SparkSession)
@@ -66,7 +67,13 @@ case class PaimonViewResolver(spark: SparkSession)
val earlyRules = Seq(CTESubstitution, SubstituteUnresolvedOrdinals)
val rewritten = earlyRules.foldLeft(parsedPlan)((plan, rule) => rule.apply(plan))
- val aliases = SparkTypeUtils.fromPaimonRowType(view.rowType()).fields.zipWithIndex.map {
+ // Spark internally replaces CharType/VarcharType with StringType during V2 table resolution,
+ // so the view's schema must also use StringType to avoid UpCast failures
+ // (e.g., "Cannot up cast from STRING to VARCHAR(50)").
+ val viewSchema = CharVarcharUtils.replaceCharVarcharWithStringInSchema(
+ SparkTypeUtils.fromPaimonRowType(view.rowType()))
+
+ val aliases = viewSchema.fields.zipWithIndex.map {
case (expected, pos) =>
val attr = GetColumnByOrdinal(pos, expected.dataType)
Alias(UpCast(attr, expected.dataType), expected.name)(explicitMetadata =
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTestBase.scala
index 7c503d98a5..a0c29bfcbc 100644
--- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTestBase.scala
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonViewTestBase.scala
@@ -242,4 +242,52 @@ abstract class PaimonViewTestBase extends PaimonHiveTestBase {
}
}
}
+
+ test("Paimon View: create view on table with VARCHAR columns") {
+ Seq(sparkCatalogName, paimonHiveCatalogName).foreach {
+ catalogName =>
+ sql(s"USE $catalogName")
+ withDatabase("test_db") {
+ sql("CREATE DATABASE test_db")
+ sql("USE test_db")
+ withTable("t") {
+ withView("v1") {
+ sql("CREATE TABLE t (id INT, name VARCHAR(50), country STRING) USING paimon")
+ sql("INSERT INTO t VALUES (1, 'alice', 'beijing'), (2, 'bob', 'shanghai')")
+ sql("CREATE VIEW v1 AS SELECT * FROM t")
+ checkAnswer(
+ sql("SELECT * FROM v1"),
+ Seq(Row(1, "alice", "beijing"), Row(2, "bob", "shanghai")))
+ }
+ }
+ }
+ }
+ }
+
+ test("Paimon View: create view on table with nested VARCHAR and CHAR columns") {
+ Seq(sparkCatalogName, paimonHiveCatalogName).foreach {
+ catalogName =>
+ sql(s"USE $catalogName")
+ withDatabase("test_db") {
+ sql("CREATE DATABASE test_db")
+ sql("USE test_db")
+ withTable("t") {
+ withView("v1") {
+ sql("""CREATE TABLE t (
+ | id INT,
+ | tags ARRAY<VARCHAR(100)>,
+ | props MAP<STRING, VARCHAR(200)>
+ |) USING paimon""".stripMargin)
+ sql("""INSERT INTO t VALUES
+ | (1, ARRAY('a', 'b'), MAP('k1', 'v1')),
+ | (2, ARRAY('c'), MAP('k2', 'v2'))""".stripMargin)
+ sql("CREATE VIEW v1 AS SELECT * FROM t")
+ checkAnswer(
+ sql("SELECT * FROM v1"),
+ Seq(Row(1, Seq("a", "b"), Map("k1" -> "v1")), Row(2, Seq("c"), Map("k2" -> "v2"))))
+ }
+ }
+ }
+ }
+ }
}
(paimon) 17/25: [flink] fix de-merge-into runtimeContext may throw NoSuchMethod error (#7607)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 8fb5b3e3bd679d33430f7fa034d78acf54311463
Author: Faiz <wx...@antgroup.com>
AuthorDate: Tue Apr 7 21:53:14 2026 +0800
[flink] fix de-merge-into runtimeContext may throw NoSuchMethod error (#7607)
---
.../org/apache/paimon/flink/dataevolution/MergeIntoUpdateChecker.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/dataevolution/MergeIntoUpdateChecker.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/dataevolution/MergeIntoUpdateChecker.java
index 94392a84ec..8b1122382a 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/dataevolution/MergeIntoUpdateChecker.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/dataevolution/MergeIntoUpdateChecker.java
@@ -23,6 +23,7 @@ import org.apache.paimon.Snapshot;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.flink.sink.Committable;
import org.apache.paimon.flink.utils.BoundedOneInputOperator;
+import org.apache.paimon.flink.utils.RuntimeContextUtils;
import org.apache.paimon.index.GlobalIndexMeta;
import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.io.CompactIncrement;
@@ -70,7 +71,7 @@ public class MergeIntoUpdateChecker extends BoundedOneInputOperator<Committable,
affectedPartitions = new HashSet<>();
Preconditions.checkState(
- getRuntimeContext().getTaskInfo().getNumberOfParallelSubtasks() == 1,
+ RuntimeContextUtils.getNumberOfParallelSubtasks(getRuntimeContext()) == 1,
"Parallelism of MergeIntoUpdateChecker must be 1.");
}
(paimon) 16/25: [docs] Add Spark SQL example for fast_forward in manage-branches (#7604)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 32b886a324e33c17bb8fa1b7e6aecc3b3c8a70ee
Author: sanshi <43...@users.noreply.github.com>
AuthorDate: Tue Apr 7 14:42:44 2026 +0800
[docs] Add Spark SQL example for fast_forward in manage-branches (#7604)
---
docs/content/maintenance/manage-branches.md | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/docs/content/maintenance/manage-branches.md b/docs/content/maintenance/manage-branches.md
index 8daef956f4..ba2c2bf708 100644
--- a/docs/content/maintenance/manage-branches.md
+++ b/docs/content/maintenance/manage-branches.md
@@ -202,6 +202,17 @@ Run the following command:
--branch_name <branch-name> \
[--catalog_conf <paimon-catalog-conf> [--catalog_conf <paimon-catalog-conf> ...]]
```
+
+{{< /tab >}}
+
+{{< tab "Spark SQL" >}}
+
+Run the following sql:
+
+```sql
+CALL sys.fast_forward('default.T', 'branch1');
+```
+
{{< /tab >}}
{{< /tabs >}}
(paimon) 07/25: [core] Extract Parquet stats from in-memory footer instead of re-reading file (#7489)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 7ce5d01bce3b23d299a678ab2e4c639a25c3e567
Author: majian <47...@users.noreply.github.com>
AuthorDate: Thu Apr 2 11:12:21 2026 +0800
[core] Extract Parquet stats from in-memory footer instead of re-reading file (#7489)
Currently, Paimon extracts Parquet column statistics by re-reading the
file footer after the writer is closed. On object stores like OSS, a
newly closed file may not be immediately visible for reading due to
eventual consistency, causing `FileNotFoundException`.
This PR caches the `ParquetMetadata` in memory during writer close (via
`ParquetWriter.getFooter()`), and passes it through the stats extraction
pipeline to avoid re-reading the file. When the in-memory metadata is
unavailable (e.g. non-Parquet formats), it falls back to the original
file-based extraction path.
---
.../org/apache/paimon/format/FormatWriter.java | 17 ++
.../apache/paimon/format/SimpleStatsExtractor.java | 20 ++
.../org/apache/paimon/io/SimpleStatsProducer.java | 18 ++
.../org/apache/paimon/io/SingleFileWriter.java | 11 ++
.../paimon/io/StatsCollectingSingleFileWriter.java | 2 +-
.../parquet/ParquetSimpleStatsExtractor.java | 26 +++
.../apache/paimon/format/parquet/ParquetUtil.java | 45 +++--
.../format/parquet/writer/ParquetBulkWriter.java | 13 ++
.../format/parquet/ParquetInMemoryStatsTest.java | 213 +++++++++++++++++++++
9 files changed, 348 insertions(+), 17 deletions(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/format/FormatWriter.java b/paimon-common/src/main/java/org/apache/paimon/format/FormatWriter.java
index 3f35135f51..60d126be21 100644
--- a/paimon-common/src/main/java/org/apache/paimon/format/FormatWriter.java
+++ b/paimon-common/src/main/java/org/apache/paimon/format/FormatWriter.java
@@ -20,6 +20,8 @@ package org.apache.paimon.format;
import org.apache.paimon.data.InternalRow;
+import javax.annotation.Nullable;
+
import java.io.Closeable;
import java.io.IOException;
@@ -49,4 +51,19 @@ public interface FormatWriter extends Closeable {
* @throws IOException Thrown if calculating the length fails.
*/
boolean reachTargetSize(boolean suggestedCheck, long targetSize) throws IOException;
+
+ /**
+ * Returns format-specific writer metadata that can be used to extract statistics without
+ * re-reading the file. This is useful for object stores (like OSS/S3) where the file may not be
+ * immediately visible after close.
+ *
+ * <p>This method should only be called after {@link #close()}. By default, returns {@code null}
+ * indicating no in-memory metadata is available.
+ *
+ * @return format-specific metadata object, or {@code null} if not available.
+ */
+ @Nullable
+ default Object writerMetadata() {
+ return null;
+ }
}
diff --git a/paimon-common/src/main/java/org/apache/paimon/format/SimpleStatsExtractor.java b/paimon-common/src/main/java/org/apache/paimon/format/SimpleStatsExtractor.java
index 9ffbf495e5..41ff84dd7c 100644
--- a/paimon-common/src/main/java/org/apache/paimon/format/SimpleStatsExtractor.java
+++ b/paimon-common/src/main/java/org/apache/paimon/format/SimpleStatsExtractor.java
@@ -22,6 +22,8 @@ import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.Path;
import org.apache.paimon.utils.Pair;
+import javax.annotation.Nullable;
+
import java.io.IOException;
/** Extracts statistics directly from file. */
@@ -29,6 +31,24 @@ public interface SimpleStatsExtractor {
SimpleColStats[] extract(FileIO fileIO, Path path, long length) throws IOException;
+ /**
+ * Extract statistics using optional in-memory writer metadata to avoid re-reading the file.
+ * When writerMetadata is non-null and the extractor supports it, the stats will be extracted
+ * from memory instead of from the file. This avoids issues with object stores where the file
+ * may not be immediately visible after close.
+ *
+ * @param fileIO the file IO
+ * @param path the file path
+ * @param length the file length
+ * @param writerMetadata optional format-specific metadata from the writer, or null
+ * @return column statistics
+ */
+ default SimpleColStats[] extract(
+ FileIO fileIO, Path path, long length, @Nullable Object writerMetadata)
+ throws IOException {
+ return extract(fileIO, path, length);
+ }
+
Pair<SimpleColStats[], FileInfo> extractWithFileInfo(FileIO fileIO, Path path, long length)
throws IOException;
diff --git a/paimon-core/src/main/java/org/apache/paimon/io/SimpleStatsProducer.java b/paimon-core/src/main/java/org/apache/paimon/io/SimpleStatsProducer.java
index 1e818ab9d0..e33dcfc46a 100644
--- a/paimon-core/src/main/java/org/apache/paimon/io/SimpleStatsProducer.java
+++ b/paimon-core/src/main/java/org/apache/paimon/io/SimpleStatsProducer.java
@@ -40,6 +40,17 @@ public interface SimpleStatsProducer {
SimpleColStats[] extract(FileIO fileIO, Path path, long length) throws IOException;
+ /**
+ * Extract statistics using optional in-memory writer metadata.
+ *
+ * @param writerMetadata optional format-specific metadata from the writer, or null
+ */
+ default SimpleColStats[] extract(
+ FileIO fileIO, Path path, long length, @Nullable Object writerMetadata)
+ throws IOException {
+ return extract(fileIO, path, length);
+ }
+
static SimpleStatsProducer disabledProducer() {
return new SimpleStatsProducer() {
@@ -92,6 +103,13 @@ public interface SimpleStatsProducer {
throws IOException {
return extractor.extract(fileIO, path, length);
}
+
+ @Override
+ public SimpleColStats[] extract(
+ FileIO fileIO, Path path, long length, @Nullable Object writerMetadata)
+ throws IOException {
+ return extractor.extract(fileIO, path, length, writerMetadata);
+ }
};
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/io/SingleFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/io/SingleFileWriter.java
index aea24be3ca..473acecea9 100644
--- a/paimon-core/src/main/java/org/apache/paimon/io/SingleFileWriter.java
+++ b/paimon-core/src/main/java/org/apache/paimon/io/SingleFileWriter.java
@@ -59,6 +59,7 @@ public abstract class SingleFileWriter<T, R> implements FileWriter<T, R> {
@Nullable private PositionOutputStream out;
@Nullable private Long outputBytes;
+ @Nullable private Object writerMetadata;
private long recordCount;
protected boolean closed;
@@ -193,6 +194,7 @@ public abstract class SingleFileWriter<T, R> implements FileWriter<T, R> {
try {
if (writer != null) {
writer.close();
+ writerMetadata = writer.writerMetadata();
writer = null;
}
if (out != null) {
@@ -216,4 +218,13 @@ public abstract class SingleFileWriter<T, R> implements FileWriter<T, R> {
}
return outputBytes;
}
+
+ /**
+ * Returns cached writer metadata from the format writer. Available after {@link #close()} is
+ * called. Can be used by stats extractors to avoid re-reading the file from object storage.
+ */
+ @Nullable
+ protected Object writerMetadata() {
+ return writerMetadata;
+ }
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/io/StatsCollectingSingleFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/io/StatsCollectingSingleFileWriter.java
index 20cebfcb27..773ca08a81 100644
--- a/paimon-core/src/main/java/org/apache/paimon/io/StatsCollectingSingleFileWriter.java
+++ b/paimon-core/src/main/java/org/apache/paimon/io/StatsCollectingSingleFileWriter.java
@@ -83,6 +83,6 @@ public abstract class StatsCollectingSingleFileWriter<T, R> extends SingleFileWr
.toArray(SimpleColStats[]::new);
}
- return statsProducer.extract(fileIO, path, fileSize);
+ return statsProducer.extract(fileIO, path, fileSize, writerMetadata());
}
}
diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSimpleStatsExtractor.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSimpleStatsExtractor.java
index a265b76d4c..f286cfce5d 100644
--- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSimpleStatsExtractor.java
+++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetSimpleStatsExtractor.java
@@ -42,8 +42,11 @@ import org.apache.parquet.column.statistics.FloatStatistics;
import org.apache.parquet.column.statistics.IntStatistics;
import org.apache.parquet.column.statistics.LongStatistics;
import org.apache.parquet.column.statistics.Statistics;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
import org.apache.parquet.schema.PrimitiveType;
+import javax.annotation.Nullable;
+
import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
@@ -74,6 +77,29 @@ public class ParquetSimpleStatsExtractor implements SimpleStatsExtractor {
return extractWithFileInfo(fileIO, path, length).getLeft();
}
+ @Override
+ public SimpleColStats[] extract(
+ FileIO fileIO, Path path, long length, @Nullable Object writerMetadata)
+ throws IOException {
+ if (writerMetadata instanceof ParquetMetadata) {
+ // Use in-memory metadata directly, avoiding re-reading the file.
+ // This is critical for object stores (OSS/S3) where the file may not be
+ // immediately visible after close.
+ Map<String, Statistics<?>> columnStats =
+ ParquetUtil.extractColumnStats((ParquetMetadata) writerMetadata);
+ SimpleColStatsCollector[] collectors = SimpleColStatsCollector.create(statsCollectors);
+ return IntStream.range(0, rowType.getFieldCount())
+ .mapToObj(
+ i -> {
+ DataField field = rowType.getFields().get(i);
+ return toFieldStats(
+ field, columnStats.get(field.name()), collectors[i]);
+ })
+ .toArray(SimpleColStats[]::new);
+ }
+ return extract(fileIO, path, length);
+ }
+
@Override
public Pair<SimpleColStats[], FileInfo> extractWithFileInfo(
FileIO fileIO, Path path, long length) throws IOException {
diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetUtil.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetUtil.java
index f3c358c375..575252095d 100644
--- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetUtil.java
+++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/ParquetUtil.java
@@ -54,25 +54,38 @@ public class ParquetUtil {
throws IOException {
try (ParquetFileReader reader = getParquetReader(fileIO, path, length, options)) {
ParquetMetadata parquetMetadata = reader.getFooter();
- List<BlockMetaData> blockMetaDataList = parquetMetadata.getBlocks();
- Map<String, Statistics<?>> resultStats = new HashMap<>();
- for (BlockMetaData blockMetaData : blockMetaDataList) {
- List<ColumnChunkMetaData> columnChunkMetaDataList = blockMetaData.getColumns();
- for (ColumnChunkMetaData columnChunkMetaData : columnChunkMetaDataList) {
- Statistics<?> stats = columnChunkMetaData.getStatistics();
- String columnName = columnChunkMetaData.getPath().toDotString();
- Statistics<?> midStats;
- if (!resultStats.containsKey(columnName)) {
- midStats = stats;
- } else {
- midStats = resultStats.get(columnName);
- midStats.mergeStatistics(stats);
- }
- resultStats.put(columnName, midStats);
+ Map<String, Statistics<?>> resultStats = extractColumnStats(parquetMetadata);
+ return Pair.of(resultStats, new SimpleStatsExtractor.FileInfo(reader.getRecordCount()));
+ }
+ }
+
+ /**
+ * Extract column stats from in-memory {@link ParquetMetadata}. This avoids re-reading the file
+ * from storage, which is critical for object stores (like OSS/S3) where the file may not be
+ * immediately visible after close.
+ *
+ * @param parquetMetadata the in-memory Parquet metadata (footer)
+ * @return result sets as map, key is column name, value is statistics
+ */
+ public static Map<String, Statistics<?>> extractColumnStats(ParquetMetadata parquetMetadata) {
+ List<BlockMetaData> blockMetaDataList = parquetMetadata.getBlocks();
+ Map<String, Statistics<?>> resultStats = new HashMap<>();
+ for (BlockMetaData blockMetaData : blockMetaDataList) {
+ List<ColumnChunkMetaData> columnChunkMetaDataList = blockMetaData.getColumns();
+ for (ColumnChunkMetaData columnChunkMetaData : columnChunkMetaDataList) {
+ Statistics<?> stats = columnChunkMetaData.getStatistics();
+ String columnName = columnChunkMetaData.getPath().toDotString();
+ Statistics<?> midStats;
+ if (!resultStats.containsKey(columnName)) {
+ midStats = stats;
+ } else {
+ midStats = resultStats.get(columnName);
+ midStats.mergeStatistics(stats);
}
+ resultStats.put(columnName, midStats);
}
- return Pair.of(resultStats, new SimpleStatsExtractor.FileInfo(reader.getRecordCount()));
}
+ return resultStats;
}
/**
diff --git a/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/ParquetBulkWriter.java b/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/ParquetBulkWriter.java
index ac11a2ef00..d7282f699f 100644
--- a/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/ParquetBulkWriter.java
+++ b/paimon-format/src/main/java/org/apache/paimon/format/parquet/writer/ParquetBulkWriter.java
@@ -22,6 +22,9 @@ import org.apache.paimon.data.InternalRow;
import org.apache.paimon.format.FormatWriter;
import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+
+import javax.annotation.Nullable;
import java.io.IOException;
@@ -33,6 +36,9 @@ public class ParquetBulkWriter implements FormatWriter {
/** The ParquetWriter to write to. */
private final ParquetWriter<InternalRow> parquetWriter;
+ /** Cached footer metadata after close, used to avoid re-reading the file for stats. */
+ @Nullable private ParquetMetadata footerMetadata;
+
/**
* Creates a new ParquetBulkWriter wrapping the given ParquetWriter.
*
@@ -50,10 +56,17 @@ public class ParquetBulkWriter implements FormatWriter {
@Override
public void close() throws IOException {
parquetWriter.close();
+ this.footerMetadata = parquetWriter.getFooter();
}
@Override
public boolean reachTargetSize(boolean suggestedCheck, long targetSize) throws IOException {
return suggestedCheck && parquetWriter.getDataSize() >= targetSize;
}
+
+ @Nullable
+ @Override
+ public Object writerMetadata() {
+ return footerMetadata;
+ }
}
diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetInMemoryStatsTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetInMemoryStatsTest.java
new file mode 100644
index 0000000000..f0d8d34f43
--- /dev/null
+++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetInMemoryStatsTest.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.format.parquet;
+
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.Decimal;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.format.FileFormat;
+import org.apache.paimon.format.FormatWriter;
+import org.apache.paimon.format.FormatWriterFactory;
+import org.apache.paimon.format.SimpleColStats;
+import org.apache.paimon.format.SimpleStatsExtractor;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.PositionOutputStream;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.statistics.SimpleColStatsCollector;
+import org.apache.paimon.types.BigIntType;
+import org.apache.paimon.types.BooleanType;
+import org.apache.paimon.types.DateType;
+import org.apache.paimon.types.DecimalType;
+import org.apache.paimon.types.DoubleType;
+import org.apache.paimon.types.FloatType;
+import org.apache.paimon.types.IntType;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.types.SmallIntType;
+import org.apache.paimon.types.TinyIntType;
+import org.apache.paimon.types.VarCharType;
+
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.math.BigDecimal;
+import java.util.stream.IntStream;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Tests that verify extracting column stats from in-memory ParquetMetadata produces the same
+ * results as extracting from the file. This is critical for object stores (like OSS/S3) where the
+ * file may not be immediately visible after close.
+ */
+public class ParquetInMemoryStatsTest {
+
+ @TempDir java.nio.file.Path tempDir;
+
+ private final FileIO fileIO = new LocalFileIO();
+
+ @Test
+ public void testInMemoryStatsMatchFileStats() throws Exception {
+ RowType rowType =
+ RowType.builder()
+ .fields(
+ new VarCharType(100),
+ new BooleanType(),
+ new TinyIntType(),
+ new SmallIntType(),
+ new IntType(),
+ new BigIntType(),
+ new FloatType(),
+ new DoubleType(),
+ new DecimalType(10, 2),
+ new DateType())
+ .build();
+
+ FileFormat format = FileFormat.fromIdentifier("parquet", new Options());
+ FormatWriterFactory writerFactory = format.createWriterFactory(rowType);
+ Path path = new Path(tempDir.toString() + "/test_inmemory_stats.parquet");
+
+ // Write test data
+ PositionOutputStream out = fileIO.newOutputStream(path, false);
+ FormatWriter writer = writerFactory.create(out, "SNAPPY");
+
+ for (int i = 0; i < 100; i++) {
+ GenericRow row = new GenericRow(10);
+ row.setField(0, BinaryString.fromString("value_" + i));
+ row.setField(1, i % 2 == 0);
+ row.setField(2, (byte) (i % 127));
+ row.setField(3, (short) (i * 10));
+ row.setField(4, i * 100);
+ row.setField(5, (long) i * 1000);
+ // Start from 1 to avoid -0.0 vs 0.0 IEEE 754 representation difference
+ // between in-memory and serialized Parquet statistics
+ row.setField(6, (i + 1) * 1.1f);
+ row.setField(7, (i + 1) * 2.2);
+ row.setField(8, Decimal.fromBigDecimal(new BigDecimal(i + ".99"), 10, 2));
+ row.setField(9, 18000 + i); // date as days since epoch
+ writer.addElement(row);
+ }
+
+ // Also add some nulls
+ for (int i = 0; i < 10; i++) {
+ GenericRow row = new GenericRow(10);
+ // Leave all fields null
+ writer.addElement(row);
+ }
+
+ writer.close();
+ out.close();
+
+ // Get in-memory metadata from writer
+ Object writerMetadata = writer.writerMetadata();
+ assertThat(writerMetadata).isNotNull();
+ assertThat(writerMetadata).isInstanceOf(ParquetMetadata.class);
+
+ // Create stats extractors
+ int fieldCount = rowType.getFieldCount();
+ SimpleColStatsCollector.Factory[] statsCollectors =
+ IntStream.range(0, fieldCount)
+ .mapToObj(p -> SimpleColStatsCollector.from("full"))
+ .toArray(SimpleColStatsCollector.Factory[]::new);
+ SimpleStatsExtractor extractor =
+ format.createStatsExtractor(rowType, statsCollectors).get();
+
+ // Extract stats from file (original path)
+ long fileSize = fileIO.getFileSize(path);
+ SimpleColStats[] fromFile = extractor.extract(fileIO, path, fileSize);
+
+ // Extract stats from in-memory metadata (new path)
+ SimpleColStats[] fromMemory = extractor.extract(fileIO, path, fileSize, writerMetadata);
+
+ // They should be exactly the same
+ assertThat(fromMemory).isEqualTo(fromFile);
+
+ // Also verify the stats are correct
+ assertThat(fromFile[0].min()).isEqualTo(BinaryString.fromString("value_0"));
+ assertThat(fromFile[0].max()).isEqualTo(BinaryString.fromString("value_99"));
+ assertThat(fromFile[0].nullCount()).isEqualTo(10L);
+
+ assertThat(fromFile[4].min()).isEqualTo(0);
+ assertThat(fromFile[4].max()).isEqualTo(9900);
+ assertThat(fromFile[4].nullCount()).isEqualTo(10L);
+ }
+
+ @Test
+ public void testInMemoryStatsFallbackWhenMetadataIsNull() throws Exception {
+ RowType rowType = RowType.builder().fields(new IntType(), new VarCharType(50)).build();
+
+ FileFormat format = FileFormat.fromIdentifier("parquet", new Options());
+ FormatWriterFactory writerFactory = format.createWriterFactory(rowType);
+ Path path = new Path(tempDir.toString() + "/test_fallback.parquet");
+
+ PositionOutputStream out = fileIO.newOutputStream(path, false);
+ FormatWriter writer = writerFactory.create(out, "SNAPPY");
+
+ for (int i = 0; i < 10; i++) {
+ GenericRow row = new GenericRow(2);
+ row.setField(0, i);
+ row.setField(1, BinaryString.fromString("text_" + i));
+ writer.addElement(row);
+ }
+ writer.close();
+ out.close();
+
+ int fieldCount = rowType.getFieldCount();
+ SimpleColStatsCollector.Factory[] statsCollectors =
+ IntStream.range(0, fieldCount)
+ .mapToObj(p -> SimpleColStatsCollector.from("full"))
+ .toArray(SimpleColStatsCollector.Factory[]::new);
+ SimpleStatsExtractor extractor =
+ format.createStatsExtractor(rowType, statsCollectors).get();
+
+ long fileSize = fileIO.getFileSize(path);
+
+ // Extract with null metadata (should fall back to file reading)
+ SimpleColStats[] fromFallback = extractor.extract(fileIO, path, fileSize, null);
+
+ // Extract from file directly
+ SimpleColStats[] fromFile = extractor.extract(fileIO, path, fileSize);
+
+ // Results should be identical
+ assertThat(fromFallback).isEqualTo(fromFile);
+ }
+
+ @Test
+ public void testWriterMetadataDefaultIsNull() throws Exception {
+ // A non-Parquet FormatWriter (or any FormatWriter that doesn't override writerMetadata())
+ // should return null by default
+ FormatWriter mockWriter =
+ new FormatWriter() {
+ @Override
+ public void addElement(org.apache.paimon.data.InternalRow element) {}
+
+ @Override
+ public boolean reachTargetSize(boolean suggestedCheck, long targetSize) {
+ return false;
+ }
+
+ @Override
+ public void close() {}
+ };
+
+ assertThat(mockWriter.writerMetadata()).isNull();
+ }
+}
(paimon) 02/25: [python] Fix REST signature mismatch by encoding query param values (#7568)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 861252a7028fce664dc9583beacc6bca6ee94731
Author: XiaoHongbo <13...@qq.com>
AuthorDate: Wed Apr 1 11:10:33 2026 +0800
[python] Fix REST signature mismatch by encoding query param values (#7568)
Requests with special characters in query parameters (e.g.
functionNamePattern=func%) fail with `401 NotAuthorizedException:
"accessKeyId validate failed with error message: current signature xxx
not match"`. This PR fixes above issue by encoding query param values
---
paimon-python/pypaimon/api/typedef.py | 17 +++++++++++++++--
paimon-python/pypaimon/tests/rest/dlf_signer_test.py | 7 +++++++
2 files changed, 22 insertions(+), 2 deletions(-)
diff --git a/paimon-python/pypaimon/api/typedef.py b/paimon-python/pypaimon/api/typedef.py
index 893b199b4b..3d05cb2fb6 100644
--- a/paimon-python/pypaimon/api/typedef.py
+++ b/paimon-python/pypaimon/api/typedef.py
@@ -15,15 +15,28 @@
# specific language governing permissions and limitations
# under the License.
-from dataclasses import dataclass
+from dataclasses import dataclass, field
from typing import Dict, TypeVar
+from urllib.parse import quote
T = TypeVar("T")
+def _encode_string(value: str) -> str:
+ if value is None:
+ return value
+ return quote(str(value), safe='')
+
+
@dataclass
class RESTAuthParameter:
method: str
path: str
data: str
- parameters: Dict[str, str]
+ parameters: Dict[str, str] = field(default_factory=dict)
+
+ def __post_init__(self):
+ if self.parameters:
+ self.parameters = {
+ k: _encode_string(v) for k, v in self.parameters.items()
+ }
diff --git a/paimon-python/pypaimon/tests/rest/dlf_signer_test.py b/paimon-python/pypaimon/tests/rest/dlf_signer_test.py
index a2e72f599e..9109f5d619 100644
--- a/paimon-python/pypaimon/tests/rest/dlf_signer_test.py
+++ b/paimon-python/pypaimon/tests/rest/dlf_signer_test.py
@@ -131,6 +131,13 @@ class DLFSignerTest(unittest.TestCase):
self.assertIn("Content-MD5", header)
self.assertEqual("UNSIGNED-PAYLOAD", header.get("x-dlf-content-sha256"))
+ def test_rest_auth_parameter_encodes_values(self):
+ param = RESTAuthParameter(
+ method="GET", path="/test",
+ data="", parameters={"functionNamePattern": "func%"})
+ self.assertEqual(
+ param.parameters["functionNamePattern"], "func%25")
+
def test_parse_signing_algo_from_uri(self):
parse = DLFAuthProviderFactory.parse_signing_algo_from_uri
(paimon) 11/25: [core] Fix partial-update sequence-group delete mismatch under projected reads (#7586)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 227d631200431813743b47aa6549ad3fb92b163e
Author: bryndenZh <49...@users.noreply.github.com>
AuthorDate: Thu Apr 2 22:58:38 2026 +0800
[core] Fix partial-update sequence-group delete mismatch under projected reads (#7586)
---
.../compact/PartialUpdateMergeFunction.java | 9 ++++-
.../compact/PartialUpdateMergeFunctionTest.java | 40 ++++++++++++++++++++++
2 files changed, 48 insertions(+), 1 deletion(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/PartialUpdateMergeFunction.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/PartialUpdateMergeFunction.java
index 2f429191c4..3d8c7098a8 100644
--- a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/PartialUpdateMergeFunction.java
+++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/PartialUpdateMergeFunction.java
@@ -496,6 +496,7 @@ public class PartialUpdateMergeFunction implements MergeFunction<KeyValue> {
RowType targetType = readType != null ? readType : rowType;
Map<Integer, FieldsComparator> projectedSeqComparators = new HashMap<>();
Map<Integer, FieldAggregator> projectedAggregators = new HashMap<>();
+ Set<Integer> projectedSequenceGroupPartialDelete = sequenceGroupPartialDelete;
if (readType != null) {
// Build index mapping from table schema to read schema
@@ -547,6 +548,12 @@ public class PartialUpdateMergeFunction implements MergeFunction<KeyValue> {
projectedAggregators.put(newIndex, fieldAggregators.get(oldIndex).get());
}
}
+
+ projectedSequenceGroupPartialDelete =
+ sequenceGroupPartialDelete.stream()
+ .filter(indexMap::containsKey)
+ .map(indexMap::get)
+ .collect(Collectors.toSet());
} else {
// Use original mappings
this.fieldSeqComparators.forEach(
@@ -563,7 +570,7 @@ public class PartialUpdateMergeFunction implements MergeFunction<KeyValue> {
projectedAggregators,
!fieldSeqComparators.isEmpty(),
removeRecordOnDelete,
- sequenceGroupPartialDelete,
+ projectedSequenceGroupPartialDelete,
ArrayUtils.toPrimitiveBoolean(
fieldTypes.stream().map(DataType::isNullable).toArray(Boolean[]::new)));
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/PartialUpdateMergeFunctionTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/PartialUpdateMergeFunctionTest.java
index 846734b555..0d4a5a2a4f 100644
--- a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/PartialUpdateMergeFunctionTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/PartialUpdateMergeFunctionTest.java
@@ -132,6 +132,46 @@ public class PartialUpdateMergeFunctionTest {
validate(func, null, null, null, null, null, null, null);
}
+ @Test
+ public void testSequenceGroupPartialDeleteWithProjection() {
+ Options options = new Options();
+ options.set("fields.f3.sequence-group", "f1,f2");
+ options.set("fields.f6.sequence-group", "f4,f5");
+ options.set("partial-update.remove-record-on-sequence-group", "f3,f6");
+ RowType rowType =
+ RowType.of(
+ DataTypes.INT(),
+ DataTypes.INT(),
+ DataTypes.INT(),
+ DataTypes.INT(),
+ DataTypes.INT(),
+ DataTypes.INT(),
+ DataTypes.INT());
+ MergeFunctionFactory<KeyValue> factory =
+ PartialUpdateMergeFunction.factory(options, rowType, ImmutableList.of("f0"));
+
+ // Reordered fields
+ RowType projectedType =
+ RowType.of(
+ new DataType[] {
+ DataTypes.INT(),
+ DataTypes.INT(),
+ DataTypes.INT(),
+ DataTypes.INT(),
+ DataTypes.INT(),
+ DataTypes.INT(),
+ DataTypes.INT()
+ },
+ new String[] {"f3", "f6", "f0", "f1", "f2", "f4", "f5"});
+ MergeFunction<KeyValue> func = factory.create(projectedType);
+
+ func.reset();
+ add(func, 11, 22, 100, 200, 1, 12, 21);
+ add(func, RowKind.DELETE, 11, 22, 100, 200, 1, 12, 21);
+
+ validate(func, null, null, null, null, null, null, null);
+ }
+
@Test
public void testMultiSequenceFields() {
Options options = new Options();
(paimon) 24/25: [python] Fix duplicate _ROW_ID when file exceeds read batch size (#7626)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit f27a002af4219768e976cce91af1d6587135fce6
Author: littlecoder04 <xi...@alibaba-inc.com>
AuthorDate: Fri Apr 10 22:11:03 2026 +0800
[python] Fix duplicate _ROW_ID when file exceeds read batch size (#7626)
When a data file contains more than 1024 rows, upsert_by_arrow_with_key
fails with: `ValueError: Input data contains duplicate _ROW_ID values`.
This PR fixes above issue by advancing first_row_id in
DataFileBatchReader._assign_row_tracking after each batch.
---
.../pypaimon/read/reader/data_file_batch_reader.py | 1 +
.../pypaimon/tests/data_evolution_test.py | 41 ++++++++++++++++++++++
2 files changed, 42 insertions(+)
diff --git a/paimon-python/pypaimon/read/reader/data_file_batch_reader.py b/paimon-python/pypaimon/read/reader/data_file_batch_reader.py
index 7f2e1c61e1..126d57f625 100644
--- a/paimon-python/pypaimon/read/reader/data_file_batch_reader.py
+++ b/paimon-python/pypaimon/read/reader/data_file_batch_reader.py
@@ -211,6 +211,7 @@ class DataFileBatchReader(RecordBatchReader):
idx = self.system_fields[SpecialFields.ROW_ID.name]
# Create a new array that fills with computed row IDs
arrays[idx] = pa.array(range(self.first_row_id, self.first_row_id + record_batch.num_rows), type=pa.int64())
+ self.first_row_id += record_batch.num_rows
# Handle _SEQUENCE_NUMBER field
if SpecialFields.SEQUENCE_NUMBER.name in self.system_fields.keys():
diff --git a/paimon-python/pypaimon/tests/data_evolution_test.py b/paimon-python/pypaimon/tests/data_evolution_test.py
index 266041eec5..b6e0b32727 100644
--- a/paimon-python/pypaimon/tests/data_evolution_test.py
+++ b/paimon-python/pypaimon/tests/data_evolution_test.py
@@ -17,6 +17,7 @@ limitations under the License.
"""
import os
import shutil
+import sys
import tempfile
import unittest
from types import SimpleNamespace
@@ -1437,3 +1438,43 @@ class DataEvolutionTest(unittest.TestCase):
]))
self.assertEqual(actual.num_rows, 2)
self.assertEqual(actual, expect)
+
+ def test_large_file_read(self):
+ pa_schema = pa.schema([
+ ('id', pa.int32()),
+ ('name', pa.string()),
+ ])
+ schema = Schema.from_pyarrow_schema(pa_schema, options={
+ 'row-tracking.enabled': 'true',
+ 'data-evolution.enabled': 'true',
+ })
+ self.catalog.create_table('default.test_large_file_row_id', schema, False)
+ table = self.catalog.get_table('default.test_large_file_row_id')
+
+ # Write >1024 rows in a single file
+ num_rows = 2000
+ data = pa.Table.from_pydict({
+ 'id': list(range(num_rows)),
+ 'name': [f'name_{i}' for i in range(num_rows)],
+ }, schema=pa_schema)
+
+ wb = table.new_batch_write_builder()
+ tw = wb.new_write()
+ tc = wb.new_commit()
+ tw.write_arrow(data)
+ tc.commit(tw.prepare_commit())
+ tw.close()
+ tc.close()
+
+ update_ids = list(range(0, 1500))
+ upsert_data = pa.Table.from_pydict({
+ 'id': update_ids,
+ 'name': [f'upsert_{i}' for i in update_ids],
+ }, schema=pa_schema)
+
+ wb = table.new_batch_write_builder()
+ tu = wb.new_update()
+ msgs = tu.upsert_by_arrow_with_key(upsert_data, ['id'])
+ tc = wb.new_commit()
+ tc.commit(msgs)
+ tc.close()
(paimon) 03/25: [core] Fix that latestSnapshotOfUser might list snapshot directory to find the earliest (#7572)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 97813f617cf4f580c798c4d1f8f460846a9f97c9
Author: yuzelin <33...@users.noreply.github.com>
AuthorDate: Wed Apr 1 15:29:03 2026 +0800
[core] Fix that latestSnapshotOfUser might list snapshot directory to find the earliest (#7572)
Previously, latestSnapshotOfUser called earliestSnapshotId() to
determine the loop bound before iterating snapshots.
During checkpoint, if the snapshot pointed to by the EARLIEST hint is
concurrently expired, the hint misses and earliestSnapshotId() falls
back to findByListFiles(), which lists the entire snapshot directory.
This method is called during every checkpoint via prepareCommit →
createWriterCleanChecker → latestCommittedIdentifier →
latestSnapshotOfUserFromFilesystem. For jobs with high parallelism and a
large number of snapshots (e.g. 10000+), this might cause massive
listStatus requests, easily triggering QPS limits on object storage
(e.g. OSS QpsLimitExceeded).
The fix removes the earliestSnapshotId() call and instead iterates
backward from the latest snapshot, stopping when a FileNotFoundException
is encountered (indicating an expired snapshot). Other exceptions are
thrown directly.
---
.../org/apache/paimon/utils/SnapshotManager.java | 28 +++++-----------------
.../apache/paimon/utils/SnapshotManagerTest.java | 4 ++--
2 files changed, 8 insertions(+), 24 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java b/paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java
index 414ef3f5b7..b6a818604e 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/SnapshotManager.java
@@ -600,31 +600,14 @@ public class SnapshotManager implements Serializable {
if (latestId == null) {
return Optional.empty();
}
- if (earliestId == null) {
- earliestId =
- Preconditions.checkNotNull(
- earliestSnapshotId(),
- "Latest snapshot id is not null, but earliest snapshot id is null. "
- + "This is unexpected.");
- }
- for (long id = latestId; id >= earliestId; id--) {
+ long searchEnd = earliestId != null ? earliestId : Snapshot.FIRST_SNAPSHOT_ID;
+ for (long id = latestId; id >= searchEnd; id--) {
Snapshot snapshot;
try {
- snapshot = snapshot(id);
- } catch (Exception e) {
- long newEarliestId =
- Preconditions.checkNotNull(
- earliestSnapshotId(),
- "Latest snapshot id is not null, but earliest snapshot id is null. "
- + "This is unexpected.");
-
- // this is a valid snapshot, should throw exception
- if (id >= newEarliestId) {
- throw e;
- }
-
- // this is an expired snapshot
+ snapshot = tryGetSnapshot(id);
+ } catch (FileNotFoundException e) {
+ // this snapshot has been expired, stop searching
LOG.warn(
"Snapshot #"
+ id
@@ -632,6 +615,7 @@ public class SnapshotManager implements Serializable {
+ user
+ ") is not found.");
break;
+ // other exceptions will be thrown
}
if (user.equals(snapshot.commitUser())) {
diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java
index 19ae865692..486f0b3fe5 100644
--- a/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/utils/SnapshotManagerTest.java
@@ -334,7 +334,7 @@ public class SnapshotManagerTest {
SnapshotManager snapshotManager =
newSnapshotManager(localFileIO, new Path(tempDir.toString()));
// create 100 snapshots using user "lastCommitUser"
- for (long i = 0; i < 100; i++) {
+ for (long i = 1; i <= 100; i++) {
Snapshot snapshot =
new Snapshot(
i,
@@ -375,7 +375,7 @@ public class SnapshotManagerTest {
Thread.sleep(100);
// expire snapshot
- localFileIO.deleteQuietly(snapshotManager.snapshotPath(0));
+ localFileIO.deleteQuietly(snapshotManager.snapshotPath(1));
thread.join();
assertThat(exception.get()).isNull();
(paimon) 08/25: [api] Fix locale-sensitive date formatting in DLFOpenApiSigner (#7577)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit de76b56763d01ace65a450f72ce088cc49606857
Author: Jiajia Li <pl...@alibaba-inc.com>
AuthorDate: Thu Apr 2 13:43:42 2026 +0800
[api] Fix locale-sensitive date formatting in DLFOpenApiSigner (#7577)
Pin DateTimeFormatter to Locale.ENGLISH so RFC 1123 date headers are
always in English regardless of JVM default locale. Add tests for
Chinese and Japanese locales.
Fix locale-sensitive date formatting in `DLFOpenApiSigner`. The
`DateTimeFormatter` for RFC 1123 dates was not pinned to
`Locale.ENGLISH`, causing day/month names to be localized on non-English
JVM locales (e.g. zh_CN, ja_JP), which breaks the Aliyun OpenAPI
signature.
---
.../apache/paimon/rest/auth/DLFOpenApiSigner.java | 3 +-
.../paimon/rest/auth/DLFRequestSignerTest.java | 41 ++++++++++++++++++++++
2 files changed, 43 insertions(+), 1 deletion(-)
diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiSigner.java b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiSigner.java
index 80f8761850..aaf3713828 100644
--- a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiSigner.java
+++ b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiSigner.java
@@ -31,6 +31,7 @@ import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.Base64;
import java.util.HashMap;
+import java.util.Locale;
import java.util.Map;
import java.util.TreeMap;
import java.util.UUID;
@@ -65,7 +66,7 @@ public class DLFOpenApiSigner implements DLFRequestSigner {
private static final String API_VERSION = "2026-01-18";
private static final DateTimeFormatter GMT_DATE_FORMATTER =
- DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss 'GMT'")
+ DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.ENGLISH)
.withZone(ZoneId.of("GMT"));
@Override
diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFRequestSignerTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFRequestSignerTest.java
index ef9d44aa5e..e1b67e3a9e 100644
--- a/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFRequestSignerTest.java
+++ b/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFRequestSignerTest.java
@@ -26,6 +26,7 @@ import java.time.ZonedDateTime;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
@@ -274,6 +275,46 @@ public class DLFRequestSignerTest {
assertEquals("2026-01-18", headers.get("x-acs-version"));
}
+ @Test
+ public void testOpenApiDateFormatWithNonEnglishLocale() {
+ Locale originalLocale = Locale.getDefault();
+ try {
+ // Simulate Chinese locale environment (e.g., zh_CN, zh_US_#Hans)
+ Locale.setDefault(Locale.CHINA);
+
+ DLFOpenApiSigner signer = new DLFOpenApiSigner();
+ Instant now = ZonedDateTime.of(2025, 4, 16, 3, 44, 46, 0, ZoneOffset.UTC).toInstant();
+ String host = "dlfnext.cn-hangzhou.aliyuncs.com";
+
+ Map<String, String> headers = signer.signHeaders(null, now, null, host);
+ String dateValue = headers.get("Date");
+
+ // Date must always be in English RFC 1123 format regardless of JVM locale
+ assertEquals("Wed, 16 Apr 2025 03:44:46 GMT", dateValue);
+ } finally {
+ Locale.setDefault(originalLocale);
+ }
+ }
+
+ @Test
+ public void testOpenApiDateFormatWithJapaneseLocale() {
+ Locale originalLocale = Locale.getDefault();
+ try {
+ Locale.setDefault(Locale.JAPAN);
+
+ DLFOpenApiSigner signer = new DLFOpenApiSigner();
+ Instant now = ZonedDateTime.of(2025, 4, 16, 3, 44, 46, 0, ZoneOffset.UTC).toInstant();
+ String host = "dlfnext.cn-hangzhou.aliyuncs.com";
+
+ Map<String, String> headers = signer.signHeaders(null, now, null, host);
+ String dateValue = headers.get("Date");
+
+ assertEquals("Wed, 16 Apr 2025 03:44:46 GMT", dateValue);
+ } finally {
+ Locale.setDefault(originalLocale);
+ }
+ }
+
@Test
public void testConcurrentNonceGeneration() throws InterruptedException {
DLFOpenApiSigner signer = new DLFOpenApiSigner();
(paimon) 18/25: [python] Add file_source assertion in row-tracking commit to fail fast on missing file_source (#7610)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit b60d1bac4d7a64187985210da62eb8fa2ee885d8
Author: littlecoder04 <xi...@gmail.com>
AuthorDate: Thu Apr 9 13:26:25 2026 +0800
[python] Add file_source assertion in row-tracking commit to fail fast on missing file_source (#7610)
if file_source is None, the condition `None == 0` silently evaluates to
False, skipping row ID assignment. This causes nextRowId not to
increment, leading to first_row_id conflicts. So this PR add file_source
check like java.
---
paimon-python/pypaimon/write/file_store_commit.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/paimon-python/pypaimon/write/file_store_commit.py b/paimon-python/pypaimon/write/file_store_commit.py
index 829bcaa0b7..a4099ac1db 100644
--- a/paimon-python/pypaimon/write/file_store_commit.py
+++ b/paimon-python/pypaimon/write/file_store_commit.py
@@ -697,6 +697,9 @@ class FileStoreCommit:
blob_start_by_field = {}
for entry in commit_entries:
+ assert entry.file.file_source is not None, \
+ f"file_source must be present for row-tracking table, file={entry.file.file_name}"
+
# Check if this is an append file that needs row ID assignment
if (entry.kind == 0 and # ADD kind
entry.file.file_source == 0 and # APPEND file source
(paimon) 12/25: [hotfix] Fix incorrect integer comparison prevent misjudgment in chain table. (#7591)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit daa619b0195d567d744468c4d9d31e22dee841ea
Author: Wenchao Wu <60...@users.noreply.github.com>
AuthorDate: Fri Apr 3 12:36:06 2026 +0800
[hotfix] Fix incorrect integer comparison prevent misjudgment in chain table. (#7591)
---
.../src/main/java/org/apache/paimon/table/ChainGroupReadTable.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/ChainGroupReadTable.java b/paimon-core/src/main/java/org/apache/paimon/table/ChainGroupReadTable.java
index b178715a2c..6267f3eb04 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/ChainGroupReadTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/ChainGroupReadTable.java
@@ -390,7 +390,7 @@ public class ChainGroupReadTable extends FallbackReadFileStoreTable {
bucketInAll = totalBuckets;
} else {
checkArgument(
- totalBuckets == bucketInAll,
+ totalBuckets.equals(bucketInAll),
"Inconsistent bucket num " + dataSplit.bucket());
}
(paimon) 25/25: [python] Add conflict detection in shard update (#7630)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 1fe4cf22aec9f4bb1f17438c02f90ea1f140c908
Author: littlecoder04 <xi...@alibaba-inc.com>
AuthorDate: Sun Apr 12 15:52:13 2026 +0800
[python] Add conflict detection in shard update (#7630)
This PR is a follow-up to #7323. PR #7323 introduced conflict detection
for Python data evolution updates, but the shard-update path was not
covered.
As a result, when shard update and compact run concurrently, the shard
update may commit successfully against a stale scan snapshot instead of
failing fast. The problem only shows up later during read, with the
error:
`All files in a field merge split should have the same row count`.This
PR extends the same conflict-detection coverage to the shard-update
path.
run_compact_conflict_test in run_mixed_tests.sh
---
.../test/java/org/apache/paimon/JavaPyE2ETest.java | 69 ++++++++++++++++++++++
paimon-python/dev/run_mixed_tests.sh | 42 ++++++++++++-
paimon-python/pypaimon/read/plan.py | 5 +-
.../pypaimon/read/scanner/file_scanner.py | 23 +++++---
.../pypaimon/read/streaming_table_scan.py | 2 +-
paimon-python/pypaimon/read/table_scan.py | 12 ++--
paimon-python/pypaimon/tests/binary_row_test.py | 8 +--
.../pypaimon/tests/e2e/java_py_read_write_test.py | 59 ++++++++++++++++++
.../pypaimon/tests/reader_append_only_test.py | 45 ++++++++++++++
.../pypaimon/write/commit/commit_scanner.py | 4 +-
paimon-python/pypaimon/write/file_store_commit.py | 2 +-
paimon-python/pypaimon/write/table_update.py | 6 +-
.../pypaimon/write/table_update_by_row_id.py | 5 +-
13 files changed, 255 insertions(+), 27 deletions(-)
diff --git a/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java b/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
index bd1e006ec7..c09bf34663 100644
--- a/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
@@ -18,6 +18,8 @@
package org.apache.paimon;
+import org.apache.paimon.append.dataevolution.DataEvolutionCompactCoordinator;
+import org.apache.paimon.append.dataevolution.DataEvolutionCompactTask;
import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.catalog.CatalogFactory;
@@ -48,9 +50,11 @@ import org.apache.paimon.table.Table;
import org.apache.paimon.table.sink.BatchTableCommit;
import org.apache.paimon.table.sink.BatchTableWrite;
import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.table.sink.CommitMessage;
import org.apache.paimon.table.sink.InnerTableCommit;
import org.apache.paimon.table.sink.StreamTableCommit;
import org.apache.paimon.table.sink.StreamTableWrite;
+import org.apache.paimon.table.source.EndOfScanException;
import org.apache.paimon.table.source.ReadBuilder;
import org.apache.paimon.table.source.Split;
import org.apache.paimon.table.source.TableRead;
@@ -937,6 +941,71 @@ public class JavaPyE2ETest {
return GenericRow.ofKind(rowKind, values[0], values[1], values[2]);
}
+ /** Step 1: Write 5 base files for compact conflict test. */
+ @Test
+ @EnabledIfSystemProperty(named = "run.e2e.tests", matches = "true")
+ public void testCompactConflictWriteBase() throws Exception {
+ Identifier id = identifier("compact_conflict_test");
+ try {
+ catalog.dropTable(id, true);
+ } catch (Exception ignore) {
+ }
+ Schema schema =
+ Schema.newBuilder()
+ .column("f0", DataTypes.INT())
+ .column("f1", DataTypes.STRING())
+ .column("f2", DataTypes.STRING())
+ .option(ROW_TRACKING_ENABLED.key(), "true")
+ .option(DATA_EVOLUTION_ENABLED.key(), "true")
+ .option(BUCKET.key(), "-1")
+ .build();
+ catalog.createTable(id, schema, false);
+
+ RowType fullType = schema.rowType().project(Arrays.asList("f0", "f1"));
+
+ for (int fileIdx = 0; fileIdx < 5; fileIdx++) {
+ int startId = fileIdx * 200;
+ FileStoreTable t = (FileStoreTable) catalog.getTable(id);
+ BatchWriteBuilder builder = t.newBatchWriteBuilder();
+ try (BatchTableWrite w = builder.newWrite().withWriteType(fullType)) {
+ for (int i = 0; i < 200; i++) {
+ w.write(
+ GenericRow.of(
+ startId + i, BinaryString.fromString("n" + (startId + i))));
+ }
+ builder.newCommit().commit(w.prepareCommit());
+ }
+ }
+ LOG.info("compact_conflict_test: 5 base files written (200 rows each, total 1000)");
+ }
+
+ /** Step 3: Run compact on compact_conflict_test table. */
+ @Test
+ @EnabledIfSystemProperty(named = "run.e2e.tests", matches = "true")
+ public void testCompactConflictRunCompact() throws Exception {
+ Identifier id = identifier("compact_conflict_test");
+ doDataEvolutionCompact((FileStoreTable) catalog.getTable(id));
+ LOG.info("compact_conflict_test: compact done, 5 files merged into 1 (1000 rows)");
+ }
+
+ private void doDataEvolutionCompact(FileStoreTable table) throws Exception {
+ DataEvolutionCompactCoordinator coordinator =
+ new DataEvolutionCompactCoordinator(table, false, false);
+ List<CommitMessage> messages = new ArrayList<>();
+ try {
+ List<DataEvolutionCompactTask> tasks;
+ while (!(tasks = coordinator.plan()).isEmpty()) {
+ for (DataEvolutionCompactTask task : tasks) {
+ messages.add(task.doCompact(table, "test-compact"));
+ }
+ }
+ } catch (EndOfScanException ignore) {
+ }
+ if (!messages.isEmpty()) {
+ table.newBatchWriteBuilder().newCommit().commit(messages);
+ }
+ }
+
private static String rowToStringWithStruct(InternalRow row, RowType type) {
StringBuilder build = new StringBuilder();
build.append(row.getRowKind().shortString()).append("[");
diff --git a/paimon-python/dev/run_mixed_tests.sh b/paimon-python/dev/run_mixed_tests.sh
index 09bcb58d2b..79bd66dd97 100755
--- a/paimon-python/dev/run_mixed_tests.sh
+++ b/paimon-python/dev/run_mixed_tests.sh
@@ -244,6 +244,32 @@ run_compressed_text_test() {
fi
}
+run_compact_conflict_test() {
+ echo -e "${YELLOW}=== Running Compact Conflict Test (Java Write Base, Python Shard Update + Java Compact) ===${NC}"
+
+ cd "$PROJECT_ROOT"
+
+ # Step 1: Java writes 5 base files
+ echo "Running Maven test for JavaPyE2ETest.testCompactConflictWriteBase..."
+ if mvn test -Dtest=org.apache.paimon.JavaPyE2ETest#testCompactConflictWriteBase -pl paimon-core -q -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java write base files completed successfully${NC}"
+ else
+ echo -e "${RED}✗ Java write base files failed${NC}"
+ return 1
+ fi
+
+ # Step 2-4: Python shard update (scan -> Java compact -> commit conflict detected)
+ cd "$PAIMON_PYTHON_DIR"
+ echo "Running Python test for JavaPyReadWriteTest.test_compact_conflict_shard_update..."
+ if python -m pytest java_py_read_write_test.py::JavaPyReadWriteTest::test_compact_conflict_shard_update -v; then
+ echo -e "${GREEN}✓ Python compact conflict test completed successfully${NC}"
+ return 0
+ else
+ echo -e "${RED}✗ Python compact conflict test failed${NC}"
+ return 1
+ fi
+}
+
run_blob_alter_compact_test() {
echo -e "${YELLOW}=== Running Blob Alter+Compact Test (Java Write+Alter+Compact, Python Read) ===${NC}"
@@ -276,6 +302,7 @@ main() {
local pk_dv_result=0
local btree_index_result=0
local compressed_text_result=0
+ local compact_conflict_result=0
local blob_alter_compact_result=0
echo -e "${YELLOW}Starting mixed language test execution...${NC}"
@@ -335,6 +362,13 @@ main() {
echo ""
+ # Run compact conflict test (Java write+compact, Python read)
+ if ! run_compact_conflict_test; then
+ compact_conflict_result=1
+ fi
+
+ echo ""
+
# Run blob alter+compact test (Java write+alter+compact, Python read)
if ! run_blob_alter_compact_test; then
blob_alter_compact_result=1
@@ -386,6 +420,12 @@ main() {
echo -e "${RED}✗ Compressed Text Test (Java Write, Python Read): FAILED${NC}"
fi
+ if [[ $compact_conflict_result -eq 0 ]]; then
+ echo -e "${GREEN}✓ Compact Conflict Test (Java Write+Compact, Python Read): PASSED${NC}"
+ else
+ echo -e "${RED}✗ Compact Conflict Test (Java Write+Compact, Python Read): FAILED${NC}"
+ fi
+
if [[ $blob_alter_compact_result -eq 0 ]]; then
echo -e "${GREEN}✓ Blob Alter+Compact Test (Java Write+Alter+Compact, Python Read): PASSED${NC}"
else
@@ -397,7 +437,7 @@ main() {
# Clean up warehouse directory after all tests
cleanup_warehouse
- if [[ $java_write_result -eq 0 && $python_read_result -eq 0 && $python_write_result -eq 0 && $java_read_result -eq 0 && $pk_dv_result -eq 0 && $btree_index_result -eq 0 && $compressed_text_result -eq 0 && $blob_alter_compact_result -eq 0 ]]; then
+ if [[ $java_write_result -eq 0 && $python_read_result -eq 0 && $python_write_result -eq 0 && $java_read_result -eq 0 && $pk_dv_result -eq 0 && $btree_index_result -eq 0 && $compressed_text_result -eq 0 && $compact_conflict_result -eq 0 && $blob_alter_compact_result -eq 0 ]]; then
echo -e "${GREEN}🎉 All tests passed! Java-Python interoperability verified.${NC}"
return 0
else
diff --git a/paimon-python/pypaimon/read/plan.py b/paimon-python/pypaimon/read/plan.py
index 8c69a41a9b..c4ebc2408f 100644
--- a/paimon-python/pypaimon/read/plan.py
+++ b/paimon-python/pypaimon/read/plan.py
@@ -16,8 +16,8 @@
# limitations under the License.
################################################################################
-from dataclasses import dataclass
-from typing import List
+from dataclasses import dataclass, field
+from typing import List, Optional
from pypaimon.read.split import Split
@@ -27,6 +27,7 @@ from pypaimon.read.split import Split
class Plan:
"""Implementation of Plan for native Python reading."""
_splits: List[Split]
+ snapshot_id: Optional[int] = field(default=None)
def splits(self) -> List[Split]:
return self._splits
diff --git a/paimon-python/pypaimon/read/scanner/file_scanner.py b/paimon-python/pypaimon/read/scanner/file_scanner.py
index aea709669e..41559a44db 100755
--- a/paimon-python/pypaimon/read/scanner/file_scanner.py
+++ b/paimon-python/pypaimon/read/scanner/file_scanner.py
@@ -18,7 +18,7 @@ limitations under the License.
import logging
import os
import time
-from typing import Callable, Dict, List, Optional, Set
+from typing import Callable, Dict, List, Optional, Set, Tuple
logger = logging.getLogger(__name__)
@@ -40,6 +40,7 @@ from pypaimon.read.scanner.data_evolution_split_generator import \
from pypaimon.read.scanner.primary_key_table_split_generator import \
PrimaryKeyTableSplitGenerator
from pypaimon.read.split import DataSplit
+from pypaimon.snapshot.snapshot import Snapshot
from pypaimon.snapshot.snapshot_manager import SnapshotManager
from pypaimon.table.bucket_mode import BucketMode
from pypaimon.table.source.deletion_file import DeletionFile
@@ -165,7 +166,7 @@ class FileScanner:
def __init__(
self,
table,
- manifest_scanner: Callable[[], List[ManifestFileMeta]],
+ manifest_scanner: Callable[[], Tuple[List[ManifestFileMeta], Optional[Snapshot]]],
predicate: Optional[Predicate] = None,
limit: Optional[int] = None
):
@@ -199,6 +200,9 @@ class FileScanner:
self.only_read_real_buckets = options.bucket() == BucketMode.POSTPONE_BUCKET.value
self.data_evolution = options.data_evolution_enabled()
self.deletion_vectors_enabled = options.deletion_vectors_enabled()
+ self._global_index_result = None
+ self._scanned_snapshot = None
+ self._scanned_snapshot_id = None
def schema_fields_func(schema_id: int):
return self.table.schema_manager.get_schema(schema_id).fields
@@ -215,7 +219,8 @@ class FileScanner:
bucket_files = set()
for e in entries:
bucket_files.add((tuple(e.partition.values), e.bucket))
- return self._scan_dv_index(self.snapshot_manager.get_latest_snapshot(), bucket_files)
+ snapshot = self._scanned_snapshot if self._scanned_snapshot else self.snapshot_manager.get_latest_snapshot()
+ return self._scan_dv_index(snapshot, bucket_files)
def scan(self) -> Plan:
start_ms = time.time() * 1000
@@ -240,7 +245,7 @@ class FileScanner:
)
if not entries:
- return Plan([])
+ return Plan([], snapshot_id=self._scanned_snapshot_id)
# Configure sharding if needed
if self.idx_of_this_subtask is not None:
@@ -257,7 +262,7 @@ class FileScanner:
"File store scan plan completed in %d ms. Files size: %d",
duration_ms, len(entries)
)
- return Plan(splits)
+ return Plan(splits, snapshot_id=self._scanned_snapshot_id)
def _create_data_evolution_split_generator(self):
row_ranges = None
@@ -270,7 +275,9 @@ class FileScanner:
if row_ranges is None and self.predicate is not None:
row_ranges = _row_ranges_from_predicate(self.predicate)
- manifest_files = self.manifest_scanner()
+ manifest_files, snapshot = self.manifest_scanner()
+ self._scanned_snapshot = snapshot
+ self._scanned_snapshot_id = snapshot.id if snapshot else None
# Filter manifest files by row ranges if available
if row_ranges is not None:
@@ -291,7 +298,9 @@ class FileScanner:
)
def plan_files(self) -> List[ManifestEntry]:
- manifest_files = self.manifest_scanner()
+ manifest_files, snapshot = self.manifest_scanner()
+ self._scanned_snapshot = snapshot
+ self._scanned_snapshot_id = snapshot.id if snapshot else None
if len(manifest_files) == 0:
return []
return self.read_manifest_entries(manifest_files)
diff --git a/paimon-python/pypaimon/read/streaming_table_scan.py b/paimon-python/pypaimon/read/streaming_table_scan.py
index 835c61dd73..f426d2064f 100644
--- a/paimon-python/pypaimon/read/streaming_table_scan.py
+++ b/paimon-python/pypaimon/read/streaming_table_scan.py
@@ -323,7 +323,7 @@ class AsyncStreamingTableScan:
def _create_initial_plan(self, snapshot: Snapshot) -> Plan:
"""Create a Plan for the initial full scan of the latest snapshot."""
def all_manifests():
- return self._manifest_list_manager.read_all(snapshot)
+ return self._manifest_list_manager.read_all(snapshot), snapshot
starting_scanner = FileScanner(
self.table,
diff --git a/paimon-python/pypaimon/read/table_scan.py b/paimon-python/pypaimon/read/table_scan.py
index 416ef1717d..53f42c0c7f 100755
--- a/paimon-python/pypaimon/read/table_scan.py
+++ b/paimon-python/pypaimon/read/table_scan.py
@@ -59,7 +59,7 @@ class TableScan:
earliest_snapshot = snapshot_manager.try_get_earliest_snapshot()
latest_snapshot = snapshot_manager.get_latest_snapshot()
if earliest_snapshot is None or latest_snapshot is None:
- return FileScanner(self.table, lambda: [])
+ return FileScanner(self.table, lambda: ([], None))
start_timestamp = int(ts[0])
end_timestamp = int(ts[1])
if start_timestamp >= end_timestamp:
@@ -67,7 +67,7 @@ class TableScan:
"Ending timestamp %s should be >= starting timestamp %s." % (end_timestamp, start_timestamp))
if (start_timestamp == end_timestamp or start_timestamp > latest_snapshot.time_millis
or end_timestamp < earliest_snapshot.time_millis):
- return FileScanner(self.table, lambda: [])
+ return FileScanner(self.table, lambda: ([], None))
starting_snapshot = snapshot_manager.earlier_or_equal_time_mills(start_timestamp)
earliest_snapshot = snapshot_manager.try_get_earliest_snapshot()
@@ -84,8 +84,10 @@ class TableScan:
def incremental_manifest():
snapshots_in_range = []
+ end_snapshot = snapshot_manager.get_snapshot_by_id(end_id) if end_id >= 1 else None
for snapshot_id in range(start_id + 1, end_id + 1):
snapshot = snapshot_manager.get_snapshot_by_id(snapshot_id)
+ end_snapshot = snapshot
if snapshot.commit_kind == "APPEND":
snapshots_in_range.append(snapshot)
@@ -94,7 +96,7 @@ class TableScan:
for snapshot in snapshots_in_range:
manifest_files = manifest_list_manager.read_delta(snapshot)
manifests.extend(manifest_files)
- return manifests
+ return manifests, end_snapshot
return FileScanner(self.table, incremental_manifest, self.predicate, self.limit)
elif options.contains(CoreOptions.SCAN_TAG_NAME): # Handle tag-based reading
@@ -104,7 +106,7 @@ class TableScan:
tag_manager = self.table.tag_manager()
tag = tag_manager.get_or_throw(tag_name)
snapshot = tag.trim_to_snapshot()
- return manifest_list_manager.read_all(snapshot)
+ return manifest_list_manager.read_all(snapshot), snapshot
return FileScanner(
self.table,
@@ -115,7 +117,7 @@ class TableScan:
def all_manifests():
snapshot = snapshot_manager.get_latest_snapshot()
- return manifest_list_manager.read_all(snapshot)
+ return manifest_list_manager.read_all(snapshot), snapshot
return FileScanner(
self.table,
diff --git a/paimon-python/pypaimon/tests/binary_row_test.py b/paimon-python/pypaimon/tests/binary_row_test.py
index b1a35226eb..607b184444 100644
--- a/paimon-python/pypaimon/tests/binary_row_test.py
+++ b/paimon-python/pypaimon/tests/binary_row_test.py
@@ -117,7 +117,7 @@ class BinaryRowTest(unittest.TestCase):
def test_is_not_null_append(self):
table = self.catalog.get_table('default.test_append')
- file_scanner = FileScanner(table, lambda: [])
+ file_scanner = FileScanner(table, lambda: ([], None))
latest_snapshot = file_scanner.snapshot_manager.get_latest_snapshot()
manifest_files = file_scanner.manifest_list_manager.read_all(latest_snapshot)
manifest_entries = file_scanner.manifest_file_manager.read(manifest_files[0].file_name)
@@ -254,7 +254,7 @@ class BinaryRowTest(unittest.TestCase):
table_write.close()
table_commit.close()
- file_scanner = FileScanner(table, lambda: [])
+ file_scanner = FileScanner(table, lambda: ([], None))
latest_snapshot = file_scanner.snapshot_manager.get_latest_snapshot()
manifest_files = file_scanner.manifest_list_manager.read_all(latest_snapshot)
manifest_entries = file_scanner.manifest_file_manager.read(manifest_files[0].file_name)
@@ -293,7 +293,7 @@ class BinaryRowTest(unittest.TestCase):
}
self.assertEqual(expected_data, actual.to_pydict())
- file_scanner = FileScanner(table, lambda: [])
+ file_scanner = FileScanner(table, lambda: ([], None))
latest_snapshot = file_scanner.snapshot_manager.get_latest_snapshot()
manifest_files = file_scanner.manifest_list_manager.read_all(latest_snapshot)
manifest_entries = file_scanner.manifest_file_manager.read(manifest_files[0].file_name)
@@ -324,7 +324,7 @@ class BinaryRowTest(unittest.TestCase):
trimmed_pk_fields)
def _overwrite_manifest_entry(self, table):
- file_scanner = FileScanner(table, lambda: [])
+ file_scanner = FileScanner(table, lambda: ([], None))
latest_snapshot = file_scanner.snapshot_manager.get_latest_snapshot()
manifest_files = file_scanner.manifest_list_manager.read_all(latest_snapshot)
manifest_entries = file_scanner.manifest_file_manager.read(manifest_files[0].file_name)
diff --git a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
index 6bbbcfc37f..5bdafafc15 100644
--- a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
+++ b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
@@ -512,3 +512,62 @@ class JavaPyReadWriteTest(unittest.TestCase):
splits = table_scan.plan().splits()
result = table_read.to_arrow(splits)
self.assertEqual(result.num_rows, 200)
+
+ def test_compact_conflict_shard_update(self):
+ """
+ 1. Java writes 5 base files (testCompactConflictWriteBase)
+ 2. pypaimon ShardTableUpdator scans table, prepares evolution
+ 3. Java runs compact (testCompactConflictRunCompact)
+ 4. pypaimon commits stale evolution -> conflict detected, raises RuntimeError
+ """
+ import subprocess
+
+ table = self.catalog.get_table('default.compact_conflict_test')
+
+ # Step 2: pypaimon shard update - scan and prepare commit
+ wb = table.new_batch_write_builder()
+ update = wb.new_update()
+ update.with_read_projection(['f0'])
+ update.with_update_type(['f2'])
+ upd = update.new_shard_updator(shard_num=0, total_shard_count=3)
+ print(f"Shard 0 row_ranges: {[(r[1].from_, r[1].to) for r in upd.row_ranges]}")
+
+ reader = upd.arrow_reader()
+ import pyarrow as pa
+ rows_read = 0
+ for batch in iter(reader.read_next_batch, None):
+ n = batch.num_rows
+ rows_read += n
+ upd.update_by_arrow_batch(
+ pa.RecordBatch.from_pydict(
+ {'f2': [f'evo_{i}' for i in range(n)]},
+ schema=pa.schema([('f2', pa.string())])
+ )
+ )
+ print(f"Shard update read {rows_read} rows")
+ stale_commit_msgs = upd.prepare_commit()
+
+ # Step 3: Java compact (compact happening between scan and commit)
+ project_root = os.path.join(self.tempdir, '..', '..', '..', '..')
+ result = subprocess.run(
+ ['mvn', 'test',
+ '-pl', 'paimon-core',
+ '-Dtest=org.apache.paimon.JavaPyE2ETest#testCompactConflictRunCompact',
+ '-Drun.e2e.tests=true',
+ '-Dsurefire.failIfNoSpecifiedTests=false',
+ '-q'],
+ cwd=os.path.abspath(project_root),
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ universal_newlines=True, timeout=120
+ )
+ self.assertEqual(result.returncode, 0,
+ f"Java compact failed:\n{result.stdout}\n{result.stderr}")
+ print("Java compact completed")
+
+ # Step 4: pypaimon commits stale evolution -> conflict detected
+ tc = wb.new_commit()
+ with self.assertRaises(RuntimeError) as ctx:
+ tc.commit(stale_commit_msgs)
+ self.assertIn("conflicts", str(ctx.exception))
+ tc.close()
+ print(f"Conflict detected as expected: {ctx.exception}")
diff --git a/paimon-python/pypaimon/tests/reader_append_only_test.py b/paimon-python/pypaimon/tests/reader_append_only_test.py
index adb0ff4f25..3ec1413ad3 100644
--- a/paimon-python/pypaimon/tests/reader_append_only_test.py
+++ b/paimon-python/pypaimon/tests/reader_append_only_test.py
@@ -102,6 +102,51 @@ class AoReaderTest(unittest.TestCase):
actual = self._read_test_table(read_builder).sort_by('user_id')
self.assertEqual(actual, self.expected)
+ def test_plan_snapshot_id_for_empty_and_non_empty_scan(self):
+ schema = Schema.from_pyarrow_schema(self.pa_schema, partition_keys=['dt'])
+ self.catalog.create_table('default.test_plan_snapshot_id', schema, False)
+ table = self.catalog.get_table('default.test_plan_snapshot_id')
+
+ empty_plan = table.new_read_builder().new_scan().plan()
+ self.assertIsNone(empty_plan.snapshot_id)
+ self.assertEqual(len(empty_plan.splits()), 0)
+
+ self._write_test_table(table)
+
+ plan = table.new_read_builder().new_scan().plan()
+ self.assertEqual(plan.snapshot_id, 2)
+ self.assertGreater(len(plan.splits()), 0)
+
+ def test_incremental_timestamp_empty_range_keeps_end_snapshot_id(self):
+ schema = Schema.from_pyarrow_schema(self.pa_schema, partition_keys=['dt'])
+ self.catalog.create_table('default.test_incremental_empty_range_snapshot', schema, False)
+ table = self.catalog.get_table('default.test_incremental_empty_range_snapshot')
+
+ write_builder = table.new_batch_write_builder()
+ table_write = write_builder.new_write()
+ table_commit = write_builder.new_commit()
+ pa_table = pa.Table.from_pydict({
+ 'user_id': [1],
+ 'item_id': [1001],
+ 'behavior': ['a'],
+ 'dt': ['p1'],
+ }, schema=self.pa_schema)
+ table_write.write_arrow(pa_table)
+ table_commit.commit(table_write.prepare_commit())
+ table_write.close()
+ table_commit.close()
+
+ snapshot_manager = SnapshotManager(table)
+ snapshot = snapshot_manager.get_latest_snapshot()
+ table_inc = table.copy({
+ CoreOptions.INCREMENTAL_BETWEEN_TIMESTAMP.key():
+ "{},{}".format(snapshot.time_millis, snapshot.time_millis + 1)
+ })
+
+ plan = table_inc.new_read_builder().new_scan().plan()
+ self.assertEqual(plan.snapshot_id, snapshot.id)
+ self.assertEqual(len(plan.splits()), 0)
+
def test_lance_ao_reader_with_filter(self):
schema = Schema.from_pyarrow_schema(self.pa_schema, partition_keys=['dt'], options={'file.format': 'lance'})
self.catalog.create_table('default.test_append_only_lance_filter', schema, False)
diff --git a/paimon-python/pypaimon/write/commit/commit_scanner.py b/paimon-python/pypaimon/write/commit/commit_scanner.py
index 6158c86f5c..d758df3675 100644
--- a/paimon-python/pypaimon/write/commit/commit_scanner.py
+++ b/paimon-python/pypaimon/write/commit/commit_scanner.py
@@ -66,7 +66,7 @@ class CommitScanner:
all_manifests = self.manifest_list_manager.read_all(latest_snapshot)
return FileScanner(
- self.table, lambda: [], partition_filter
+ self.table, lambda: ([], None), partition_filter
).read_manifest_entries(all_manifests)
def read_incremental_entries_from_changed_partitions(self, snapshot: Snapshot,
@@ -92,7 +92,7 @@ class CommitScanner:
partition_filter = self._build_partition_filter_from_entries(commit_entries)
return FileScanner(
- self.table, lambda: [], partition_filter
+ self.table, lambda: ([], None), partition_filter
).read_manifest_entries(delta_manifests)
def _build_partition_filter_from_entries(self, entries: List[ManifestEntry]):
diff --git a/paimon-python/pypaimon/write/file_store_commit.py b/paimon-python/pypaimon/write/file_store_commit.py
index a4099ac1db..8937842680 100644
--- a/paimon-python/pypaimon/write/file_store_commit.py
+++ b/paimon-python/pypaimon/write/file_store_commit.py
@@ -516,7 +516,7 @@ class FileStoreCommit:
"""Generate commit entries for OVERWRITE mode based on latest snapshot."""
entries = []
current_entries = [] if latest_snapshot is None \
- else (FileScanner(self.table, lambda: [], partition_filter).
+ else (FileScanner(self.table, lambda: ([], None), partition_filter).
read_manifest_entries(self.manifest_list_manager.read_all(latest_snapshot)))
for entry in current_entries:
entry.kind = 1 # DELETE
diff --git a/paimon-python/pypaimon/write/table_update.py b/paimon-python/pypaimon/write/table_update.py
index b26194eb55..ec192a98a5 100644
--- a/paimon-python/pypaimon/write/table_update.py
+++ b/paimon-python/pypaimon/write/table_update.py
@@ -171,7 +171,9 @@ class ShardTableUpdator:
self.dict = defaultdict(list)
scanner = self.table.new_read_builder().new_scan()
- splits = scanner.plan().splits()
+ plan = scanner.plan()
+ self.snapshot_id = plan.snapshot_id if plan.snapshot_id is not None else -1
+ splits = plan.splits()
splits = _filter_by_whole_file_shard(splits, shard_num, total_shard_count)
self.splits = splits
@@ -197,7 +199,7 @@ class ShardTableUpdator:
def prepare_commit(self) -> List[CommitMessage]:
commit_messages = []
for (partition, files) in self.dict.items():
- commit_messages.append(CommitMessage(partition, 0, files))
+ commit_messages.append(CommitMessage(partition, 0, files, self.snapshot_id))
return commit_messages
def update_by_arrow_batch(self, data: pa.RecordBatch):
diff --git a/paimon-python/pypaimon/write/table_update_by_row_id.py b/paimon-python/pypaimon/write/table_update_by_row_id.py
index 089fde047c..2e2efe44f7 100644
--- a/paimon-python/pypaimon/write/table_update_by_row_id.py
+++ b/paimon-python/pypaimon/write/table_update_by_row_id.py
@@ -65,7 +65,8 @@ class TableUpdateByRowId:
read_builder = self.table.new_read_builder()
scan = read_builder.new_scan()
- splits = scan.plan().splits()
+ plan = scan.plan()
+ splits = plan.splits()
for split in splits:
for file in split.files:
@@ -77,7 +78,7 @@ class TableUpdateByRowId:
total_row_count = sum(first_row_id_to_row_count_map.values())
- snapshot_id = self.table.snapshot_manager().get_latest_snapshot().id
+ snapshot_id = plan.snapshot_id if plan.snapshot_id is not None else -1
return (snapshot_id,
sorted(list(set(first_row_ids))),
first_row_id_to_partition_map,
(paimon) 23/25: [core] Fix blob read failure after alter table and compaction (#7618)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit c258aea8e931506f409f4c301b7df91fbb3ca68a
Author: littlecoder04 <xi...@alibaba-inc.com>
AuthorDate: Fri Apr 10 16:18:23 2026 +0800
[core] Fix blob read failure after alter table and compaction (#7618)
Reading a blob table fails after ALTER TABLE SET and compaction with:
`java.lang.IllegalArgumentException: All files in this bunch should have
the same schema id.` This PR fixes the above issue by do not check
schemaId for blob file and do not allow rename for blob col.
---
.../paimon/operation/DataEvolutionSplitRead.java | 8 +-
.../org/apache/paimon/schema/SchemaManager.java | 15 ++
.../test/java/org/apache/paimon/JavaPyE2ETest.java | 80 +++++++
.../org/apache/paimon/append/BlobTableTest.java | 239 +++++++++++++++++++++
.../paimon/operation/DataEvolutionReadTest.java | 44 ++++
paimon-python/dev/run_mixed_tests.sh | 41 +++-
paimon-python/pypaimon/read/reader/field_bunch.py | 9 +-
paimon-python/pypaimon/schema/schema_manager.py | 13 ++
paimon-python/pypaimon/tests/blob_table_test.py | 25 +++
.../pypaimon/tests/e2e/java_py_read_write_test.py | 9 +
10 files changed, 474 insertions(+), 9 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java
index dd8433b1b4..c0e70ff4f8 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java
@@ -548,9 +548,11 @@ public class DataEvolutionSplitRead implements SplitRead<InternalRow> {
}
}
if (!files.isEmpty()) {
- checkArgument(
- file.schemaId() == files.get(0).schemaId(),
- "All files in this bunch should have the same schema id.");
+ if (!isBlobFile(file.fileName())) {
+ checkArgument(
+ file.schemaId() == files.get(0).schemaId(),
+ "All files in this bunch should have the same schema id.");
+ }
checkArgument(
file.writeCols().equals(files.get(0).writeCols()),
"All files in this bunch should have the same write columns.");
diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
index 8667f2271d..3aca4b762b 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
@@ -42,6 +42,7 @@ import org.apache.paimon.types.ArrayType;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypeCasts;
+import org.apache.paimon.types.DataTypeRoot;
import org.apache.paimon.types.MapType;
import org.apache.paimon.types.ReassignFieldId;
import org.apache.paimon.types.RowType;
@@ -404,6 +405,7 @@ public class SchemaManager implements Serializable {
} else if (change instanceof RenameColumn) {
RenameColumn rename = (RenameColumn) change;
assertNotUpdatingPartitionKeys(oldTableSchema, rename.fieldNames(), "rename");
+ assertNotRenamingBlobColumn(newFields, rename.fieldNames());
new NestedColumnModifier(rename.fieldNames(), lazyIdentifier) {
@Override
protected void updateLastColumn(
@@ -908,6 +910,19 @@ public class SchemaManager implements Serializable {
}
}
+ private static void assertNotRenamingBlobColumn(List<DataField> fields, String[] fieldNames) {
+ if (fieldNames.length > 1) {
+ return;
+ }
+ String fieldName = fieldNames[0];
+ for (DataField field : fields) {
+ if (field.name().equals(fieldName) && field.type().is(DataTypeRoot.BLOB)) {
+ throw new UnsupportedOperationException(
+ String.format("Cannot rename BLOB column: [%s]", fieldName));
+ }
+ }
+ }
+
private abstract static class NestedColumnModifier {
private final String[] updateFieldNames;
diff --git a/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java b/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
index ca640aa68d..bd1e006ec7 100644
--- a/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
@@ -791,6 +791,86 @@ public class JavaPyE2ETest {
}
}
+ @Test
+ @EnabledIfSystemProperty(named = "run.e2e.tests", matches = "true")
+ public void testBlobWriteAlterCompact() throws Exception {
+ Identifier identifier = identifier("blob_alter_compact_test");
+ catalog.dropTable(identifier, true);
+ Schema schema =
+ Schema.newBuilder()
+ .column("f0", DataTypes.INT())
+ .column("f1", DataTypes.STRING())
+ .column("f2", DataTypes.BLOB())
+ .option("target-file-size", "100 MB")
+ .option("row-tracking.enabled", "true")
+ .option("data-evolution.enabled", "true")
+ .option("compaction.min.file-num", "2")
+ .option("bucket", "-1")
+ .build();
+ catalog.createTable(identifier, schema, false);
+
+ byte[] blobBytes = new byte[1024];
+ new java.util.Random(42).nextBytes(blobBytes);
+
+ // Batch 1: write with schemaId=0
+ FileStoreTable table = (FileStoreTable) catalog.getTable(identifier);
+ StreamTableWrite write =
+ table.newStreamWriteBuilder().withCommitUser(commitUser).newWrite();
+ StreamTableCommit commit = table.newCommit(commitUser);
+ for (int i = 0; i < 100; i++) {
+ write.write(
+ GenericRow.of(
+ 1,
+ BinaryString.fromString("batch1"),
+ new org.apache.paimon.data.BlobData(blobBytes)));
+ }
+ commit.commit(0, write.prepareCommit(false, 0));
+
+ // ALTER TABLE SET -> schemaId becomes 1
+ catalog.alterTable(
+ identifier,
+ org.apache.paimon.schema.SchemaChange.setOption("snapshot.num-retained.min", "5"),
+ false);
+
+ // Batch 2: write with schemaId=1
+ table = (FileStoreTable) catalog.getTable(identifier);
+ write = table.newStreamWriteBuilder().withCommitUser(commitUser).newWrite();
+ commit = table.newCommit(commitUser);
+ for (int i = 0; i < 100; i++) {
+ write.write(
+ GenericRow.of(
+ 2,
+ BinaryString.fromString("batch2"),
+ new org.apache.paimon.data.BlobData(blobBytes)));
+ }
+ commit.commit(1, write.prepareCommit(false, 1));
+ write.close();
+ commit.close();
+
+ // Compact
+ table = (FileStoreTable) catalog.getTable(identifier);
+ org.apache.paimon.append.dataevolution.DataEvolutionCompactCoordinator coordinator =
+ new org.apache.paimon.append.dataevolution.DataEvolutionCompactCoordinator(
+ table, false, false);
+ List<org.apache.paimon.append.dataevolution.DataEvolutionCompactTask> tasks =
+ coordinator.plan();
+ assertThat(tasks.size()).isGreaterThan(0);
+ List<org.apache.paimon.table.sink.CommitMessage> compactMessages = new ArrayList<>();
+ for (org.apache.paimon.append.dataevolution.DataEvolutionCompactTask task : tasks) {
+ compactMessages.add(task.doCompact(table, commitUser));
+ }
+ StreamTableCommit compactCommit = table.newCommit(commitUser);
+ compactCommit.commit(2, compactMessages);
+ compactCommit.close();
+
+ FileStoreTable readTable = (FileStoreTable) catalog.getTable(identifier);
+ List<Split> splits = new ArrayList<>(readTable.newSnapshotReader().read().dataSplits());
+ TableRead read = readTable.newRead();
+ RowType rowType = readTable.rowType();
+ List<String> res = getResult(read, splits, row -> internalRowToString(row, rowType));
+ assertThat(res).hasSize(200);
+ }
+
// Helper method from TableTestBase
protected Identifier identifier(String tableName) {
return new Identifier(database, tableName);
diff --git a/paimon-core/src/test/java/org/apache/paimon/append/BlobTableTest.java b/paimon-core/src/test/java/org/apache/paimon/append/BlobTableTest.java
index 736ce904cf..e3019485bb 100644
--- a/paimon-core/src/test/java/org/apache/paimon/append/BlobTableTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/append/BlobTableTest.java
@@ -19,6 +19,8 @@
package org.apache.paimon.append;
import org.apache.paimon.CoreOptions;
+import org.apache.paimon.append.dataevolution.DataEvolutionCompactCoordinator;
+import org.apache.paimon.append.dataevolution.DataEvolutionCompactTask;
import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.Blob;
import org.apache.paimon.data.BlobData;
@@ -33,10 +35,14 @@ import org.apache.paimon.manifest.ManifestEntry;
import org.apache.paimon.operation.DataEvolutionSplitRead;
import org.apache.paimon.reader.RecordReader;
import org.apache.paimon.schema.Schema;
+import org.apache.paimon.schema.SchemaChange;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.Table;
import org.apache.paimon.table.TableTestBase;
import org.apache.paimon.table.sink.BatchTableWrite;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.table.sink.StreamTableWrite;
+import org.apache.paimon.table.sink.StreamWriteBuilder;
import org.apache.paimon.table.source.ReadBuilder;
import org.apache.paimon.table.system.RowTrackingTable;
import org.apache.paimon.types.DataField;
@@ -45,6 +51,7 @@ import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.Range;
import org.apache.paimon.utils.UriReader;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
@@ -512,6 +519,238 @@ public class BlobTableTest extends TableTestBase {
assertThat(projectedCount.get()).isEqualTo(1);
}
+ @Test
+ void testReadBlobAfterAlterTableAndCompaction() throws Exception {
+ Schema.Builder schemaBuilder = Schema.newBuilder();
+ schemaBuilder.column("f0", DataTypes.INT());
+ schemaBuilder.column("f1", DataTypes.STRING());
+ schemaBuilder.column("f2", DataTypes.BLOB());
+ schemaBuilder.option(CoreOptions.TARGET_FILE_SIZE.key(), "100 MB");
+ schemaBuilder.option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true");
+ schemaBuilder.option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true");
+ schemaBuilder.option(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "2");
+ catalog.createTable(identifier(), schemaBuilder.build(), true);
+
+ // Step 1: write data with schemaId=0
+ commitDefault(writeDataDefault(100, 1));
+
+ // Step 2: ALTER TABLE SET an unrelated option -> schemaId becomes 1
+ catalog.alterTable(
+ identifier(), SchemaChange.setOption("snapshot.num-retained.min", "5"), false);
+
+ // Step 3: write more data with schemaId=1
+ commitDefault(writeDataDefault(100, 1));
+
+ // Step 4: compact blob table using DataEvolutionCompactCoordinator
+ FileStoreTable table = getTableDefault();
+ DataEvolutionCompactCoordinator coordinator =
+ new DataEvolutionCompactCoordinator(table, false, false);
+ List<DataEvolutionCompactTask> tasks = coordinator.plan();
+ assertThat(tasks.size()).isGreaterThan(0);
+ List<CommitMessage> compactMessages = new ArrayList<>();
+ for (DataEvolutionCompactTask task : tasks) {
+ compactMessages.add(task.doCompact(table, commitUser));
+ }
+ commitDefault(compactMessages);
+
+ // Step 5: read after compaction
+ readDefault(row -> assertThat(row.getBlob(2).toData()).isEqualTo(blobBytes));
+ }
+
+ @Test
+ void testReadBlobAfterAddColumnAndCompaction() throws Exception {
+ Schema.Builder schemaBuilder = Schema.newBuilder();
+ schemaBuilder.column("f0", DataTypes.INT());
+ schemaBuilder.column("f1", DataTypes.STRING());
+ schemaBuilder.column("f2", DataTypes.BLOB());
+ schemaBuilder.option(CoreOptions.TARGET_FILE_SIZE.key(), "100 MB");
+ schemaBuilder.option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true");
+ schemaBuilder.option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true");
+ schemaBuilder.option(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "2");
+ catalog.createTable(identifier(), schemaBuilder.build(), true);
+
+ {
+ FileStoreTable t = getTableDefault();
+ StreamWriteBuilder b = t.newStreamWriteBuilder().withCommitUser(commitUser);
+ try (StreamTableWrite w = b.newWrite()) {
+ for (int j = 0; j < 100; j++) {
+ w.write(
+ GenericRow.of(
+ 1, BinaryString.fromString("batch1"), new BlobData(blobBytes)));
+ }
+ commitDefault(w.prepareCommit(false, Long.MAX_VALUE));
+ }
+ }
+
+ catalog.alterTable(identifier(), SchemaChange.addColumn("f3", DataTypes.STRING()), false);
+
+ {
+ FileStoreTable t = getTableDefault();
+ StreamWriteBuilder b = t.newStreamWriteBuilder().withCommitUser(commitUser);
+ try (StreamTableWrite w = b.newWrite()) {
+ for (int j = 0; j < 100; j++) {
+ w.write(
+ GenericRow.of(
+ 2,
+ BinaryString.fromString("batch2"),
+ new BlobData(blobBytes),
+ BinaryString.fromString("after-add")));
+ }
+ commitDefault(w.prepareCommit(false, Long.MAX_VALUE));
+ }
+ }
+
+ FileStoreTable table = getTableDefault();
+ DataEvolutionCompactCoordinator coordinator =
+ new DataEvolutionCompactCoordinator(table, false, false);
+ List<DataEvolutionCompactTask> tasks = coordinator.plan();
+ assertThat(tasks.size()).isGreaterThan(0);
+ List<CommitMessage> compactMessages = new ArrayList<>();
+ for (DataEvolutionCompactTask task : tasks) {
+ compactMessages.add(task.doCompact(table, commitUser));
+ }
+ commitDefault(compactMessages);
+
+ AtomicInteger batch1Count = new AtomicInteger(0);
+ AtomicInteger batch2Count = new AtomicInteger(0);
+ readDefault(
+ row -> {
+ assertThat(row.getBlob(2).toData()).isEqualTo(blobBytes);
+ if (row.getInt(0) == 1) {
+ batch1Count.incrementAndGet();
+ } else if (row.getInt(0) == 2) {
+ batch2Count.incrementAndGet();
+ }
+ });
+ assertThat(batch1Count.get()).isEqualTo(100);
+ assertThat(batch2Count.get()).isEqualTo(100);
+ }
+
+ @Test
+ void testReadBlobAfterDropColumnAndCompaction() throws Exception {
+ Schema.Builder schemaBuilder = Schema.newBuilder();
+ schemaBuilder.column("f0", DataTypes.INT());
+ schemaBuilder.column("f1", DataTypes.STRING());
+ schemaBuilder.column("f2", DataTypes.BLOB());
+ schemaBuilder.column("f3", DataTypes.STRING());
+ schemaBuilder.option(CoreOptions.TARGET_FILE_SIZE.key(), "100 MB");
+ schemaBuilder.option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true");
+ schemaBuilder.option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true");
+ schemaBuilder.option(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "2");
+ catalog.createTable(identifier(), schemaBuilder.build(), true);
+
+ {
+ FileStoreTable t = getTableDefault();
+ StreamWriteBuilder b = t.newStreamWriteBuilder().withCommitUser(commitUser);
+ try (StreamTableWrite w = b.newWrite()) {
+ for (int j = 0; j < 100; j++) {
+ w.write(
+ GenericRow.of(
+ 1,
+ BinaryString.fromString("batch1"),
+ new BlobData(blobBytes),
+ BinaryString.fromString("before-drop")));
+ }
+ commitDefault(w.prepareCommit(false, Long.MAX_VALUE));
+ }
+ }
+
+ catalog.alterTable(identifier(), SchemaChange.dropColumn("f3"), false);
+
+ {
+ FileStoreTable t = getTableDefault();
+ StreamWriteBuilder b = t.newStreamWriteBuilder().withCommitUser(commitUser);
+ try (StreamTableWrite w = b.newWrite()) {
+ for (int j = 0; j < 100; j++) {
+ w.write(
+ GenericRow.of(
+ 2, BinaryString.fromString("batch2"), new BlobData(blobBytes)));
+ }
+ commitDefault(w.prepareCommit(false, Long.MAX_VALUE));
+ }
+ }
+
+ FileStoreTable table = getTableDefault();
+ DataEvolutionCompactCoordinator coordinator =
+ new DataEvolutionCompactCoordinator(table, false, false);
+ List<DataEvolutionCompactTask> tasks = coordinator.plan();
+ assertThat(tasks.size()).isGreaterThan(0);
+ List<CommitMessage> compactMessages = new ArrayList<>();
+ for (DataEvolutionCompactTask task : tasks) {
+ compactMessages.add(task.doCompact(table, commitUser));
+ }
+ commitDefault(compactMessages);
+
+ AtomicInteger batch1Count = new AtomicInteger(0);
+ AtomicInteger batch2Count = new AtomicInteger(0);
+ readDefault(
+ row -> {
+ assertThat(row.getBlob(2).toData()).isEqualTo(blobBytes);
+ if (row.getInt(0) == 1) {
+ batch1Count.incrementAndGet();
+ } else if (row.getInt(0) == 2) {
+ batch2Count.incrementAndGet();
+ }
+ });
+ assertThat(batch1Count.get()).isEqualTo(100);
+ assertThat(batch2Count.get()).isEqualTo(100);
+ }
+
+ @Disabled("Reproduce: rename blob column causes read failure after compaction")
+ @Test
+ void testRenameBlobColumnReadFailure() throws Exception {
+ Schema.Builder schemaBuilder = Schema.newBuilder();
+ schemaBuilder.column("f0", DataTypes.INT());
+ schemaBuilder.column("f1", DataTypes.STRING());
+ schemaBuilder.column("f2", DataTypes.BLOB());
+ schemaBuilder.option(CoreOptions.TARGET_FILE_SIZE.key(), "100 MB");
+ schemaBuilder.option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true");
+ schemaBuilder.option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true");
+ schemaBuilder.option(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "2");
+ catalog.createTable(identifier(), schemaBuilder.build(), true);
+
+ // Step 1: write blob data — blob files record writeCols=["f2"]
+ commitDefault(writeDataDefault(100, 1));
+
+ // Step 2: rename blob column f2 -> f2_renamed
+ catalog.alterTable(identifier(), SchemaChange.renameColumn("f2", "f2_renamed"), false);
+
+ // Step 3: write more data — new blob files have writeCols=["f2_renamed"]
+ commitDefault(writeDataDefault(100, 1));
+
+ // Step 4: compact merges files into the same split
+ FileStoreTable table = getTableDefault();
+ DataEvolutionCompactCoordinator coordinator =
+ new DataEvolutionCompactCoordinator(table, false, false);
+ List<DataEvolutionCompactTask> tasks = coordinator.plan();
+ assertThat(tasks.size()).isGreaterThan(0);
+ List<CommitMessage> compactMessages = new ArrayList<>();
+ for (DataEvolutionCompactTask task : tasks) {
+ compactMessages.add(task.doCompact(table, commitUser));
+ }
+ commitDefault(compactMessages);
+
+ // Step 5: read fails —
+ assertThatThrownBy(() -> readDefault(row -> {}))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("All files in this bunch should have the same write columns");
+ }
+
+ @Test
+ void testRenameBlobColumnShouldFail() throws Exception {
+ createTableDefault();
+ commitDefault(writeDataDefault(10, 1));
+
+ assertThatThrownBy(
+ () ->
+ catalog.alterTable(
+ identifier(),
+ SchemaChange.renameColumn("f2", "f2_renamed"),
+ false))
+ .isInstanceOf(UnsupportedOperationException.class)
+ .hasMessageContaining("Cannot rename BLOB column");
+ }
+
private void createExternalStorageTable() throws Exception {
Schema.Builder schemaBuilder = Schema.newBuilder();
schemaBuilder.column("f0", DataTypes.INT());
diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionReadTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionReadTest.java
index 4e85b1e3dd..9bb9a7274e 100644
--- a/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionReadTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionReadTest.java
@@ -297,6 +297,36 @@ public class DataEvolutionReadTest {
fileName, firstRowId, rowCount, maxSequenceNumber, Arrays.asList("blob_col"));
}
+ /** Creates a blob file with a specified schemaId. */
+ private DataFileMeta createBlobFileWithSchema(
+ String fileName,
+ long firstRowId,
+ long rowCount,
+ long maxSequenceNumber,
+ long schemaId) {
+ return DataFileMeta.create(
+ fileName + ".blob",
+ rowCount,
+ rowCount,
+ DataFileMeta.EMPTY_MIN_KEY,
+ DataFileMeta.EMPTY_MAX_KEY,
+ SimpleStats.EMPTY_STATS,
+ SimpleStats.EMPTY_STATS,
+ 0,
+ maxSequenceNumber,
+ schemaId,
+ DataFileMeta.DUMMY_LEVEL,
+ Collections.emptyList(),
+ Timestamp.fromEpochMillis(System.currentTimeMillis()),
+ rowCount,
+ null,
+ FileSource.APPEND,
+ null,
+ null,
+ firstRowId,
+ Arrays.asList("blob_col"));
+ }
+
/** Creates a blob file with custom write columns. */
private DataFileMeta createBlobFileWithCols(
String fileName,
@@ -327,6 +357,20 @@ public class DataEvolutionReadTest {
writeCols);
}
+ @Test
+ void testAddBlobFilesWithDifferentSchemaId() {
+ DataFileMeta blobEntry1 = createBlobFileWithSchema("blob1", 0, 100, 1, 0L);
+ DataFileMeta blobEntry2 = createBlobFileWithSchema("blob2", 100, 200, 1, 1L);
+
+ blobBunch.add(blobEntry1);
+ assertThatCode(() -> blobBunch.add(blobEntry2)).doesNotThrowAnyException();
+
+ assertThat(blobBunch.files).hasSize(2);
+ assertThat(blobBunch.files.get(0).schemaId()).isEqualTo(0L);
+ assertThat(blobBunch.files.get(1).schemaId()).isEqualTo(1L);
+ assertThat(blobBunch.rowCount()).isEqualTo(300);
+ }
+
@Test
public void testRowIdPushDown() {
SpecialFieldBunch blobBunch = new SpecialFieldBunch(Long.MAX_VALUE, true);
diff --git a/paimon-python/dev/run_mixed_tests.sh b/paimon-python/dev/run_mixed_tests.sh
index a7e5d5dd76..09bcb58d2b 100755
--- a/paimon-python/dev/run_mixed_tests.sh
+++ b/paimon-python/dev/run_mixed_tests.sh
@@ -244,6 +244,29 @@ run_compressed_text_test() {
fi
}
+run_blob_alter_compact_test() {
+ echo -e "${YELLOW}=== Running Blob Alter+Compact Test (Java Write+Alter+Compact, Python Read) ===${NC}"
+
+ cd "$PROJECT_ROOT"
+
+ echo "Running Maven test for JavaPyE2ETest.testBlobWriteAlterCompact..."
+ if mvn test -Dtest=org.apache.paimon.JavaPyE2ETest#testBlobWriteAlterCompact -pl paimon-core -q -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java blob write+alter+compact test completed successfully${NC}"
+ else
+ echo -e "${RED}✗ Java blob write+alter+compact test failed${NC}"
+ return 1
+ fi
+ cd "$PAIMON_PYTHON_DIR"
+ echo "Running Python test for JavaPyReadWriteTest.test_read_blob_after_alter_and_compact..."
+ if python -m pytest java_py_read_write_test.py::JavaPyReadWriteTest::test_read_blob_after_alter_and_compact -v; then
+ echo -e "${GREEN}✓ Python blob read test completed successfully${NC}"
+ return 0
+ else
+ echo -e "${RED}✗ Python blob read test failed${NC}"
+ return 1
+ fi
+}
+
# Main execution
main() {
local java_write_result=0
@@ -253,6 +276,7 @@ main() {
local pk_dv_result=0
local btree_index_result=0
local compressed_text_result=0
+ local blob_alter_compact_result=0
echo -e "${YELLOW}Starting mixed language test execution...${NC}"
echo ""
@@ -311,6 +335,13 @@ main() {
echo ""
+ # Run blob alter+compact test (Java write+alter+compact, Python read)
+ if ! run_blob_alter_compact_test; then
+ blob_alter_compact_result=1
+ fi
+
+ echo ""
+
echo -e "${YELLOW}=== Test Results Summary ===${NC}"
if [[ $java_write_result -eq 0 ]]; then
@@ -355,12 +386,18 @@ main() {
echo -e "${RED}✗ Compressed Text Test (Java Write, Python Read): FAILED${NC}"
fi
+ if [[ $blob_alter_compact_result -eq 0 ]]; then
+ echo -e "${GREEN}✓ Blob Alter+Compact Test (Java Write+Alter+Compact, Python Read): PASSED${NC}"
+ else
+ echo -e "${RED}✗ Blob Alter+Compact Test (Java Write+Alter+Compact, Python Read): FAILED${NC}"
+ fi
+
echo ""
# Clean up warehouse directory after all tests
cleanup_warehouse
- if [[ $java_write_result -eq 0 && $python_read_result -eq 0 && $python_write_result -eq 0 && $java_read_result -eq 0 && $pk_dv_result -eq 0 && $btree_index_result -eq 0 && $compressed_text_result -eq 0 ]]; then
+ if [[ $java_write_result -eq 0 && $python_read_result -eq 0 && $python_write_result -eq 0 && $java_read_result -eq 0 && $pk_dv_result -eq 0 && $btree_index_result -eq 0 && $compressed_text_result -eq 0 && $blob_alter_compact_result -eq 0 ]]; then
echo -e "${GREEN}🎉 All tests passed! Java-Python interoperability verified.${NC}"
return 0
else
@@ -370,4 +407,4 @@ main() {
}
# Run main function
-main "$@"
\ No newline at end of file
+main "$@"
diff --git a/paimon-python/pypaimon/read/reader/field_bunch.py b/paimon-python/pypaimon/read/reader/field_bunch.py
index fccf2cb325..f9f2dca3ce 100644
--- a/paimon-python/pypaimon/read/reader/field_bunch.py
+++ b/paimon-python/pypaimon/read/reader/field_bunch.py
@@ -99,10 +99,11 @@ class BlobBunch(FieldBunch):
)
if self._files:
- if file.schema_id != self._files[0].schema_id:
- raise ValueError(
- "All files in a blob bunch should have the same schema id."
- )
+ if not DataFileMeta.is_blob_file(file.file_name):
+ if file.schema_id != self._files[0].schema_id:
+ raise ValueError(
+ "All files in a blob bunch should have the same schema id."
+ )
if file.write_cols != self._files[0].write_cols:
raise ValueError(
"All files in a blob bunch should have the same write columns."
diff --git a/paimon-python/pypaimon/schema/schema_manager.py b/paimon-python/pypaimon/schema/schema_manager.py
index 0a18c77630..721925dc33 100644
--- a/paimon-python/pypaimon/schema/schema_manager.py
+++ b/paimon-python/pypaimon/schema/schema_manager.py
@@ -136,6 +136,18 @@ def _assert_not_updating_primary_keys(
raise ValueError(f"Cannot {operation} primary key")
+def _assert_not_renaming_blob_column(
+ new_fields: List[DataField], field_names: List[str]):
+ if len(field_names) > 1:
+ return
+ field_name = field_names[0]
+ for field in new_fields:
+ if field.name == field_name and str(field.type) == 'BLOB':
+ raise ValueError(
+ f"Cannot rename BLOB column: [{field_name}]"
+ )
+
+
def _handle_rename_column(change: RenameColumn, new_fields: List[DataField]):
field_name = change.field_names[-1]
new_name = change.new_name
@@ -355,6 +367,7 @@ class SchemaManager:
_assert_not_updating_partition_keys(
old_table_schema, change.field_names, "rename"
)
+ _assert_not_renaming_blob_column(new_fields, change.field_names)
_handle_rename_column(change, new_fields)
elif isinstance(change, DropColumn):
_drop_column_validation(old_table_schema, change)
diff --git a/paimon-python/pypaimon/tests/blob_table_test.py b/paimon-python/pypaimon/tests/blob_table_test.py
index 0d3bf49cbd..7e4d6c6d26 100755
--- a/paimon-python/pypaimon/tests/blob_table_test.py
+++ b/paimon-python/pypaimon/tests/blob_table_test.py
@@ -24,6 +24,7 @@ import unittest
import pyarrow as pa
from pypaimon import CatalogFactory, Schema
+from pypaimon.schema.schema_change import SchemaChange
from pypaimon.table.file_store_table import FileStoreTable
from pypaimon.write.commit_message import CommitMessage
@@ -3024,6 +3025,30 @@ class DataBlobWriterTest(unittest.TestCase):
result = read.to_arrow(splits)
self.assertEqual(result.num_rows, 1)
+ def test_rename_blob_column_should_fail(self):
+ pa_schema = pa.schema([
+ ('id', pa.int32()),
+ ('name', pa.string()),
+ ('blob_col', pa.large_binary()),
+ ])
+
+ schema = Schema.from_pyarrow_schema(
+ pa_schema,
+ options={
+ 'row-tracking.enabled': 'true',
+ 'data-evolution.enabled': 'true',
+ }
+ )
+ self.catalog.create_table('test_db.blob_rename_test', schema, False)
+
+ with self.assertRaises(RuntimeError) as ctx:
+ self.catalog.alter_table(
+ 'test_db.blob_rename_test',
+ [SchemaChange.rename_column('blob_col', 'blob_col_renamed')],
+ False
+ )
+ self.assertIn('Cannot rename BLOB column', str(ctx.exception))
+
if __name__ == '__main__':
unittest.main()
diff --git a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
index 1f906c656d..6bbbcfc37f 100644
--- a/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
+++ b/paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py
@@ -503,3 +503,12 @@ class JavaPyReadWriteTest(unittest.TestCase):
table_read.to_arrow(splits)
self.assertIn(file_format, str(ctx.exception))
self.assertIn("not yet supported", str(ctx.exception))
+
+ def test_read_blob_after_alter_and_compact(self):
+ table = self.catalog.get_table('default.blob_alter_compact_test')
+ read_builder = table.new_read_builder()
+ table_scan = read_builder.new_scan()
+ table_read = read_builder.new_read()
+ splits = table_scan.plan().splits()
+ result = table_read.to_arrow(splits)
+ self.assertEqual(result.num_rows, 200)
(paimon) 15/25: [python] Fix stats evolution index mapping to use field ID instead of field name (#7593)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 06f3a6d17229497590dcc4878e2f09b63f25291d
Author: Jiajia Li <pl...@alibaba-inc.com>
AuthorDate: Tue Apr 7 11:44:33 2026 +0800
[python] Fix stats evolution index mapping to use field ID instead of field name (#7593)
SimpleStatsEvolutions._create_index_cast_mapping was using field.name to
build the mapping between table schema and data schema. This breaks
after column renames because names change while field IDs remain stable.
Changed to use field.id, aligning with Java's
SchemaEvolutionUtil.createIndexMapping.
---
.../pypaimon/manifest/simple_stats_evolutions.py | 10 +-
.../tests/manifest/simple_stats_evolutions_test.py | 156 +++++++++++++++++++++
2 files changed, 162 insertions(+), 4 deletions(-)
diff --git a/paimon-python/pypaimon/manifest/simple_stats_evolutions.py b/paimon-python/pypaimon/manifest/simple_stats_evolutions.py
index f7f5946162..a3c235aff9 100644
--- a/paimon-python/pypaimon/manifest/simple_stats_evolutions.py
+++ b/paimon-python/pypaimon/manifest/simple_stats_evolutions.py
@@ -55,13 +55,15 @@ class SimpleStatsEvolutions:
Create index and cast mapping between table fields and data fields.
This is a simplified implementation.
"""
- # Create a mapping from field names to indices in data_fields
- data_field_map = {field.name: i for i, field in enumerate(data_fields)}
+ # Create a mapping from field IDs to indices in data_fields.
+ # Field IDs are stable across schema evolution (including column renames),
+ # while field names may change.
+ data_field_map = {field.id: i for i, field in enumerate(data_fields)}
index_mapping = []
for table_field in table_fields:
- if table_field.name in data_field_map:
- index_mapping.append(data_field_map[table_field.name])
+ if table_field.id in data_field_map:
+ index_mapping.append(data_field_map[table_field.id])
else:
index_mapping.append(-1) # Field not found in data schema
diff --git a/paimon-python/pypaimon/tests/manifest/simple_stats_evolutions_test.py b/paimon-python/pypaimon/tests/manifest/simple_stats_evolutions_test.py
new file mode 100644
index 0000000000..cfc811eaab
--- /dev/null
+++ b/paimon-python/pypaimon/tests/manifest/simple_stats_evolutions_test.py
@@ -0,0 +1,156 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import unittest
+
+from pypaimon.manifest.simple_stats_evolutions import SimpleStatsEvolutions
+from pypaimon.schema.data_types import DataField, AtomicType
+from pypaimon.manifest.schema.simple_stats import SimpleStats
+from pypaimon.table.row.generic_row import GenericRow
+
+
+class SimpleStatsEvolutionsTest(unittest.TestCase):
+
+ def _make_fields(self, field_defs):
+ """Helper: field_defs is list of (id, name, type_str)."""
+ return [DataField(fid, fname, AtomicType(ftype)) for fid, fname, ftype in field_defs]
+
+ def _make_stats(self, fields, min_vals, max_vals, null_counts):
+ """Helper: build SimpleStats from parallel lists."""
+ df = [DataField(fid, fname, AtomicType(ftype)) for fid, fname, ftype in fields]
+ return SimpleStats(GenericRow(min_vals, df), GenericRow(max_vals, df), null_counts)
+
+ def test_same_schema_no_mapping(self):
+ """When data schema == table schema, index_mapping should be None."""
+ fields = self._make_fields([(0, 'a', 'INT'), (1, 'b', 'STRING')])
+ schemas = {0: fields}
+ evolutions = SimpleStatsEvolutions(lambda sid: schemas[sid], 0)
+ evolution = evolutions.get_or_create(0)
+ self.assertIsNone(evolution.index_mapping)
+
+ def test_added_column(self):
+ """New column: mapping is -1, null_count = row_count."""
+ data_fields = self._make_fields([(0, 'a', 'INT'), (1, 'b', 'INT')])
+ table_fields = self._make_fields([(0, 'a', 'INT'), (1, 'b', 'INT'), (2, 'c', 'INT')])
+ schemas = {0: data_fields, 1: table_fields}
+ evolutions = SimpleStatsEvolutions(lambda sid: schemas[sid], 1)
+ evolution = evolutions.get_or_create(0)
+
+ self.assertEqual(evolution.index_mapping, [0, 1, -1])
+
+ stats = self._make_stats([(0, 'a', 'INT'), (1, 'b', 'INT')],
+ [1, 2], [10, 20], [0, 5])
+ evolved = evolution.evolution(stats, row_count=500, stats_fields=None)
+ self.assertIsNone(evolved.min_values.get_field(2))
+ self.assertIsNone(evolved.max_values.get_field(2))
+ self.assertEqual(evolved.null_counts, [0, 5, 500])
+
+ def test_dropped_column(self):
+ """Dropped column is excluded from mapping."""
+ data_fields = self._make_fields([(0, 'a', 'INT'), (1, 'b', 'STRING'), (2, 'c', 'BIGINT')])
+ table_fields = self._make_fields([(0, 'a', 'INT'), (2, 'c', 'BIGINT')])
+ schemas = {0: data_fields, 1: table_fields}
+ evolutions = SimpleStatsEvolutions(lambda sid: schemas[sid], 1)
+ evolution = evolutions.get_or_create(0)
+ # a->0, c->2 (b dropped)
+ self.assertEqual(evolution.index_mapping, [0, 2])
+
+ def test_renamed_column(self):
+ """Column rename: matched by field ID, not name."""
+ data_fields = self._make_fields([(0, 'old_name', 'INT'), (1, 'b', 'INT')])
+ table_fields = self._make_fields([(0, 'new_name', 'INT'), (1, 'b', 'INT')])
+ schemas = {0: data_fields, 1: table_fields}
+ evolutions = SimpleStatsEvolutions(lambda sid: schemas[sid], 1)
+ evolution = evolutions.get_or_create(0)
+
+ self.assertEqual(evolution.index_mapping, [0, 1])
+ self.assertNotEqual(evolution.index_mapping[0], -1) # not missing
+
+ stats = self._make_stats([(0, 'old_name', 'INT'), (1, 'b', 'INT')],
+ [10, 20], [100, 200], [0, 5])
+ evolved = evolution.evolution(stats, row_count=1000, stats_fields=None)
+ self.assertEqual(evolved.min_values.get_field(0), 10)
+ self.assertEqual(evolved.min_values.get_field(1), 20)
+ self.assertEqual(evolved.max_values.get_field(0), 100)
+ self.assertEqual(evolved.max_values.get_field(1), 200)
+ self.assertEqual(evolved.null_counts, [0, 5])
+
+ def test_reordered_columns(self):
+ """Column order differs between schemas, matched by field ID."""
+ data_fields = self._make_fields([(0, 'a', 'INT'), (1, 'b', 'STRING'), (2, 'c', 'BIGINT')])
+ table_fields = self._make_fields([(2, 'c', 'BIGINT'), (0, 'a', 'INT'), (1, 'b', 'STRING')])
+ schemas = {0: data_fields, 1: table_fields}
+ evolutions = SimpleStatsEvolutions(lambda sid: schemas[sid], 1)
+ evolution = evolutions.get_or_create(0)
+ self.assertEqual(evolution.index_mapping, [2, 0, 1])
+
+ def test_caching(self):
+ """Evolution objects are cached by schema ID."""
+ fields = self._make_fields([(0, 'a', 'INT')])
+ schemas = {0: fields, 1: fields}
+ evolutions = SimpleStatsEvolutions(lambda sid: schemas[sid], 1)
+ e1 = evolutions.get_or_create(0)
+ e2 = evolutions.get_or_create(0)
+ self.assertIs(e1, e2)
+
+ def test_complex_evolution(self):
+ """Reorder + add + drop + rename combined.
+
+ Data schema (id=0): a(0), b(1), c(2), d(3)
+ Table schema (id=1): cc(2), a(0), d(3), e(4, new), bb(1)
+
+ Verifies index_mapping, min/max projection, and null count evolution.
+ """
+ data_fields = self._make_fields([
+ (0, 'a', 'INT'), (1, 'b', 'INT'), (2, 'c', 'INT'), (3, 'd', 'INT')
+ ])
+ table_fields = self._make_fields([
+ (2, 'cc', 'INT'), (0, 'a', 'INT'), (3, 'd', 'INT'), (4, 'e', 'INT'), (1, 'bb', 'INT')
+ ])
+ schemas = {0: data_fields, 1: table_fields}
+ evolutions = SimpleStatsEvolutions(lambda sid: schemas[sid], 1)
+ evolution = evolutions.get_or_create(0)
+
+ # Mapping: cc(id=2)->2, a(id=0)->0, d(id=3)->3, e(id=4)->-1, bb(id=1)->1
+ self.assertEqual(evolution.index_mapping, [2, 0, 3, -1, 1])
+
+ stats = self._make_stats(
+ [(0, 'a', 'INT'), (1, 'b', 'INT'), (2, 'c', 'INT'), (3, 'd', 'INT')],
+ [10, 20, 30, 40], [100, 200, 300, 400], [1, 2, 3, 4])
+ evolved = evolution.evolution(stats, row_count=1000, stats_fields=None)
+
+ # min: cc->30, a->10, d->40, e->None, bb->20
+ self.assertEqual(evolved.min_values.get_field(0), 30)
+ self.assertEqual(evolved.min_values.get_field(1), 10)
+ self.assertEqual(evolved.min_values.get_field(2), 40)
+ self.assertIsNone(evolved.min_values.get_field(3))
+ self.assertEqual(evolved.min_values.get_field(4), 20)
+
+ # max: cc->300, a->100, d->400, e->None, bb->200
+ self.assertEqual(evolved.max_values.get_field(0), 300)
+ self.assertEqual(evolved.max_values.get_field(1), 100)
+ self.assertEqual(evolved.max_values.get_field(2), 400)
+ self.assertIsNone(evolved.max_values.get_field(3))
+ self.assertEqual(evolved.max_values.get_field(4), 200)
+
+ # null counts: cc->3, a->1, d->4, e->1000(row_count), bb->2
+ self.assertEqual(evolved.null_counts, [3, 1, 4, 1000, 2])
+
+
+if __name__ == '__main__':
+ unittest.main()
(paimon) 04/25: [python] Fix token cache isolation by adding user identity to cache key (#7571)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 500c9c921939ac8f6c5e9fca6494939ab0e1fd0e
Author: shyjsarah <44...@users.noreply.github.com>
AuthorDate: Wed Apr 1 01:44:22 2026 -0700
[python] Fix token cache isolation by adding user identity to cache key (#7571)
Revert #7562 and fix token cache pollution with a minimal approach.
**Problem:**
In #7562, the class-level `_TOKEN_CACHE` was completely removed to fix
token pollution across different catalog instances. However, this also
removed the ability to reuse tokens for the same user on the same table,
causing unnecessary token refresh requests.
The root cause of the original pollution was that `_TOKEN_CACHE` was
keyed only by table identifier (`str(self.identifier)`), so different
users (different AK/SK credentials) operating on the same table within
one process would share the same cached data token.
**Fix:**
Instead of removing the cache entirely, this PR adds **user identity**
to the cache key to isolate tokens across different credentials while
preserving token reuse for the same user.
---
.../pypaimon/catalog/rest/rest_token_file_io.py | 76 +++++-
.../pypaimon/tests/rest/rest_token_file_io_test.py | 254 ++++++++++++++-------
2 files changed, 233 insertions(+), 97 deletions(-)
diff --git a/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py b/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py
index 16b16e6972..7d4553175b 100644
--- a/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py
+++ b/paimon-python/pypaimon/catalog/rest/rest_token_file_io.py
@@ -27,6 +27,8 @@ from pypaimon.api.rest_util import RESTUtil
from pypaimon.catalog.rest.rest_token import RESTToken
from pypaimon.common.file_io import FileIO
from pypaimon.filesystem.pyarrow_file_io import PyArrowFileIO
+from pypaimon.api.auth.bearer import BearTokenAuthProvider
+from pypaimon.api.auth.dlf_provider import DLFAuthProvider
from pypaimon.common.identifier import Identifier, SYSTEM_TABLE_SPLITTER
from pypaimon.common.options import Options
from pypaimon.common.options.config import CatalogOptions, OssOptions
@@ -44,6 +46,10 @@ class RESTTokenFileIO(FileIO):
_FILE_IO_CACHE: TTLCache = None
_FILE_IO_CACHE_LOCK = threading.Lock()
+ _TOKEN_CACHE: dict = {}
+ _TOKEN_LOCKS: dict = {}
+ _TOKEN_LOCKS_LOCK = threading.Lock()
+
@classmethod
def _get_file_io_cache(cls) -> TTLCache:
if cls._FILE_IO_CACHE is None:
@@ -69,14 +75,12 @@ class RESTTokenFileIO(FileIO):
self.properties = self.catalog_options or Options({}) # For compatibility with refresh_token()
self.token: Optional[RESTToken] = None
self.api_instance: Optional[RESTApi] = None
- self.lock = threading.Lock()
self.log = logging.getLogger(__name__)
self._uri_reader_factory_cache: Optional[UriReaderFactory] = None
def __getstate__(self):
state = self.__dict__.copy()
# Remove non-serializable objects
- state.pop('lock', None)
state.pop('api_instance', None)
state.pop('_uri_reader_factory_cache', None)
# token can be serialized, but we'll refresh it on deserialization
@@ -84,8 +88,6 @@ class RESTTokenFileIO(FileIO):
def __setstate__(self, state):
self.__dict__.update(state)
- # Recreate lock after deserialization
- self.lock = threading.Lock()
self._uri_reader_factory_cache = None
# api_instance will be recreated when needed
self.api_instance = None
@@ -201,17 +203,69 @@ class RESTTokenFileIO(FileIO):
def filesystem(self):
return self.file_io().filesystem
+ def _build_cache_key(self) -> str:
+ """Build cache key with user identity to isolate tokens across different credentials."""
+ return f"{self._get_user_identity()}:{self.identifier}"
+
+ def _get_user_identity(self) -> str:
+ """Get user identity for token cache isolation.
+
+ For DLF auth (AK/SK, STS, ECS Role): returns access_key_id via get_token().
+ For Bear Token auth: returns the bearer token string.
+ For unknown auth: returns 'anonymous'.
+ """
+ if self.api_instance is None:
+ self.api_instance = RESTApi(self.properties, False)
+ auth_provider = self.api_instance.rest_auth_function.auth_provider
+ if isinstance(auth_provider, DLFAuthProvider):
+ return auth_provider.get_token().access_key_id or 'anonymous'
+ if isinstance(auth_provider, BearTokenAuthProvider):
+ return auth_provider.token or 'anonymous'
+ return 'anonymous'
+
def try_to_refresh_token(self):
- if self._should_refresh():
- with self.lock:
- if self._should_refresh():
- self.refresh_token()
+ identifier_str = self._build_cache_key()
- def _should_refresh(self):
- if self.token is None:
+ if self.token is not None and not self._is_token_expired(self.token):
+ return
+
+ cached_token = self._get_cached_token(identifier_str)
+ if cached_token and not self._is_token_expired(cached_token):
+ self.token = cached_token
+ return
+
+ global_lock = self._get_global_token_lock(identifier_str)
+
+ with global_lock:
+ cached_token = self._get_cached_token(identifier_str)
+ if cached_token and not self._is_token_expired(cached_token):
+ self.token = cached_token
+ return
+
+ token_to_check = cached_token if cached_token else self.token
+ if token_to_check is None or self._is_token_expired(token_to_check):
+ self.refresh_token()
+ self._set_cached_token(identifier_str, self.token)
+
+ def _get_cached_token(self, identifier_str: str) -> Optional[RESTToken]:
+ with self._TOKEN_LOCKS_LOCK:
+ return self._TOKEN_CACHE.get(identifier_str)
+
+ def _set_cached_token(self, identifier_str: str, token: RESTToken):
+ with self._TOKEN_LOCKS_LOCK:
+ self._TOKEN_CACHE[identifier_str] = token
+
+ def _is_token_expired(self, token: Optional[RESTToken]) -> bool:
+ if token is None:
return True
current_time = int(time.time() * 1000)
- return (self.token.expire_at_millis - current_time) < RESTApi.TOKEN_EXPIRATION_SAFE_TIME_MILLIS
+ return (token.expire_at_millis - current_time) < RESTApi.TOKEN_EXPIRATION_SAFE_TIME_MILLIS
+
+ def _get_global_token_lock(self, identifier_str: str) -> threading.Lock:
+ with self._TOKEN_LOCKS_LOCK:
+ if identifier_str not in self._TOKEN_LOCKS:
+ self._TOKEN_LOCKS[identifier_str] = threading.Lock()
+ return self._TOKEN_LOCKS[identifier_str]
def refresh_token(self):
self.log.info(f"begin refresh data token for identifier [{self.identifier}]")
diff --git a/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py b/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py
index cdcd5ed36c..b94ad58b24 100644
--- a/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py
+++ b/paimon-python/pypaimon/tests/rest/rest_token_file_io_test.py
@@ -148,9 +148,6 @@ class RESTTokenFileIOTest(unittest.TestCase):
self.catalog_options
)
- self.assertTrue(hasattr(original_file_io, 'lock'))
- self.assertIsNotNone(original_file_io.lock)
-
pickled = pickle.dumps(original_file_io)
deserialized_file_io = pickle.loads(pickled)
@@ -159,10 +156,6 @@ class RESTTokenFileIOTest(unittest.TestCase):
self.assertEqual(deserialized_file_io.path, original_file_io.path)
self.assertEqual(deserialized_file_io.properties.data, original_file_io.properties.data)
- self.assertTrue(hasattr(deserialized_file_io, 'lock'))
- self.assertIsNotNone(deserialized_file_io.lock)
- self.assertIsNot(deserialized_file_io.lock, original_file_io.lock)
-
self.assertIsNone(deserialized_file_io.api_instance)
test_file_path = f"file://{self.temp_dir}/pickle_test.txt"
@@ -306,56 +299,183 @@ class RESTTokenFileIOTest(unittest.TestCase):
self.assertIsNotNone(restored_file_io.uri_reader_factory,
"uri_reader_factory should work after deserialization")
- def test_should_refresh_when_token_is_none(self):
- """_should_refresh() returns True when token is None."""
- with patch.object(RESTTokenFileIO, 'try_to_refresh_token'):
- file_io = RESTTokenFileIO(
- self.identifier, self.warehouse_path, self.catalog_options)
- self.assertIsNone(file_io.token)
- self.assertTrue(file_io._should_refresh())
+ def test_build_cache_key_with_dlf_auth(self):
+ """Test that _build_cache_key includes DLF access_key_id for cache isolation."""
+ from pypaimon.api.auth.dlf_provider import DLFAuthProvider
+ from pypaimon.api.token_loader import DLFToken
+
+ catalog_options = Options({
+ CatalogOptions.URI.key(): "http://test-uri",
+ CatalogOptions.TOKEN_PROVIDER.key(): "dlf",
+ CatalogOptions.DLF_REGION.key(): "cn-hangzhou",
+ CatalogOptions.DLF_ACCESS_KEY_ID.key(): "test-ak-123",
+ CatalogOptions.DLF_ACCESS_KEY_SECRET.key(): "test-sk-456",
+ })
- def test_should_refresh_when_token_not_expired(self):
- """_should_refresh() returns False when token is far from expiry."""
with patch.object(RESTTokenFileIO, 'try_to_refresh_token'):
file_io = RESTTokenFileIO(
- self.identifier, self.warehouse_path, self.catalog_options)
- # Token that expires 2 hours from now (well beyond the 1-hour safe margin)
- future_millis = int(time.time() * 1000) + 7200_000
- file_io.token = RESTToken({'ak': 'v'}, future_millis)
- self.assertFalse(file_io._should_refresh())
-
- def test_should_refresh_when_token_expired(self):
- """_should_refresh() returns True when token is already expired."""
+ self.identifier,
+ self.warehouse_path,
+ catalog_options
+ )
+
+ mock_api = MagicMock()
+ mock_dlf_provider = MagicMock(spec=DLFAuthProvider)
+ mock_dlf_provider.get_token.return_value = DLFToken(
+ access_key_id="test-ak-123",
+ access_key_secret="test-sk-456",
+ security_token=None
+ )
+ mock_api.rest_auth_function.auth_provider = mock_dlf_provider
+ file_io.api_instance = mock_api
+
+ cache_key = file_io._build_cache_key()
+ self.assertTrue(cache_key.startswith("test-ak-123:"),
+ f"Cache key should start with access_key_id, got: {cache_key}")
+ self.assertIn(str(self.identifier), cache_key)
+
+ def test_build_cache_key_with_bear_auth(self):
+ """Test that _build_cache_key includes bearer token for cache isolation."""
+ from pypaimon.api.auth.bearer import BearTokenAuthProvider
+
+ catalog_options = Options({
+ CatalogOptions.URI.key(): "http://test-uri",
+ CatalogOptions.TOKEN_PROVIDER.key(): "bear",
+ CatalogOptions.TOKEN.key(): "my-bearer-token",
+ })
+
with patch.object(RESTTokenFileIO, 'try_to_refresh_token'):
file_io = RESTTokenFileIO(
- self.identifier, self.warehouse_path, self.catalog_options)
- # Token that expired 1 second ago
- past_millis = int(time.time() * 1000) - 1000
- file_io.token = RESTToken({'ak': 'v'}, past_millis)
- self.assertTrue(file_io._should_refresh())
-
- def test_try_to_refresh_token_calls_refresh_once(self):
- """try_to_refresh_token() calls refresh_token() exactly once via double-check."""
- file_io = RESTTokenFileIO(
- self.identifier, self.warehouse_path, self.catalog_options)
- self.assertIsNone(file_io.token)
-
- mock_response = MagicMock()
- mock_response.token = {'ak': 'test-ak'}
- mock_response.expires_at_millis = int(time.time() * 1000) + 7200_000
+ self.identifier,
+ self.warehouse_path,
+ catalog_options
+ )
mock_api = MagicMock()
- mock_api.load_table_token.return_value = mock_response
+ mock_bear_provider = MagicMock(spec=BearTokenAuthProvider)
+ mock_bear_provider.token = "my-bearer-token"
+ mock_api.rest_auth_function.auth_provider = mock_bear_provider
file_io.api_instance = mock_api
- file_io.try_to_refresh_token()
+ cache_key = file_io._build_cache_key()
+ self.assertTrue(cache_key.startswith("my-bearer-token:"),
+ f"Cache key should start with bearer token, got: {cache_key}")
+ self.assertIn(str(self.identifier), cache_key)
+
+ def test_different_ak_same_table_token_isolation(self):
+ """Test that two RESTTokenFileIO instances with different AKs on the same table
+ do not share cached tokens."""
+ from pypaimon.api.auth.dlf_provider import DLFAuthProvider
+ from pypaimon.api.token_loader import DLFToken
+
+ # Clear class-level token cache before test
+ RESTTokenFileIO._TOKEN_CACHE.clear()
+ RESTTokenFileIO._TOKEN_LOCKS.clear()
- mock_api.load_table_token.assert_called_once()
- self.assertIsNotNone(file_io.token)
+ identifier = Identifier.from_string("db.same_table")
+ future_expiry = int(time.time() * 1000) + 7_200_000 # 2 hours from now
- # Second call should NOT trigger refresh again (token is valid)
- file_io.try_to_refresh_token()
- mock_api.load_table_token.assert_called_once()
+ # Create file_io_1 with AK "read-ak"
+ with patch.object(RESTTokenFileIO, 'try_to_refresh_token'):
+ file_io_1 = RESTTokenFileIO(
+ identifier, self.warehouse_path,
+ Options({
+ CatalogOptions.URI.key(): "http://test-uri",
+ CatalogOptions.TOKEN_PROVIDER.key(): "dlf",
+ CatalogOptions.DLF_REGION.key(): "cn-hangzhou",
+ CatalogOptions.DLF_ACCESS_KEY_ID.key(): "read-ak",
+ CatalogOptions.DLF_ACCESS_KEY_SECRET.key(): "read-sk",
+ })
+ )
+
+ mock_api_1 = MagicMock()
+ mock_dlf_1 = MagicMock(spec=DLFAuthProvider)
+ mock_dlf_1.get_token.return_value = DLFToken("read-ak", "read-sk", None)
+ mock_api_1.rest_auth_function.auth_provider = mock_dlf_1
+ file_io_1.api_instance = mock_api_1
+
+ read_token = RESTToken({"fs.oss.accessKeyId": "read-data-ak"}, future_expiry)
+ file_io_1.token = read_token
+ cache_key_1 = file_io_1._build_cache_key()
+ file_io_1._set_cached_token(cache_key_1, read_token)
+
+ # Create file_io_2 with AK "write-ak"
+ with patch.object(RESTTokenFileIO, 'try_to_refresh_token'):
+ file_io_2 = RESTTokenFileIO(
+ identifier, self.warehouse_path,
+ Options({
+ CatalogOptions.URI.key(): "http://test-uri",
+ CatalogOptions.TOKEN_PROVIDER.key(): "dlf",
+ CatalogOptions.DLF_REGION.key(): "cn-hangzhou",
+ CatalogOptions.DLF_ACCESS_KEY_ID.key(): "write-ak",
+ CatalogOptions.DLF_ACCESS_KEY_SECRET.key(): "write-sk",
+ })
+ )
+
+ mock_api_2 = MagicMock()
+ mock_dlf_2 = MagicMock(spec=DLFAuthProvider)
+ mock_dlf_2.get_token.return_value = DLFToken("write-ak", "write-sk", None)
+ mock_api_2.rest_auth_function.auth_provider = mock_dlf_2
+ file_io_2.api_instance = mock_api_2
+
+ cache_key_2 = file_io_2._build_cache_key()
+
+ # Cache keys should be different
+ self.assertNotEqual(cache_key_1, cache_key_2,
+ "Different AKs on the same table should produce different cache keys")
+
+ # file_io_2 should NOT get file_io_1's cached token
+ cached_for_2 = file_io_2._get_cached_token(cache_key_2)
+ self.assertIsNone(cached_for_2,
+ "file_io_2 should not see file_io_1's cached token")
+
+ def test_same_ak_same_table_token_reuse(self):
+ """Test that two RESTTokenFileIO instances with the same AK on the same table
+ can share cached tokens."""
+ from pypaimon.api.auth.dlf_provider import DLFAuthProvider
+ from pypaimon.api.token_loader import DLFToken
+
+ # Clear class-level token cache before test
+ RESTTokenFileIO._TOKEN_CACHE.clear()
+ RESTTokenFileIO._TOKEN_LOCKS.clear()
+
+ identifier = Identifier.from_string("db.same_table")
+ future_expiry = int(time.time() * 1000) + 7_200_000
+
+ def make_file_io():
+ with patch.object(RESTTokenFileIO, 'try_to_refresh_token'):
+ fio = RESTTokenFileIO(
+ identifier, self.warehouse_path,
+ Options({
+ CatalogOptions.URI.key(): "http://test-uri",
+ CatalogOptions.TOKEN_PROVIDER.key(): "dlf",
+ CatalogOptions.DLF_REGION.key(): "cn-hangzhou",
+ CatalogOptions.DLF_ACCESS_KEY_ID.key(): "same-ak",
+ CatalogOptions.DLF_ACCESS_KEY_SECRET.key(): "same-sk",
+ })
+ )
+ mock_api = MagicMock()
+ mock_dlf = MagicMock(spec=DLFAuthProvider)
+ mock_dlf.get_token.return_value = DLFToken("same-ak", "same-sk", None)
+ mock_api.rest_auth_function.auth_provider = mock_dlf
+ fio.api_instance = mock_api
+ return fio
+
+ file_io_1 = make_file_io()
+ file_io_2 = make_file_io()
+
+ # Cache keys should be the same
+ self.assertEqual(file_io_1._build_cache_key(), file_io_2._build_cache_key(),
+ "Same AK on the same table should produce the same cache key")
+
+ # Set token via file_io_1
+ shared_token = RESTToken({"fs.oss.accessKeyId": "shared-data-ak"}, future_expiry)
+ file_io_1._set_cached_token(file_io_1._build_cache_key(), shared_token)
+
+ # file_io_2 should see the same token
+ cached = file_io_2._get_cached_token(file_io_2._build_cache_key())
+ self.assertIsNotNone(cached, "file_io_2 should see file_io_1's cached token")
+ self.assertEqual(cached.token.get("fs.oss.accessKeyId"), "shared-data-ak")
def test_refresh_token_strips_system_table_suffix(self):
"""refresh_token() strips $snapshots suffix before requesting token."""
@@ -365,7 +485,7 @@ class RESTTokenFileIOTest(unittest.TestCase):
mock_response = MagicMock()
mock_response.token = {'ak': 'test-ak'}
- mock_response.expires_at_millis = int(time.time() * 1000) + 7200_000
+ mock_response.expires_at_millis = int(time.time() * 1000) + 7_200_000
mock_api = MagicMock()
mock_api.load_table_token.return_value = mock_response
@@ -373,7 +493,6 @@ class RESTTokenFileIOTest(unittest.TestCase):
file_io.refresh_token()
- # Verify load_table_token was called with base table identifier (no $snapshots)
called_identifier = mock_api.load_table_token.call_args[0][0]
self.assertEqual(called_identifier.get_database_name(), "db")
self.assertEqual(called_identifier.get_object_name(), "my_table")
@@ -386,7 +505,7 @@ class RESTTokenFileIOTest(unittest.TestCase):
mock_response = MagicMock()
mock_response.token = {'ak': 'test-ak'}
- mock_response.expires_at_millis = int(time.time() * 1000) + 7200_000
+ mock_response.expires_at_millis = int(time.time() * 1000) + 7_200_000
mock_api = MagicMock()
mock_api.load_table_token.return_value = mock_response
@@ -397,43 +516,6 @@ class RESTTokenFileIOTest(unittest.TestCase):
called_identifier = mock_api.load_table_token.call_args[0][0]
self.assertEqual(called_identifier.get_object_name(), "my_table")
- def test_different_instances_do_not_share_token(self):
- """Two instances with same identifier get independent tokens (no class-level cache)."""
- same_identifier = Identifier.from_string("db.shared_table")
-
- file_io_a = RESTTokenFileIO(
- same_identifier, self.warehouse_path, self.catalog_options)
- file_io_b = RESTTokenFileIO(
- same_identifier, self.warehouse_path, self.catalog_options)
-
- token_a = RESTToken({'ak': 'ak-A'}, int(time.time() * 1000) + 7200_000)
- token_b = RESTToken({'ak': 'ak-B'}, int(time.time() * 1000) + 7200_000)
-
- mock_response_a = MagicMock()
- mock_response_a.token = token_a.token
- mock_response_a.expires_at_millis = token_a.expire_at_millis
-
- mock_response_b = MagicMock()
- mock_response_b.token = token_b.token
- mock_response_b.expires_at_millis = token_b.expire_at_millis
-
- mock_api_a = MagicMock()
- mock_api_a.load_table_token.return_value = mock_response_a
- file_io_a.api_instance = mock_api_a
-
- mock_api_b = MagicMock()
- mock_api_b.load_table_token.return_value = mock_response_b
- file_io_b.api_instance = mock_api_b
-
- # Refresh both
- file_io_a.try_to_refresh_token()
- file_io_b.try_to_refresh_token()
-
- # Each instance should hold its own token
- self.assertEqual(file_io_a.token.token['ak'], 'ak-A')
- self.assertEqual(file_io_b.token.token['ak'], 'ak-B')
- self.assertIsNot(file_io_a.token, file_io_b.token)
-
if __name__ == '__main__':
unittest.main()
(paimon) 22/25: [core] When scan.primary-branch is set, allow primary branch to be an append only table (#7614)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit a352fcfc2cba7de0c3404a459a392412eed0e698
Author: tsreaper <ts...@gmail.com>
AuthorDate: Fri Apr 10 13:06:47 2026 +0800
[core] When scan.primary-branch is set, allow primary branch to be an append only table (#7614)
In `scan.fallback-branch`, we allow main branch to be append only, and
fallback branch to have primary key. So in `scan.primary-branch`, we
should allow main branch to have primary key, and primary branch to be
append only.
---
.../paimon/table/FallbackReadFileStoreTable.java | 24 -------------
.../org/apache/paimon/flink/BranchSqlITCase.java | 42 ++++++++++++++++++++++
2 files changed, 42 insertions(+), 24 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java
index 5c839a5581..1d1c6214ce 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/FallbackReadFileStoreTable.java
@@ -224,30 +224,6 @@ public class FallbackReadFileStoreTable extends DelegatedFileStoreTable {
mainRowType,
otherBranch,
otherRowType);
-
- List<String> mainPrimaryKeys = wrapped.schema().primaryKeys();
- List<String> otherPrimaryKeys = other.schema().primaryKeys();
- if (!mainPrimaryKeys.isEmpty()) {
- if (otherPrimaryKeys.isEmpty()) {
- throw new IllegalArgumentException(
- "Branch "
- + mainBranch
- + " has primary keys while branch "
- + otherBranch
- + " does not. This is not allowed.");
- }
- Preconditions.checkArgument(
- mainPrimaryKeys.equals(otherPrimaryKeys),
- "Branch %s and %s both have primary keys but are not the same.\n"
- + "Primary keys of %s are %s.\n"
- + "Primary keys of %s are %s.",
- mainBranch,
- otherBranch,
- mainBranch,
- mainPrimaryKeys,
- otherBranch,
- otherPrimaryKeys);
- }
}
private boolean sameRowTypeIgnoreNullable(RowType mainRowType, RowType otherRowType) {
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java
index 641efb7335..7ecfbf77b2 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java
@@ -923,6 +923,48 @@ public class BranchSqlITCase extends CatalogITCaseBase {
.hasMessageContaining("Branch");
}
+ @Test
+ public void testPrimaryBranchBatchRead() throws Exception {
+ // Create non-PK table, then create branch, then ALTER main to add PKs.
+ // This results in main = PK table, branch = non-PK table.
+ sql(
+ "CREATE TABLE t ( pt INT NOT NULL, k INT NOT NULL, v STRING ) "
+ + "PARTITIONED BY (pt) WITH ( 'bucket' = '-1' )");
+
+ sql("CALL sys.create_branch('default.t', 'nb')");
+ sql("ALTER TABLE t SET ( 'primary-key' = 'pt, k', 'bucket' = '2' )");
+ sql("ALTER TABLE t SET ( 'scan.primary-branch' = 'nb' )");
+
+ // Insert into non-PK branch (primary, has priority)
+ sql("INSERT INTO `t$branch_nb` VALUES (1, 20, 'cat'), (1, 30, 'dog')");
+ // Insert overlapping partition into PK main (fallback)
+ sql("INSERT INTO t VALUES (1, 10, 'apple'), (1, 20, 'banana')");
+
+ // pt=1 exists in primary branch → read from branch
+ assertThat(collectResult("SELECT v, k FROM t"))
+ .containsExactlyInAnyOrder("+I[cat, 20]", "+I[dog, 30]");
+ assertThat(collectResult("SELECT v, k FROM `t$branch_nb`"))
+ .containsExactlyInAnyOrder("+I[cat, 20]", "+I[dog, 30]");
+
+ // Insert pt=2 into primary branch, pt=3 only into main
+ sql("INSERT INTO `t$branch_nb` VALUES (2, 10, 'tiger'), (2, 20, 'wolf')");
+ sql("INSERT INTO t VALUES (3, 10, 'horse')");
+
+ // pt=1,2 from primary branch; pt=3 from main (fallback)
+ assertThat(collectResult("SELECT v, k FROM t"))
+ .containsExactlyInAnyOrder(
+ "+I[cat, 20]",
+ "+I[dog, 30]",
+ "+I[tiger, 10]",
+ "+I[wolf, 20]",
+ "+I[horse, 10]");
+
+ // Unset scan.primary-branch, main table should show its own data
+ sql("ALTER TABLE t RESET ( 'scan.primary-branch' )");
+ assertThat(collectResult("SELECT v, k FROM t"))
+ .containsExactlyInAnyOrder("+I[apple, 10]", "+I[banana, 20]", "+I[horse, 10]");
+ }
+
private List<String> collectResult(String sql) throws Exception {
List<String> result = new ArrayList<>();
try (CloseableIterator<Row> it = tEnv.executeSql(sql).collect()) {
(paimon) 20/25: [core] Add hashCode to AbstractFileStoreTable for consistency with equals (#7616)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 3579ba16634a3c7a0b127ccbd96443db8952eb43
Author: Weitai Li <l8...@gmail.com>
AuthorDate: Thu Apr 9 22:23:54 2026 +0800
[core] Add hashCode to AbstractFileStoreTable for consistency with equals (#7616)
---
.../paimon/table/AbstractFileStoreTable.java | 5 ++++
.../paimon/table/AppendOnlySimpleTableTest.java | 27 ++++++++++++++++++++++
.../paimon/table/PrimaryKeySimpleTableTest.java | 27 ++++++++++++++++++++++
3 files changed, 59 insertions(+)
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
index f8a6b1722f..522d25c7d3 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java
@@ -796,4 +796,9 @@ abstract class AbstractFileStoreTable implements FileStoreTable {
AbstractFileStoreTable that = (AbstractFileStoreTable) o;
return Objects.equals(path, that.path) && Objects.equals(tableSchema, that.tableSchema);
}
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(path, tableSchema);
+ }
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlySimpleTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlySimpleTableTest.java
index ed6f8349e0..30ccc652a0 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlySimpleTableTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlySimpleTableTest.java
@@ -1565,6 +1565,33 @@ public class AppendOnlySimpleTableTest extends SimpleTableTestBase {
}
}
+ @Test
+ public void testEqualsAndHashCode() throws Exception {
+ // Test same table equals and hashCode consistency
+ FileStoreTable table1 = createFileStoreTable();
+ FileStoreTable table2 = table1.copy(table1.schema());
+ assertThat(table1.equals(table2)).isTrue();
+ assertThat(table1.hashCode()).isEqualTo(table2.hashCode());
+
+ // Test with different options
+ Map<String, String> optionsWithMock = new HashMap<>(table1.schema().options());
+ optionsWithMock.put("mockKey", "mockValue");
+ TableSchema schemaWithMock = table1.schema().copy(optionsWithMock);
+ FileStoreTable tableWithMock = table1.copy(schemaWithMock);
+
+ assertThat(table1.equals(tableWithMock)).isFalse();
+ assertThat(table1.hashCode()).isNotEqualTo(tableWithMock.hashCode());
+
+ // Test same options should be equal
+ Map<String, String> sameOptionsWithMock = new HashMap<>(table1.schema().options());
+ sameOptionsWithMock.put("mockKey", "mockValue");
+ TableSchema sameSchemaWithMock = table1.schema().copy(sameOptionsWithMock);
+ FileStoreTable sameTableWithMock = table1.copy(sameSchemaWithMock);
+
+ assertThat(tableWithMock.equals(sameTableWithMock)).isTrue();
+ assertThat(tableWithMock.hashCode()).isEqualTo(sameTableWithMock.hashCode());
+ }
+
private void writeData() throws Exception {
writeData(options -> {});
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeySimpleTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeySimpleTableTest.java
index d920cd0a07..9401cce832 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeySimpleTableTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeySimpleTableTest.java
@@ -2473,6 +2473,33 @@ public class PrimaryKeySimpleTableTest extends SimpleTableTestBase {
"1|10|100|binary|varbinary|mapKey:mapVal|multiset"));
}
+ @Test
+ public void testEqualsAndHashCode() throws Exception {
+ // Test same table equals and hashCode consistency
+ FileStoreTable table1 = createFileStoreTable();
+ FileStoreTable table2 = table1.copy(table1.schema());
+ assertThat(table1.equals(table2)).isTrue();
+ assertThat(table1.hashCode()).isEqualTo(table2.hashCode());
+
+ // Test with different options
+ Map<String, String> optionsWithMock = new HashMap<>(table1.schema().options());
+ optionsWithMock.put("mockKey", "mockValue");
+ TableSchema schemaWithMock = table1.schema().copy(optionsWithMock);
+ FileStoreTable tableWithMock = table1.copy(schemaWithMock);
+
+ assertThat(table1.equals(tableWithMock)).isFalse();
+ assertThat(table1.hashCode()).isNotEqualTo(tableWithMock.hashCode());
+
+ // Test same options should be equal
+ Map<String, String> sameOptionsWithMock = new HashMap<>(table1.schema().options());
+ sameOptionsWithMock.put("mockKey", "mockValue");
+ TableSchema sameSchemaWithMock = table1.schema().copy(sameOptionsWithMock);
+ FileStoreTable sameTableWithMock = table1.copy(sameSchemaWithMock);
+
+ assertThat(tableWithMock.equals(sameTableWithMock)).isTrue();
+ assertThat(tableWithMock.hashCode()).isEqualTo(sameTableWithMock.hashCode());
+ }
+
private void assertReadChangelog(int id, FileStoreTable table) throws Exception {
// read the changelog at #{id}
table = table.copy(singletonMap(CoreOptions.SCAN_SNAPSHOT_ID.key(), String.valueOf(id)));
(paimon) 19/25: [spark] Fix RowTrackingTable reading by add _ROW_ID existing check before adding _ROW_ID (#7606)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch release-1.4
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 14b0816b6cea7b2319018f30ceec2ef3d1cbdf75
Author: littlecoder04 <xi...@gmail.com>
AuthorDate: Thu Apr 9 20:49:39 2026 +0800
[spark] Fix RowTrackingTable reading by add _ROW_ID existing check before adding _ROW_ID (#7606)
---
.../org/apache/paimon/spark/PaimonBaseScanBuilder.scala | 5 ++++-
.../scala/org/apache/paimon/spark/read/BaseScan.scala | 5 ++++-
.../apache/paimon/spark/sql/RowTrackingTestBase.scala | 17 +++++++++++++++++
3 files changed, 25 insertions(+), 2 deletions(-)
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScanBuilder.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScanBuilder.scala
index 14a090ff14..c249169ae7 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScanBuilder.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonBaseScanBuilder.scala
@@ -69,7 +69,10 @@ abstract class PaimonBaseScanBuilder
val postScan = mutable.ArrayBuffer.empty[SparkPredicate]
var newRowType = rowType
- if (coreOptions.rowTrackingEnabled() && coreOptions.dataEvolutionEnabled()) {
+ if (
+ coreOptions.rowTrackingEnabled() && coreOptions
+ .dataEvolutionEnabled() && !rowType.containsField(SpecialFields.ROW_ID.name())
+ ) {
newRowType = SpecialFields.rowTypeWithRowTracking(newRowType);
}
val converter = SparkV2FilterConverter(newRowType)
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/BaseScan.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/BaseScan.scala
index 42f724d3e6..3237475d69 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/BaseScan.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/BaseScan.scala
@@ -68,7 +68,10 @@ trait BaseScan extends Scan with SupportsReportStatistics with Logging {
val coreOptions: CoreOptions = CoreOptions.fromMap(table.options())
lazy val tableRowType: RowType = {
- if (coreOptions.rowTrackingEnabled()) {
+ if (
+ coreOptions
+ .rowTrackingEnabled() && !table.rowType().containsField(SpecialFields.ROW_ID.name())
+ ) {
SpecialFields.rowTypeWithRowTracking(table.rowType())
} else {
table.rowType()
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/RowTrackingTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/RowTrackingTestBase.scala
index 0ea9d21bcf..e012656cc8 100644
--- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/RowTrackingTestBase.scala
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/RowTrackingTestBase.scala
@@ -795,6 +795,22 @@ abstract class RowTrackingTestBase extends PaimonSparkTestBase {
}
}
+ test("Row Tracking: query row_tracking system table with filter pushdown") {
+ withTable("t") {
+ sql("CREATE TABLE t (a INT, b INT) TBLPROPERTIES ('row-tracking.enabled' = 'true')")
+ sql("INSERT INTO t VALUES (1, 10), (2, 20), (3, 30)")
+
+ val query = s"SELECT a, b FROM `t$$row_tracking` WHERE a > 1 ORDER BY a"
+ checkAnswer(sql(query), Seq(Row(2, 20), Row(3, 30)))
+
+ val scan = getScan(query)
+ assert(
+ scan.description().contains("DataFilters"),
+ s"Expected predicate pushdown (DataFilters) in scan description, but got: ${scan.description()}"
+ )
+ }
+ }
+
test("Data Evolution: compact fields action") {
withTable("s", "t") {
sql("CREATE TABLE s (id INT, b INT)")
@@ -952,4 +968,5 @@ abstract class RowTrackingTestBase extends PaimonSparkTestBase {
assert(!indexEntries.exists(entry => entry.partition().getString(0).toString.equals("p1")))
}
}
+
}