You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2026/05/03 00:02:17 UTC
(paimon) branch master updated: [python] Fix FileSystemBranchManager from-tag and fast-forward path computation (#7756)
This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 718f292c95 [python] Fix FileSystemBranchManager from-tag and fast-forward path computation (#7756)
718f292c95 is described below
commit 718f292c95c43a9bf8cf022c0ab50e301bd9a4a4
Author: chaoyang <ch...@apache.org>
AuthorDate: Sun May 3 08:02:12 2026 +0800
[python] Fix FileSystemBranchManager from-tag and fast-forward path computation (#7756)
Fix a pre-existing bug in `FileSystemBranchManager._copy_with_branch`
that made `create_branch(tag_name=...)` and `fast_forward` raise
`SameFileError`: the `SnapshotManager` it produced still pointed at the
main-branch snapshot directory, so `copy_file(src, dst)` collapsed to
`src == dst`.
The fix mirrors Java `SnapshotManager.copyWithBranch`
(paimon-core/.../utils/SnapshotManager.java:89-95): the Python
`SnapshotManager` now carries an explicit `branch` field and a
`copy_with_branch(branch_name)` factory, so its `snapshot_dir` /
`get_snapshot_path(...)` resolve to
`{table_path}/branch/branch-{name}/snapshot/...` for non-main branches.
`FileSystemBranchManager._copy_with_branch` then dispatches to the
per-manager factories (`SnapshotManager.copy_with_branch` /
`SchemaManager.copy_with_branch`) instead of blindly reconstructing a
main-branch `SnapshotManager`.
`SnapshotLoader` is rebranched in lockstep so REST-path catalog loads
target the requested branch rather than falling back to the main-branch
identifier (mirrors Java `SnapshotLoaderImpl.copyWithBranch`).
---
.../pypaimon/branch/filesystem_branch_manager.py | 21 ++-
paimon-python/pypaimon/snapshot/snapshot_loader.py | 14 ++
.../pypaimon/snapshot/snapshot_manager.py | 26 +++-
.../pypaimon/tests/branch_manager_test.py | 141 +++++++++++++++++++++
.../pypaimon/tests/snapshot_manager_test.py | 3 +
5 files changed, 191 insertions(+), 14 deletions(-)
diff --git a/paimon-python/pypaimon/branch/filesystem_branch_manager.py b/paimon-python/pypaimon/branch/filesystem_branch_manager.py
index ca8111b62f..45fe6e4c43 100644
--- a/paimon-python/pypaimon/branch/filesystem_branch_manager.py
+++ b/paimon-python/pypaimon/branch/filesystem_branch_manager.py
@@ -83,22 +83,21 @@ class FileSystemBranchManager(BranchManager):
def _copy_with_branch(self, manager, branch: str):
"""
- Create a copy of the manager for a different branch.
-
- Args:
- manager: The manager to copy (TagManager, SnapshotManager, etc.)
- branch: The target branch name
-
- Returns:
- A new manager instance for the specified branch
+ Return a new manager whose path accessors land under
+ ``{table_path}/branch/branch-{name}``.
+
+ Equivalent to the per-manager rebranch calls inlined in Java
+ ``FileSystemBranchManager.createBranch(name, tagName)`` (e.g.
+ ``snapshotManager.copyWithBranch(branchName)``). ``TagManager``
+ is rebuilt directly because the Python ``TagManager`` has no
+ ``copy_with_branch`` factory yet.
"""
if isinstance(manager, TagManager):
return TagManager(self.file_io, self.table_path, branch)
elif isinstance(manager, SchemaManager):
- # SchemaManager doesn't support branch parameter, use branch path instead
- return SchemaManager(self.file_io, self.branch_path(branch))
+ return manager.copy_with_branch(branch)
elif isinstance(manager, SnapshotManager):
- return SnapshotManager(self.snapshot_manager.table)
+ return manager.copy_with_branch(branch)
else:
return manager
diff --git a/paimon-python/pypaimon/snapshot/snapshot_loader.py b/paimon-python/pypaimon/snapshot/snapshot_loader.py
index aba0a66c58..37f8fa7a91 100644
--- a/paimon-python/pypaimon/snapshot/snapshot_loader.py
+++ b/paimon-python/pypaimon/snapshot/snapshot_loader.py
@@ -36,6 +36,20 @@ class SnapshotLoader:
self.catalog_loader = catalog_loader
self.identifier = identifier
+ def copy_with_branch(self, branch: str) -> 'SnapshotLoader':
+ """Return a new loader whose identifier carries ``branch``.
+
+ Mirrors Java ``SnapshotLoaderImpl.copyWithBranch``
+ (tag/SnapshotLoaderImpl.java).
+ """
+ from pypaimon.common.identifier import Identifier
+ rebranched = Identifier(
+ database=self.identifier.get_database_name(),
+ object=self.identifier.get_table_name(),
+ branch=branch,
+ )
+ return SnapshotLoader(self.catalog_loader, rebranched)
+
def load(self) -> Optional[str]:
"""Load the latest snapshot from the catalog.
diff --git a/paimon-python/pypaimon/snapshot/snapshot_manager.py b/paimon-python/pypaimon/snapshot/snapshot_manager.py
index e0b4b08bc8..074fc844cd 100644
--- a/paimon-python/pypaimon/snapshot/snapshot_manager.py
+++ b/paimon-python/pypaimon/snapshot/snapshot_manager.py
@@ -29,17 +29,37 @@ from pypaimon.snapshot.snapshot_loader import SnapshotLoader
class SnapshotManager:
"""Manager for snapshot files using unified FileIO."""
- def __init__(self, table):
+ def __init__(self, table, branch: Optional[str] = None):
+ # Lazy imports to avoid a cycle: pypaimon.branch.__init__
+ # eagerly loads FileSystemBranchManager, which imports
+ # SnapshotManager.
+ from pypaimon.branch.branch_manager import BranchManager
+ from pypaimon.common.identifier import DEFAULT_MAIN_BRANCH
from pypaimon.table.file_store_table import FileStoreTable
self.table: FileStoreTable = table
self.file_io: FileIO = self.table.file_io
self.snapshot_loader: Optional[SnapshotLoader] = self.table.catalog_environment.snapshot_loader()
- snapshot_path = self.table.table_path.rstrip('/')
- self.snapshot_dir = f"{snapshot_path}/snapshot"
+ if branch is None:
+ branch = self.table.current_branch() or DEFAULT_MAIN_BRANCH
+ self.branch = BranchManager.normalize_branch(branch)
+
+ table_root = self.table.table_path.rstrip('/')
+ branch_root = BranchManager.branch_path(table_root, self.branch)
+ self.snapshot_dir = f"{branch_root}/snapshot"
self.latest_file = f"{self.snapshot_dir}/LATEST"
+ def copy_with_branch(self, branch_name: str) -> 'SnapshotManager':
+ # Mirrors Java SnapshotManager.copyWithBranch: the new manager
+ # carries a SnapshotLoader rebranched to ``branch_name`` so REST
+ # loads target the correct branch instead of falling back to the
+ # main-branch identifier.
+ new_manager = SnapshotManager(self.table, branch_name)
+ if self.snapshot_loader is not None:
+ new_manager.snapshot_loader = self.snapshot_loader.copy_with_branch(branch_name)
+ return new_manager
+
def get_latest_snapshot(self) -> Optional[Snapshot]:
"""
Get the latest snapshot with loader priority.
diff --git a/paimon-python/pypaimon/tests/branch_manager_test.py b/paimon-python/pypaimon/tests/branch_manager_test.py
index 897f5f5388..7f79eb1d38 100644
--- a/paimon-python/pypaimon/tests/branch_manager_test.py
+++ b/paimon-python/pypaimon/tests/branch_manager_test.py
@@ -20,8 +20,12 @@ import shutil
import tempfile
import unittest
+import pyarrow as pa
+
+from pypaimon import CatalogFactory, Schema
from pypaimon.branch.branch_manager import BranchManager, DEFAULT_MAIN_BRANCH
from pypaimon.branch.filesystem_branch_manager import FileSystemBranchManager
+from pypaimon.common.identifier import Identifier
class BranchManagerTest(unittest.TestCase):
@@ -179,5 +183,142 @@ class FileSystemBranchManagerTest(unittest.TestCase):
self.assertFalse(manager.branch_exists("develop"))
+class SnapshotManagerBranchAwarenessTest(unittest.TestCase):
+ """Pin down SnapshotManager's branch-aware path computation.
+
+ Mirrors Java ``SnapshotManager.copyWithBranch`` (utils/SnapshotManager.java):
+ a non-main branch must produce paths under
+ ``{table_path}/branch/branch-{name}/snapshot/...``.
+ """
+
+ def setUp(self):
+ self.temp_dir = tempfile.mkdtemp(prefix="unittest_snapshot_branch_")
+ warehouse = os.path.join(self.temp_dir, "warehouse")
+ os.makedirs(warehouse, exist_ok=True)
+ self.catalog = CatalogFactory.create({"warehouse": warehouse})
+ self.catalog.create_database("default", True)
+
+ self.pa_schema = pa.schema([("id", pa.int64()), ("value", pa.string())])
+ self.identifier = Identifier.from_string("default.snapshot_branch_table")
+ self.catalog.create_table(
+ self.identifier, Schema.from_pyarrow_schema(self.pa_schema), False)
+ self.table = self.catalog.get_table(self.identifier)
+
+ def tearDown(self):
+ shutil.rmtree(self.temp_dir, ignore_errors=True)
+
+ def test_default_constructor_targets_main_branch(self):
+ sm = self.table.snapshot_manager()
+ self.assertEqual(sm.branch, "main")
+ self.assertEqual(sm.snapshot_dir, f"{self.table.table_path.rstrip('/')}/snapshot")
+
+ def test_copy_with_branch_returns_branch_aware_paths(self):
+ sm = self.table.snapshot_manager()
+ branch_sm = sm.copy_with_branch("b1")
+
+ self.assertIsNot(branch_sm, sm)
+ self.assertEqual(branch_sm.branch, "b1")
+
+ expected_dir = f"{self.table.table_path.rstrip('/')}/branch/branch-b1/snapshot"
+ self.assertEqual(branch_sm.snapshot_dir, expected_dir)
+ self.assertEqual(branch_sm.get_snapshot_path(7), f"{expected_dir}/snapshot-7")
+ self.assertNotEqual(sm.get_snapshot_path(7), branch_sm.get_snapshot_path(7))
+
+ def test_copy_with_branch_rebranches_snapshot_loader(self):
+ from pypaimon.common.identifier import Identifier
+ from pypaimon.snapshot.snapshot_loader import SnapshotLoader
+
+ sm = self.table.snapshot_manager()
+ # FileSystem path has no loader; inject one to exercise the
+ # rebranch code path. Mirrors Java SnapshotLoaderImpl.copyWithBranch.
+ sm.snapshot_loader = SnapshotLoader(
+ catalog_loader=object(),
+ identifier=Identifier(database="default",
+ object="snapshot_branch_table"),
+ )
+
+ branch_sm = sm.copy_with_branch("b1")
+
+ self.assertIsNotNone(branch_sm.snapshot_loader)
+ self.assertIsNot(branch_sm.snapshot_loader, sm.snapshot_loader)
+ self.assertEqual(branch_sm.snapshot_loader.identifier.branch, "b1")
+ self.assertEqual(
+ branch_sm.snapshot_loader.identifier.database,
+ sm.snapshot_loader.identifier.database)
+ self.assertEqual(
+ branch_sm.snapshot_loader.identifier.object,
+ sm.snapshot_loader.identifier.object)
+ # Original loader's identifier untouched.
+ self.assertIsNone(sm.snapshot_loader.identifier.branch)
+
+
+class FileSystemBranchManagerEndToEndTest(unittest.TestCase):
+ """Catalog-driven end-to-end tests that exercise ``_copy_with_branch``.
+
+ These regress the from-tag and fast-forward paths that previously
+ raised ``SameFileError``: the SnapshotManager produced by
+ ``_copy_with_branch`` still pointed at the main-branch snapshot dir,
+ so ``copy_file(src, dst)`` collapsed to ``src == dst``.
+ """
+
+ def setUp(self):
+ self.temp_dir = tempfile.mkdtemp(prefix="unittest_fs_branch_e2e_")
+ warehouse = os.path.join(self.temp_dir, "warehouse")
+ os.makedirs(warehouse, exist_ok=True)
+ self.catalog = CatalogFactory.create({"warehouse": warehouse})
+ self.catalog.create_database("default", True)
+
+ self.pa_schema = pa.schema([("id", pa.int64()), ("value", pa.string())])
+ self.identifier = Identifier.from_string("default.fs_branch_e2e_table")
+ self.catalog.create_table(
+ self.identifier, Schema.from_pyarrow_schema(self.pa_schema), False)
+
+ table = self.catalog.get_table(self.identifier)
+ wb = table.new_batch_write_builder()
+ w = wb.new_write()
+ w.write_arrow(pa.Table.from_pydict(
+ {"id": [1, 2, 3], "value": ["a", "b", "c"]}, schema=self.pa_schema))
+ wb.new_commit().commit(w.prepare_commit())
+ w.close()
+
+ self.table = self.catalog.get_table(self.identifier)
+ self.table_root = self.table.table_path.rstrip('/')
+
+ def tearDown(self):
+ shutil.rmtree(self.temp_dir, ignore_errors=True)
+
+ def _latest_snapshot_id(self) -> int:
+ latest = self.table.snapshot_manager().get_latest_snapshot()
+ self.assertIsNotNone(latest)
+ return latest.id
+
+ def test_create_branch_from_tag_lands_files_under_branch_dir(self):
+ snapshot_id = self._latest_snapshot_id()
+
+ self.table.create_tag("t1")
+ bm = self.table.branch_manager()
+ bm.create_branch("b1", tag_name="t1")
+
+ branch_root = f"{self.table_root}/branch/branch-b1"
+ self.assertTrue(os.path.isdir(branch_root))
+ self.assertTrue(
+ os.path.isfile(f"{branch_root}/snapshot/snapshot-{snapshot_id}"))
+ self.assertTrue(os.path.isfile(f"{branch_root}/tag/tag-t1"))
+ # At least one schema file (schema-0) must have been copied.
+ schema_dir = f"{branch_root}/schema"
+ self.assertTrue(os.path.isdir(schema_dir))
+ self.assertTrue(
+ any(name.startswith("schema-") for name in os.listdir(schema_dir)))
+
+ def test_fast_forward_after_create_branch_from_tag(self):
+ self.table.create_tag("t1")
+ bm = self.table.branch_manager()
+ bm.create_branch("b1", tag_name="t1")
+ # Must not raise: previously fast_forward's copy_files(src=branch_dir,
+ # dst=main_dir) collapsed to src == dst because the branch-side
+ # SnapshotManager still pointed at the main snapshot dir.
+ bm.fast_forward("b1")
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/paimon-python/pypaimon/tests/snapshot_manager_test.py b/paimon-python/pypaimon/tests/snapshot_manager_test.py
index ccdaff2684..0d1f1b1388 100644
--- a/paimon-python/pypaimon/tests/snapshot_manager_test.py
+++ b/paimon-python/pypaimon/tests/snapshot_manager_test.py
@@ -42,6 +42,7 @@ class SnapshotManagerTest(unittest.TestCase):
table = Mock()
table.table_path = "/tmp/test_table"
+ table.current_branch.return_value = "main"
table.file_io = Mock()
table.file_io.exists_batch.return_value = {
"/tmp/test_table/snapshot/snapshot-5": True,
@@ -82,6 +83,7 @@ class SnapshotManagerTest(unittest.TestCase):
table = Mock()
table.table_path = "/tmp/test_table"
+ table.current_branch.return_value = "main"
table.file_io = Mock()
# All paths return False (no files exist)
table.file_io.exists_batch.return_value = {}
@@ -105,6 +107,7 @@ class SnapshotManagerTest(unittest.TestCase):
table = Mock()
table.table_path = "/tmp/test_table"
+ table.current_branch.return_value = "main"
table.file_io = Mock()
# All 3 snapshots exist but are COMPACT (will be skipped)