You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2026/05/03 00:02:17 UTC

(paimon) branch master updated: [python] Fix FileSystemBranchManager from-tag and fast-forward path computation (#7756)

This is an automated email from the ASF dual-hosted git repository.

JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new 718f292c95 [python] Fix FileSystemBranchManager from-tag and fast-forward path computation (#7756)
718f292c95 is described below

commit 718f292c95c43a9bf8cf022c0ab50e301bd9a4a4
Author: chaoyang <ch...@apache.org>
AuthorDate: Sun May 3 08:02:12 2026 +0800

    [python] Fix FileSystemBranchManager from-tag and fast-forward path computation (#7756)
    
    Fix a pre-existing bug in `FileSystemBranchManager._copy_with_branch`
    that made `create_branch(tag_name=...)` and `fast_forward` raise
    `SameFileError`: the `SnapshotManager` it produced still pointed at the
    main-branch snapshot directory, so `copy_file(src, dst)` collapsed to
    `src == dst`.
    
    The fix mirrors Java `SnapshotManager.copyWithBranch`
    (paimon-core/.../utils/SnapshotManager.java:89-95): the Python
    `SnapshotManager` now carries an explicit `branch` field and a
    `copy_with_branch(branch_name)` factory, so its `snapshot_dir` /
    `get_snapshot_path(...)` resolve to
    `{table_path}/branch/branch-{name}/snapshot/...` for non-main branches.
    `FileSystemBranchManager._copy_with_branch` then dispatches to the
    per-manager factories (`SnapshotManager.copy_with_branch` /
    `SchemaManager.copy_with_branch`) instead of blindly reconstructing a
    main-branch `SnapshotManager`.
    
    `SnapshotLoader` is rebranched in lockstep so REST-path catalog loads
    target the requested branch rather than falling back to the main-branch
    identifier (mirrors Java `SnapshotLoaderImpl.copyWithBranch`).
---
 .../pypaimon/branch/filesystem_branch_manager.py   |  21 ++-
 paimon-python/pypaimon/snapshot/snapshot_loader.py |  14 ++
 .../pypaimon/snapshot/snapshot_manager.py          |  26 +++-
 .../pypaimon/tests/branch_manager_test.py          | 141 +++++++++++++++++++++
 .../pypaimon/tests/snapshot_manager_test.py        |   3 +
 5 files changed, 191 insertions(+), 14 deletions(-)

diff --git a/paimon-python/pypaimon/branch/filesystem_branch_manager.py b/paimon-python/pypaimon/branch/filesystem_branch_manager.py
index ca8111b62f..45fe6e4c43 100644
--- a/paimon-python/pypaimon/branch/filesystem_branch_manager.py
+++ b/paimon-python/pypaimon/branch/filesystem_branch_manager.py
@@ -83,22 +83,21 @@ class FileSystemBranchManager(BranchManager):
 
     def _copy_with_branch(self, manager, branch: str):
         """
-        Create a copy of the manager for a different branch.
-
-        Args:
-            manager: The manager to copy (TagManager, SnapshotManager, etc.)
-            branch: The target branch name
-
-        Returns:
-            A new manager instance for the specified branch
+        Return a new manager whose path accessors land under
+        ``{table_path}/branch/branch-{name}``.
+
+        Equivalent to the per-manager rebranch calls inlined in Java
+        ``FileSystemBranchManager.createBranch(name, tagName)`` (e.g.
+        ``snapshotManager.copyWithBranch(branchName)``). ``TagManager``
+        is rebuilt directly because the Python ``TagManager`` has no
+        ``copy_with_branch`` factory yet.
         """
         if isinstance(manager, TagManager):
             return TagManager(self.file_io, self.table_path, branch)
         elif isinstance(manager, SchemaManager):
-            # SchemaManager doesn't support branch parameter, use branch path instead
-            return SchemaManager(self.file_io, self.branch_path(branch))
+            return manager.copy_with_branch(branch)
         elif isinstance(manager, SnapshotManager):
-            return SnapshotManager(self.snapshot_manager.table)
+            return manager.copy_with_branch(branch)
         else:
             return manager
 
diff --git a/paimon-python/pypaimon/snapshot/snapshot_loader.py b/paimon-python/pypaimon/snapshot/snapshot_loader.py
index aba0a66c58..37f8fa7a91 100644
--- a/paimon-python/pypaimon/snapshot/snapshot_loader.py
+++ b/paimon-python/pypaimon/snapshot/snapshot_loader.py
@@ -36,6 +36,20 @@ class SnapshotLoader:
         self.catalog_loader = catalog_loader
         self.identifier = identifier
 
+    def copy_with_branch(self, branch: str) -> 'SnapshotLoader':
+        """Return a new loader whose identifier carries ``branch``.
+
+        Mirrors Java ``SnapshotLoaderImpl.copyWithBranch``
+        (tag/SnapshotLoaderImpl.java).
+        """
+        from pypaimon.common.identifier import Identifier
+        rebranched = Identifier(
+            database=self.identifier.get_database_name(),
+            object=self.identifier.get_table_name(),
+            branch=branch,
+        )
+        return SnapshotLoader(self.catalog_loader, rebranched)
+
     def load(self) -> Optional[str]:
         """Load the latest snapshot from the catalog.
         
diff --git a/paimon-python/pypaimon/snapshot/snapshot_manager.py b/paimon-python/pypaimon/snapshot/snapshot_manager.py
index e0b4b08bc8..074fc844cd 100644
--- a/paimon-python/pypaimon/snapshot/snapshot_manager.py
+++ b/paimon-python/pypaimon/snapshot/snapshot_manager.py
@@ -29,17 +29,37 @@ from pypaimon.snapshot.snapshot_loader import SnapshotLoader
 class SnapshotManager:
     """Manager for snapshot files using unified FileIO."""
 
-    def __init__(self, table):
+    def __init__(self, table, branch: Optional[str] = None):
+        # Lazy imports to avoid a cycle: pypaimon.branch.__init__
+        # eagerly loads FileSystemBranchManager, which imports
+        # SnapshotManager.
+        from pypaimon.branch.branch_manager import BranchManager
+        from pypaimon.common.identifier import DEFAULT_MAIN_BRANCH
         from pypaimon.table.file_store_table import FileStoreTable
 
         self.table: FileStoreTable = table
         self.file_io: FileIO = self.table.file_io
         self.snapshot_loader: Optional[SnapshotLoader] = self.table.catalog_environment.snapshot_loader()
 
-        snapshot_path = self.table.table_path.rstrip('/')
-        self.snapshot_dir = f"{snapshot_path}/snapshot"
+        if branch is None:
+            branch = self.table.current_branch() or DEFAULT_MAIN_BRANCH
+        self.branch = BranchManager.normalize_branch(branch)
+
+        table_root = self.table.table_path.rstrip('/')
+        branch_root = BranchManager.branch_path(table_root, self.branch)
+        self.snapshot_dir = f"{branch_root}/snapshot"
         self.latest_file = f"{self.snapshot_dir}/LATEST"
 
+    def copy_with_branch(self, branch_name: str) -> 'SnapshotManager':
+        # Mirrors Java SnapshotManager.copyWithBranch: the new manager
+        # carries a SnapshotLoader rebranched to ``branch_name`` so REST
+        # loads target the correct branch instead of falling back to the
+        # main-branch identifier.
+        new_manager = SnapshotManager(self.table, branch_name)
+        if self.snapshot_loader is not None:
+            new_manager.snapshot_loader = self.snapshot_loader.copy_with_branch(branch_name)
+        return new_manager
+
     def get_latest_snapshot(self) -> Optional[Snapshot]:
         """
         Get the latest snapshot with loader priority.
diff --git a/paimon-python/pypaimon/tests/branch_manager_test.py b/paimon-python/pypaimon/tests/branch_manager_test.py
index 897f5f5388..7f79eb1d38 100644
--- a/paimon-python/pypaimon/tests/branch_manager_test.py
+++ b/paimon-python/pypaimon/tests/branch_manager_test.py
@@ -20,8 +20,12 @@ import shutil
 import tempfile
 import unittest
 
+import pyarrow as pa
+
+from pypaimon import CatalogFactory, Schema
 from pypaimon.branch.branch_manager import BranchManager, DEFAULT_MAIN_BRANCH
 from pypaimon.branch.filesystem_branch_manager import FileSystemBranchManager
+from pypaimon.common.identifier import Identifier
 
 
 class BranchManagerTest(unittest.TestCase):
@@ -179,5 +183,142 @@ class FileSystemBranchManagerTest(unittest.TestCase):
         self.assertFalse(manager.branch_exists("develop"))
 
 
+class SnapshotManagerBranchAwarenessTest(unittest.TestCase):
+    """Pin down SnapshotManager's branch-aware path computation.
+
+    Mirrors Java ``SnapshotManager.copyWithBranch`` (utils/SnapshotManager.java):
+    a non-main branch must produce paths under
+    ``{table_path}/branch/branch-{name}/snapshot/...``.
+    """
+
+    def setUp(self):
+        self.temp_dir = tempfile.mkdtemp(prefix="unittest_snapshot_branch_")
+        warehouse = os.path.join(self.temp_dir, "warehouse")
+        os.makedirs(warehouse, exist_ok=True)
+        self.catalog = CatalogFactory.create({"warehouse": warehouse})
+        self.catalog.create_database("default", True)
+
+        self.pa_schema = pa.schema([("id", pa.int64()), ("value", pa.string())])
+        self.identifier = Identifier.from_string("default.snapshot_branch_table")
+        self.catalog.create_table(
+            self.identifier, Schema.from_pyarrow_schema(self.pa_schema), False)
+        self.table = self.catalog.get_table(self.identifier)
+
+    def tearDown(self):
+        shutil.rmtree(self.temp_dir, ignore_errors=True)
+
+    def test_default_constructor_targets_main_branch(self):
+        sm = self.table.snapshot_manager()
+        self.assertEqual(sm.branch, "main")
+        self.assertEqual(sm.snapshot_dir, f"{self.table.table_path.rstrip('/')}/snapshot")
+
+    def test_copy_with_branch_returns_branch_aware_paths(self):
+        sm = self.table.snapshot_manager()
+        branch_sm = sm.copy_with_branch("b1")
+
+        self.assertIsNot(branch_sm, sm)
+        self.assertEqual(branch_sm.branch, "b1")
+
+        expected_dir = f"{self.table.table_path.rstrip('/')}/branch/branch-b1/snapshot"
+        self.assertEqual(branch_sm.snapshot_dir, expected_dir)
+        self.assertEqual(branch_sm.get_snapshot_path(7), f"{expected_dir}/snapshot-7")
+        self.assertNotEqual(sm.get_snapshot_path(7), branch_sm.get_snapshot_path(7))
+
+    def test_copy_with_branch_rebranches_snapshot_loader(self):
+        from pypaimon.common.identifier import Identifier
+        from pypaimon.snapshot.snapshot_loader import SnapshotLoader
+
+        sm = self.table.snapshot_manager()
+        # FileSystem path has no loader; inject one to exercise the
+        # rebranch code path. Mirrors Java SnapshotLoaderImpl.copyWithBranch.
+        sm.snapshot_loader = SnapshotLoader(
+            catalog_loader=object(),
+            identifier=Identifier(database="default",
+                                  object="snapshot_branch_table"),
+        )
+
+        branch_sm = sm.copy_with_branch("b1")
+
+        self.assertIsNotNone(branch_sm.snapshot_loader)
+        self.assertIsNot(branch_sm.snapshot_loader, sm.snapshot_loader)
+        self.assertEqual(branch_sm.snapshot_loader.identifier.branch, "b1")
+        self.assertEqual(
+            branch_sm.snapshot_loader.identifier.database,
+            sm.snapshot_loader.identifier.database)
+        self.assertEqual(
+            branch_sm.snapshot_loader.identifier.object,
+            sm.snapshot_loader.identifier.object)
+        # Original loader's identifier untouched.
+        self.assertIsNone(sm.snapshot_loader.identifier.branch)
+
+
+class FileSystemBranchManagerEndToEndTest(unittest.TestCase):
+    """Catalog-driven end-to-end tests that exercise ``_copy_with_branch``.
+
+    These regress the from-tag and fast-forward paths that previously
+    raised ``SameFileError``: the SnapshotManager produced by
+    ``_copy_with_branch`` still pointed at the main-branch snapshot dir,
+    so ``copy_file(src, dst)`` collapsed to ``src == dst``.
+    """
+
+    def setUp(self):
+        self.temp_dir = tempfile.mkdtemp(prefix="unittest_fs_branch_e2e_")
+        warehouse = os.path.join(self.temp_dir, "warehouse")
+        os.makedirs(warehouse, exist_ok=True)
+        self.catalog = CatalogFactory.create({"warehouse": warehouse})
+        self.catalog.create_database("default", True)
+
+        self.pa_schema = pa.schema([("id", pa.int64()), ("value", pa.string())])
+        self.identifier = Identifier.from_string("default.fs_branch_e2e_table")
+        self.catalog.create_table(
+            self.identifier, Schema.from_pyarrow_schema(self.pa_schema), False)
+
+        table = self.catalog.get_table(self.identifier)
+        wb = table.new_batch_write_builder()
+        w = wb.new_write()
+        w.write_arrow(pa.Table.from_pydict(
+            {"id": [1, 2, 3], "value": ["a", "b", "c"]}, schema=self.pa_schema))
+        wb.new_commit().commit(w.prepare_commit())
+        w.close()
+
+        self.table = self.catalog.get_table(self.identifier)
+        self.table_root = self.table.table_path.rstrip('/')
+
+    def tearDown(self):
+        shutil.rmtree(self.temp_dir, ignore_errors=True)
+
+    def _latest_snapshot_id(self) -> int:
+        latest = self.table.snapshot_manager().get_latest_snapshot()
+        self.assertIsNotNone(latest)
+        return latest.id
+
+    def test_create_branch_from_tag_lands_files_under_branch_dir(self):
+        snapshot_id = self._latest_snapshot_id()
+
+        self.table.create_tag("t1")
+        bm = self.table.branch_manager()
+        bm.create_branch("b1", tag_name="t1")
+
+        branch_root = f"{self.table_root}/branch/branch-b1"
+        self.assertTrue(os.path.isdir(branch_root))
+        self.assertTrue(
+            os.path.isfile(f"{branch_root}/snapshot/snapshot-{snapshot_id}"))
+        self.assertTrue(os.path.isfile(f"{branch_root}/tag/tag-t1"))
+        # At least one schema file (schema-0) must have been copied.
+        schema_dir = f"{branch_root}/schema"
+        self.assertTrue(os.path.isdir(schema_dir))
+        self.assertTrue(
+            any(name.startswith("schema-") for name in os.listdir(schema_dir)))
+
+    def test_fast_forward_after_create_branch_from_tag(self):
+        self.table.create_tag("t1")
+        bm = self.table.branch_manager()
+        bm.create_branch("b1", tag_name="t1")
+        # Must not raise: previously fast_forward's copy_files(src=branch_dir,
+        # dst=main_dir) collapsed to src == dst because the branch-side
+        # SnapshotManager still pointed at the main snapshot dir.
+        bm.fast_forward("b1")
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/paimon-python/pypaimon/tests/snapshot_manager_test.py b/paimon-python/pypaimon/tests/snapshot_manager_test.py
index ccdaff2684..0d1f1b1388 100644
--- a/paimon-python/pypaimon/tests/snapshot_manager_test.py
+++ b/paimon-python/pypaimon/tests/snapshot_manager_test.py
@@ -42,6 +42,7 @@ class SnapshotManagerTest(unittest.TestCase):
 
         table = Mock()
         table.table_path = "/tmp/test_table"
+        table.current_branch.return_value = "main"
         table.file_io = Mock()
         table.file_io.exists_batch.return_value = {
             "/tmp/test_table/snapshot/snapshot-5": True,
@@ -82,6 +83,7 @@ class SnapshotManagerTest(unittest.TestCase):
 
         table = Mock()
         table.table_path = "/tmp/test_table"
+        table.current_branch.return_value = "main"
         table.file_io = Mock()
         # All paths return False (no files exist)
         table.file_io.exists_batch.return_value = {}
@@ -105,6 +107,7 @@ class SnapshotManagerTest(unittest.TestCase):
 
         table = Mock()
         table.table_path = "/tmp/test_table"
+        table.current_branch.return_value = "main"
         table.file_io = Mock()
 
         # All 3 snapshots exist but are COMPACT (will be skipped)