You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by gi...@apache.org on 2026/09/13 23:04:22 UTC
(iceberg-python) branch main updated: Add support for environment context (#3441)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new 0e4ac51c6 Add support for environment context (#3441)
0e4ac51c6 is described below
commit 0e4ac51c6d17c18d2ab0ddfb31a037eb6b924447
Author: Yuya Ebihara <eb...@gmail.com>
AuthorDate: Sun Sep 13 22:51:15 2026 +0000
Add support for environment context (#3441)
* Add support for environment context
* fixup! Add support for environment context
* Keep environment context fixed and isolate legacy tests
Return fresh engine metadata, copy only engine fields into new snapshot
summaries, and cover writes and property precedence without changing
existing summary assertions.
Co-authored-by: Copilot App <22...@users.noreply.github.com>
---------
Co-authored-by: Kevin Liu <ke...@gmail.com>
Co-authored-by: Copilot App <22...@users.noreply.github.com>
---
mkdocs/docs/api.md | 3 +++
pyiceberg/environment_context.py | 30 ++++++++++++++++++++++++++++
pyiceberg/table/snapshots.py | 6 ++++++
tests/conftest.py | 18 +++++++++++++++++
tests/table/test_snapshots.py | 42 +++++++++++++++++++++++++++++++++++++++
tests/test_environment_context.py | 32 +++++++++++++++++++++++++++++
6 files changed, 131 insertions(+)
diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md
index 1f5d4d669..5ab064390 100644
--- a/mkdocs/docs/api.md
+++ b/mkdocs/docs/api.md
@@ -1405,6 +1405,9 @@ tbl.overwrite(df, snapshot_properties={"abc": "def"})
assert tbl.metadata.snapshots[-1].summary["abc"] == "def"
```
+New snapshot summaries automatically include `engine-name` (`pyiceberg`) and `engine-version`
+(the installed PyIceberg version). These values override same-named entries in `snapshot_properties`.
+
## Snapshot Management
Manage snapshots with operations through the `Table` API:
diff --git a/pyiceberg/environment_context.py b/pyiceberg/environment_context.py
new file mode 100644
index 000000000..0a4f4ecca
--- /dev/null
+++ b/pyiceberg/environment_context.py
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pyiceberg import __version__
+
+
+class EnvironmentContext:
+ """Environment context carrying the engine name and version for snapshot summaries."""
+
+ @staticmethod
+ def get() -> dict[str, str]:
+ """Return a new dictionary containing only the engine name and version."""
+ return {
+ "engine-name": "pyiceberg",
+ "engine-version": __version__,
+ }
diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py
index f862529e8..0450df286 100644
--- a/pyiceberg/table/snapshots.py
+++ b/pyiceberg/table/snapshots.py
@@ -25,6 +25,7 @@ from typing import TYPE_CHECKING, Any
from pydantic import Field, PrivateAttr, model_serializer
+from pyiceberg.environment_context import EnvironmentContext
from pyiceberg.io import FileIO
from pyiceberg.manifest import DataFile, DataFileContent, ManifestFile, _manifests
from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec
@@ -413,6 +414,11 @@ def update_snapshot_summaries(summary: Summary, previous_summary: Mapping[str, s
removed_property=REMOVED_EQUALITY_DELETES,
)
+ if context := EnvironmentContext.get():
+ # Defensively select only engine fields so future context additions cannot overwrite snapshot metadata.
+ summary["engine-name"] = context["engine-name"]
+ summary["engine-version"] = context["engine-version"]
+
return summary
diff --git a/tests/conftest.py b/tests/conftest.py
index 27505a9cb..a9b2a7693 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -48,6 +48,7 @@ from pydantic_core import to_json
from pytest_lazy_fixtures import lf
from pyiceberg.catalog import Catalog, load_catalog
+from pyiceberg.environment_context import EnvironmentContext
from pyiceberg.expressions import BoundReference
from pyiceberg.io import (
ADLS_ACCOUNT_KEY,
@@ -107,12 +108,29 @@ if TYPE_CHECKING:
NON_UNIT_TEST_MARKERS = {"integration", "s3", "adls", "gcs", "notebook", "benchmark"}
+_original_environment_context_get = EnvironmentContext.get
+
+
def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:
for item in items:
if not any(marker.name in NON_UNIT_TEST_MARKERS for marker in item.iter_markers()):
item.add_marker("unit")
+@pytest.fixture(autouse=True, scope="session")
+def _disable_environment_context() -> Generator[None, None, None]:
+ """Disable engine metadata for existing tests, including session-scoped fixtures."""
+ with pytest.MonkeyPatch.context() as monkeypatch:
+ monkeypatch.setattr(EnvironmentContext, "get", staticmethod(dict))
+ yield
+
+
+@pytest.fixture
+def enable_environment_context(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Restore real engine metadata for tests that explicitly request it."""
+ monkeypatch.setattr(EnvironmentContext, "get", staticmethod(_original_environment_context_get))
+
+
@pytest.fixture(autouse=True, scope="session")
def _isolate_pyiceberg_config() -> None:
"""Make test runs ignore your local PyIceberg config.
diff --git a/tests/table/test_snapshots.py b/tests/table/test_snapshots.py
index 439375cc6..efb414592 100644
--- a/tests/table/test_snapshots.py
+++ b/tests/table/test_snapshots.py
@@ -24,6 +24,7 @@ from urllib.parse import urlparse
import pyarrow as pa
import pytest
+from pyiceberg import __version__
from pyiceberg.catalog import Catalog
from pyiceberg.exceptions import ValidationException
from pyiceberg.io.pyarrow import _dataframe_to_data_files
@@ -709,6 +710,47 @@ def overwrite_table(catalog: Catalog, arrow_table_simple: pa.Table) -> Table:
return table
+def test_snapshot_writes_include_engine_metadata(
+ enable_environment_context: None, catalog: Catalog, arrow_table_simple: pa.Table
+) -> None:
+ catalog.create_namespace("default")
+ table = catalog.create_table("default.engine_metadata", arrow_table_simple.schema)
+ table.append(arrow_table_simple)
+ table.overwrite(arrow_table_simple)
+ table.delete()
+
+ table.refresh()
+ snapshots = table.snapshots()
+ assert snapshots
+ for snapshot in snapshots:
+ assert snapshot.summary is not None
+ assert snapshot.summary["engine-name"] == "pyiceberg"
+ assert snapshot.summary["engine-version"] == __version__
+
+
+def test_snapshot_engine_metadata_overrides_snapshot_properties(
+ enable_environment_context: None, catalog: Catalog, arrow_table_simple: pa.Table
+) -> None:
+ catalog.create_namespace("default")
+ table = catalog.create_table("default.engine_metadata", arrow_table_simple.schema)
+ table.append(
+ arrow_table_simple,
+ snapshot_properties={
+ "engine-name": "custom-engine",
+ "engine-version": "custom-version",
+ "job-id": "snapshot-job",
+ },
+ )
+
+ table.refresh()
+ snapshot = table.current_snapshot()
+ assert snapshot is not None
+ assert snapshot.summary is not None
+ assert snapshot.summary["engine-name"] == "pyiceberg"
+ assert snapshot.summary["engine-version"] == __version__
+ assert snapshot.summary["job-id"] == "snapshot-job"
+
+
def _write_data_file(table: Table, rows: pa.Table) -> DataFile:
return next(
iter(
diff --git a/tests/test_environment_context.py b/tests/test_environment_context.py
new file mode 100644
index 000000000..09eea1c86
--- /dev/null
+++ b/tests/test_environment_context.py
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from pyiceberg import __version__
+from pyiceberg.environment_context import EnvironmentContext
+
+
+def test_get_returns_fresh_engine_metadata(enable_environment_context: None) -> None:
+ first = EnvironmentContext.get()
+ second = EnvironmentContext.get()
+ assert first is not second
+
+ first.clear()
+
+ assert second == {
+ "engine-name": "pyiceberg",
+ "engine-version": __version__,
+ }
+ assert EnvironmentContext.get() == second