You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ke...@apache.org on 2025/06/22 18:07:09 UTC
(iceberg-python) branch main updated: perf: optimize `table.add_files` and `inspect.files` (#2133)
This is an automated email from the ASF dual-hosted git repository.
kevinjqliu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new e937f6a1 perf: optimize `table.add_files` and `inspect.files` (#2133)
e937f6a1 is described below
commit e937f6a1811c9e090552a4ae2015a8032e7ea910
Author: Jayce Slesar <47...@users.noreply.github.com>
AuthorDate: Sun Jun 22 14:07:04 2025 -0400
perf: optimize `table.add_files` and `inspect.files` (#2133)
Should help with #2130 and #2132
Modifies `Table.add_files` to explicitly use `inspect.data_files` and
also parallelize `inspect._files`
I didn't see anywhere else where looping over manifest entries was
parallelized, so seems better to parallelize across manifests than
within.
No changes here but should be faster.
---------
Co-authored-by: Kevin Liu <ke...@users.noreply.github.com>
---
pyiceberg/table/__init__.py | 2 +-
pyiceberg/table/inspect.py | 11 +++++++----
2 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py
index f82658a1..2eebd0e4 100644
--- a/pyiceberg/table/__init__.py
+++ b/pyiceberg/table/__init__.py
@@ -847,7 +847,7 @@ class Transaction:
import pyarrow.compute as pc
expr = pc.field("file_path").isin(file_paths)
- referenced_files = [file["file_path"] for file in self._table.inspect.files().filter(expr).to_pylist()]
+ referenced_files = [file["file_path"] for file in self._table.inspect.data_files().filter(expr).to_pylist()]
if referenced_files:
raise ValueError(f"Cannot add files that are already referenced by table, files: {', '.join(referenced_files)}")
diff --git a/pyiceberg/table/inspect.py b/pyiceberg/table/inspect.py
index cce5250a..3bb0268a 100644
--- a/pyiceberg/table/inspect.py
+++ b/pyiceberg/table/inspect.py
@@ -650,11 +650,14 @@ class InspectTable:
snapshot = self._get_snapshot(snapshot_id)
io = self.tbl.io
- files_table: list[pa.Table] = []
- for manifest_list in snapshot.manifests(io):
- files_table.append(self._get_files_from_manifest(manifest_list, data_file_filter))
- return pa.concat_tables(files_table)
+ executor = ExecutorFactory.get_or_create()
+ results = list(
+ executor.map(
+ lambda manifest_list: self._get_files_from_manifest(manifest_list, data_file_filter), snapshot.manifests(io)
+ )
+ )
+ return pa.concat_tables(results)
def files(self, snapshot_id: Optional[int] = None) -> "pa.Table":
return self._files(snapshot_id)