You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by fo...@apache.org on 2024/04/14 04:47:57 UTC
(iceberg-python) 01/03: Read: fetch file_schema directly from pyarrow_to_schema (#597)
This is an automated email from the ASF dual-hosted git repository.
fokko pushed a commit to branch pyiceberg-0.6.x
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
commit bc6bea1b2e4600d650c67fedcdf28f074b5d28b8
Author: Honah J <ho...@apache.org>
AuthorDate: Fri Apr 12 17:53:22 2024 -0700
Read: fetch file_schema directly from pyarrow_to_schema (#597)
---
pyiceberg/io/pyarrow.py | 10 ++--------
tests/conftest.py | 4 +++-
tests/io/test_pyarrow.py | 6 +++---
3 files changed, 8 insertions(+), 12 deletions(-)
diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py
index 038414ee..357fa340 100644
--- a/pyiceberg/io/pyarrow.py
+++ b/pyiceberg/io/pyarrow.py
@@ -120,7 +120,6 @@ from pyiceberg.schema import (
pre_order_visit,
promote,
prune_columns,
- sanitize_column_names,
visit,
visit_with_partner,
)
@@ -950,12 +949,7 @@ def _task_to_table(
with fs.open_input_file(path) as fin:
fragment = arrow_format.make_fragment(fin)
physical_schema = fragment.physical_schema
- schema_raw = None
- if metadata := physical_schema.metadata:
- schema_raw = metadata.get(ICEBERG_SCHEMA)
- file_schema = (
- Schema.model_validate_json(schema_raw) if schema_raw is not None else pyarrow_to_schema(physical_schema, name_mapping)
- )
+ file_schema = pyarrow_to_schema(physical_schema, name_mapping)
pyarrow_filter = None
if bound_row_filter is not AlwaysTrue():
@@ -963,7 +957,7 @@ def _task_to_table(
bound_file_filter = bind(file_schema, translated_row_filter, case_sensitive=case_sensitive)
pyarrow_filter = expression_to_pyarrow(bound_file_filter)
- file_project_schema = sanitize_column_names(prune_columns(file_schema, projected_field_ids, select_full_types=False))
+ file_project_schema = prune_columns(file_schema, projected_field_ids, select_full_types=False)
if file_schema is None:
raise ValueError(f"Missing Iceberg schema in Metadata for file: {path}")
diff --git a/tests/conftest.py b/tests/conftest.py
index 62444b45..46fa4a8e 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1913,9 +1913,11 @@ def data_file(table_schema_simple: Schema, tmp_path: str) -> str:
import pyarrow as pa
from pyarrow import parquet as pq
+ from pyiceberg.io.pyarrow import schema_to_pyarrow
+
table = pa.table(
{"foo": ["a", "b", "c"], "bar": [1, 2, 3], "baz": [True, False, None]},
- metadata={"iceberg.schema": table_schema_simple.model_dump_json()},
+ schema=schema_to_pyarrow(table_schema_simple),
)
file_path = f"{tmp_path}/0000-data.parquet"
diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py
index 2acffdfd..98e6bf5e 100644
--- a/tests/io/test_pyarrow.py
+++ b/tests/io/test_pyarrow.py
@@ -1383,7 +1383,7 @@ def test_delete(deletes_file: str, example_task: FileScanTask, table_schema_simp
str(with_deletes)
== """pyarrow.Table
foo: string
-bar: int64 not null
+bar: int32 not null
baz: bool
----
foo: [["a","c"]]
@@ -1426,7 +1426,7 @@ def test_delete_duplicates(deletes_file: str, example_task: FileScanTask, table_
str(with_deletes)
== """pyarrow.Table
foo: string
-bar: int64 not null
+bar: int32 not null
baz: bool
----
foo: [["a","c"]]
@@ -1462,7 +1462,7 @@ def test_pyarrow_wrap_fsspec(example_task: FileScanTask, table_schema_simple: Sc
str(projection)
== """pyarrow.Table
foo: string
-bar: int64 not null
+bar: int32 not null
baz: bool
----
foo: [["a","b","c"]]