You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by fo...@apache.org on 2024/04/14 04:47:56 UTC
(iceberg-python) branch pyiceberg-0.6.x updated (20093c34 -> 86ff66e3)
This is an automated email from the ASF dual-hosted git repository.
fokko pushed a change to branch pyiceberg-0.6.x
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
from 20093c34 Fix Glue Integration test (#536)
new bc6bea1b Read: fetch file_schema directly from pyarrow_to_schema (#597)
new a8cc4a63 UUID literal to binary and fixed (#529)
new 86ff66e3 fix integration test
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
pyiceberg/expressions/literals.py | 13 +++++++++++++
pyiceberg/io/pyarrow.py | 10 ++--------
tests/conftest.py | 4 +++-
tests/expressions/test_literals.py | 20 ++++++++++++++++++-
tests/integration/test_writes.py | 39 +++++++++++++++++++++++++++++++++-----
tests/io/test_pyarrow.py | 6 +++---
6 files changed, 74 insertions(+), 18 deletions(-)
(iceberg-python) 02/03: UUID literal to binary and fixed (#529)
Posted by fo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
fokko pushed a commit to branch pyiceberg-0.6.x
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
commit a8cc4a63b4c5f337c3e3b59abd42e8e8cd3089eb
Author: Seb Pretzer <24...@users.noreply.github.com>
AuthorDate: Mon Mar 18 16:40:51 2024 -0500
UUID literal to binary and fixed (#529)
* poetry lock --no-update
* adding uuid literal to fixed/binary type
* simplifying uuid generation
* editted wrong line of test
* adding assert_type for uuid literal
* assertation doesn't add much
* mypy no longer failing
* old code got into PR...oops
* cleaning up empty lines
---
pyiceberg/expressions/literals.py | 13 +++++++++++++
tests/expressions/test_literals.py | 20 +++++++++++++++++++-
2 files changed, 32 insertions(+), 1 deletion(-)
diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py
index bf76be36..d9f66ae2 100644
--- a/pyiceberg/expressions/literals.py
+++ b/pyiceberg/expressions/literals.py
@@ -609,6 +609,19 @@ class UUIDLiteral(Literal[bytes]):
def _(self, _: UUIDType) -> Literal[bytes]:
return self
+ @to.register(FixedType)
+ def _(self, type_var: FixedType) -> Literal[bytes]:
+ if len(type_var) == UUID_BYTES_LENGTH:
+ return FixedLiteral(self.value)
+ else:
+ raise TypeError(
+ f"Cannot convert UUIDLiteral into {type_var}, different length: {len(type_var)} <> {UUID_BYTES_LENGTH}"
+ )
+
+ @to.register(BinaryType)
+ def _(self, _: BinaryType) -> Literal[bytes]:
+ return BinaryLiteral(self.value)
+
class FixedLiteral(Literal[bytes]):
def __init__(self, value: bytes) -> None:
diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py
index 309bd28c..c974fc8b 100644
--- a/tests/expressions/test_literals.py
+++ b/tests/expressions/test_literals.py
@@ -758,7 +758,6 @@ def test_invalid_uuid_conversions() -> None:
DecimalType(9, 2),
StringType(),
FixedType(1),
- BinaryType(),
],
)
@@ -882,6 +881,25 @@ def test_uuid_literal_initialization() -> None:
assert test_uuid.bytes == uuid_literal.value
+def test_uuid_to_fixed() -> None:
+ test_uuid = uuid.uuid4()
+ uuid_literal = literal(test_uuid)
+ fixed_literal = uuid_literal.to(FixedType(16))
+ assert test_uuid.bytes == fixed_literal.value
+ with pytest.raises(TypeError) as e:
+ uuid_literal.to(FixedType(15))
+ assert "Cannot convert UUIDLiteral into fixed[15], different length: 15 <> 16" in str(e.value)
+ assert isinstance(fixed_literal, FixedLiteral) # type: ignore
+
+
+def test_uuid_to_binary() -> None:
+ test_uuid = uuid.uuid4()
+ uuid_literal = literal(test_uuid)
+ binary_literal = uuid_literal.to(BinaryType())
+ assert test_uuid.bytes == binary_literal.value
+ assert isinstance(binary_literal, BinaryLiteral) # type: ignore
+
+
# __ __ ___
# | \/ |_ _| _ \_ _
# | |\/| | || | _/ || |
(iceberg-python) 03/03: fix integration test
Posted by fo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
fokko pushed a commit to branch pyiceberg-0.6.x
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
commit 86ff66e3f770d9ed0996138503e17533ef1d8290
Author: HonahX <ho...@apache.org>
AuthorDate: Sat Apr 13 16:53:42 2024 -0700
fix integration test
---
tests/integration/test_writes.py | 39 ++++++++++++++++++++++++++++++++++-----
1 file changed, 34 insertions(+), 5 deletions(-)
diff --git a/tests/integration/test_writes.py b/tests/integration/test_writes.py
index 887a5194..425e4cee 100644
--- a/tests/integration/test_writes.py
+++ b/tests/integration/test_writes.py
@@ -20,7 +20,7 @@ import time
import uuid
from datetime import date, datetime
from pathlib import Path
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
from urllib.parse import urlparse
import pyarrow as pa
@@ -170,15 +170,22 @@ def arrow_table_with_only_nulls(pa_schema: pa.Schema) -> pa.Table:
return pa.Table.from_pylist([{}, {}], schema=pa_schema)
-def _create_table(session_catalog: Catalog, identifier: str, properties: Properties, data: List[pa.Table]) -> Table:
+def _create_table(
+ session_catalog: Catalog,
+ identifier: str,
+ properties: Properties,
+ data: Optional[List[pa.Table]] = None,
+ schema: Schema = TABLE_SCHEMA,
+) -> Table:
try:
session_catalog.drop_table(identifier=identifier)
except NoSuchTableError:
pass
- tbl = session_catalog.create_table(identifier=identifier, schema=TABLE_SCHEMA, properties=properties)
- for d in data:
- tbl.append(d)
+ tbl = session_catalog.create_table(identifier=identifier, schema=schema, properties=properties)
+ if data:
+ for d in data:
+ tbl.append(d)
return tbl
@@ -425,6 +432,28 @@ def test_data_files(spark: SparkSession, session_catalog: Catalog, arrow_table_w
assert [row.deleted_data_files_count for row in rows] == [0, 0, 1, 0, 0]
+@pytest.mark.integration
+@pytest.mark.parametrize("format_version", [1, 2])
+def test_python_writes_special_character_column_with_spark_reads(
+ spark: SparkSession, session_catalog: Catalog, format_version: int
+) -> None:
+ identifier = "default.python_writes_special_character_column_with_spark_reads"
+ column_name_with_special_character = "letter/abc"
+ TEST_DATA_WITH_SPECIAL_CHARACTER_COLUMN = {
+ column_name_with_special_character: ['a', None, 'z'],
+ }
+ pa_schema = pa.schema([
+ (column_name_with_special_character, pa.string()),
+ ])
+ arrow_table_with_special_character_column = pa.Table.from_pydict(TEST_DATA_WITH_SPECIAL_CHARACTER_COLUMN, schema=pa_schema)
+ tbl = _create_table(session_catalog, identifier, {"format-version": str(format_version)}, schema=pa_schema)
+
+ tbl.overwrite(arrow_table_with_special_character_column)
+ spark_df = spark.sql(f"SELECT * FROM {identifier}").toPandas()
+ pyiceberg_df = tbl.scan().to_pandas()
+ assert spark_df.equals(pyiceberg_df)
+
+
@pytest.mark.integration
@pytest.mark.parametrize("format_version", ["1", "2"])
@pytest.mark.parametrize(
(iceberg-python) 01/03: Read: fetch file_schema directly from pyarrow_to_schema (#597)
Posted by fo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
fokko pushed a commit to branch pyiceberg-0.6.x
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
commit bc6bea1b2e4600d650c67fedcdf28f074b5d28b8
Author: Honah J <ho...@apache.org>
AuthorDate: Fri Apr 12 17:53:22 2024 -0700
Read: fetch file_schema directly from pyarrow_to_schema (#597)
---
pyiceberg/io/pyarrow.py | 10 ++--------
tests/conftest.py | 4 +++-
tests/io/test_pyarrow.py | 6 +++---
3 files changed, 8 insertions(+), 12 deletions(-)
diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py
index 038414ee..357fa340 100644
--- a/pyiceberg/io/pyarrow.py
+++ b/pyiceberg/io/pyarrow.py
@@ -120,7 +120,6 @@ from pyiceberg.schema import (
pre_order_visit,
promote,
prune_columns,
- sanitize_column_names,
visit,
visit_with_partner,
)
@@ -950,12 +949,7 @@ def _task_to_table(
with fs.open_input_file(path) as fin:
fragment = arrow_format.make_fragment(fin)
physical_schema = fragment.physical_schema
- schema_raw = None
- if metadata := physical_schema.metadata:
- schema_raw = metadata.get(ICEBERG_SCHEMA)
- file_schema = (
- Schema.model_validate_json(schema_raw) if schema_raw is not None else pyarrow_to_schema(physical_schema, name_mapping)
- )
+ file_schema = pyarrow_to_schema(physical_schema, name_mapping)
pyarrow_filter = None
if bound_row_filter is not AlwaysTrue():
@@ -963,7 +957,7 @@ def _task_to_table(
bound_file_filter = bind(file_schema, translated_row_filter, case_sensitive=case_sensitive)
pyarrow_filter = expression_to_pyarrow(bound_file_filter)
- file_project_schema = sanitize_column_names(prune_columns(file_schema, projected_field_ids, select_full_types=False))
+ file_project_schema = prune_columns(file_schema, projected_field_ids, select_full_types=False)
if file_schema is None:
raise ValueError(f"Missing Iceberg schema in Metadata for file: {path}")
diff --git a/tests/conftest.py b/tests/conftest.py
index 62444b45..46fa4a8e 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1913,9 +1913,11 @@ def data_file(table_schema_simple: Schema, tmp_path: str) -> str:
import pyarrow as pa
from pyarrow import parquet as pq
+ from pyiceberg.io.pyarrow import schema_to_pyarrow
+
table = pa.table(
{"foo": ["a", "b", "c"], "bar": [1, 2, 3], "baz": [True, False, None]},
- metadata={"iceberg.schema": table_schema_simple.model_dump_json()},
+ schema=schema_to_pyarrow(table_schema_simple),
)
file_path = f"{tmp_path}/0000-data.parquet"
diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py
index 2acffdfd..98e6bf5e 100644
--- a/tests/io/test_pyarrow.py
+++ b/tests/io/test_pyarrow.py
@@ -1383,7 +1383,7 @@ def test_delete(deletes_file: str, example_task: FileScanTask, table_schema_simp
str(with_deletes)
== """pyarrow.Table
foo: string
-bar: int64 not null
+bar: int32 not null
baz: bool
----
foo: [["a","c"]]
@@ -1426,7 +1426,7 @@ def test_delete_duplicates(deletes_file: str, example_task: FileScanTask, table_
str(with_deletes)
== """pyarrow.Table
foo: string
-bar: int64 not null
+bar: int32 not null
baz: bool
----
foo: [["a","c"]]
@@ -1462,7 +1462,7 @@ def test_pyarrow_wrap_fsspec(example_task: FileScanTask, table_schema_simple: Sc
str(projection)
== """pyarrow.Table
foo: string
-bar: int64 not null
+bar: int32 not null
baz: bool
----
foo: [["a","b","c"]]