You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by fo...@apache.org on 2023/01/24 22:31:56 UTC
[iceberg] branch master updated: Python: Parallelize IO (#6645)
This is an automated email from the ASF dual-hosted git repository.
fokko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 2671621565 Python: Parallelize IO (#6645)
2671621565 is described below
commit 2671621565cde8adda27b81d1699663f71d9b3d4
Author: Fokko Driesprong <fo...@apache.org>
AuthorDate: Tue Jan 24 23:31:48 2023 +0100
Python: Parallelize IO (#6645)
* Python: Parallelize IO
Alternative for https://github.com/apache/iceberg/pull/6590
This uses the ThreadPool approach instead of ThreadPoolExecutor.
The ThreadPoolExecutor is more flexible and works well
with heterogeneous tasks. This allows the user to handle
exceptions per task and able to cancel individual tasks.
But the ThreadPoolExecutor also has some limitation
such as not able to forcefully terminate all the tasks.
For reading tasks I think the ThreadPool might be more appriopriate,
but for writing the ThreadPoolExecutor might be more applicable.
A very nice writeup of the differences is available in this blog:
https://superfastpython.com/threadpool-vs-threadpoolexecutor/
Before:
```
➜ python git:(fd-threadpool) time python3 /tmp/test.py
python3 /tmp/test.py 3.45s user 2.84s system 2% cpu 3:34.19 total
```
After:
```
➜ python git:(fd-threadpool) ✗ time python3 /tmp/test.py
python3 /tmp/test.py 3.13s user 2.83s system 19% cpu 31.369 total
➜ python git:(fd-threadpool) ✗ time python3 /tmp/test.py
python3 /tmp/test.py 2.94s user 3.08s system 18% cpu 32.538 total
➜ python git:(fd-threadpool) ✗ time python3 /tmp/test.py
python3 /tmp/test.py 2.84s user 3.14s system 20% cpu 29.033 total
```
Longlining the requests from EU to the USA, so this might impact
the results a bit, but makes IO more dominant.
* Set read options
---
python/pyiceberg/expressions/literals.py | 4 ++
python/pyiceberg/io/pyarrow.py | 97 +++++++++++++++++++-------------
python/pyiceberg/table/__init__.py | 26 ++++++---
3 files changed, 80 insertions(+), 47 deletions(-)
diff --git a/python/pyiceberg/expressions/literals.py b/python/pyiceberg/expressions/literals.py
index cd53ab6d41..fcfaaf7fe9 100644
--- a/python/pyiceberg/expressions/literals.py
+++ b/python/pyiceberg/expressions/literals.py
@@ -420,6 +420,10 @@ class TimestampLiteral(Literal[int]):
def _(self, _: TimestampType) -> Literal[int]:
return self
+ @to.register(TimestamptzType)
+ def _(self, _: TimestamptzType) -> Literal[int]:
+ return self
+
@to.register(DateType)
def _(self, _: DateType) -> Literal[int]:
return DateLiteral(micros_to_days(self.value))
diff --git a/python/pyiceberg/io/pyarrow.py b/python/pyiceberg/io/pyarrow.py
index 12edc6e3e2..848ec4e0c8 100644
--- a/python/pyiceberg/io/pyarrow.py
+++ b/python/pyiceberg/io/pyarrow.py
@@ -26,6 +26,7 @@ from __future__ import annotations
import os
from functools import lru_cache
+from multiprocessing.pool import ThreadPool
from typing import (
TYPE_CHECKING,
Any,
@@ -470,13 +471,62 @@ def expression_to_pyarrow(expr: BooleanExpression) -> pc.Expression:
return boolean_expression_visit(expr, _ConvertToArrowExpression())
+def _file_to_table(
+ fs: FileSystem,
+ task: FileScanTask,
+ bound_row_filter: BooleanExpression,
+ projected_schema: Schema,
+ projected_field_ids: Set[int],
+ case_sensitive: bool,
+) -> pa.Table:
+ _, path = PyArrowFileIO.parse_location(task.file.file_path)
+
+ # Get the schema
+ with fs.open_input_file(path) as fout:
+ parquet_schema = pq.read_schema(fout)
+ schema_raw = None
+ if metadata := parquet_schema.metadata:
+ schema_raw = metadata.get(ICEBERG_SCHEMA)
+ if schema_raw is None:
+ raise ValueError(
+ "Iceberg schema is not embedded into the Parquet file, see https://github.com/apache/iceberg/issues/6505"
+ )
+ file_schema = Schema.parse_raw(schema_raw)
+
+ pyarrow_filter = None
+ if bound_row_filter is not AlwaysTrue():
+ translated_row_filter = translate_column_names(bound_row_filter, file_schema, case_sensitive=case_sensitive)
+ bound_file_filter = bind(file_schema, translated_row_filter, case_sensitive=case_sensitive)
+ pyarrow_filter = expression_to_pyarrow(bound_file_filter)
+
+ file_project_schema = prune_columns(file_schema, projected_field_ids, select_full_types=False)
+
+ if file_schema is None:
+ raise ValueError(f"Missing Iceberg schema in Metadata for file: {path}")
+
+ # Prune the stuff that we don't need anyway
+ file_project_schema_arrow = schema_to_pyarrow(file_project_schema)
+
+ read_options = {
+ "pre_buffer": True,
+ "use_buffered_stream": True,
+ "buffer_size": 8388608,
+ }
+
+ arrow_table = ds.dataset(
+ source=[path], schema=file_project_schema_arrow, format=ds.ParquetFileFormat(**read_options), filesystem=fs
+ ).to_table(filter=pyarrow_filter)
+
+ return to_requested_schema(projected_schema, file_project_schema, arrow_table)
+
+
def project_table(
- files: Iterable[FileScanTask], table: Table, row_filter: BooleanExpression, projected_schema: Schema, case_sensitive: bool
+ tasks: Iterable[FileScanTask], table: Table, row_filter: BooleanExpression, projected_schema: Schema, case_sensitive: bool
) -> pa.Table:
"""Resolves the right columns based on the identifier
Args:
- files(Iterable[FileScanTask]): A URI or a path to a local file
+ tasks(Iterable[FileScanTask]): A URI or a path to a local file
table(Table): The table that's being queried
row_filter(BooleanExpression): The expression for filtering rows
projected_schema(Schema): The output schema
@@ -487,7 +537,7 @@ def project_table(
"""
if isinstance(table.io, PyArrowFileIO):
- scheme, path = PyArrowFileIO.parse_location(table.location())
+ scheme, _ = PyArrowFileIO.parse_location(table.location())
fs = table.io.get_fs(scheme)
else:
raise ValueError(f"Expected PyArrowFileIO, got: {table.io}")
@@ -498,41 +548,12 @@ def project_table(
id for id in projected_schema.field_ids if not isinstance(projected_schema.find_type(id), (MapType, ListType))
}.union(extract_field_ids(bound_row_filter))
- tables = []
- for task in files:
- _, path = PyArrowFileIO.parse_location(task.file.file_path)
-
- # Get the schema
- with fs.open_input_file(path) as fout:
- parquet_schema = pq.read_schema(fout)
- schema_raw = None
- if metadata := parquet_schema.metadata:
- schema_raw = metadata.get(ICEBERG_SCHEMA)
- if schema_raw is None:
- raise ValueError(
- "Iceberg schema is not embedded into the Parquet file, see https://github.com/apache/iceberg/issues/6505"
- )
- file_schema = Schema.parse_raw(schema_raw)
-
- pyarrow_filter = None
- if row_filter is not AlwaysTrue():
- translated_row_filter = translate_column_names(bound_row_filter, file_schema, case_sensitive=case_sensitive)
- bound_file_filter = bind(file_schema, translated_row_filter, case_sensitive=case_sensitive)
- pyarrow_filter = expression_to_pyarrow(bound_file_filter)
-
- file_project_schema = prune_columns(file_schema, projected_field_ids, select_full_types=False)
-
- if file_schema is None:
- raise ValueError(f"Missing Iceberg schema in Metadata for file: {path}")
-
- # Prune the stuff that we don't need anyway
- file_project_schema_arrow = schema_to_pyarrow(file_project_schema)
-
- arrow_table = ds.dataset(
- source=[path], schema=file_project_schema_arrow, format=ds.ParquetFileFormat(), filesystem=fs
- ).to_table(filter=pyarrow_filter)
-
- tables.append(to_requested_schema(projected_schema, file_project_schema, arrow_table))
+ with ThreadPool() as pool:
+ tables = pool.starmap(
+ func=_file_to_table,
+ iterable=[(fs, task, bound_row_filter, projected_schema, projected_field_ids, case_sensitive) for task in tasks],
+ chunksize=None, # we could use this to control how to materialize the generator of tasks (we should also make the expression above lazy)
+ )
if len(tables) > 1:
return pa.concat_tables(tables)
diff --git a/python/pyiceberg/table/__init__.py b/python/pyiceberg/table/__init__.py
index 910ea7f0e0..db42540496 100644
--- a/python/pyiceberg/table/__init__.py
+++ b/python/pyiceberg/table/__init__.py
@@ -19,6 +19,8 @@ from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass
from functools import cached_property
+from itertools import chain
+from multiprocessing.pool import ThreadPool
from typing import (
TYPE_CHECKING,
Any,
@@ -272,6 +274,13 @@ def _check_content(file: DataFile) -> DataFile:
return file
+def _open_manifest(io: FileIO, manifest: ManifestFile, partition_filter: Callable[[DataFile], bool]) -> List[FileScanTask]:
+ all_files = files(io.new_input(manifest.manifest_path))
+ matching_partition_files = filter(partition_filter, all_files)
+ matching_partition_data_files = map(_check_content, matching_partition_files)
+ return [FileScanTask(file) for file in matching_partition_data_files]
+
+
class DataScan(TableScan["DataScan"]):
def __init__(
self,
@@ -308,7 +317,7 @@ class DataScan(TableScan["DataScan"]):
def plan_files(self) -> Iterator[FileScanTask]:
snapshot = self.snapshot()
if not snapshot:
- return
+ return iter([])
io = self.table.io
@@ -328,14 +337,13 @@ class DataScan(TableScan["DataScan"]):
partition_evaluators: Dict[int, Callable[[DataFile], bool]] = KeyDefaultDict(self._build_partition_evaluator)
- for manifest in manifests:
- partition_filter = partition_evaluators[manifest.partition_spec_id]
- all_files = files(io.new_input(manifest.manifest_path))
- matching_partition_files = filter(partition_filter, all_files)
-
- matching_partition_data_files = map(_check_content, matching_partition_files)
-
- yield from (FileScanTask(file) for file in matching_partition_data_files)
+ with ThreadPool() as pool:
+ return chain(
+ *pool.starmap(
+ func=_open_manifest,
+ iterable=[(io, manifest, partition_evaluators[manifest.partition_spec_id]) for manifest in manifests],
+ )
+ )
def to_arrow(self) -> pa.Table:
return project_table(