You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by dw...@apache.org on 2020/10/06 23:17:30 UTC
[iceberg] branch master updated: Parallelize reading manifest list
files in metadata tables. (#1440)
This is an automated email from the ASF dual-hosted git repository.
dweeks pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 84c4f39 Parallelize reading manifest list files in metadata tables. (#1440)
84c4f39 is described below
commit 84c4f39a8403cad60b40708da8fb60106c083e0b
Author: Ryan Blue <bl...@apache.org>
AuthorDate: Tue Oct 6 16:17:18 2020 -0700
Parallelize reading manifest list files in metadata tables. (#1440)
---
core/src/main/java/org/apache/iceberg/AllDataFilesTable.java | 3 ++-
core/src/main/java/org/apache/iceberg/AllEntriesTable.java | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/AllDataFilesTable.java b/core/src/main/java/org/apache/iceberg/AllDataFilesTable.java
index 12114f1..0c6d59a 100644
--- a/core/src/main/java/org/apache/iceberg/AllDataFilesTable.java
+++ b/core/src/main/java/org/apache/iceberg/AllDataFilesTable.java
@@ -136,7 +136,8 @@ public class AllDataFilesTable extends BaseMetadataTable {
private static CloseableIterable<ManifestFile> allDataManifestFiles(List<Snapshot> snapshots) {
try (CloseableIterable<ManifestFile> iterable = new ParallelIterable<>(
- Iterables.transform(snapshots, Snapshot::dataManifests), ThreadPools.getWorkerPool())) {
+ Iterables.transform(snapshots, snapshot -> (Iterable<ManifestFile>) () -> snapshot.dataManifests().iterator()),
+ ThreadPools.getWorkerPool())) {
return CloseableIterable.withNoopClose(Sets.newHashSet(iterable));
} catch (IOException e) {
throw new RuntimeIOException(e, "Failed to close parallel iterable");
diff --git a/core/src/main/java/org/apache/iceberg/AllEntriesTable.java b/core/src/main/java/org/apache/iceberg/AllEntriesTable.java
index 209f6ec..7bfbb2c 100644
--- a/core/src/main/java/org/apache/iceberg/AllEntriesTable.java
+++ b/core/src/main/java/org/apache/iceberg/AllEntriesTable.java
@@ -121,7 +121,8 @@ public class AllEntriesTable extends BaseMetadataTable {
private static CloseableIterable<ManifestFile> allManifestFiles(List<Snapshot> snapshots) {
try (CloseableIterable<ManifestFile> iterable = new ParallelIterable<>(
- Iterables.transform(snapshots, Snapshot::allManifests), ThreadPools.getWorkerPool())) {
+ Iterables.transform(snapshots, snapshot -> (Iterable<ManifestFile>) () -> snapshot.allManifests().iterator()),
+ ThreadPools.getWorkerPool())) {
return CloseableIterable.withNoopClose(Sets.newHashSet(iterable));
} catch (IOException e) {
throw new RuntimeIOException(e, "Failed to close parallel iterable");