You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by am...@apache.org on 2015/10/11 21:36:49 UTC
drill git commit: DRILL-3918: During expansion save the metadata for
future use.
Repository: drill
Updated Branches:
refs/heads/master 8197ba8f9 -> b4d47c56b
DRILL-3918: During expansion save the metadata for future use.
close apache/drill#196
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/b4d47c56
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/b4d47c56
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/b4d47c56
Branch: refs/heads/master
Commit: b4d47c56b6484eedbf3f44516338da4cdbfbb7b4
Parents: 8197ba8
Author: Aman Sinha <as...@maprtech.com>
Authored: Sun Oct 11 08:02:30 2015 -0700
Committer: Aman Sinha <as...@maprtech.com>
Committed: Sun Oct 11 12:30:38 2015 -0700
----------------------------------------------------------------------
.../drill/exec/store/dfs/FileSelection.java | 22 +++++++++++++++++++
.../exec/store/parquet/ParquetFormatPlugin.java | 2 +-
.../exec/store/parquet/ParquetGroupScan.java | 23 ++++++++++++++++----
3 files changed, 42 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/b4d47c56/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
index d5deefa..be3e57d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.drill.exec.store.parquet.Metadata.ParquetTableMetadata_v1;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
@@ -44,6 +45,10 @@ public class FileSelection {
public List<String> files;
public String selectionRoot;
+ // this is a temporary location for the reference to Parquet metadata
+ // TODO: ideally this should be in a Parquet specific derived class.
+ private ParquetTableMetadata_v1 parquetMeta = null;
+
public FileSelection() {
}
@@ -60,6 +65,13 @@ public class FileSelection {
this(statuses, null);
}
+ public FileSelection(List<String> files, String selectionRoot,
+ ParquetTableMetadata_v1 meta) {
+ this.files = files;
+ this.selectionRoot = selectionRoot;
+ this.parquetMeta = meta;
+ }
+
public FileSelection(List<FileStatus> statuses, String selectionRoot) {
this.statuses = statuses;
this.files = Lists.newArrayList();
@@ -128,6 +140,16 @@ public class FileSelection {
return statuses;
}
+ /**
+ * Return the parquet table metadata that may have been read
+ * from a metadata cache file during creation of this file selection.
+ * It will always be null for non-parquet files and null for cases
+ * where no metadata cache was created.
+ */
+ public ParquetTableMetadata_v1 getParquetMetadata() {
+ return parquetMeta;
+ }
+
private static String commonPath(FileStatus... paths) {
String commonPath = "";
String[][] folders = new String[paths.length][];
http://git-wip-us.apache.org/repos/asf/drill/blob/b4d47c56/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
index ac0d8e3..c30806c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFormatPlugin.java
@@ -230,7 +230,7 @@ public class ParquetFormatPlugin implements FormatPlugin{
// /a/b/c.parquet and the format of the selection root must match that of the file names
// otherwise downstream operations such as partition pruning can break.
Path metaRootPath = Path.getPathWithoutSchemeAndAuthority(metaRootDir.getPath());
- return new FileSelection(fileNames, metaRootPath.toString(), true);
+ return new FileSelection(fileNames, metaRootPath.toString(), metadata /* save metadata for future use */);
} else {
// don't expand yet; ParquetGroupScan's metadata gathering operation
// does that.
http://git-wip-us.apache.org/repos/asf/drill/blob/b4d47c56/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
index 094f00f..55eed43 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
@@ -119,6 +119,12 @@ public class ParquetGroupScan extends AbstractFileGroupScan {
private List<SchemaPath> columns;
private ListMultimap<Integer, RowGroupInfo> mappings;
private List<RowGroupInfo> rowGroupInfos;
+ /**
+ * The parquet table metadata may have already been read
+ * from a metadata cache file earlier; we can re-use during
+ * the ParquetGroupScan and avoid extra loading time.
+ */
+ private ParquetTableMetadata_v1 parquetTableMetadata = null;
/*
* total number of rows (obtained from parquet footer)
@@ -177,6 +183,7 @@ public class ParquetGroupScan extends AbstractFileGroupScan {
}
this.selectionRoot = selectionRoot;
+ this.parquetTableMetadata = selection.getParquetMetadata();
init();
}
@@ -201,6 +208,7 @@ public class ParquetGroupScan extends AbstractFileGroupScan {
this.partitionValueMap = that.partitionValueMap == null ? null : new HashMap(that.partitionValueMap);
this.fileSet = that.fileSet == null ? null : new HashSet(that.fileSet);
this.usedMetadataCache = that.usedMetadataCache;
+ this.parquetTableMetadata = that.parquetTableMetadata;
}
@@ -486,7 +494,6 @@ public class ParquetGroupScan extends AbstractFileGroupScan {
}
private void init() throws IOException {
- ParquetTableMetadata_v1 parquetTableMetadata;
List<FileStatus> fileStatuses = null;
if (entries.size() == 1) {
Path p = Path.getPathWithoutSchemeAndAuthority(new Path(entries.get(0).getPath()));
@@ -498,7 +505,9 @@ public class ParquetGroupScan extends AbstractFileGroupScan {
}
if (metaPath != null && fs.exists(metaPath)) {
usedMetadataCache = true;
- parquetTableMetadata = Metadata.readBlockMeta(fs, metaPath.toString());
+ if (parquetTableMetadata == null) {
+ parquetTableMetadata = Metadata.readBlockMeta(fs, metaPath.toString());
+ }
} else {
parquetTableMetadata = Metadata.getParquetTableMetadata(fs, p.toString());
}
@@ -508,9 +517,15 @@ public class ParquetGroupScan extends AbstractFileGroupScan {
if (fs.isDirectory(new Path(selectionRoot)) && fs.exists(metaPath)) {
usedMetadataCache = true;
if (fileSet != null) {
- parquetTableMetadata = removeUnneededRowGroups(Metadata.readBlockMeta(fs, metaPath.toString()));
+ if (parquetTableMetadata == null) {
+ parquetTableMetadata = removeUnneededRowGroups(Metadata.readBlockMeta(fs, metaPath.toString()));
+ } else {
+ parquetTableMetadata = removeUnneededRowGroups(parquetTableMetadata);
+ }
} else {
- parquetTableMetadata = Metadata.readBlockMeta(fs, metaPath.toString());
+ if (parquetTableMetadata == null) {
+ parquetTableMetadata = Metadata.readBlockMeta(fs, metaPath.toString());
+ }
}
} else {
fileStatuses = Lists.newArrayList();