You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by am...@apache.org on 2016/09/09 15:08:49 UTC
drill git commit: DRILL-4877: If pruning was not applicable only keep
the selectionRoot in the entries field.
Repository: drill
Updated Branches:
refs/heads/master f11f32129 -> 18866d5b0
DRILL-4877: If pruning was not applicable only keep the selectionRoot in the entries field.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/18866d5b
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/18866d5b
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/18866d5b
Branch: refs/heads/master
Commit: 18866d5b0652ad540bc0cc3889c3360441870670
Parents: f11f321
Author: Aman Sinha <as...@maprtech.com>
Authored: Wed Sep 7 22:56:14 2016 -0700
Committer: Aman Sinha <as...@maprtech.com>
Committed: Thu Sep 8 14:33:37 2016 -0700
----------------------------------------------------------------------
.../drill/exec/store/dfs/MetadataContext.java | 8 ++-----
.../exec/store/parquet/ParquetGroupScan.java | 11 +++++----
.../store/parquet/TestParquetMetadataCache.java | 25 ++++++++++++++++++++
3 files changed, 33 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/18866d5b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MetadataContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MetadataContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MetadataContext.java
index d115bee..aff8367 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MetadataContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/MetadataContext.java
@@ -67,12 +67,8 @@ public class MetadataContext {
pruneStatus = status;
}
- public boolean wasPruningStarted() {
- return pruneStatus != PruneStatus.NOT_STARTED;
- }
-
- public boolean wasPruned() {
- return pruneStatus == PruneStatus.PRUNED;
+ public PruneStatus getPruneStatus() {
+ return pruneStatus;
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/18866d5b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
index 8b255cf..ec34e7a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
@@ -49,6 +49,7 @@ import org.apache.drill.exec.store.dfs.DrillFileSystem;
import org.apache.drill.exec.store.dfs.DrillPathFilter;
import org.apache.drill.exec.store.dfs.FileSelection;
import org.apache.drill.exec.store.dfs.MetadataContext;
+import org.apache.drill.exec.store.dfs.MetadataContext.PruneStatus;
import org.apache.drill.exec.store.dfs.ReadEntryFromHDFS;
import org.apache.drill.exec.store.dfs.ReadEntryWithPath;
import org.apache.drill.exec.store.dfs.easy.FileWork;
@@ -174,11 +175,11 @@ public class ParquetGroupScan extends AbstractFileGroupScan {
this.entries = Lists.newArrayList();
if (fileSelection.getMetaContext() != null &&
- (fileSelection.getMetaContext().wasPruningStarted() &&
- ! fileSelection.getMetaContext().wasPruned())) {
- // if pruning was attempted and nothing was pruned, initialize the entries with just
- // the selection root instead of the fully expanded list to reduce overhead. The fully
- // expanded list is already stored as part of the fileSet.
+ (fileSelection.getMetaContext().getPruneStatus() == PruneStatus.NOT_STARTED ||
+ fileSelection.getMetaContext().getPruneStatus() == PruneStatus.NOT_PRUNED)) {
+ // if pruning was not applicable or was attempted and nothing was pruned, initialize the
+ // entries with just the selection root instead of the fully expanded list to reduce overhead.
+ // The fully expanded list is already stored as part of the fileSet.
// TODO: at some point we should examine whether the list of entries is absolutely needed.
entries.add(new ReadEntryWithPath(fileSelection.getSelectionRoot()));
} else {
http://git-wip-us.apache.org/repos/asf/drill/blob/18866d5b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java
index 342ee91..e199ba5 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java
@@ -371,7 +371,32 @@ public class TestParquetMetadataCache extends PlanTestBase {
}
+ @Test // DRILL-4877
+ public void testDrill4877() throws Exception {
+ // create metadata cache
+ test(String.format("refresh table metadata dfs_test.`%s/%s`", getDfsTestTmpSchemaLocation(), tableName2));
+ checkForMetadataFile(tableName2);
+
+ // run query and check correctness
+ String query1 = String.format("select max(dir0) as max0, max(dir1) as max1 from dfs_test.`%s/%s` ",
+ getDfsTestTmpSchemaLocation(), tableName2);
+
+ testBuilder()
+ .sqlQuery(query1)
+ .unOrdered()
+ .baselineColumns("max0", "max1")
+ .baselineValues("1995", "Q4")
+ .go();
+
+ int expectedNumFiles = 1; // point to selectionRoot since no pruning is done in this query
+ String numFilesPattern = "numFiles=" + expectedNumFiles;
+ String usedMetaPattern = "usedMetadataFile=true";
+ String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", getDfsTestTmpSchemaLocation(), tableName2);
+ PlanTestBase.testPlanMatchingPatterns(query1, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern},
+ new String[] {});
+
+ }
private void checkForMetadataFile(String table) throws Exception {
String tmpDir = getDfsTestTmpSchemaLocation();