You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by am...@apache.org on 2016/06/23 18:10:46 UTC
[6/7] drill git commit: DRILL-4733: max(dir0) reading more columns
than necessary
DRILL-4733: max(dir0) reading more columns than necessary
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/19cdcd95
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/19cdcd95
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/19cdcd95
Branch: refs/heads/1.7.0
Commit: 19cdcd95808ddb6f9e19bbccb753017993ad78d6
Parents: 4687a8b
Author: Arina Ielchiieva <ar...@gmail.com>
Authored: Tue Jun 21 12:33:32 2016 +0000
Committer: Aman Sinha <as...@maprtech.com>
Committed: Thu Jun 23 10:29:38 2016 -0700
----------------------------------------------------------------------
.../drill/exec/store/ImplicitColumnExplorer.java | 6 ------
.../exec/store/dfs/easy/EasyFormatPlugin.java | 7 ++++++-
.../drill/exec/store/TestImplicitFileColumns.java | 17 +++++++++++++++++
.../parquetWithSchemaChange/voter5/voter5.parquet | Bin 0 -> 1250 bytes
.../voter50/voter50.parquet | Bin 0 -> 2960 bytes
5 files changed, 23 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/19cdcd95/exec/java-exec/src/main/java/org/apache/drill/exec/store/ImplicitColumnExplorer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ImplicitColumnExplorer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ImplicitColumnExplorer.java
index 94a0dca..af74eb7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ImplicitColumnExplorer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ImplicitColumnExplorer.java
@@ -135,12 +135,6 @@ public class ImplicitColumnExplorer {
tableColumns.add(column);
}
}
-
- // We must make sure to pass a table column(not to be confused with partition column) to the underlying record
- // reader.
- if (tableColumns.size() == 0) {
- tableColumns.add(AbstractRecordReader.STAR_COLUMN);
- }
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/19cdcd95/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
index 5881d33..f56f445 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
@@ -41,6 +41,7 @@ import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.record.CloseableRecordBatch;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.store.AbstractRecordReader;
import org.apache.drill.exec.store.ImplicitColumnExplorer;
import org.apache.drill.exec.store.RecordReader;
import org.apache.drill.exec.store.RecordWriter;
@@ -126,8 +127,12 @@ public abstract class EasyFormatPlugin<T extends FormatPluginConfig> implements
final ImplicitColumnExplorer columnExplorer = new ImplicitColumnExplorer(context, scan.getColumns());
if (!columnExplorer.isSelectAllColumns()) {
+ // We must make sure to pass a table column (not to be confused with implicit column) to the underlying record reader.
+ List<SchemaPath> tableColumns =
+ columnExplorer.getTableColumns().size() == 0 ?
+ Lists.<SchemaPath>newArrayList(AbstractRecordReader.STAR_COLUMN) : columnExplorer.getTableColumns();
scan = new EasySubScan(scan.getUserName(), scan.getWorkUnits(), scan.getFormatPlugin(),
- columnExplorer.getTableColumns(), scan.getSelectionRoot());
+ tableColumns, scan.getSelectionRoot());
scan.setOperatorId(scan.getOperatorId());
}
http://git-wip-us.apache.org/repos/asf/drill/blob/19cdcd95/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java
index 6900da9..ce0f5e9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.store;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
import org.apache.drill.BaseTestQuery;
+import org.apache.drill.common.util.TestTools;
import org.apache.drill.exec.util.JsonStringArrayList;
import org.apache.drill.exec.util.Text;
import org.apache.hadoop.fs.Path;
@@ -110,4 +111,20 @@ public class TestImplicitFileColumns extends BaseTestQuery {
.go();
}
+ @Test // DRILL-4733
+ public void testMultilevelParquetWithSchemaChange() throws Exception {
+ try {
+ test("alter session set `planner.enable_decimal_data_type` = true");
+ testBuilder()
+ .sqlQuery(String.format("select max(dir0) as max_dir from dfs_test.`%s/src/test/resources/multilevel/parquetWithSchemaChange`",
+ TestTools.getWorkingPath()))
+ .unOrdered()
+ .baselineColumns("max_dir")
+ .baselineValues("voter50")
+ .go();
+ } finally {
+ test("alter session set `planner.enable_decimal_data_type` = false");
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/drill/blob/19cdcd95/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter5/voter5.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter5/voter5.parquet b/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter5/voter5.parquet
new file mode 100644
index 0000000..cc76280
Binary files /dev/null and b/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter5/voter5.parquet differ
http://git-wip-us.apache.org/repos/asf/drill/blob/19cdcd95/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter50/voter50.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter50/voter50.parquet b/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter50/voter50.parquet
new file mode 100644
index 0000000..ff66b42
Binary files /dev/null and b/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter50/voter50.parquet differ