You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by jn...@apache.org on 2015/07/24 20:06:28 UTC
drill git commit: DRILL-3533: Fix incorrect query result when
querying parquet for fields that do not exists.
Repository: drill
Updated Branches:
refs/heads/master 66a30f12b -> 1b69869d9
DRILL-3533: Fix incorrect query result when querying parquet for fields that do not exists.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/1b69869d
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/1b69869d
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/1b69869d
Branch: refs/heads/master
Commit: 1b69869d981eece6c23a2e52a413133f993f3125
Parents: 66a30f1
Author: Jinfeng Ni <jn...@apache.org>
Authored: Tue Jul 21 22:00:40 2015 -0700
Committer: Jinfeng Ni <jn...@apache.org>
Committed: Thu Jul 23 15:54:37 2015 -0700
----------------------------------------------------------------------
.../exec/store/parquet2/DrillParquetReader.java | 5 ++++-
.../exec/store/parquet/TestParquetComplex.java | 14 ++++++++++++++
.../resources/store/parquet/complex/baseline8.json | 16 ++++++++++++++++
3 files changed, 34 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/1b69869d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
index 4e7d628..4c49def 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.PathSegment;
@@ -259,7 +260,9 @@ public class DrillParquetReader extends AbstractRecordReader {
if(!noColumnsFound) {
writer = new VectorContainerWriter(output);
- recordMaterializer = new DrillParquetRecordMaterializer(output, writer, projection, getColumns(),
+ // Discard the columns not found in the schema when create DrillParquetRecordMaterializer, since they have been added to output already.
+ final Collection<SchemaPath> columns = columnsNotFound == null || columnsNotFound.size() == 0 ? getColumns(): CollectionUtils.subtract(getColumns(), columnsNotFound);
+ recordMaterializer = new DrillParquetRecordMaterializer(output, writer, projection, columns,
fragmentContext.getOptions());
primitiveVectors = writer.getMapVector().getPrimitiveVectors();
recordReader = columnIO.getRecordReader(pageReadStore, recordMaterializer);
http://git-wip-us.apache.org/repos/asf/drill/blob/1b69869d/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java
index baf5e82..6397ef7 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java
@@ -177,4 +177,18 @@ public class TestParquetComplex extends BaseTestQuery {
.build()
.run();
}
+
+ @Test //DRILL-3533
+ public void notxistsField() throws Exception {
+ String query = String.format("select t.`marketing_info`.notexists as notexists, t.`marketing_info`.camp_id as id from %s t", DATAFILE);
+ String[] columns = {"notexists", "id"};
+ testBuilder()
+ .sqlQuery(query)
+ .unOrdered()
+ .jsonBaselineFile("store/parquet/complex/baseline8.json")
+ .baselineColumns(columns)
+ .build()
+ .run();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/drill/blob/1b69869d/exec/java-exec/src/test/resources/store/parquet/complex/baseline8.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/store/parquet/complex/baseline8.json b/exec/java-exec/src/test/resources/store/parquet/complex/baseline8.json
new file mode 100644
index 0000000..290d896
--- /dev/null
+++ b/exec/java-exec/src/test/resources/store/parquet/complex/baseline8.json
@@ -0,0 +1,16 @@
+{
+ "notexists" : null,
+ "id" : 4
+} {
+ "notexists" : null,
+ "id" : 6
+} {
+ "notexists" : null,
+ "id" : 17
+} {
+ "notexists" : null,
+ "id" : 17
+} {
+ "notexists" : null,
+ "id" : 8
+}
\ No newline at end of file