You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by pa...@apache.org on 2014/12/12 17:24:13 UTC
[07/10] drill git commit: DRILL-1845: Do not pass partition columns
to parquet reader
DRILL-1845: Do not pass partition columns to parquet reader
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/e735808c
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/e735808c
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/e735808c
Branch: refs/heads/master
Commit: e735808cd743e65f55cb9a5650afc8d334fb37fc
Parents: 08e555e
Author: Hanifi Gunes <hg...@maprtech.com>
Authored: Thu Dec 11 15:06:51 2014 -0800
Committer: Parth Chandra <pc...@maprtech.com>
Committed: Thu Dec 11 16:58:34 2014 -0800
----------------------------------------------------------------------
.../org/apache/drill/exec/physical/base/GroupScan.java | 3 ++-
.../exec/store/parquet/ParquetScanBatchCreator.java | 13 +++++++++----
2 files changed, 11 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/e735808c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
index 935d817..23860a3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.physical.base;
import java.util.List;
+import com.google.common.collect.ImmutableList;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.physical.PhysicalOperatorSetupException;
@@ -33,7 +34,7 @@ import com.google.common.collect.Lists;
*/
public interface GroupScan extends Scan, HasAffinity{
- public static final List<SchemaPath> ALL_COLUMNS = Lists.<SchemaPath>newArrayList(SchemaPath.getSimplePath("*"));
+ public static final List<SchemaPath> ALL_COLUMNS = ImmutableList.of(SchemaPath.getSimplePath("*"));
public static final long NO_COLUMN_STATS = -1;
public abstract void applyAssignments(List<DrillbitEndpoint> endpoints) throws PhysicalOperatorSetupException;
http://git-wip-us.apache.org/repos/asf/drill/blob/e735808c/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
index 53a6ffc..dc1d892 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
@@ -29,6 +29,7 @@ import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.base.GroupScan;
import org.apache.drill.exec.physical.impl.BatchCreator;
import org.apache.drill.exec.physical.impl.ScanBatch;
import org.apache.drill.exec.record.RecordBatch;
@@ -68,20 +69,24 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan
List<Integer> selectedPartitionColumns = Lists.newArrayList();
boolean selectAllColumns = AbstractRecordReader.isStarQuery(columns);
+ List<SchemaPath> newColumns = columns;
if (!selectAllColumns) {
- List<SchemaPath> newColums = Lists.newArrayList();
+ newColumns = Lists.newArrayList();
Pattern pattern = Pattern.compile(String.format("%s[0-9]+", partitionDesignator));
for (SchemaPath column : columns) {
Matcher m = pattern.matcher(column.getAsUnescapedPath());
if (m.matches()) {
selectedPartitionColumns.add(Integer.parseInt(column.getAsUnescapedPath().toString().substring(partitionDesignator.length())));
} else {
- newColums.add(column);
+ newColumns.add(column);
}
}
+ if (newColumns.isEmpty()) {
+ newColumns = GroupScan.ALL_COLUMNS;
+ }
final int id = rowGroupScan.getOperatorId();
// Create the new row group scan with the new columns
- rowGroupScan = new ParquetRowGroupScan(rowGroupScan.getStorageEngine(), rowGroupScan.getRowGroupReadEntries(), newColums, rowGroupScan.getSelectionRoot());
+ rowGroupScan = new ParquetRowGroupScan(rowGroupScan.getStorageEngine(), rowGroupScan.getRowGroupReadEntries(), newColumns, rowGroupScan.getSelectionRoot());
rowGroupScan.setOperatorId(id);
}
@@ -118,7 +123,7 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan
);
} else {
ParquetMetadata footer = footers.get(e.getPath());
- readers.add(new DrillParquetReader(context, footer, e, columns, conf));
+ readers.add(new DrillParquetReader(context, footer, e, newColumns, conf));
}
if (rowGroupScan.getSelectionRoot() != null) {
String[] r = rowGroupScan.getSelectionRoot().split("/");