You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ja...@apache.org on 2014/08/07 05:55:55 UTC
[04/20] git commit: DRILL-1232: Fix unsupported exception while using
'where' clause with partition names
DRILL-1232: Fix unsupported exception while using 'where' clause with partition names
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/07483f22
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/07483f22
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/07483f22
Branch: refs/heads/master
Commit: 07483f2226f2b07b0ebc98cfc213077de3adcb22
Parents: 3da5d70
Author: Mehant Baid <me...@gmail.com>
Authored: Fri Aug 1 17:35:14 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Wed Aug 6 16:44:21 2014 -0700
----------------------------------------------------------------------
.../apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java | 8 +++++++-
.../drill/exec/store/parquet/ParquetScanBatchCreator.java | 8 +++++---
2 files changed, 12 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/07483f22/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
index bdab07f..bf8e301 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.store.dfs.easy;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.regex.Matcher;
@@ -126,15 +127,20 @@ public abstract class EasyFormatPlugin<T extends FormatPluginConfig> implements
if (columns == null || columns.size() == 0) {
selectAllColumns = true;
} else {
+ List<SchemaPath> newColumns = Lists.newArrayList();
Pattern pattern = Pattern.compile(String.format("%s[0-9]+", partitionDesignator));
for (SchemaPath column : columns) {
Matcher m = pattern.matcher(column.getAsUnescapedPath());
if (m.matches()) {
- scan.getColumns().remove(column);
selectedPartitionColumns.add(Integer.parseInt(column.getAsUnescapedPath().toString().substring(partitionDesignator.length())));
+ } else {
+ newColumns.add(column);
}
}
+ // Create a new sub scan object with the new set of columns;
+ scan = new EasySubScan(scan.getWorkUnits(), scan.getFormatPlugin(), newColumns, scan.getSelectionRoot());
}
+
int numParts = 0;
for(FileWork work : scan.getWorkUnits()){
readers.add(getRecordReader(context, work, scan.getColumns()));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/07483f22/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
index ae73af3..5edd3c5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
@@ -61,7 +61,6 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan
Preconditions.checkArgument(children.isEmpty());
String partitionDesignator = context.getConfig().getString(ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL);
List<SchemaPath> columns = rowGroupScan.getColumns();
-
List<RecordReader> readers = Lists.newArrayList();
List<String[]> partitionColumns = Lists.newArrayList();
@@ -71,17 +70,20 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan
if (columns == null || columns.size() == 0) {
selectAllColumns = true;
} else {
+ List<SchemaPath> newColums = Lists.newArrayList();
Pattern pattern = Pattern.compile(String.format("%s[0-9]+", partitionDesignator));
for (SchemaPath column : columns) {
Matcher m = pattern.matcher(column.getAsUnescapedPath());
if (m.matches()) {
- columns.remove(column);
selectedPartitionColumns.add(Integer.parseInt(column.getAsUnescapedPath().toString().substring(partitionDesignator.length())));
+ } else {
+ newColums.add(column);
}
}
+ // Create the new row group scan with the new columns
+ rowGroupScan = new ParquetRowGroupScan(rowGroupScan.getStorageEngine(), rowGroupScan.getRowGroupReadEntries(), newColums, rowGroupScan.getSelectionRoot());
}
-
FileSystem fs = rowGroupScan.getStorageEngine().getFileSystem().getUnderlying();
Configuration conf = fs.getConf();
conf.setBoolean(ENABLE_BYTES_READ_COUNTER, false);