You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/09/02 00:22:07 UTC
[06/18] git commit: DRILL-191 ParquetGroupScan and AffinityCalculator
do not properly handle directories or multiple files
DRILL-191 ParquetGroupScan and AffinityCalculator do not properly handle directories or multiple files
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/f77c5258
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/f77c5258
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/f77c5258
Branch: refs/heads/master
Commit: f77c5258fd58e1ecf6f8cb76d0ecc29b8529965e
Parents: 1a2352c
Author: Steven Phillips <sp...@maprtech.com>
Authored: Thu Aug 29 17:08:37 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Sep 1 15:16:21 2013 -0700
----------------------------------------------------------------------
.../drill/exec/store/AffinityCalculator.java | 24 ++++++++-----
.../exec/store/parquet/ParquetGroupScan.java | 38 ++++++++++----------
.../exec/store/TestAffinityCalculator.java | 6 ++--
.../exec/store/TestParquetPhysicalPlan.java | 13 +++++--
4 files changed, 49 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f77c5258/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java
index b341ea4..6210059 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java
@@ -20,26 +20,25 @@ public class AffinityCalculator {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AffinityCalculator.class);
- BlockLocation[] blocks;
- ImmutableRangeMap<Long,BlockLocation> blockMap;
+ HashMap<String,ImmutableRangeMap<Long,BlockLocation>> blockMapMap = new HashMap<>();
FileSystem fs;
String fileName;
Collection<DrillbitEndpoint> endpoints;
HashMap<String,DrillbitEndpoint> endPointMap;
- Stopwatch watch = new Stopwatch();
- public AffinityCalculator(String fileName, FileSystem fs, Collection<DrillbitEndpoint> endpoints) {
+ public AffinityCalculator(FileSystem fs, Collection<DrillbitEndpoint> endpoints) {
this.fs = fs;
- this.fileName = fileName;
this.endpoints = endpoints;
- buildBlockMap();
buildEndpointMap();
}
/**
* Builds a mapping of block locations to file byte range
*/
- private void buildBlockMap() {
+ private void buildBlockMap(String fileName) {
+ Stopwatch watch = new Stopwatch();
+ BlockLocation[] blocks;
+ ImmutableRangeMap<Long,BlockLocation> blockMap;
try {
watch.start();
FileStatus file = fs.getFileStatus(new Path(fileName));
@@ -60,6 +59,7 @@ public class AffinityCalculator {
blockMap = blockMapBuilder.build();
watch.stop();
logger.debug("Took {} ms to build block map", watch.elapsed(TimeUnit.MILLISECONDS));
+ blockMapMap.put(fileName, blockMap);
}
/**
* For a given RowGroup, calculate how many bytes are available on each on drillbit endpoint
@@ -67,8 +67,14 @@ public class AffinityCalculator {
* @param rowGroup the RowGroup to calculate endpoint bytes for
*/
public void setEndpointBytes(ParquetGroupScan.RowGroupInfo rowGroup) {
- watch.reset();
+ Stopwatch watch = new Stopwatch();
watch.start();
+ String fileName = rowGroup.getPath();
+ if (!blockMapMap.containsKey(fileName)) {
+ buildBlockMap(fileName);
+ }
+
+ ImmutableRangeMap<Long,BlockLocation> blockMap = blockMapMap.get(fileName);
HashMap<String,Long> hostMap = new HashMap<>();
HashMap<DrillbitEndpoint,Long> endpointByteMap = new HashMap();
long start = rowGroup.getStart();
@@ -116,7 +122,7 @@ public class AffinityCalculator {
* Builds a mapping of drillbit endpoints to hostnames
*/
private void buildEndpointMap() {
- watch.reset();
+ Stopwatch watch = new Stopwatch();
watch.start();
endPointMap = new HashMap<String, DrillbitEndpoint>();
for (DrillbitEndpoint d : endpoints) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f77c5258/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
index 54618c1..2dc5102 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
@@ -46,6 +46,7 @@ import org.apache.drill.exec.store.StorageEngineRegistry;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import parquet.hadoop.Footer;
import parquet.hadoop.ParquetFileReader;
import parquet.hadoop.metadata.BlockMetaData;
import parquet.hadoop.metadata.ColumnChunkMetaData;
@@ -87,7 +88,6 @@ public class ParquetGroupScan extends AbstractGroupScan {
private StorageEngineRegistry engineRegistry;
private ParquetStorageEngineConfig engineConfig;
private FileSystem fs;
- private String fileName;
private final FieldReference ref;
private List<EndpointAffinity> endpointAffinities;
@@ -106,7 +106,6 @@ public class ParquetGroupScan extends AbstractGroupScan {
this.entries = entries;
this.ref = ref;
readFooter();
- this.fileName = rowGroupInfos.get(0).getPath();
calculateEndpointBytes();
}
@@ -118,7 +117,6 @@ public class ParquetGroupScan extends AbstractGroupScan {
this.entries = entries;
this.ref = ref;
readFooter();
- this.fileName = rowGroupInfos.get(0).getPath();
calculateEndpointBytes();
}
@@ -130,23 +128,27 @@ public class ParquetGroupScan extends AbstractGroupScan {
ColumnChunkMetaData columnChunkMetaData;
for (ReadEntryWithPath readEntryWithPath : entries){
Path path = new Path(readEntryWithPath.getPath());
- ParquetMetadata footer = ParquetFileReader.readFooter(this.storageEngine.getHadoopConfig(), path);
+ List<Footer> footers = ParquetFileReader.readFooters(this.storageEngine.getHadoopConfig(), path);
readEntryWithPath.getPath();
- int i = 0;
- for (BlockMetaData rowGroup : footer.getBlocks()){
- // need to grab block information from HDFS
- columnChunkMetaData = rowGroup.getColumns().iterator().next();
- start = columnChunkMetaData.getFirstDataPageOffset();
- // this field is not being populated correctly, but the column chunks know their sizes, just summing them for now
- //end = start + rowGroup.getTotalByteSize();
- length = 0;
- for (ColumnChunkMetaData col : rowGroup.getColumns()){
- length += col.getTotalSize();
+ for (Footer footer : footers) {
+ int index = 0;
+ ParquetMetadata metadata = footer.getParquetMetadata();
+ for (BlockMetaData rowGroup : metadata.getBlocks()){
+ // need to grab block information from HDFS
+ columnChunkMetaData = rowGroup.getColumns().iterator().next();
+ start = columnChunkMetaData.getFirstDataPageOffset();
+ // this field is not being populated correctly, but the column chunks know their sizes, just summing them for now
+ //end = start + rowGroup.getTotalByteSize();
+ length = 0;
+ for (ColumnChunkMetaData col : rowGroup.getColumns()){
+ length += col.getTotalSize();
+ }
+ String filePath = footer.getFile().toUri().getPath();
+ rowGroupInfos.add(new ParquetGroupScan.RowGroupInfo(filePath, start, length, index));
+ logger.debug("rowGroupInfo path: {} start: {} length {}", filePath, start, length);
+ index++;
}
- rowGroupInfos.add(new ParquetGroupScan.RowGroupInfo(readEntryWithPath.getPath(), start, length, i));
- logger.debug("rowGroupInfo path: {} start: {} length {}", readEntryWithPath.getPath(), start, length);
- i++;
}
}
watch.stop();
@@ -156,7 +158,7 @@ public class ParquetGroupScan extends AbstractGroupScan {
private void calculateEndpointBytes() {
watch.reset();
watch.start();
- AffinityCalculator ac = new AffinityCalculator(fileName, fs, availableEndpoints);
+ AffinityCalculator ac = new AffinityCalculator(fs, availableEndpoints);
for (RowGroupInfo e : rowGroupInfos) {
ac.setEndpointBytes(e);
totalBytes += e.getLength();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f77c5258/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java
index cff7c86..b25c805 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java
@@ -103,7 +103,7 @@ public class TestAffinityCalculator {
}};
- AffinityCalculator ac = new AffinityCalculator(path, fs, endPoints);
+ AffinityCalculator ac = new AffinityCalculator(fs, endPoints);
for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) {
ac.setEndpointBytes(rowGroup);
}
@@ -115,7 +115,7 @@ public class TestAffinityCalculator {
buildRowGroups(rowGroups, numberOfBlocks, blockSize, 2);
- ac = new AffinityCalculator(path, fs, endPoints);
+ ac = new AffinityCalculator(fs, endPoints);
for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) {
ac.setEndpointBytes(rowGroup);
}
@@ -127,7 +127,7 @@ public class TestAffinityCalculator {
buildRowGroups(rowGroups, numberOfBlocks, blockSize, 6);
- ac = new AffinityCalculator(path, fs, endPoints);
+ ac = new AffinityCalculator(fs, endPoints);
for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) {
ac.setEndpointBytes(rowGroup);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f77c5258/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
index 18ac294..c58bc71 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
@@ -44,7 +44,7 @@ public class TestParquetPhysicalPlan {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestParquetPhysicalPlan.class);
//public String fileName = "/physical_test2.json";
- public String fileName = "parquet_scan_union_screen_physical.json";
+ public String fileName = "parquet/parquet_scan_union_screen_physical.json";
// public String fileName = "parquet-sample.json";
@@ -59,19 +59,28 @@ public class TestParquetPhysicalPlan {
client.connect();
List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, Resources.toString(Resources.getResource(fileName),Charsets.UTF_8));
RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
+ int count = 0;
for (QueryResultBatch b : results) {
System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
+ count += b.getHeader().getRowCount();
loader.load(b.getHeader().getDef(), b.getData());
for (VectorWrapper vw : loader) {
+// System.out.println(vw.getValueVector().getField().getName() + vw.getValueVector().getField().getType());
System.out.println(vw.getValueVector().getField().getName());
ValueVector vv = vw.getValueVector();
for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
Object o = vv.getAccessor().getObject(i);
- System.out.println(vv.getAccessor().getObject(i));
+ if (o instanceof byte[]) {
+ System.out.println(new String((byte[]) o));
+ } else {
+ System.out.println(vv.getAccessor().getObject(i));
+ }
+ break;
}
}
}
client.close();
+ System.out.println(String.format("Got %d total results", count));
}
}