You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/09/02 00:22:07 UTC

[06/18] git commit: DRILL-191 ParquetGroupScan and AffinityCalculator do not properly handle directories or multiple files

DRILL-191 ParquetGroupScan and AffinityCalculator do not properly handle directories or multiple files


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/f77c5258
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/f77c5258
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/f77c5258

Branch: refs/heads/master
Commit: f77c5258fd58e1ecf6f8cb76d0ecc29b8529965e
Parents: 1a2352c
Author: Steven Phillips <sp...@maprtech.com>
Authored: Thu Aug 29 17:08:37 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Sep 1 15:16:21 2013 -0700

----------------------------------------------------------------------
 .../drill/exec/store/AffinityCalculator.java    | 24 ++++++++-----
 .../exec/store/parquet/ParquetGroupScan.java    | 38 ++++++++++----------
 .../exec/store/TestAffinityCalculator.java      |  6 ++--
 .../exec/store/TestParquetPhysicalPlan.java     | 13 +++++--
 4 files changed, 49 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f77c5258/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java
index b341ea4..6210059 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java
@@ -20,26 +20,25 @@ public class AffinityCalculator {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AffinityCalculator.class);
 
 
-  BlockLocation[] blocks;
-  ImmutableRangeMap<Long,BlockLocation> blockMap;
+  HashMap<String,ImmutableRangeMap<Long,BlockLocation>> blockMapMap = new HashMap<>();
   FileSystem fs;
   String fileName;
   Collection<DrillbitEndpoint> endpoints;
   HashMap<String,DrillbitEndpoint> endPointMap;
-  Stopwatch watch = new Stopwatch();
 
-  public AffinityCalculator(String fileName, FileSystem fs, Collection<DrillbitEndpoint> endpoints) {
+  public AffinityCalculator(FileSystem fs, Collection<DrillbitEndpoint> endpoints) {
     this.fs = fs;
-    this.fileName = fileName;
     this.endpoints = endpoints;
-    buildBlockMap();
     buildEndpointMap();
   }
 
   /**
    * Builds a mapping of block locations to file byte range
    */
-  private void buildBlockMap() {
+  private void buildBlockMap(String fileName) {
+    Stopwatch watch = new Stopwatch();
+    BlockLocation[] blocks;
+    ImmutableRangeMap<Long,BlockLocation> blockMap;
     try {
       watch.start();
       FileStatus file = fs.getFileStatus(new Path(fileName));
@@ -60,6 +59,7 @@ public class AffinityCalculator {
     blockMap = blockMapBuilder.build();
     watch.stop();
     logger.debug("Took {} ms to build block map", watch.elapsed(TimeUnit.MILLISECONDS));
+    blockMapMap.put(fileName, blockMap);
   }
   /**
    * For a given RowGroup, calculate how many bytes are available on each on drillbit endpoint
@@ -67,8 +67,14 @@ public class AffinityCalculator {
    * @param rowGroup the RowGroup to calculate endpoint bytes for
    */
   public void setEndpointBytes(ParquetGroupScan.RowGroupInfo rowGroup) {
-    watch.reset();
+    Stopwatch watch = new Stopwatch();
     watch.start();
+    String fileName = rowGroup.getPath();
+    if (!blockMapMap.containsKey(fileName)) {
+      buildBlockMap(fileName);
+    }
+
+    ImmutableRangeMap<Long,BlockLocation> blockMap = blockMapMap.get(fileName);
     HashMap<String,Long> hostMap = new HashMap<>();
     HashMap<DrillbitEndpoint,Long> endpointByteMap = new HashMap();
     long start = rowGroup.getStart();
@@ -116,7 +122,7 @@ public class AffinityCalculator {
    * Builds a mapping of drillbit endpoints to hostnames
    */
   private void buildEndpointMap() {
-    watch.reset();
+    Stopwatch watch = new Stopwatch();
     watch.start();
     endPointMap = new HashMap<String, DrillbitEndpoint>();
     for (DrillbitEndpoint d : endpoints) {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f77c5258/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
index 54618c1..2dc5102 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
@@ -46,6 +46,7 @@ import org.apache.drill.exec.store.StorageEngineRegistry;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
+import parquet.hadoop.Footer;
 import parquet.hadoop.ParquetFileReader;
 import parquet.hadoop.metadata.BlockMetaData;
 import parquet.hadoop.metadata.ColumnChunkMetaData;
@@ -87,7 +88,6 @@ public class ParquetGroupScan extends AbstractGroupScan {
   private StorageEngineRegistry engineRegistry;
   private ParquetStorageEngineConfig engineConfig;
   private FileSystem fs;
-  private String fileName;
   private final FieldReference ref;
   private List<EndpointAffinity> endpointAffinities;
 
@@ -106,7 +106,6 @@ public class ParquetGroupScan extends AbstractGroupScan {
     this.entries = entries;
     this.ref = ref;
     readFooter();
-    this.fileName = rowGroupInfos.get(0).getPath();
     calculateEndpointBytes();
   }
 
@@ -118,7 +117,6 @@ public class ParquetGroupScan extends AbstractGroupScan {
     this.entries = entries;
     this.ref = ref;
     readFooter();
-    this.fileName = rowGroupInfos.get(0).getPath();
     calculateEndpointBytes();
   }
 
@@ -130,23 +128,27 @@ public class ParquetGroupScan extends AbstractGroupScan {
     ColumnChunkMetaData columnChunkMetaData;
     for (ReadEntryWithPath readEntryWithPath : entries){
       Path path = new Path(readEntryWithPath.getPath());
-      ParquetMetadata footer = ParquetFileReader.readFooter(this.storageEngine.getHadoopConfig(), path);
+      List<Footer> footers = ParquetFileReader.readFooters(this.storageEngine.getHadoopConfig(), path);
       readEntryWithPath.getPath();
 
-      int i = 0;
-      for (BlockMetaData rowGroup : footer.getBlocks()){
-        // need to grab block information from HDFS
-        columnChunkMetaData = rowGroup.getColumns().iterator().next();
-        start = columnChunkMetaData.getFirstDataPageOffset();
-        // this field is not being populated correctly, but the column chunks know their sizes, just summing them for now
-        //end = start + rowGroup.getTotalByteSize();
-        length = 0;
-        for (ColumnChunkMetaData col : rowGroup.getColumns()){
-          length += col.getTotalSize();
+      for (Footer footer : footers) {
+        int index = 0;
+        ParquetMetadata metadata = footer.getParquetMetadata();
+        for (BlockMetaData rowGroup : metadata.getBlocks()){
+          // need to grab block information from HDFS
+          columnChunkMetaData = rowGroup.getColumns().iterator().next();
+          start = columnChunkMetaData.getFirstDataPageOffset();
+          // this field is not being populated correctly, but the column chunks know their sizes, just summing them for now
+          //end = start + rowGroup.getTotalByteSize();
+          length = 0;
+          for (ColumnChunkMetaData col : rowGroup.getColumns()){
+            length += col.getTotalSize();
+          }
+          String filePath = footer.getFile().toUri().getPath();
+          rowGroupInfos.add(new ParquetGroupScan.RowGroupInfo(filePath, start, length, index));
+          logger.debug("rowGroupInfo path: {} start: {} length {}", filePath, start, length);
+          index++;
         }
-        rowGroupInfos.add(new ParquetGroupScan.RowGroupInfo(readEntryWithPath.getPath(), start, length, i));
-        logger.debug("rowGroupInfo path: {} start: {} length {}", readEntryWithPath.getPath(), start, length);
-        i++;
       }
     }
     watch.stop();
@@ -156,7 +158,7 @@ public class ParquetGroupScan extends AbstractGroupScan {
   private void calculateEndpointBytes() {
     watch.reset();
     watch.start();
-    AffinityCalculator ac = new AffinityCalculator(fileName, fs, availableEndpoints);
+    AffinityCalculator ac = new AffinityCalculator(fs, availableEndpoints);
     for (RowGroupInfo e : rowGroupInfos) {
       ac.setEndpointBytes(e);
       totalBytes += e.getLength();

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f77c5258/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java
index cff7c86..b25c805 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java
@@ -103,7 +103,7 @@ public class TestAffinityCalculator {
     }};
 
 
-    AffinityCalculator ac = new AffinityCalculator(path, fs, endPoints);
+    AffinityCalculator ac = new AffinityCalculator(fs, endPoints);
     for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) {
       ac.setEndpointBytes(rowGroup);
     }
@@ -115,7 +115,7 @@ public class TestAffinityCalculator {
 
     buildRowGroups(rowGroups, numberOfBlocks, blockSize, 2);
 
-    ac = new AffinityCalculator(path, fs, endPoints);
+    ac = new AffinityCalculator(fs, endPoints);
     for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) {
       ac.setEndpointBytes(rowGroup);
     }
@@ -127,7 +127,7 @@ public class TestAffinityCalculator {
 
     buildRowGroups(rowGroups, numberOfBlocks, blockSize, 6);
 
-    ac = new AffinityCalculator(path, fs, endPoints);
+    ac = new AffinityCalculator(fs, endPoints);
     for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) {
       ac.setEndpointBytes(rowGroup);
     }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f77c5258/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
index 18ac294..c58bc71 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
@@ -44,7 +44,7 @@ public class TestParquetPhysicalPlan {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestParquetPhysicalPlan.class);
 
   //public String fileName = "/physical_test2.json";
-  public String fileName = "parquet_scan_union_screen_physical.json";
+  public String fileName = "parquet/parquet_scan_union_screen_physical.json";
 //  public String fileName = "parquet-sample.json";
 
 
@@ -59,19 +59,28 @@ public class TestParquetPhysicalPlan {
       client.connect();
       List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, Resources.toString(Resources.getResource(fileName),Charsets.UTF_8));
       RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
+      int count = 0;
       for (QueryResultBatch b : results) {
         System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
+        count += b.getHeader().getRowCount();
         loader.load(b.getHeader().getDef(), b.getData());
         for (VectorWrapper vw : loader) {
+//          System.out.println(vw.getValueVector().getField().getName() + vw.getValueVector().getField().getType());
           System.out.println(vw.getValueVector().getField().getName());
           ValueVector vv = vw.getValueVector();
           for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
             Object o = vv.getAccessor().getObject(i);
-            System.out.println(vv.getAccessor().getObject(i));
+            if (o instanceof byte[]) {
+              System.out.println(new String((byte[]) o));
+            } else {
+              System.out.println(vv.getAccessor().getObject(i));
+            }
+            break;
           }
         }
       }
       client.close();
+      System.out.println(String.format("Got %d total results", count));
     }
   }