You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2014/03/11 22:57:29 UTC

[1/5] git commit: DRILL-396: QuerySubmitter enhancements

Repository: incubator-drill
Updated Branches:
  refs/heads/master 08a9a90d9 -> 2d10afd4b


DRILL-396: QuerySubmitter enhancements


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/2ed7a678
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/2ed7a678
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/2ed7a678

Branch: refs/heads/master
Commit: 2ed7a678b1caef67279a775d7e8b217e3e18041f
Parents: 08a9a90
Author: Steven Phillips <sp...@maprtech.com>
Authored: Tue Mar 11 08:26:13 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Mar 11 08:26:13 2014 -0700

----------------------------------------------------------------------
 .../drill/exec/client/QuerySubmitter.java       | 101 +++++++++++++++----
 .../org/apache/drill/exec/util/VectorUtil.java  |  34 +++++++
 2 files changed, 116 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2ed7a678/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
index b07b3ae..0253069 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
@@ -20,10 +20,13 @@ package org.apache.drill.exec.client;
 import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.exec.coord.ZKClusterCoordinator;
 import org.apache.drill.exec.exception.SchemaChangeException;
@@ -64,33 +67,47 @@ public class QuerySubmitter {
       jc.usage();
       System.exit(0);
     }
-    System.exit(submitter.submitQuery(o.location, o.planType, o.zk, o.local, o.bits));
+
+    System.exit(submitter.submitQuery(o.location, o.queryString, o.planType, o.zk, o.local, o.bits, o.format));
   }
 
   static class Options {
-    @Parameter(names = {"-f"}, description = "file containing plan", required=true)
+    @Parameter(names = {"-f, --file"}, description = "file containing plan", required=false)
     public String location = null;
 
-    @Parameter(names = {"-t"}, description = "type of plan, logical/physical", required=true)
+    @Parameter(names = {"-q", "-e", "--query"}, description = "query string", required = false)
+    public String queryString = null;
+
+    @Parameter(names = {"-t", "--type"}, description = "type of query, sql/logical/physical", required=true)
     public String planType;
 
-    @Parameter(names = {"-zk"}, description = "zookeeper connect string.", required=false)
+    @Parameter(names = {"-z", "--zookeeper"}, description = "zookeeper connect string.", required=false)
     public String zk = "localhost:2181";
 
-    @Parameter(names = {"-local"}, description = "run query in local mode", required=false)
+    @Parameter(names = {"-l", "--local"}, description = "run query in local mode", required=false)
     public boolean local;
 
-    @Parameter(names = "-bits", description = "number of drillbits to run. local mode only", required=false)
+    @Parameter(names = {"-b", "--bits"}, description = "number of drillbits to run. local mode only", required=false)
     public int bits = 1;
 
-    @Parameter(names = {"-h", "-help", "--help"}, description = "show usage", help=true)
+    @Parameter(names = {"-h", "--help"}, description = "show usage", help=true)
     public boolean help = false;
+
+    @Parameter(names = {"--format"}, description = "output format, csv,tsv,table", required = false)
+    public String format = "table";
   }
 
-  public int submitQuery(String planLocation, String type, String zkQuorum, boolean local, int bits) throws Exception {
+  public enum Format {
+    TSV, CSV, TABLE
+  }
+
+  public int submitQuery(String planLocation, String queryString, String type, String zkQuorum, boolean local, int bits, String format) throws Exception {
     DrillConfig config = DrillConfig.create();
     DrillClient client = null;
-    
+
+    Preconditions.checkArgument(!(planLocation == null && queryString == null), "Must provide either query file or query string");
+    Preconditions.checkArgument(!(planLocation != null && queryString != null), "Must provide either query file or query string, not both");
+
     try{
       if (local) {
         RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
@@ -106,29 +123,62 @@ public class QuerySubmitter {
         client = new DrillClient(config, clusterCoordinator);
       }
       client.connect();
-      QueryResultsListener listener = new QueryResultsListener();
-      String plan = Charsets.UTF_8.decode(ByteBuffer.wrap(Files.readAllBytes(Paths.get(planLocation)))).toString();
+      QueryResultsListener listener;
+      String plan;
+      if (queryString == null) {
+        plan = Charsets.UTF_8.decode(ByteBuffer.wrap(Files.readAllBytes(Paths.get(planLocation)))).toString();
+      } else {
+        plan = queryString;
+      }
+      String[] queries;
       UserProtos.QueryType queryType;
       type = type.toLowerCase();
       switch(type) {
         case "sql":
           queryType = UserProtos.QueryType.SQL;
+          queries = plan.split(";");
           break;
         case "logical":
           queryType = UserProtos.QueryType.LOGICAL;
+          queries = new String[]{ plan };
           break;
         case "physical":
           queryType = UserProtos.QueryType.PHYSICAL;
+          queries = new String[]{ plan };
           break;
         default:
           System.out.println("Invalid query type: " + type);
           return -1;
       }
+      Format outputFormat;
+      format = format.toLowerCase();
+      switch(format) {
+        case "csv":
+          outputFormat = Format.CSV;
+          break;
+        case "tsv":
+          outputFormat = Format.TSV;
+          break;
+        case "table":
+          outputFormat = Format.TABLE;
+          break;
+        default:
+          System.out.println("Invalid format type: " + format);
+          return -1;
+      }
       Stopwatch watch = new Stopwatch();
-      watch.start();
-      client.runQuery(queryType, plan, listener);
-      int rows = listener.await();
-      System.out.println(String.format("Got %d record%s in %f seconds", rows, rows > 1 ? "s" : "", (float)watch.elapsed(TimeUnit.MILLISECONDS) / (float)1000));
+      for (String query : queries) {
+        listener = new QueryResultsListener(outputFormat);
+        watch.start();
+        client.runQuery(queryType, query, listener);
+        int rows = listener.await();
+        System.out.println(String.format("%d record%s selected (%f seconds)", rows, rows > 1 ? "s" : "", (float) watch.elapsed(TimeUnit.MILLISECONDS) / (float) 1000));
+        if (query != queries[queries.length - 1]) {
+          System.out.println();
+        }
+        watch.stop();
+        watch.reset();
+      }
       return 0;
     }finally{
       if(client != null) client.close();
@@ -139,7 +189,11 @@ public class QuerySubmitter {
     AtomicInteger count = new AtomicInteger();
     private CountDownLatch latch = new CountDownLatch(1);
     RecordBatchLoader loader = new RecordBatchLoader(new BootStrapContext(DrillConfig.create()).getAllocator());
-    int width;
+    Format format;
+
+    public QueryResultsListener(Format format) {
+      this.format = format;
+    }
 
     @Override
     public void submissionFailed(RpcException ex) {
@@ -157,12 +211,21 @@ public class QuerySubmitter {
         } catch (SchemaChangeException e) {
           submissionFailed(new RpcException(e));
         }
-        
-        VectorUtil.showVectorAccessibleContent(loader);
+
+        switch(format) {
+          case TABLE:
+            VectorUtil.showVectorAccessibleContent(loader);
+            break;
+          case TSV:
+            VectorUtil.showVectorAccessibleContent(loader, "\t");
+            break;
+          case CSV:
+            VectorUtil.showVectorAccessibleContent(loader, ",");
+            break;
+        }
       }
       
       if (result.getHeader().getIsLastChunk()) {
-        //System.out.println(StringUtils.repeat("-", width*17 + 1));
         latch.countDown();
       }
       result.release();

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2ed7a678/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
index 553e50d..a35bb5f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
@@ -29,6 +29,40 @@ import com.beust.jcommander.internal.Lists;
 
 public class VectorUtil {
 
+  public static void showVectorAccessibleContent(VectorAccessible va, final String delimiter) {
+
+    int rows = va.getRecordCount();
+    List<String> columns = Lists.newArrayList();
+    for (VectorWrapper vw : va) {
+      columns.add(vw.getValueVector().getField().getName());
+    }
+
+    int width = columns.size();
+    for (String column : columns) {
+      System.out.printf("%s%s",column, column == columns.get(width - 1) ? "\n" : delimiter);
+    }
+    for (int row = 0; row < rows; row++) {
+      int columnCounter = 0;
+      for (VectorWrapper vw : va) {
+        boolean lastColumn = columnCounter == width - 1;
+        Object o = vw.getValueVector().getAccessor().getObject(row);
+        if (o == null) {
+          //null value
+          String value = "null";
+          System.out.printf("%s%s", value, lastColumn ? "\n" : delimiter);
+        }
+        else if (o instanceof byte[]) {
+          String value = new String((byte[]) o);
+          System.out.printf("%s%s", value, lastColumn ? "\n" : delimiter);
+        } else {
+          String value = o.toString();
+          System.out.printf("%s%s", value, lastColumn ? "\n" : delimiter);
+        }
+        columnCounter++;
+      }
+    }
+  }
+
   public static void showVectorAccessibleContent(VectorAccessible va) {
 
     int rows = va.getRecordCount();


[4/5] git commit: DRILL-398: Fix build/packaging issues with storage-hive jars 1. Remove hive-exec dependencies from root pom.xml and move hive-hbase dependency to contrib/storage-hive/pom.xml. Idea is to download all required hive dependencies through s

Posted by ja...@apache.org.
DRILL-398: Fix build/packaging issues with storage-hive jars
1. Remove hive-exec dependencies from root pom.xml and move hive-hbase dependency to contrib/storage-hive/pom.xml. Idea is to download all required hive dependencies through storage-hive artifact.
2. In distribution/pom.xml exclude hive-exec jar from storage-hive dependency. Storage-hive jar already contains the shaded hive-exec.
3. In contrib/storage-hive/pom.xml add a target to unpack the classes in storage-hive.jar into contrib/storage-hive/target/classes directory. Unpacking is needed because when you run 'mvn test' from root directory, dependencies within the project (such as protocol or storage-hive) are resolved to target/classes directory instead of jar. This is by design in Maven. If we don't unpack, we get class not found errors for classes in storage-hive.jar while running tests only.


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/48146084
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/48146084
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/48146084

Branch: refs/heads/master
Commit: 481460846a5c22d9c9899f5d93d443190c40061c
Parents: 17ea543
Author: vkorukanti <ve...@gmail.com>
Authored: Fri Mar 7 10:22:11 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Mar 11 14:30:22 2014 -0700

----------------------------------------------------------------------
 contrib/storage-hive/pom.xml | 43 ++++++++++++++++++++++++++++++++++-
 distribution/pom.xml         |  7 +++++-
 pom.xml                      | 48 ---------------------------------------
 3 files changed, 48 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/48146084/contrib/storage-hive/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/pom.xml b/contrib/storage-hive/pom.xml
index 237d6d7..314e016 100644
--- a/contrib/storage-hive/pom.xml
+++ b/contrib/storage-hive/pom.xml
@@ -82,6 +82,21 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-hbase-handler</artifactId>
+      <version>0.12.0</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hbase</groupId>
+          <artifactId>hbase</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
   </dependencies>
 
   <build>
@@ -114,7 +129,33 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <version>2.8</version>
+        <executions>
+          <execution>
+            <id>unpack</id>
+            <phase>package</phase>
+            <goals>
+              <goal>unpack</goal>
+            </goals>
+            <configuration>
+              <artifactItems>
+                <artifactItem>
+                  <groupId>org.apache.drill.contrib</groupId>
+                  <artifactId>storage-hive</artifactId>
+                  <version>${project.version}</version>
+                  <type>jar</type>
+                  <overWrite>true</overWrite>
+                  <outputDirectory>${project.build.directory}/classes</outputDirectory>
+                  <includes>**/**</includes>
+                </artifactItem>
+              </artifactItems>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
-
   </build>
 </project>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/48146084/distribution/pom.xml
----------------------------------------------------------------------
diff --git a/distribution/pom.xml b/distribution/pom.xml
index ffbac0f..06065f4 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -32,6 +32,12 @@
       <groupId>org.apache.drill.contrib</groupId>
       <artifactId>storage-hive</artifactId>
       <version>${project.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.apache.hive</groupId>
+          <artifactId>hive-exec</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.drill</groupId>
@@ -86,7 +92,6 @@
             <exclude>**/control</exclude>
             <exclude>**/*.json</exclude>
             <exclude>**/hadoop-excludes.txt</exclude>
-            <exclude>**/drill_hive_db/*</exclude>
           </excludes>
         </configuration>
       </plugin>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/48146084/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 9d8b542..967e1da 100644
--- a/pom.xml
+++ b/pom.xml
@@ -118,8 +118,6 @@
             <exclude>**/*.tdd</exclude>
             <exclude>**/*.iml</exclude>
             <exclude>**/target/**</exclude>
-            <exclude>**/drill_hive_db/**</exclude>
-            <exclude>**/drill_hive_wh/**</exclude>
           </excludes>
         </configuration>
       </plugin>
@@ -360,52 +358,6 @@
       <version>${dep.slf4j.version}</version>
     </dependency>
 
-    <dependency>
-      <groupId>org.apache.hive</groupId>
-      <artifactId>hive-exec</artifactId>
-      <version>0.12.0</version>
-      <exclusions>
-        <exclusion>
-          <groupId>commons-logging</groupId>
-          <artifactId>commons-logging</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>commons-logging</groupId>
-          <artifactId>commons-logging-api</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>jline</groupId>
-          <artifactId>jline</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.slf4j</groupId>
-          <artifactId>slf4j-log4j12</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>log4j</groupId>
-          <artifactId>log4j</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>com.google.guava</groupId>
-          <artifactId>guava</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hive</groupId>
-      <artifactId>hive-hbase-handler</artifactId>
-      <version>0.12.0</version>
-      <exclusions>
-        <exclusion>
-          <groupId>org.slf4j</groupId>
-          <artifactId>slf4j-log4j12</artifactId>
-        </exclusion>
-        <exclusion>
-          <groupId>org.apache.hbase</groupId>
-          <artifactId>hbase</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
     <!-- Test Dependencies -->
     <dependency>
       <groupId>com.googlecode.jmockit</groupId>


[2/5] git commit: Drill 342/238: Clean up Parquet tests.

Posted by ja...@apache.org.
Drill 342/238: Clean up Parquet tests.


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/822bd9c2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/822bd9c2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/822bd9c2

Branch: refs/heads/master
Commit: 822bd9c2dd02724385e4140cc0ab2960e8902d39
Parents: 2ed7a67
Author: Jason Altekruse <al...@gmail.com>
Authored: Tue Mar 11 08:27:53 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Mar 11 08:27:53 2014 -0700

----------------------------------------------------------------------
 .../store/parquet/ParquetRecordReaderTest.java  |  23 +-
 .../store/parquet/ParquetRecordReaderTest2.java | 742 -------------------
 .../resources/parquet/parquet_nullable.json     |  42 ++
 .../parquet/parquet_nullable_varlen.json        |  42 ++
 .../parquet/parquet_selective_column_read.json  |  27 +
 .../src/test/resources/parquet_nullable.json    |  41 -
 .../test/resources/parquet_nullable_varlen.json |  41 -
 .../src/test/resources/parquet_scan_screen.json |  40 -
 .../parquet_scan_screen_read_entry_replace.json |  40 -
 .../parquet_scan_union_screen_physical.json     |  35 -
 .../parquet_selective_column_read.json          |  27 -
 pom.xml                                         |   2 +
 12 files changed, 124 insertions(+), 978 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
index 9e999e3..2b043aa 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
@@ -30,7 +30,6 @@ import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 import mockit.Injectable;
-
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.expression.ExpressionPosition;
 import org.apache.drill.common.expression.FieldReference;
@@ -42,7 +41,7 @@ import org.apache.drill.exec.exception.SchemaChangeException;
 import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.physical.impl.OutputMutator;
-import org.apache.drill.exec.proto.BitControl.PlanFragment;
+import org.apache.drill.exec.proto.BitControl;
 import org.apache.drill.exec.proto.UserProtos;
 import org.apache.drill.exec.record.MaterializedField;
 import org.apache.drill.exec.record.RecordBatchLoader;
@@ -282,7 +281,7 @@ public class ParquetRecordReaderTest {
     HashMap<String, FieldInfo> fields = new HashMap<>();
     ParquetTestProperties props = new ParquetTestProperties(3, 3000, DEFAULT_BYTES_PER_PAGE, fields);
     populateFieldInfoMap(props);
-    testParquetFullEngineEventBased(true, "/parquet_scan_screen.json", "/tmp/test.parquet", 1, props);
+    testParquetFullEngineEventBased(true, "/parquet/parquet_scan_screen.json", "/tmp/test.parquet", 1, props);
   }
 
   // TODO - Test currently marked ignore to prevent breaking of the build process, requires a binary file that was
@@ -295,7 +294,7 @@ public class ParquetRecordReaderTest {
     ParquetTestProperties props = new ParquetTestProperties(1, 3000000, DEFAULT_BYTES_PER_PAGE, fields);
     Object[] boolVals = {true, null, null};
     props.fields.put("a", new FieldInfo("boolean", "a", 1, boolVals, TypeProtos.MinorType.BIT, props));
-    testParquetFullEngineEventBased(false, "/parquet_nullable.json", "/tmp/nullable_test.parquet", 1, props);
+    testParquetFullEngineEventBased(false, "/parquet/parquet_nullable.json", "/tmp/nullable_test.parquet", 1, props);
   }
 
   @Ignore
@@ -308,7 +307,7 @@ public class ParquetRecordReaderTest {
     byte[] val3 = { 'l','o','n','g','e','r',' ','s','t','r','i','n','g'};
     Object[] boolVals = { val, val2, val3};
     props.fields.put("a", new FieldInfo("boolean", "a", 1, boolVals, TypeProtos.MinorType.BIT, props));
-    testParquetFullEngineEventBased(false, "/parquet_nullable_varlen.json", "/tmp/nullable_varlen.parquet", 1, props);
+    testParquetFullEngineEventBased(false, "/parquet/parquet_nullable_varlen.json", "/tmp/nullable_varlen.parquet", 1, props);
   }
 
   @Test
@@ -335,15 +334,15 @@ public class ParquetRecordReaderTest {
     HashMap<String, FieldInfo> fields = new HashMap<>();
     ParquetTestProperties props = new ParquetTestProperties(5, 300000, DEFAULT_BYTES_PER_PAGE, fields);
     TestFileGenerator.populatePigTPCHCustomerFields(props);
-    String readEntries = "{path: \"/tmp/tpc-h/customer\"}";
-    testParquetFullEngineEventBased(false, false, "/parquet_scan_screen_read_entry_replace.json", readEntries,
+    String readEntries = "\"/tmp/tpc-h/customer\"";
+    testParquetFullEngineEventBased(false, false, "/parquet/parquet_scan_screen_read_entry_replace.json", readEntries,
         "unused, no file is generated", 1, props, true);
 
     fields = new HashMap();
     props = new ParquetTestProperties(5, 300000, DEFAULT_BYTES_PER_PAGE, fields);
     TestFileGenerator.populatePigTPCHSupplierFields(props);
-    readEntries = "{path: \"/tmp/tpc-h/supplier\"}";
-    testParquetFullEngineEventBased(false, false, "/parquet_scan_screen_read_entry_replace.json", readEntries,
+    readEntries = "\"/tmp/tpc-h/supplier\"";
+    testParquetFullEngineEventBased(false, false, "/parquet/parquet_scan_screen_read_entry_replace.json", readEntries,
         "unused, no file is generated", 1, props, true);
   }
 
@@ -352,7 +351,7 @@ public class ParquetRecordReaderTest {
     HashMap<String, FieldInfo> fields = new HashMap<>();
     ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
     populateFieldInfoMap(props);
-    testParquetFullEngineEventBased(true, "/parquet_scan_screen.json", "/tmp/test.parquet", 1, props);
+    testParquetFullEngineEventBased(true, "/parquet/parquet_scan_screen.json", "/tmp/test.parquet", 1, props);
   }
 
 
@@ -378,7 +377,7 @@ public class ParquetRecordReaderTest {
     props.fields.put("bigInt", new FieldInfo("int64", "bigInt", 64, TestFileGenerator.longVals, TypeProtos.MinorType.BIGINT, props));
     props.fields.put("bin", new FieldInfo("binary", "bin", -1, TestFileGenerator.binVals, TypeProtos.MinorType.VARBINARY, props));
     props.fields.put("bin2", new FieldInfo("binary", "bin2", -1, TestFileGenerator.bin2Vals, TypeProtos.MinorType.VARBINARY, props));
-    testParquetFullEngineEventBased(true, false, "/parquet_selective_column_read.json", null, "/tmp/test.parquet", 1, props, false);
+    testParquetFullEngineEventBased(true, false, "/parquet/parquet_selective_column_read.json", null, "/tmp/test.parquet", 1, props, false);
   }
 
   public static void main(String[] args) throws Exception{
@@ -392,7 +391,7 @@ public class ParquetRecordReaderTest {
                               @Injectable UserServer.UserClientConnection connection) throws Exception {
     DrillConfig c = DrillConfig.create();
     FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
-    FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
+    FragmentContext context = new FragmentContext(bitContext, BitControl.PlanFragment.getDefaultInstance(), connection, registry);
 
 //    new NonStrictExpectations() {
 //      {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest2.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest2.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest2.java
deleted file mode 100644
index 2ab12cb..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest2.java
+++ /dev/null
@@ -1,742 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.parquet;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static parquet.column.Encoding.PLAIN;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.physical.impl.OutputMutator;
-import org.apache.drill.exec.proto.UserBitShared.QueryId;
-import org.apache.drill.exec.proto.UserProtos;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.RecordBatchLoader;
-import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.rpc.RpcException;
-import org.apache.drill.exec.rpc.user.ConnectionThrottle;
-import org.apache.drill.exec.rpc.user.QueryResultBatch;
-import org.apache.drill.exec.rpc.user.UserResultsListener;
-import org.apache.drill.exec.server.Drillbit;
-import org.apache.drill.exec.server.RemoteServiceSet;
-import org.apache.drill.exec.store.ByteArrayUtil;
-import org.apache.drill.exec.vector.BaseDataValueVector;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import parquet.bytes.BytesInput;
-import parquet.column.ColumnDescriptor;
-import parquet.hadoop.ParquetFileWriter;
-import parquet.hadoop.metadata.CompressionCodecName;
-import parquet.schema.MessageType;
-import parquet.schema.MessageTypeParser;
-
-import com.beust.jcommander.internal.Lists;
-import com.google.common.base.Charsets;
-import com.google.common.io.Files;
-import com.google.common.util.concurrent.SettableFuture;
-
-public class ParquetRecordReaderTest2 {
-  org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRecordReaderTest2.class);
-
-  private static final boolean VERBOSE_DEBUG = false;
-
-  // { 00000001, 00000010, 00000100, 00001000, 00010000, ... }
-  byte[] bitFields = {1, 2, 4, 8, 16, 32, 64, -128};
-  byte allBitsTrue = -1;
-  byte allBitsFalse = 0;
-  int DEFAULT_BYTES_PER_PAGE = 1024 * 1024 * 1;
-  static Object[] intVals = {-200, 100, Integer.MAX_VALUE };
-  static Object[] longVals = { -5000l, 5000l, Long.MAX_VALUE};
-  static Object[] floatVals = { 1.74f, Float.MAX_VALUE, Float.MIN_VALUE};
-  static Object[] doubleVals = {100.45d, Double.MAX_VALUE, Double.MIN_VALUE,};
-  static Object[] boolVals = {false, false, true};
-  static byte[] varLen1 = {50, 51, 52, 53, 54, 55, 56, 57, 58, 59};
-  static byte[] varLen2 = {15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1};
-  static byte[] varLen3 = {100, 99, 98};
-  static Object[] binVals = { varLen3, varLen2, varLen3};
-  static Object[] bin2Vals = { varLen3, varLen2, varLen1};
-
-  private void populateFieldInfoMap(ParquetTestProperties props){
-    props.fields.put("integer", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props));
-    props.fields.put("bigInt", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props));
-    props.fields.put("f", new FieldInfo("float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4, props));
-    props.fields.put("d", new FieldInfo("double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8, props));
-    props.fields.put("b", new FieldInfo("boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT, props));
-    props.fields.put("bin", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props));
-    props.fields.put("bin2", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
-  }
-
-  private void populatePigTPCHCustomerFields(ParquetTestProperties props){
-    // all of the data in the fieldInfo constructors doesn't matter because the file is generated outside the test
-    props.fields.put("C_CUSTKEY", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props));
-    props.fields.put("C_NATIONKEY", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props));
-    props.fields.put("C_ACCTBAL", new FieldInfo("float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4, props));
-    props.fields.put("C_NAME", new FieldInfo("double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8, props));
-    props.fields.put("C_ADDRESS", new FieldInfo("boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT, props));
-    props.fields.put("C_PHONE", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props));
-    props.fields.put("C_MKTSEGMENT", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
-    props.fields.put("C_COMMENT", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
-  }
-
-  private void populatePigTPCHSupplierFields(ParquetTestProperties props){
-    // all of the data in the fieldInfo constructors doesn't matter because the file is generated outside the test
-    props.fields.put("S_SUPPKEY", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props));
-    props.fields.put("S_NATIONKEY", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props));
-    props.fields.put("S_ACCTBAL", new FieldInfo("float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4, props));
-    props.fields.put("S_NAME", new FieldInfo("double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8, props));
-    props.fields.put("S_ADDRESS", new FieldInfo("boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT, props));
-    props.fields.put("S_PHONE", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props));
-    props.fields.put("S_COMMENT", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
-  }
-
-  @Test
-  public void testMultipleRowGroups() throws Exception {
-    HashMap<String, FieldInfo> fields = new HashMap<>();
-    ParquetTestProperties props = new ParquetTestProperties(3, 3000, DEFAULT_BYTES_PER_PAGE, fields);
-    populateFieldInfoMap(props);
-    testParquetFullEngine(true, "/parquet_scan_screen.json", "/tmp/test.parquet", 1, props);
-  }
-
-  // TODO - Test currently marked ignore to prevent breaking of the build process, requires a binary file that was
-  // generated using pig. Will need to find a good place to keep files like this.
-  // For now I will upload it to the JIRA as an attachment.
-  @Ignore
-  @Test
-  public void testNullableColumns() throws Exception {
-    HashMap<String, FieldInfo> fields = new HashMap<>();
-    ParquetTestProperties props = new ParquetTestProperties(1, 3000000, DEFAULT_BYTES_PER_PAGE, fields);
-    Object[] boolVals = {true, null, null};
-    props.fields.put("a", new FieldInfo("boolean", "a", 1, boolVals, TypeProtos.MinorType.BIT, props));
-    testParquetFullEngine(false, "/parquet_nullable.json", "/tmp/nullable.parquet", 1, props);
-  }
-
-  @Ignore
-  @Test
-  public void testNullableColumnsVarLen() throws Exception {
-    HashMap<String, FieldInfo> fields = new HashMap<>();
-    ParquetTestProperties props = new ParquetTestProperties(1, 3000000, DEFAULT_BYTES_PER_PAGE, fields);
-    byte[] val = {'b'};
-//    Object[] boolVals = { val, null, null};
-//    Object[] boolVals = { null, null, null};
-    Object[] boolVals = { val, val, val};
-    props.fields.put("a", new FieldInfo("boolean", "a", 1, boolVals, TypeProtos.MinorType.BIT, props));
-    testParquetFullEngine(false, "/parquet_nullable_varlen.json", "/tmp/nullable.parquet", 1, props);
-  }
-
-  /**
-   * Tests the attribute in a scan node to limit the columns read by a scan.
-   *
-   * The functionality of selecting all columns is tested in all of the other tests that leave out the attribute.
-   * @throws Exception
-   */
-  @Test
-  public void testSelectColumnRead() throws Exception {
-    HashMap<String, FieldInfo> fields = new HashMap<>();
-    ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
-    // generate metatdata for a series of test columns, these columns are all generated in the test file
-    populateFieldInfoMap(props);
-    generateParquetFile("/tmp/test.parquet", props);
-    fields.clear();
-    // create a new object to describe the dataset expected out of the scan operation
-    // the fields added below match those requested in the plan specified in parquet_selective_column_read.json
-    // that is used below in the test query
-    props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
-    props.fields.put("integer", new FieldInfo("int32", "integer", 32, intVals, TypeProtos.MinorType.INT, props));
-    props.fields.put("bigInt", new FieldInfo("int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT, props));
-    props.fields.put("bin", new FieldInfo("binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY, props));
-    props.fields.put("bin2", new FieldInfo("binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY, props));
-    testParquetFullEngineEventBased(false, "/parquet_selective_column_read.json", null, "/tmp/test.parquet", 1, props, false);
-  }
-
-
-  @Test
-  public void testMultipleRowGroupsAndReads() throws Exception {
-    HashMap<String, FieldInfo> fields = new HashMap<>();
-    ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
-    populateFieldInfoMap(props);
-    String readEntries = "";
-    // number of times to read the file
-    int i = 3;
-    for (int j = 0; j < i; j++){
-      readEntries += "{path: \"/tmp/test.parquet\"}";
-      if (j < i - 1)
-        readEntries += ",";
-    }
-    testParquetFullEngineEventBased(true, "/parquet_scan_screen_read_entry_replace.json", readEntries,
-        "/tmp/test.parquet", i, props, true);
-  }
-
-  // requires binary file generated by pig from TPCH data, also have to disable assert where data is coming in
-  @Ignore
-  @Test
-  public void testMultipleRowGroupsAndReadsPigError() throws Exception {
-    HashMap<String, FieldInfo> fields = new HashMap<>();
-    ParquetTestProperties props = new ParquetTestProperties(4, 3000, DEFAULT_BYTES_PER_PAGE, fields);
-    populatePigTPCHCustomerFields(props);
-//    populatePigTPCHSupplierFields(props);
-    String readEntries = "";
-    // number of times to read the file
-    int i = 1;
-    for (int j = 0; j < i; j++){
-      readEntries += "{path: \"/tmp/tpc-h/customer\"}";
-      if (j < i - 1)
-        readEntries += ",";
-    }
-    testParquetFullEngineEventBased(false, "/parquet_scan_screen_read_entry_replace.json", readEntries,
-        "/tmp/test.parquet", i, props, true);
-  }
-/*
-  @Test
-  public void testMultipleRowGroupsEvent() throws Exception {
-    HashMap<String, FieldInfo> fields = new HashMap<>();
-    ParquetTestProperties props = new ParquetTestProperties(10, 30000, DEFAULT_BYTES_PER_PAGE, fields);
-    props.fields.put("a", new FieldInfo("a", "asdf", 1, new Object[3], TypeProtos.MinorType.BIGINT, props));
-    testParquetFullEngineEventBased(false, "/parquet_scan_screen.json", "/tmp/out", 1, props);
-  }
-
-*/
-  private class ParquetTestProperties{
-    int numberRowGroups;
-    int recordsPerRowGroup;
-    int bytesPerPage = 1024 * 1024 * 1;
-    HashMap<String, FieldInfo> fields = new HashMap<>();
-
-    public ParquetTestProperties(int numberRowGroups, int recordsPerRowGroup, int bytesPerPage,
-                                 HashMap<String, FieldInfo> fields){
-      this.numberRowGroups = numberRowGroups;
-      this.recordsPerRowGroup = recordsPerRowGroup;
-      this.bytesPerPage = bytesPerPage;
-      this.fields = fields;
-    }
-
-  }
-
-  private static class FieldInfo {
-
-    String parquetType;
-    String name;
-    int bitLength;
-    int numberOfPages;
-    Object[] values;
-    TypeProtos.MinorType type;
-
-    FieldInfo(String parquetType, String name, int bitLength, Object[] values, TypeProtos.MinorType type, ParquetTestProperties props){
-      this.parquetType = parquetType;
-      this.name = name;
-      this.bitLength  = bitLength;
-      this.numberOfPages = Math.max(1, (int) Math.ceil( ((long) props.recordsPerRowGroup) * bitLength / 8.0 / props.bytesPerPage));
-      this.values = values;
-      // generator is designed to use 3 values
-      assert values.length == 3;
-      this.type = type;
-    }
-  }
-
-  private String getResource(String resourceName) {
-    return "resource:" + resourceName;
-  }
-
-  public void generateParquetFile(String filename, ParquetTestProperties props) throws Exception {
-
-    int currentBooleanByte = 0;
-    WrapAroundCounter booleanBitCounter = new WrapAroundCounter(7);
-
-    Configuration configuration = new Configuration();
-    configuration.set("fs.default.name", "file:///");
-    //"message m { required int32 integer; required int64 integer64; required boolean b; required float f; required double d;}"
-
-    FileSystem fs = FileSystem.get(configuration);
-    Path path = new Path(filename);
-    if (fs.exists(path)) fs.delete(path, false);
-
-
-    String messageSchema = "message m {";
-    for (FieldInfo fieldInfo : props.fields.values()) {
-      messageSchema += " required " + fieldInfo.parquetType + " " + fieldInfo.name + ";";
-    }
-    // remove the last semicolon, java really needs a join method for strings...
-    // TODO - nvm apparently it requires a semicolon after every field decl, might want to file a bug
-    //messageSchema = messageSchema.substring(schemaType, messageSchema.length() - 1);
-    messageSchema += "}";
-
-    MessageType schema = MessageTypeParser.parseMessageType(messageSchema);
-
-    CompressionCodecName codec = CompressionCodecName.UNCOMPRESSED;
-    ParquetFileWriter w = new ParquetFileWriter(configuration, schema, path);
-    w.start();
-    HashMap<String, Integer> columnValuesWritten = new HashMap();
-    int valsWritten;
-    for (int k = 0; k < props.numberRowGroups; k++){
-      w.startBlock(1);
-      currentBooleanByte = 0;
-      booleanBitCounter.reset();
-
-      for (FieldInfo fieldInfo : props.fields.values()) {
-
-        if ( ! columnValuesWritten.containsKey(fieldInfo.name)){
-          columnValuesWritten.put((String) fieldInfo.name, 0);
-          valsWritten = 0;
-        } else {
-          valsWritten = columnValuesWritten.get(fieldInfo.name);
-        }
-
-        String[] path1 = {(String) fieldInfo.name};
-        ColumnDescriptor c1 = schema.getColumnDescription(path1);
-
-        w.startColumn(c1, props.recordsPerRowGroup, codec);
-        int valsPerPage = (int) Math.ceil(props.recordsPerRowGroup / (float) fieldInfo.numberOfPages);
-        byte[] bytes;
-        // for variable length binary fields
-        int bytesNeededToEncodeLength = 4;
-        if ((int) fieldInfo.bitLength > 0) {
-          bytes = new byte[(int) Math.ceil(valsPerPage * (int) fieldInfo.bitLength / 8.0)];
-        } else {
-          // the twelve at the end is to account for storing a 4 byte length with each value
-          int totalValLength = ((byte[]) fieldInfo.values[0]).length + ((byte[]) fieldInfo.values[1]).length + ((byte[]) fieldInfo.values[2]).length + 3 * bytesNeededToEncodeLength;
-          // used for the case where there is a number of values in this row group that is not divisible by 3
-          int leftOverBytes = 0;
-          if ( valsPerPage % 3 > 0 ) leftOverBytes += ((byte[])fieldInfo.values[1]).length + bytesNeededToEncodeLength;
-          if ( valsPerPage % 3 > 1 ) leftOverBytes += ((byte[])fieldInfo.values[2]).length + bytesNeededToEncodeLength;
-          bytes = new byte[valsPerPage / 3 * totalValLength + leftOverBytes];
-        }
-        int bytesPerPage = (int) (valsPerPage * ((int) fieldInfo.bitLength / 8.0));
-        int bytesWritten = 0;
-        for (int z = 0; z < (int) fieldInfo.numberOfPages; z++, bytesWritten = 0) {
-          for (int i = 0; i < valsPerPage; i++) {
-            //System.out.print(i + ", " + (i % 25 == 0 ? "\n gen " + fieldInfo.name + ": " : ""));
-            if (fieldInfo.values[0] instanceof Boolean) {
-
-              bytes[currentBooleanByte] |= bitFields[booleanBitCounter.val] & ((boolean) fieldInfo.values[valsWritten % 3]
-                  ? allBitsTrue : allBitsFalse);
-              booleanBitCounter.increment();
-              if (booleanBitCounter.val == 0) {
-                currentBooleanByte++;
-              }
-              valsWritten++;
-              if (currentBooleanByte > bytesPerPage) break;
-            } else {
-              if (fieldInfo.values[valsWritten % 3] instanceof byte[]){
-                System.arraycopy(ByteArrayUtil.toByta(((byte[])fieldInfo.values[valsWritten % 3]).length),
-                    0, bytes, bytesWritten, bytesNeededToEncodeLength);
-                try{
-                System.arraycopy(fieldInfo.values[valsWritten % 3],
-                    0, bytes, bytesWritten + bytesNeededToEncodeLength, ((byte[])fieldInfo.values[valsWritten % 3]).length);
-                }
-                catch (Exception ex){
-                  Math.min(4, 5);
-                }
-                bytesWritten += ((byte[])fieldInfo.values[valsWritten % 3]).length + bytesNeededToEncodeLength;
-              }
-              else{
-                System.arraycopy( ByteArrayUtil.toByta(fieldInfo.values[valsWritten % 3]),
-                    0, bytes, i * ((int) fieldInfo.bitLength / 8), (int) fieldInfo.bitLength / 8);
-              }
-              valsWritten++;
-            }
-
-          }
-          w.writeDataPage((int) (props.recordsPerRowGroup / (int) fieldInfo.numberOfPages), bytes.length, BytesInput.from(bytes), PLAIN, PLAIN, PLAIN);
-          currentBooleanByte = 0;
-        }
-        w.endColumn();
-        columnValuesWritten.remove((String) fieldInfo.name);
-        columnValuesWritten.put((String) fieldInfo.name, valsWritten);
-      }
-
-      w.endBlock();
-    }
-    w.end(new HashMap<String, String>());
-    logger.debug("Finished generating parquet file.");
-  }
-
-  private class ParquetResultListener implements UserResultsListener {
-    private SettableFuture<Void> future = SettableFuture.create();
-    int count = 0;
-    RecordBatchLoader batchLoader;
-
-    int batchCounter = 1;
-    HashMap<String, Integer> valuesChecked = new HashMap();
-    ParquetTestProperties props;
-
-    ParquetResultListener(RecordBatchLoader batchLoader, ParquetTestProperties props){
-      this.batchLoader = batchLoader;
-      this.props = props;
-    }
-
-    @Override
-    public void submissionFailed(RpcException ex) {
-      logger.debug("Submission failed.", ex);
-      future.setException(ex);
-    }
-
-    @Override
-    public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) {
-      logger.debug("result arrived in test batch listener.");
-      if(result.getHeader().getIsLastChunk()){
-        future.set(null);
-      }
-      int columnValCounter = 0;
-      int i = 0;
-      FieldInfo currentField;
-      count += result.getHeader().getRowCount();
-      boolean schemaChanged = false;
-      try {
-        schemaChanged = batchLoader.load(result.getHeader().getDef(), result.getData());
-      } catch (SchemaChangeException e) {
-        throw new RuntimeException(e);
-      }
-
-      int recordCount = 0;
-      // print headers.
-      if (schemaChanged) {
-      } // do not believe any change is needed for when the schema changes, with the current mock scan use case
-
-      for (VectorWrapper vw : batchLoader) {
-        ValueVector vv = vw.getValueVector();
-        currentField = props.fields.get(vv.getField().getName());
-        if (VERBOSE_DEBUG){
-          System.out.println("\n" + (String) currentField.name);
-        }
-        if ( ! valuesChecked.containsKey(vv.getField().getName())){
-          valuesChecked.put(vv.getField().getName(), 0);
-          columnValCounter = 0;
-        } else {
-          columnValCounter = valuesChecked.get(vv.getField().getName());
-        }
-        for (int j = 0; j < ((BaseDataValueVector)vv).getAccessor().getValueCount(); j++) {
-          if (VERBOSE_DEBUG){
-            System.out.print(vv.getAccessor().getObject(j) + ", " + (j % 25 == 0 ? "\n batch:" + batchCounter + " v:" + j + " - " : ""));
-          }
-          assertField(vv, j, (TypeProtos.MinorType) currentField.type,
-              currentField.values[columnValCounter % 3], (String) currentField.name + "/");
-          columnValCounter++;
-        }
-        if (VERBOSE_DEBUG){
-          System.out.println("\n" + ((BaseDataValueVector)vv).getAccessor().getValueCount());
-        }
-        valuesChecked.remove(vv.getField().getName());
-        valuesChecked.put(vv.getField().getName(), columnValCounter);
-      }
-
-      if (VERBOSE_DEBUG){
-        for (i = 0; i < batchLoader.getRecordCount(); i++) {
-          recordCount++;
-          if (i % 50 == 0){
-            System.out.println();
-            for (VectorWrapper vw : batchLoader) {
-              ValueVector v = vw.getValueVector();
-              System.out.print(pad(v.getField().getName(), 20) + " ");
-
-            }
-            System.out.println();
-            System.out.println();
-          }
-
-          for (VectorWrapper vw : batchLoader) {
-            ValueVector v = vw.getValueVector();
-            System.out.print(pad(v.getAccessor().getObject(i).toString(), 20) + " ");
-          }
-          System.out.println(
-
-          );
-        }
-      }
-      batchCounter++;
-      if(result.getHeader().getIsLastChunk()){
-        future.set(null);
-      }
-    }
-
-    public void getResults() throws RpcException{
-      try{
-        future.get();
-      }catch(Throwable t){
-        throw RpcException.mapException(t);
-      }
-    }
-
-    @Override
-    public void queryIdArrived(QueryId queryId) {
-    }
-  }
-
-
-
-  // specific tests should call this method, but it is not marked as a test itself intentionally
-  public void testParquetFullEngineEventBased(boolean generateNew, String plan, String filename, int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props) throws Exception{
-    testParquetFullEngineEventBased(generateNew, plan, null, filename, numberOfTimesRead, props, true);
-  }
-
-  // specific tests should call this method, but it is not marked as a test itself intentionally
-  public void testParquetFullEngineEventBased(boolean generateNew, String plan, String readEntries, String filename,
-                                              int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props,
-                                              boolean runAsLogicalPlan) throws Exception{
-    RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
-
-    if (generateNew) generateParquetFile(filename, props);
-
-    DrillConfig config = DrillConfig.create();
-
-    try(Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator());){
-      bit1.run();
-      client.connect();
-      RecordBatchLoader batchLoader = new RecordBatchLoader(bit1.getContext().getAllocator());
-      ParquetResultListener resultListener = new ParquetResultListener(batchLoader, props);
-      long C = System.nanoTime();
-      String planText = Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8);
-      // substitute in the string for the read entries, allows reuse of the plan file for several tests
-      if (readEntries != null) {
-        planText = planText.replaceFirst( "&REPLACED_IN_PARQUET_TEST&", readEntries);
-      }
-      if (runAsLogicalPlan)
-        client.runQuery(UserProtos.QueryType.LOGICAL, planText, resultListener);
-      else
-        client.runQuery(UserProtos.QueryType.PHYSICAL, planText, resultListener);
-      resultListener.getResults();
-      for (String s : resultListener.valuesChecked.keySet()) {
-        assertEquals("Record count incorrect for column: " + s,
-            props.recordsPerRowGroup * props.numberRowGroups * numberOfTimesRead, (long) resultListener.valuesChecked.get(s));
-        logger.debug("Column {}, Values read:{}", s, resultListener.valuesChecked.get(s));
-      }
-      long D = System.nanoTime();
-      System.out.println(String.format("Took %f s to run query", (float)(D-C) / 1E9));
-    }
-  }
-
-  // specific tests should call this method, but it is not marked as a test itself intentionally
-  public void testParquetFullEngine(boolean generateNew, String plan, String filename, int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props) throws Exception{
-    testParquetFullEngine(generateNew, plan, null, filename, numberOfTimesRead, props);
-  }
-
-  // specific tests should call this method, but it is not marked as a test itself intentionally
-  public void testParquetFullEngine(boolean generateNew, String plan, String readEntries, String filename,
-                                    int numberOfTimesRead /* specified in json plan */, ParquetTestProperties props) throws Exception{
-    RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
-
-    if (generateNew) generateParquetFile(filename, props);
-
-    DrillConfig config = DrillConfig.create();
-
-    try(Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator())) {
-      long A = System.nanoTime();
-      bit1.run();
-      long B = System.nanoTime();
-      client.connect();
-      long C = System.nanoTime();
-      List<QueryResultBatch> results;
-      // insert a variable number of reads
-      if (readEntries != null){
-        results = client.runQuery(UserProtos.QueryType.LOGICAL, (Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8).replaceFirst( "&REPLACED_IN_PARQUET_TEST&", readEntries)));
-      }
-      else{
-        results = client.runQuery(UserProtos.QueryType.LOGICAL, Files.toString(FileUtils.getResourceAsFile(plan), Charsets.UTF_8));
-      }
-//      List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/parquet_scan_union_screen_physical.json"), Charsets.UTF_8));
-      long D = System.nanoTime();
-      System.out.println(String.format("Took %f s to start drillbit", (float)(B-A) / 1E9));
-      System.out.println(String.format("Took %f s to connect", (float)(C-B) / 1E9));
-      System.out.println(String.format("Took %f s to run query", (float)(D-C) / 1E9));
-      //List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/parquet_scan_union_screen_physical.json"), Charsets.UTF_8));
-      int count = 0;
-//      RecordBatchLoader batchLoader = new RecordBatchLoader(new BootStrapContext(config).getAllocator());
-      RecordBatchLoader batchLoader = new RecordBatchLoader(bit1.getContext().getAllocator());
-      byte[] bytes;
-
-      int batchCounter = 1;
-      int columnValCounter = 0;
-      int i = 0;
-      FieldInfo currentField;
-      HashMap<String, Integer> valuesChecked = new HashMap();
-      for(QueryResultBatch b : results){
-
-        count += b.getHeader().getRowCount();
-        boolean schemaChanged = batchLoader.load(b.getHeader().getDef(), b.getData());
-
-        int recordCount = 0;
-        // print headers.
-        if (schemaChanged) {
-        } // do not believe any change is needed for when the schema changes, with the current mock scan use case
-
-        for (VectorWrapper vw : batchLoader) {
-          ValueVector vv = vw.getValueVector();
-          currentField = props.fields.get(vv.getField().getName());
-          if (VERBOSE_DEBUG){
-            System.out.println("\n" + (String) currentField.name);
-          }
-          if ( ! valuesChecked.containsKey(vv.getField().getName())){
-            valuesChecked.put(vv.getField().getName(), 0);
-            columnValCounter = 0;
-          } else {
-            columnValCounter = valuesChecked.get(vv.getField().getName());
-          }
-          for (int j = 0; j < vv.getAccessor().getValueCount(); j++) {
-            if (VERBOSE_DEBUG){
-              System.out.print(vv.getAccessor().getObject(j) + ", " + (j % 25 == 0 ? "\n batch:" + batchCounter + " v:" + j + " - " : ""));
-            }
-            assertField(vv, j, currentField.type,
-                currentField.values[columnValCounter % 3], currentField.name + "/");
-            columnValCounter++;
-          }
-          if (VERBOSE_DEBUG){
-            System.out.println("\n" + vv.getAccessor().getValueCount());
-          }
-          valuesChecked.remove(vv.getField().getName());
-          valuesChecked.put(vv.getField().getName(), columnValCounter);
-        }
-
-        if (VERBOSE_DEBUG){
-          for (i = 0; i < batchLoader.getRecordCount(); i++) {
-            recordCount++;
-            if (i % 50 == 0){
-              System.out.println();
-              for (VectorWrapper vw : batchLoader) {
-                ValueVector v = vw.getValueVector();
-                System.out.print(pad(v.getField().getName(), 20) + " ");
-
-              }
-              System.out.println();
-              System.out.println();
-            }
-
-            for (VectorWrapper vw : batchLoader) {
-              ValueVector v = vw.getValueVector();
-              System.out.print(pad(v.getAccessor().getObject(i) + "", 20) + " ");
-            }
-            System.out.println(
-
-            );
-          }
-        }
-        batchCounter++;
-      }
-      for (String s : valuesChecked.keySet()) {
-        assertEquals("Record count incorrect for column: " + s, props.recordsPerRowGroup * props.numberRowGroups * numberOfTimesRead, (long) valuesChecked.get(s));
-      }
-      assert valuesChecked.keySet().size() > 0;
-    }
-  }
-
-  public String pad(String value, int length) {
-    return pad(value, length, " ");
-  }
-
-  public String pad(String value, int length, String with) {
-    StringBuilder result = new StringBuilder(length);
-    result.append(value);
-
-    while (result.length() < length) {
-      result.insert(0, with);
-    }
-
-    return result.toString();
-  }
-
-  class MockOutputMutator implements OutputMutator {
-    List<MaterializedField> removedFields = Lists.newArrayList();
-    List<ValueVector> addFields = Lists.newArrayList();
-
-    @Override
-    public void removeField(MaterializedField field) throws SchemaChangeException {
-      removedFields.add(field);
-    }
-
-    @Override
-    public void addField(ValueVector vector) throws SchemaChangeException {
-      addFields.add(vector);
-    }
-
-    @Override
-    public void removeAllFields() {
-      addFields.clear();
-    }
-
-    @Override
-    public void setNewSchema() throws SchemaChangeException {
-    }
-
-    List<MaterializedField> getRemovedFields() {
-      return removedFields;
-    }
-
-    List<ValueVector> getAddFields() {
-      return addFields;
-    }
-  }
-
-  private <T> void assertField(ValueVector valueVector, int index, TypeProtos.MinorType expectedMinorType, Object value, String name) {
-    assertField(valueVector, index, expectedMinorType, value, name, 0);
-  }
-
-  private <T> void assertField(ValueVector valueVector, int index, TypeProtos.MinorType expectedMinorType, T value, String name, int parentFieldId) {
-//    UserBitShared.FieldMetadata metadata = valueVector.getMetadata();
-//    SchemaDefProtos.FieldDef def = metadata.getDef();
-//    assertEquals(expectedMinorType, def.getMajorType().getMinorType());
-//    assertEquals(name, def.getNameList().get(0).getName());
-//    assertEquals(parentFieldId, def.getParentId());
-
-    if (expectedMinorType == TypeProtos.MinorType.MAP) {
-      return;
-    }
-
-    T val = (T) valueVector.getAccessor().getObject(index);
-    if (val instanceof String){
-      assertEquals(value, val);
-    }
-    else if (val instanceof byte[]) {
-      assertTrue(Arrays.equals((byte[]) value, (byte[]) val));
-    } else {
-      assertEquals(value, val);
-    }
-  }
-
-  private class WrapAroundCounter {
-
-    int maxVal;
-    int val;
-
-    public WrapAroundCounter(int maxVal) {
-      this.maxVal = maxVal;
-    }
-
-    public int increment() {
-      val++;
-      if (val > maxVal) {
-        val = 0;
-      }
-      return val;
-    }
-
-    public void reset() {
-      val = 0;
-    }
-
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/resources/parquet/parquet_nullable.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/parquet_nullable.json b/exec/java-exec/src/test/resources/parquet/parquet_nullable.json
new file mode 100644
index 0000000..3e09f83
--- /dev/null
+++ b/exec/java-exec/src/test/resources/parquet/parquet_nullable.json
@@ -0,0 +1,42 @@
+{
+  head:{
+    type:"APACHE_DRILL_LOGICAL",
+    version:"1",
+    generator:{
+      type:"manual",
+      info:"na"
+    }
+  },
+  storage:{
+    "dfs" :
+      {
+        "type":"named",
+        "name":"dfs"
+      }
+  },
+  query:[
+    {
+      @id:"1",
+      op:"scan",
+      memo:"initial_scan",
+      storageengine:"dfs",
+      selection: {
+          format: {type: "parquet"},
+          files: [
+            "/tmp/nullable_test.parquet"
+          ]
+       }
+    },
+    {
+      @id:"2",
+      input: 1,
+      op: "store",
+      memo: "output sink",
+      target: {
+        file: "console:///stdout"
+      }
+
+    }
+
+  ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/resources/parquet/parquet_nullable_varlen.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/parquet_nullable_varlen.json b/exec/java-exec/src/test/resources/parquet/parquet_nullable_varlen.json
new file mode 100644
index 0000000..9547f00
--- /dev/null
+++ b/exec/java-exec/src/test/resources/parquet/parquet_nullable_varlen.json
@@ -0,0 +1,42 @@
+{
+  head:{
+    type:"APACHE_DRILL_LOGICAL",
+    version:"1",
+    generator:{
+      type:"manual",
+      info:"na"
+    }
+  },
+  storage:{
+    "dfs" :
+      {
+        "type":"named",
+        "name":"dfs"
+      }
+  },
+  query:[
+    {
+      @id:"1",
+      op:"scan",
+      memo:"initial_scan",
+      storageengine:"dfs",
+      selection: {
+          format: {type: "parquet"},
+          files: [
+            "/tmp/nullable_varlen.parquet"
+          ]
+       }
+    },
+    {
+      @id:"2",
+      input: 1,
+      op: "store",
+      memo: "output sink",
+      target: {
+        file: "console:///stdout"
+      }
+
+    }
+
+  ]
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/resources/parquet/parquet_selective_column_read.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/parquet_selective_column_read.json b/exec/java-exec/src/test/resources/parquet/parquet_selective_column_read.json
new file mode 100644
index 0000000..515affc
--- /dev/null
+++ b/exec/java-exec/src/test/resources/parquet/parquet_selective_column_read.json
@@ -0,0 +1,27 @@
+{
+  head : {
+    version : 1,
+    generator : {
+      type : "manual",
+      info : "na"
+    },
+    type : "APACHE_DRILL_PHYSICAL"
+  },
+  graph : [ {
+    pop : "parquet-scan",
+    @id : 1,
+    entries : [ {
+      path : "/tmp/test.parquet"
+    } ],
+    storage : {
+      type : "file",
+      connection : "file:///"
+    },
+    columns: [ "integer", "bigInt", "bin", "bin2"]
+  }, {
+    pop : "screen",
+    @id : 2,
+    child : 1
+  } ]
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/resources/parquet_nullable.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet_nullable.json b/exec/java-exec/src/test/resources/parquet_nullable.json
deleted file mode 100644
index 5822bdb..0000000
--- a/exec/java-exec/src/test/resources/parquet_nullable.json
+++ /dev/null
@@ -1,41 +0,0 @@
-{
-  head:{
-    type:"APACHE_DRILL_LOGICAL",
-    version:"1",
-    generator:{
-      type:"manual",
-      info:"na"
-    }
-  },
-  storage:{
-    "parquet" :
-      {
-        "type":"parquet",
-        "dfsName" : "file:///"
-      }
-  },
-  query:[
-    {
-      @id:"1",
-      op:"scan",
-      memo:"initial_scan",
-      storageengine:"parquet",
-      selection: [
-        {
-            path: "/tmp/nullable_test.parquet"
-        }
-      ]
-    },
-    {
-      @id:"2",
-      input: 1,
-      op: "store",
-      memo: "output sink",
-      target: {
-        file: "console:///stdout"
-      }
-
-    }
-
-  ]
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/resources/parquet_nullable_varlen.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet_nullable_varlen.json b/exec/java-exec/src/test/resources/parquet_nullable_varlen.json
deleted file mode 100644
index c21beec..0000000
--- a/exec/java-exec/src/test/resources/parquet_nullable_varlen.json
+++ /dev/null
@@ -1,41 +0,0 @@
-{
-  head:{
-    type:"APACHE_DRILL_LOGICAL",
-    version:"1",
-    generator:{
-      type:"manual",
-      info:"na"
-    }
-  },
-  storage:{
-    "parquet" :
-      {
-        "type":"parquet",
-        "dfsName" : "file:///"
-      }
-  },
-  query:[
-    {
-      @id:"1",
-      op:"scan",
-      memo:"initial_scan",
-      storageengine:"parquet",
-      selection: [
-        {
-            path: "/tmp/nullable_varlen.parquet"
-        }
-      ]
-    },
-    {
-      @id:"2",
-      input: 1,
-      op: "store",
-      memo: "output sink",
-      target: {
-        file: "console:///stdout"
-      }
-
-    }
-
-  ]
-}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/resources/parquet_scan_screen.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet_scan_screen.json b/exec/java-exec/src/test/resources/parquet_scan_screen.json
deleted file mode 100644
index 3b3f86a..0000000
--- a/exec/java-exec/src/test/resources/parquet_scan_screen.json
+++ /dev/null
@@ -1,40 +0,0 @@
-{
-  head:{
-    type:"APACHE_DRILL_LOGICAL",
-    version:"1",
-    generator:{
-      type:"manual",
-      info:"na"
-    }
-  },
-  storage:{
-    "dfs" :
-      {
-        "type":"file",
-        "connection" : "file:///"
-      }
-  },
-  query:[
-    {
-      @id:"1",
-      op:"scan",
-      memo:"initial_scan",
-      storageengine:"dfs",
-      selection: {
-        format: {type: "parquet"},
-        files: ["/tmp/test.parquet"]
-      }
-    },
-    {
-      @id:"2",
-      input: 1,
-      op: "store",
-      memo: "output sink",
-      target: {
-        file: "console:///stdout"
-      }
-
-    }
-
-  ]
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/resources/parquet_scan_screen_read_entry_replace.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet_scan_screen_read_entry_replace.json b/exec/java-exec/src/test/resources/parquet_scan_screen_read_entry_replace.json
deleted file mode 100644
index ab11a5f..0000000
--- a/exec/java-exec/src/test/resources/parquet_scan_screen_read_entry_replace.json
+++ /dev/null
@@ -1,40 +0,0 @@
-{
-  head:{
-    type:"APACHE_DRILL_LOGICAL",
-    version:"1",
-    generator:{
-      type:"manual",
-      info:"na"
-    }
-  },
-  storage:{
-    "fs" :
-      {
-        "type":"file",
-        "connection" : "file:///"
-      }
-  },
-  query:[
-    {
-      @id:"1",
-      op:"scan",
-      memo:"initial_scan",
-      storageengine:"fs",
-      selection: {
-        format: {type: "parquet"}
-        files: [ &REPLACED_IN_PARQUET_TEST& ]
-      }
-    },
-    {
-      @id:"2",
-      input: 1,
-      op: "store",
-      memo: "output sink",
-      target: {
-        file: "console:///stdout"
-      }
-
-    }
-
-  ]
-}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/resources/parquet_scan_union_screen_physical.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet_scan_union_screen_physical.json b/exec/java-exec/src/test/resources/parquet_scan_union_screen_physical.json
deleted file mode 100644
index 954082c..0000000
--- a/exec/java-exec/src/test/resources/parquet_scan_union_screen_physical.json
+++ /dev/null
@@ -1,35 +0,0 @@
-{
-  head : {
-    type : "APACHE_DRILL_PHYSICAL",
-    version : 1,
-    generator : {
-      type : "manual"
-    }
-  },
-  graph : [ {
-    pop : "parquet-scan",
-    @id : 1,
-    entries : [
-    {
-        path : "/tmp/testParquetFile_many_types_3"
-    },
-    {
-        path : "/tmp/testParquetFile_many_types_3"
-    }
-    ],
-    storageengine:{
-                         "type":"parquet",
-                         "dfsName" : "maprfs:///"
-                   }
-  },
-  {
-     "@id": 2,
-     "child": 1,
-     "pop": "union-exchange"
-  },
-  {
-    pop : "screen",
-    @id : 3,
-    child : 2
-  } ]
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/exec/java-exec/src/test/resources/parquet_selective_column_read.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet_selective_column_read.json b/exec/java-exec/src/test/resources/parquet_selective_column_read.json
deleted file mode 100644
index 515affc..0000000
--- a/exec/java-exec/src/test/resources/parquet_selective_column_read.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
-  head : {
-    version : 1,
-    generator : {
-      type : "manual",
-      info : "na"
-    },
-    type : "APACHE_DRILL_PHYSICAL"
-  },
-  graph : [ {
-    pop : "parquet-scan",
-    @id : 1,
-    entries : [ {
-      path : "/tmp/test.parquet"
-    } ],
-    storage : {
-      type : "file",
-      connection : "file:///"
-    },
-    columns: [ "integer", "bigInt", "bin", "bin2"]
-  }, {
-    pop : "screen",
-    @id : 2,
-    child : 1
-  } ]
-}
-

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822bd9c2/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1bb58c0..9d8b542 100644
--- a/pom.xml
+++ b/pom.xml
@@ -116,6 +116,8 @@
             <exclude>**/*.proto</exclude>
             <exclude>**/*.fmpp</exclude>
             <exclude>**/*.tdd</exclude>
+            <exclude>**/*.iml</exclude>
+            <exclude>**/target/**</exclude>
             <exclude>**/drill_hive_db/**</exclude>
             <exclude>**/drill_hive_wh/**</exclude>
           </excludes>


[5/5] git commit: Fix group by, removing aggregate and segment remnants. Add new simple test queries to make sure parsing isn't broken in future. Fix binary tree so that more than 2 binary arguments are correctly translated from RexNode. Update binary f

Posted by ja...@apache.org.
Fix group by, removing aggregate and segment remnants.  Add new simple test queries to make sure parsing isn't broken in future.
Fix binary tree so that more than 2 binary arguments are correctly translated from RexNode.
Update binary function lookup to use lowercase.


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/2d10afd4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/2d10afd4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/2d10afd4

Branch: refs/heads/master
Commit: 2d10afd4b1499b84084123eda1e85e59602ac475
Parents: 4814608
Author: Jacques Nadeau <ja...@apache.org>
Authored: Tue Mar 11 14:33:57 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Mar 11 14:57:10 2014 -0700

----------------------------------------------------------------------
 .../logical/data/CollapsingAggregate.java       | 73 -------------------
 .../drill/common/logical/data/Segment.java      | 62 ----------------
 .../data/visitors/AbstractLogicalVisitor.java   | 10 ---
 .../logical/data/visitors/LogicalVisitor.java   |  4 -
 exec/java-exec/pom.xml                          |  6 ++
 .../drill/exec/client/QuerySubmitter.java       |  4 +-
 .../apache/drill/exec/opt/BasicOptimizer.java   | 54 +++++---------
 .../drill/exec/planner/logical/DrillOptiq.java  | 15 +++-
 .../org/apache/drill/TestExampleQueries.java    | 51 +++++++++++++
 .../src/test/resources/storage-engines.json     |  4 +
 pom.xml                                         |  2 +-
 .../org/apache/drill/jdbc/test/JdbcTest.java    | 77 ++++++++++----------
 12 files changed, 132 insertions(+), 230 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/common/src/main/java/org/apache/drill/common/logical/data/CollapsingAggregate.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/CollapsingAggregate.java b/common/src/main/java/org/apache/drill/common/logical/data/CollapsingAggregate.java
deleted file mode 100644
index f4258fd..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/CollapsingAggregate.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import com.google.common.collect.Iterators;
-import org.apache.drill.common.expression.FieldReference;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import java.util.Iterator;
-import java.util.List;
-
-@JsonTypeName("collapsingaggregate")
-public class CollapsingAggregate extends SingleInputOperator{
-  
-  private final FieldReference within;
-  private final FieldReference target;
-  private final FieldReference[] carryovers;
-  private final NamedExpression[]  aggregations;
-
-  @JsonCreator
-  public CollapsingAggregate(@JsonProperty("within") FieldReference within, @JsonProperty("target") FieldReference target, @JsonProperty("carryovers") FieldReference[] carryovers, @JsonProperty("aggregations") NamedExpression[] aggregations) {
-    super();
-    this.within = within;
-    this.target = target;
-    this.carryovers = carryovers;
-    this.aggregations = aggregations;
-  }
-
-  public FieldReference getWithin() {
-    return within;
-  }
-
-  public FieldReference getTarget() {
-    return target;
-  }
-
-  public FieldReference[] getCarryovers() {
-    return carryovers;
-  }
-
-  public NamedExpression[] getAggregations() {
-    return aggregations;
-  }
-
-    @Override
-    public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
-        return logicalVisitor.visitCollapsingAggregate(this, value);
-    }
-
-    @Override
-    public Iterator<LogicalOperator> iterator() {
-        return Iterators.singletonIterator(getInput());
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/common/src/main/java/org/apache/drill/common/logical/data/Segment.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Segment.java b/common/src/main/java/org/apache/drill/common/logical/data/Segment.java
deleted file mode 100644
index 3898944..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/data/Segment.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical.data;
-
-import com.google.common.collect.Iterators;
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.LogicalExpression;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-import org.apache.drill.common.logical.data.visitors.LogicalVisitor;
-
-import java.util.Iterator;
-
-@JsonTypeName("segment")
-public class Segment extends SingleInputOperator{
-  private final LogicalExpression[] exprs;
-  private final FieldReference name;
-  
-  @JsonCreator
-  public Segment(@JsonProperty("exprs") LogicalExpression[] exprs, @JsonProperty("ref") FieldReference name) {
-    super();
-    this.exprs = exprs;
-    this.name = name;
-  }
-
-  public LogicalExpression[] getExprs() {
-    return exprs;
-  }
-
-  public FieldReference getName() {
-    return name;
-  }
-
-    @Override
-    public <T, X, E extends Throwable> T accept(LogicalVisitor<T, X, E> logicalVisitor, X value) throws E {
-        return logicalVisitor.visitSegment(this, value);
-    }
-
-    @Override
-    public Iterator<LogicalOperator> iterator() {
-        return Iterators.singletonIterator(getInput());
-    }
-
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java b/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
index bb37a14..79fc34e 100644
--- a/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
+++ b/common/src/main/java/org/apache/drill/common/logical/data/visitors/AbstractLogicalVisitor.java
@@ -74,11 +74,6 @@ public abstract class AbstractLogicalVisitor<T, X, E extends Throwable> implemen
     }
 
     @Override
-    public T visitSegment(Segment segment, X value) throws E {
-        return visitOp(segment, value);
-    }
-
-    @Override
     public T visitGroupingAggregate(GroupingAggregate groupBy, X value) throws E {
       return visitOp(groupBy, value);
     }
@@ -99,11 +94,6 @@ public abstract class AbstractLogicalVisitor<T, X, E extends Throwable> implemen
     }
 
     @Override
-    public T visitCollapsingAggregate(CollapsingAggregate collapsingAggregate, X value) throws E {
-        return visitOp(collapsingAggregate, value);
-    }
-
-    @Override
     public T visitWindowFrame(WindowFrame windowFrame, X value) throws E {
         return visitOp(windowFrame, value);
     }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/common/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java b/common/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java
index f409453..d523541 100644
--- a/common/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java
+++ b/common/src/main/java/org/apache/drill/common/logical/data/visitors/LogicalVisitor.java
@@ -18,7 +18,6 @@
 package org.apache.drill.common.logical.data.visitors;
 
 
-import org.apache.drill.common.logical.data.CollapsingAggregate;
 import org.apache.drill.common.logical.data.Constant;
 import org.apache.drill.common.logical.data.Filter;
 import org.apache.drill.common.logical.data.Flatten;
@@ -29,7 +28,6 @@ import org.apache.drill.common.logical.data.Order;
 import org.apache.drill.common.logical.data.Project;
 import org.apache.drill.common.logical.data.RunningAggregate;
 import org.apache.drill.common.logical.data.Scan;
-import org.apache.drill.common.logical.data.Segment;
 import org.apache.drill.common.logical.data.Sequence;
 import org.apache.drill.common.logical.data.Store;
 import org.apache.drill.common.logical.data.Transform;
@@ -48,7 +46,6 @@ public interface LogicalVisitor<RETURN, EXTRA, EXCEP extends Throwable> {
 
     public RETURN visitScan(Scan scan, EXTRA value) throws EXCEP;
     public RETURN visitStore(Store store, EXTRA value) throws EXCEP;
-    public RETURN visitCollapsingAggregate(CollapsingAggregate collapsingAggregate, EXTRA value) throws EXCEP;
     public RETURN visitGroupingAggregate(GroupingAggregate groupBy, EXTRA value) throws EXCEP;
     public RETURN visitFilter(Filter filter, EXTRA value) throws EXCEP;
     public RETURN visitFlatten(Flatten flatten, EXTRA value) throws EXCEP;
@@ -59,7 +56,6 @@ public interface LogicalVisitor<RETURN, EXTRA, EXCEP extends Throwable> {
     public RETURN visitJoin(Join join, EXTRA value) throws EXCEP;
     public RETURN visitLimit(Limit limit, EXTRA value) throws EXCEP;
     public RETURN visitRunningAggregate(RunningAggregate runningAggregate, EXTRA value) throws EXCEP;
-    public RETURN visitSegment(Segment segment, EXTRA value) throws EXCEP;
     public RETURN visitSequence(Sequence sequence, EXTRA value) throws EXCEP;
     public RETURN visitTransform(Transform transform, EXTRA value) throws EXCEP;
     public RETURN visitUnion(Union union, EXTRA value) throws EXCEP;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index aba336e..bb044ad 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -27,6 +27,12 @@
       <version>4.1</version>
     </dependency>
     <dependency>
+      <groupId>pentaho</groupId>
+      <artifactId>mondrian-data-foodmart-json</artifactId>
+      <version>0.3.2</version>
+      <scope>test</scope>
+    </dependency>    
+    <dependency>
       <groupId>org.ow2.asm</groupId>
       <artifactId>asm-commons</artifactId>
       <version>4.1</version>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
index 0253069..6449d93 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
@@ -190,6 +190,7 @@ public class QuerySubmitter {
     private CountDownLatch latch = new CountDownLatch(1);
     RecordBatchLoader loader = new RecordBatchLoader(new BootStrapContext(DrillConfig.create()).getAllocator());
     Format format;
+    volatile Exception exception;
 
     public QueryResultsListener(Format format) {
       this.format = format;
@@ -197,7 +198,7 @@ public class QuerySubmitter {
 
     @Override
     public void submissionFailed(RpcException ex) {
-      System.out.println(String.format("Query failed: %s", ex));
+      exception = ex;
       latch.countDown();
     }
 
@@ -233,6 +234,7 @@ public class QuerySubmitter {
 
     public int await() throws Exception {
       latch.await();
+      if(exception != null) throw exception;
       return count.get();
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
index aed4802..4ea87e0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
@@ -25,14 +25,11 @@ import java.util.List;
 
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.logical.LogicalPlan;
 import org.apache.drill.common.logical.PlanProperties;
 import org.apache.drill.common.logical.StoragePluginConfig;
-import org.apache.drill.common.logical.data.CollapsingAggregate;
 import org.apache.drill.common.logical.data.Filter;
+import org.apache.drill.common.logical.data.GroupingAggregate;
 import org.apache.drill.common.logical.data.Join;
 import org.apache.drill.common.logical.data.JoinCondition;
 import org.apache.drill.common.logical.data.NamedExpression;
@@ -40,7 +37,6 @@ import org.apache.drill.common.logical.data.Order;
 import org.apache.drill.common.logical.data.Order.Ordering;
 import org.apache.drill.common.logical.data.Project;
 import org.apache.drill.common.logical.data.Scan;
-import org.apache.drill.common.logical.data.Segment;
 import org.apache.drill.common.logical.data.SinkOperator;
 import org.apache.drill.common.logical.data.Store;
 import org.apache.drill.common.logical.data.visitors.AbstractLogicalVisitor;
@@ -121,11 +117,23 @@ public class BasicOptimizer extends Optimizer{
       this.logicalPlan = logicalPlan;
     }
 
-    
-
     @Override
-    public PhysicalOperator visitSegment(Segment segment, Object value) throws OptimizerException {
-      throw new OptimizerException("Segment operators aren't currently supported besides next to a collapsing aggregate operator.");
+    public PhysicalOperator visitGroupingAggregate(GroupingAggregate groupBy, Object value) throws OptimizerException {
+      
+      List<Ordering> orderDefs = Lists.newArrayList();
+
+      
+      PhysicalOperator input = groupBy.getInput().accept(this, value);
+
+      if(groupBy.getKeys().length > 0){
+        for(NamedExpression e : groupBy.getKeys()){
+          orderDefs.add(new Ordering(Direction.Ascending, e.getExpr(), NullDirection.FIRST));
+        }
+        input = new Sort(input, orderDefs, false);
+      }
+      
+      StreamingAggregate sa = new StreamingAggregate(input, groupBy.getKeys(), groupBy.getExprs(), 1.0f);
+      return sa;
     }
 
 
@@ -146,34 +154,6 @@ public class BasicOptimizer extends Optimizer{
       return new SelectionVectorRemover(new Limit(input, limit.getFirst(), limit.getLast()));
     }
 
-    @Override
-    public PhysicalOperator visitCollapsingAggregate(CollapsingAggregate agg, Object value)
-        throws OptimizerException {
-
-      if( !(agg.getInput() instanceof Segment) ){
-        throw new OptimizerException(String.format("Currently, Drill only supports CollapsingAggregate immediately preceded by a Segment.  The input of this operator is %s.", agg.getInput()));
-      }
-      Segment segment = (Segment) agg.getInput();
-
-      if(!agg.getWithin().equals(segment.getName())){
-        throw new OptimizerException(String.format("Currently, Drill only supports CollapsingAggregate immediately preceded by a Segment where the CollapsingAggregate works on the defined segments.  In this case, the segment has been defined based on the name %s but the collapsing aggregate is working within the field %s.", segment.getName(), agg.getWithin()));
-      }
-      
-      // a collapsing aggregate is a currently implemented as a sort followed by a streaming aggregate.
-      List<Ordering> orderDefs = Lists.newArrayList();
-      
-      List<NamedExpression> keys = Lists.newArrayList();
-      for(LogicalExpression e : segment.getExprs()){
-        if( !(e instanceof SchemaPath)) throw new OptimizerException("The basic optimizer doesn't currently support collapsing aggregate where the segment value is something other than a SchemaPath.");
-        keys.add(new NamedExpression(e, new FieldReference((SchemaPath) e)));
-        orderDefs.add(new Ordering(Direction.Ascending, e, NullDirection.FIRST));
-      }
-      Sort sort = new Sort(segment.getInput().accept(this, value), orderDefs, false);
-      
-      StreamingAggregate sa = new StreamingAggregate(sort, keys.toArray(new NamedExpression[keys.size()]), agg.getAggregations(), 1.0f);
-      return sa;
-    }
-
 
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
index 2e29bd4..13f5494 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
@@ -84,9 +84,18 @@ public class DrillOptiq {
       switch (syntax) {
       case BINARY:
         logger.debug("Binary");
-        LogicalExpression op1 = call.getOperands().get(0).accept(this);
-        LogicalExpression op2 = call.getOperands().get(1).accept(this);
-        return context.getRegistry().createExpression(call.getOperator().getName(), Lists.newArrayList(op1, op2));
+        final String funcName = call.getOperator().getName().toLowerCase();
+        List<LogicalExpression> args = Lists.newArrayList();
+        for(RexNode r : call.getOperands()){
+          args.add(r.accept(this));
+        }
+        args = Lists.reverse(args);
+        LogicalExpression lastArg = args.get(0);
+        for(int i = 1; i < args.size(); i++){
+          lastArg = context.getRegistry().createExpression(funcName, Lists.newArrayList(args.get(i), lastArg));
+        }
+
+        return lastArg;
       case FUNCTION:
         logger.debug("Function");
         List<LogicalExpression> exprs = Lists.newArrayList();

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
new file mode 100644
index 0000000..c7fde56
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill;
+
+import org.apache.drill.common.util.TestTools;
+import org.apache.drill.exec.client.QuerySubmitter;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+
+public class TestExampleQueries {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestExampleQueries.class);
+  
+  @Rule public TestRule TIMEOUT = TestTools.getTimeoutRule(10000);
+  
+  @Test
+  public void testSelectWithLimit() throws Exception{
+    test("select * from cp.`employee.json` limit 5");
+  }
+  
+  @Test
+  public void testWhere() throws Exception{
+    test("select * from cp.`employee.json` where employee_id > 10 and employee_id < 20");
+  }
+
+  @Test
+  public void testGroupBy() throws Exception{
+    test("select marital_status, COUNT(1) as cnt from cp.`employee.json` group by marital_status");
+  }
+  
+  private void test(String sql) throws Exception{
+    QuerySubmitter s = new QuerySubmitter();
+    s.submitQuery(null, sql, "sql", null, true, 1, "tsv");
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/exec/java-exec/src/test/resources/storage-engines.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/storage-engines.json b/exec/java-exec/src/test/resources/storage-engines.json
index 58f091e..6e4d23e 100644
--- a/exec/java-exec/src/test/resources/storage-engines.json
+++ b/exec/java-exec/src/test/resources/storage-engines.json
@@ -3,6 +3,10 @@
     dfs: {
       type: "file",
       connection: "file:///"
+    },
+    cp: {
+      type: "file",
+      connection: "classpath:///"
     }
   }  
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 967e1da..fc5158d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -211,7 +211,7 @@
           <artifactId>maven-surefire-plugin</artifactId>
           <version>2.15</version>
           <configuration>
-            <argLine>-XX:MaxDirectMemorySize=4096M </argLine>
+            <argLine>-XX:MaxDirectMemorySize=6096M </argLine>
             <additionalClasspathElements>
               <additionalClasspathElement>./sqlparser/src/test/resources/storage-engines.json</additionalClasspathElement>
             </additionalClasspathElements>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2d10afd4/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java b/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
index 8810b6b..8cd682b 100644
--- a/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
+++ b/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
@@ -26,7 +26,6 @@ import java.util.Map;
 import org.apache.drill.common.logical.LogicalPlan;
 import org.apache.drill.common.logical.PlanProperties;
 import org.apache.drill.common.logical.StoragePluginConfig;
-import org.apache.drill.common.logical.data.CollapsingAggregate;
 import org.apache.drill.common.logical.data.Filter;
 import org.apache.drill.common.logical.data.Join;
 import org.apache.drill.common.logical.data.Limit;
@@ -284,44 +283,44 @@ public class JdbcTest {
     // .planContains("store");
   }
 
-  @Test
-  public void testDistinct() throws Exception {
-    JdbcAssert.withModel(MODEL, "HR").sql("select distinct deptId from emp")
-        .returnsUnordered("DEPTID=null", "DEPTID=31", "DEPTID=34", "DEPTID=33")
-        .planContains(CollapsingAggregate.class);
-  }
-
-  @Test
-  public void testCountNoGroupBy() throws Exception {
-    // 5 out of 6 employees have a not-null deptId
-    JdbcAssert.withModel(MODEL, "HR").sql("select count(deptId) as cd, count(*) as c from emp").returns("CD=5; C=6\n")
-        .planContains(CollapsingAggregate.class);
-  }
-
-  @Test
-  public void testDistinctCountNoGroupBy() throws Exception {
-    JdbcAssert.withModel(MODEL, "HR").sql("select count(distinct deptId) as c from emp").returns("C=3\n")
-        .planContains(CollapsingAggregate.class);
-  }
-
-  @Test
-  public void testDistinctCountGroupByEmpty() throws Exception {
-    JdbcAssert.withModel(MODEL, "HR").sql("select count(distinct deptId) as c from emp group by ()").returns("C=3\n")
-        .planContains(CollapsingAggregate.class);
-  }
-
-  @Test
-  public void testCountNull() throws Exception {
-    JdbcAssert.withModel(MODEL, "HR").sql("select count(distinct deptId) as c from emp group by ()").returns("C=3\n")
-        .planContains(CollapsingAggregate.class);
-  }
-
-  @Test
-  public void testCount() throws Exception {
-    JdbcAssert.withModel(MODEL, "HR").sql("select deptId, count(*) as c from emp group by deptId")
-        .returnsUnordered("DEPTID=31; C=1", "DEPTID=33; C=2", "DEPTID=34; C=2", "DEPTID=null; C=1")
-        .planContains(CollapsingAggregate.class); // make sure using drill
-  }
+//  @Test
+//  public void testDistinct() throws Exception {
+//    JdbcAssert.withModel(MODEL, "HR").sql("select distinct deptId from emp")
+//        .returnsUnordered("DEPTID=null", "DEPTID=31", "DEPTID=34", "DEPTID=33")
+//        .planContains(CollapsingAggregate.class);
+//  }
+//
+//  @Test
+//  public void testCountNoGroupBy() throws Exception {
+//    // 5 out of 6 employees have a not-null deptId
+//    JdbcAssert.withModel(MODEL, "HR").sql("select count(deptId) as cd, count(*) as c from emp").returns("CD=5; C=6\n")
+//        .planContains(CollapsingAggregate.class);
+//  }
+//
+//  @Test
+//  public void testDistinctCountNoGroupBy() throws Exception {
+//    JdbcAssert.withModel(MODEL, "HR").sql("select count(distinct deptId) as c from emp").returns("C=3\n")
+//        .planContains(CollapsingAggregate.class);
+//  }
+//
+//  @Test
+//  public void testDistinctCountGroupByEmpty() throws Exception {
+//    JdbcAssert.withModel(MODEL, "HR").sql("select count(distinct deptId) as c from emp group by ()").returns("C=3\n")
+//        .planContains(CollapsingAggregate.class);
+//  }
+//
+//  @Test
+//  public void testCountNull() throws Exception {
+//    JdbcAssert.withModel(MODEL, "HR").sql("select count(distinct deptId) as c from emp group by ()").returns("C=3\n")
+//        .planContains(CollapsingAggregate.class);
+//  }
+//
+//  @Test
+//  public void testCount() throws Exception {
+//    JdbcAssert.withModel(MODEL, "HR").sql("select deptId, count(*) as c from emp group by deptId")
+//        .returnsUnordered("DEPTID=31; C=1", "DEPTID=33; C=2", "DEPTID=34; C=2", "DEPTID=null; C=1")
+//        .planContains(CollapsingAggregate.class); // make sure using drill
+//  }
 
   @Test
   public void testJoin() throws Exception {


[3/5] git commit: Update Hive test data generator so that it won't fail on subsequent runs.

Posted by ja...@apache.org.
Update Hive test data generator so that it won't fail on subsequent runs.


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/17ea5432
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/17ea5432
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/17ea5432

Branch: refs/heads/master
Commit: 17ea5432c35ba8dc6858b940fbe349955ecb74c6
Parents: 822bd9c
Author: Jacques Nadeau <ja...@apache.org>
Authored: Tue Mar 11 08:43:00 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Mar 11 08:43:00 2014 -0700

----------------------------------------------------------------------
 .../exec/store/hive/HiveTestDataGenerator.java  | 22 +++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/17ea5432/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
index 7155a52..87b14ed 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
@@ -22,6 +22,7 @@ import java.io.File;
 import java.io.IOException;
 import java.io.PrintWriter;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.CommandNeedRetryException;
 import org.apache.hadoop.hive.ql.Driver;
@@ -33,18 +34,33 @@ public class HiveTestDataGenerator {
 
   static int RETRIES = 5;
   private Driver hiveDriver = null;
-
+  private static final String DB_DIR = "/tmp/drill_hive_db";
+  private static final String WH_DIR = "/tmp/drill_hive_wh";
+  
   public static void main(String[] args) throws Exception {
     HiveTestDataGenerator htd = new HiveTestDataGenerator();
     htd.generateTestData();
   }
 
+  private void cleanDir(String dir) throws IOException{
+    File f = new File(dir);
+    if(f.exists()){
+      FileUtils.cleanDirectory(f);
+      FileUtils.forceDelete(f);
+    }
+  }
+  
   public void generateTestData() throws Exception {
+    
+    // remove data from previous runs.
+    cleanDir(DB_DIR);
+    cleanDir(WH_DIR);
+    
     HiveConf conf = new HiveConf();
 
-    conf.set("javax.jdo.option.ConnectionURL", "jdbc:derby:;databaseName=/tmp/drill_hive_db;create=true");
+    conf.set("javax.jdo.option.ConnectionURL", String.format("jdbc:derby:;databaseName=%s;create=true", DB_DIR));
     conf.set("fs.default.name", "file:///");
-    conf.set("hive.metastore.warehouse.dir", "/tmp/drill_hive_wh");
+    conf.set("hive.metastore.warehouse.dir", WH_DIR);
 
     String tableName = "kv";