You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/09/02 00:22:02 UTC

[01/18] DRILL-167, DRILL-80, DRILL-59 start make changes to support sqlline module and ensure no test jars on class path

Updated Branches:
  refs/heads/master 4515263d3 -> 651b3e9fd


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/test/resources/logical_json_scan.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/logical_json_scan.json b/sandbox/prototype/exec/java-exec/src/test/resources/logical_json_scan.json
new file mode 100644
index 0000000..f9a8ffa
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/logical_json_scan.json
@@ -0,0 +1,36 @@
+{
+  head : {
+    type : "APACHE_DRILL_LOGICAL",
+    version : 1,
+    generator : {
+      type : "optiq",
+      info : "na"
+    }
+  },
+  storage : {
+    jsonl : {
+      type : "json",
+      dfsName : "file:///"
+    }
+  },
+  query : [ {
+    op : "scan",
+    @id : 1,
+    memo : "initial_scan",
+    storageengine : "jsonl",
+    selection : [ {
+      path : "/src/drill/sandbox/prototype/exec/ref/src/test/resources/employees.json"
+    } ],
+    ref : "_MAP"
+  }, {
+    op : "store",
+    @id : 2,
+    memo : "output sink",
+    input : 1,
+    target : {
+      number : 0
+    },
+    partition : null,
+    storageEngine : "queue"
+  } ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/test/resources/logical_order.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/logical_order.json b/sandbox/prototype/exec/java-exec/src/test/resources/logical_order.json
new file mode 100644
index 0000000..3e7fdfa
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/logical_order.json
@@ -0,0 +1,57 @@
+{
+  head : {
+    type : "APACHE_DRILL_LOGICAL",
+    version : 1,
+    generator : {
+      type : "optiq",
+      info : "na"
+    }
+  },
+  storage : {
+    parquet-local : {
+      type : "parquet",
+      dfsName : "file:///"
+    }
+  },
+  query : [ {
+    op : "scan",
+    @id : 1,
+    memo : "initial_scan",
+    storageengine : "parquet-local",
+    selection : [ {
+      path : "/tmp/parquet"
+    } ],
+    ref : "_MAP"
+  }, {
+    op : "project",
+    @id : 2,
+    input : 1,
+    projections : [ {
+      ref : "EXPR$0",
+      expr : "_MAP.d"
+    }, {
+      ref : "C",
+      expr : "_MAP.b"
+    } ]
+  }, {
+    op : "order",
+    @id : 3,
+    input : 2,
+    within : null,
+    orderings : [ {
+      order : "asc",
+      expr : "EXPR$0",
+      nullCollation : "NULLS_LAST"
+    } ]
+  }, {
+    op : "store",
+    @id : 4,
+    memo : "output sink",
+    input : 3,
+    target : {
+      number : 0
+    },
+    partition : null,
+    storageEngine : "queue"
+  } ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/test/resources/logical_string_filter.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/logical_string_filter.json b/sandbox/prototype/exec/java-exec/src/test/resources/logical_string_filter.json
new file mode 100644
index 0000000..28cc5e7
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/logical_string_filter.json
@@ -0,0 +1,52 @@
+{
+  head : {
+    type : "APACHE_DRILL_LOGICAL",
+    version : 1,
+    generator : {
+      type : "optiq",
+      info : "na"
+    }
+  },
+  storage : {
+    parquet-local : {
+      type : "parquet",
+      dfsName : "file:///"
+    }
+  },
+  query : [ {
+    op : "scan",
+    @id : 1,
+    memo : "initial_scan",
+    storageengine : "parquet-local",
+    selection : [ {
+      path : "/tmp/region.parquet"
+    } ],
+    ref : "_MAP"
+  }, {
+    op : "filter",
+    @id : 2,
+    input : 1,
+    expr : " (_MAP.R_NAME)  == (\"AFRICA\") "
+  }, {
+    op : "project",
+    @id : 3,
+    input : 2,
+    projections : [ {
+      ref : "EXPR$0",
+      expr : "_MAP.R_REGIONKEY"
+    }, {
+      ref : "NAME",
+      expr : "_MAP.R_NAME"
+    } ]
+  }, {
+    op : "store",
+    @id : 4,
+    memo : "output sink",
+    input : 3,
+    target : {
+      number : 0
+    },
+    partition : null,
+    storageEngine : "queue"
+  } ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/test/resources/physical_filter.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/physical_filter.json b/sandbox/prototype/exec/java-exec/src/test/resources/physical_filter.json
new file mode 100644
index 0000000..878f79d
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/physical_filter.json
@@ -0,0 +1,44 @@
+{
+  head : {
+    type : "APACHE_DRILL_PHYSICAL",
+    version : 1,
+    generator : {
+      type : "optiq",
+      info : "na"
+    }
+  },
+  graph : [ {
+    pop : "parquet-scan",
+    @id : 1,
+    entries : [ {
+      path : "/tmp/parquet"
+    } ],
+    storageengine: {
+      type: "parquet",
+      dfsName: "file:///"
+    },
+    ref : "_MAP",
+    fragmentPointer : 0
+  }, {
+    pop : "filter",
+    @id : 2,
+    child : 1,
+    expr : " (_MAP.bigInt)  < (0) "
+  }, {
+    pop : "selection-vector-remover",
+    @id : 3,
+    child : 2
+  }, {
+    pop : "project",
+    @id : 4,
+    exprs : [ {
+      ref : "KEY",
+      expr : "_MAP.bigInt"
+    } ],
+    child : 3
+  }, {
+    pop : "screen",
+    @id : 5,
+    child : 4
+  } ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/test/resources/physical_join.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/physical_join.json b/sandbox/prototype/exec/java-exec/src/test/resources/physical_join.json
new file mode 100644
index 0000000..7914ac3
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/physical_join.json
@@ -0,0 +1,117 @@
+{
+  head : {
+    type : "APACHE_DRILL_PHYSICAL",
+    version : 1,
+    generator : {
+      type : "optiq",
+      info : "na"
+    }
+  },
+  graph : [ {
+    pop : "parquet-scan",
+    @id : 1,
+    entries : [ {
+      path : "/tmp/region.parquet"
+    } ],
+    ref : "_MAP",
+    storageengine: {type: "parquet", dfsName: "file:///" },
+    fragmentPointer : 0
+  }, {
+    pop : "project",
+    @id : 2,
+    exprs : [ {
+      ref : "REGIONKEY",
+      expr : "_MAP.R_REGIONKEY"
+    }, {
+      ref : "NAME",
+      expr : "_MAP.R_NAME"
+    } ],
+    child : 1
+  }, {
+    pop : "project",
+    @id : 3,
+    exprs : [ {
+      ref : "REGIONKEY0",
+      expr : "REGIONKEY"
+    }, {
+      ref : "NAME0",
+      expr : "NAME"
+    } ],
+    child : 2
+  }, {
+    pop : "sort",
+    @id : 4,
+    child : 3,
+    orderings : [ {
+      order : "ASC",
+      expr : "NAME0"
+    } ],
+    reverse : false
+  }, {
+    pop : "selection-vector-remover",
+    @id : 5,
+    child : 4
+  }, {
+    pop : "parquet-scan",
+    @id : 6,
+    entries : [ {
+      path : "/tmp/nation.parquet"
+    } ],
+    storageengine: {type: "parquet", dfsName: "file:///" },    
+    ref : "_MAP",
+    fragmentPointer : 0
+  }, {
+    pop : "project",
+    @id : 7,
+    exprs : [ {
+      ref : "NAME",
+      expr : "_MAP.N_NAME"
+    }, {
+      ref : "REGIONKEY",
+      expr : "_MAP.N_REGIONKEY"
+    } ],
+    child : 6
+  }, {
+    pop : "sort",
+    @id : 8,
+    child : 7,
+    orderings : [ {
+      order : "ASC",
+      expr : "REGIONKEY"
+    } ],
+    reverse : false
+  }, {
+    pop : "selection-vector-remover",
+    @id : 9,
+    child : 8
+  }, {
+    pop : "merge-join",
+    @id : 10,
+    left: 9,
+    right: 5,
+    conditions : [ {
+      relationship : "==",
+      left : "REGIONKEY",
+      right : "NAME0"
+    } ]
+  }, {
+    pop : "selection-vector-remover",
+    @id : 11,
+    child : 10
+  }, {
+    pop : "project",
+    @id : 12,
+    exprs : [ {
+      ref : "NAME",
+      expr : "NAME"
+    }, {
+      ref : "NAME0",
+      expr : "NAME0"
+    } ],
+    child : 11
+  }, {
+    pop : "screen",
+    @id : 13,
+    child : 12
+  } ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/test/resources/physical_order_varbinary.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/physical_order_varbinary.json b/sandbox/prototype/exec/java-exec/src/test/resources/physical_order_varbinary.json
new file mode 100644
index 0000000..3493278
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/physical_order_varbinary.json
@@ -0,0 +1,48 @@
+{
+  head : {
+    type : "APACHE_DRILL_PHYSICAL",
+    version : 1,
+    generator : {
+      type : "optiq",
+      info : "na"
+    }
+  },
+  graph : [ {
+    pop : "parquet-scan",
+    @id : 1,
+    entries : [ {
+      path : "/tmp/nation.parquet"
+    } ],
+    storageengine: {type: "parquet", dfsName: "file:///" },
+    ref : "_MAP",
+    fragmentPointer : 0
+  }, {
+    pop : "project",
+    @id : 2,
+    exprs : [ {
+      ref : "REGIONKEY",
+      expr : "_MAP.N_REGIONKEY"
+    }, {
+      ref : "NAME",
+      expr : "_MAP.N_NAME"
+    } ],
+    child : 1
+  }, {
+    pop : "sort",
+    @id : 3,
+    child : 2,
+    orderings : [ {
+      order : "ASC",
+      expr : "NAME"
+    } ],
+    reverse : false
+  }, {
+    pop : "selection-vector-remover",
+    @id : 4,
+    child : 3
+  }, {
+    pop : "screen",
+    @id : 5,
+    child : 4
+  } ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/ref/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/ref/pom.xml b/sandbox/prototype/exec/ref/pom.xml
index 5724015..ec664d5 100644
--- a/sandbox/prototype/exec/ref/pom.xml
+++ b/sandbox/prototype/exec/ref/pom.xml
@@ -27,11 +27,11 @@
 			<version>0.4.2</version>
 		</dependency>
 
-		<dependency>
-			<groupId>org.apache.hadoop</groupId>
-			<artifactId>hadoop-examples</artifactId>
-			<version>1.1.1</version>
-		</dependency>
+<!-- 		<dependency> -->
+<!-- 			<groupId>org.apache.hadoop</groupId> -->
+<!-- 			<artifactId>hadoop-examples</artifactId> -->
+<!-- 			<version>1.2.1</version> -->
+<!-- 		</dependency> -->
 	</dependencies>
 
   <profiles>
@@ -78,7 +78,6 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>
-        <version>2.4</version>
 
         <executions>
           <execution>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlline
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlline b/sandbox/prototype/sqlline
index 98fc569..7cec974 100755
--- a/sandbox/prototype/sqlline
+++ b/sandbox/prototype/sqlline
@@ -7,10 +7,11 @@
 
 # Build classpath on first call. (To force rebuild, remove .classpath.)
 if [ ! -f .classpath ]; then
-    (cd sqlparser; mvn dependency:build-classpath -Dmdep.outputFile=../.classpath)
-    echo -n :sqlparser/target/classes/ >> .classpath
+    (cd contrib/sqlline; mvn dependency:build-classpath -Dmdep.outputFile=../../.classpath)
 fi
 
-exec java -cp "$(cat .classpath)" sqlline.SqlLine "$@"
+#exec java -Drebel.log=true -javaagent:/src/jrebel/jrebel.jar -Dlogback.configurationFile=/src/drill/sandbox/prototype/sqlparser/src/test/resources/logback.xml -cp ./:"$(cat .classpath)" sqlline.SqlLine "$@"
+exec java -Dlogback.configurationFile=/src/drill/sandbox/prototype/sqlparser/src/test/resources/logback.xml -cp ./:"$(cat .classpath)" sqlline.SqlLine "$@"
+
 
 # End sqlline

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/pom.xml b/sandbox/prototype/sqlparser/pom.xml
index c15e93c..212362c 100644
--- a/sandbox/prototype/sqlparser/pom.xml
+++ b/sandbox/prototype/sqlparser/pom.xml
@@ -29,7 +29,7 @@
     <dependency>
       <groupId>net.hydromatic</groupId>
       <artifactId>optiq</artifactId>
-      <version>0.4.9</version>
+      <version>0.4.10</version>
     </dependency>
     <dependency>
       <groupId>org.apache.drill</groupId>
@@ -60,7 +60,6 @@
       <scope>test</scope>
     </dependency>
 
-
     <dependency>
       <groupId>org.apache.drill.exec</groupId>
       <artifactId>ref</artifactId>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
new file mode 100644
index 0000000..f2265f8
--- /dev/null
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
@@ -0,0 +1,147 @@
+package org.apache.drill.jdbc;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.Map;
+import java.util.Properties;
+
+import net.hydromatic.linq4j.QueryProvider;
+import net.hydromatic.linq4j.expressions.Expression;
+import net.hydromatic.optiq.MutableSchema;
+import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.impl.java.JavaTypeFactory;
+import net.hydromatic.optiq.impl.java.MapSchema;
+import net.hydromatic.optiq.jdbc.HandlerImpl;
+import net.hydromatic.optiq.jdbc.OptiqConnection;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.logical.StorageEngineConfig;
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.coord.ClusterCoordinator;
+import org.apache.drill.exec.coord.ZKClusterCoordinator;
+import org.apache.drill.exec.server.Drillbit;
+import org.apache.drill.exec.server.RemoteServiceSet;
+import org.apache.drill.exec.store.SchemaProvider;
+import org.apache.drill.exec.store.SchemaProviderRegistry;
+import org.apache.drill.sql.client.full.FileSystemSchema;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import com.google.common.io.Resources;
+
+public class DrillHandler extends HandlerImpl {
+
+  private ClusterCoordinator coordinator;
+  private volatile DrillClient client;
+  private Drillbit bit;
+  private DrillConfig config = DrillConfig.create();
+  private SchemaProviderRegistry registry;
+
+  public void onConnectionInit(OptiqConnection connection) throws SQLException {
+    super.onConnectionInit(connection);
+
+    registry = new SchemaProviderRegistry(config);
+    final Properties p = connection.getProperties();
+    Preconditions.checkArgument(bit == null);
+    Preconditions.checkArgument(client == null);
+    Preconditions.checkArgument(coordinator == null);
+    // final String model = p.getProperty("model");
+    // if (model != null) {
+    // if (model != null) {
+    // try {
+    // new ModelHandler(connection, model);
+    // } catch (IOException e) {
+    // throw new SQLException(e);
+    // }
+    // }
+    // }
+
+    final String zk = connection.getProperties().getProperty("zk");
+
+    try {
+      String enginesData = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8);
+
+      StorageEngines engines = config.getMapper().readValue(enginesData, StorageEngines.class);
+      MutableSchema rootSchema = connection.getRootSchema();
+
+      for (Map.Entry<String, StorageEngineConfig> entry : engines) {
+        SchemaProvider provider = registry.getSchemaProvider(entry.getValue());
+        FileSystemSchema schema = new FileSystemSchema(client, entry.getValue(), provider, rootSchema.getTypeFactory(),
+            rootSchema, entry.getKey(), rootSchema.getExpression(), rootSchema.getQueryProvider());
+        rootSchema.addSchema(entry.getKey(), schema);
+      }
+
+      rootSchema.addSchema("--FAKE--", new FakeSchema(rootSchema, rootSchema.getQueryProvider(), rootSchema.getTypeFactory(), "fake", rootSchema.getExpression()));
+      
+      if (zk != null) {
+        coordinator = new ZKClusterCoordinator(config, zk);
+        coordinator.start(10000);
+        DrillClient cl = new DrillClient(config, coordinator);
+        cl.connect();
+        client = cl;
+      } else {
+
+        RemoteServiceSet local = RemoteServiceSet.getLocalServiceSet();
+        this.coordinator = local.getCoordinator();
+        bit = new Drillbit(config, local);
+        bit.run();
+        
+        DrillClient cl = new DrillClient(config, coordinator);
+        cl.connect();
+        client = cl;
+      }
+    } catch (Exception ex) {
+      throw new SQLException("Failure trying to connect to Drill.", ex);
+    }
+
+    // The "schema" parameter currently gives a name to the schema. In future
+    // it will choose a schema that (presumably) already exists.
+    final String schemaName = connection.getProperties().getProperty("schema");
+    if (schemaName != null) {
+      connection.setSchema(schemaName);
+    }
+
+  }
+  
+  public class FakeSchema extends MapSchema{
+
+    public FakeSchema(Schema parentSchema, QueryProvider queryProvider, JavaTypeFactory typeFactory, String name,
+        Expression expression) {
+      super(parentSchema, queryProvider, typeFactory, name, expression);
+      
+    }
+    public DrillClient getClient() {
+      return client;
+    }
+  }
+  
+  public DrillClient getClient() {
+    return client;
+  }
+
+
+  @Override
+  public void onConnectionClose(OptiqConnection connection) throws RuntimeException {
+    super.onConnectionClose(connection);
+    if (client != null)
+      client.close();
+    if (bit != null)
+      bit.close();
+    if (coordinator != null)
+      try {
+        coordinator.close();
+      } catch (IOException e) {
+        throw new RuntimeException("Failure closing coordinator.", e);
+      }
+  }
+
+}
+/*
+ * 
+ * optiq work ==========
+ * 
+ * 1. todo: test can cast(<any> as varchar) (or indeed to any type)
+ * 
+ * 2. We declare a variant record by adding a '_MAP any' field. It's nice that there can be some declared fields, and
+ * some undeclared. todo: Better syntactic sugar.
+ */
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillTable.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillTable.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillTable.java
index dd86372..6dfea48 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillTable.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillTable.java
@@ -17,12 +17,8 @@
  ******************************************************************************/
 package org.apache.drill.jdbc;
 
-import java.io.IOException;
 import java.lang.reflect.Type;
-import java.util.Arrays;
 import java.util.Collections;
-import java.util.List;
-import java.util.Map;
 
 import net.hydromatic.linq4j.BaseQueryable;
 import net.hydromatic.linq4j.Enumerator;
@@ -36,18 +32,10 @@ import net.hydromatic.optiq.MutableSchema;
 import net.hydromatic.optiq.Schema;
 import net.hydromatic.optiq.Statistic;
 import net.hydromatic.optiq.Statistics;
-import net.hydromatic.optiq.TableFactory;
 import net.hydromatic.optiq.TranslatableTable;
-import net.hydromatic.optiq.impl.java.JavaTypeFactory;
 
-import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.logical.StorageEngineConfig;
 import org.apache.drill.exec.client.DrillClient;
-import org.apache.drill.exec.ref.rops.DataWriter;
-import org.apache.drill.exec.ref.rse.ClasspathRSE;
-import org.apache.drill.exec.ref.rse.ClasspathRSE.ClasspathInputConfig;
-import org.apache.drill.exec.server.Drillbit;
-import org.apache.drill.exec.server.RemoteServiceSet;
 import org.apache.drill.optiq.DrillRel;
 import org.apache.drill.optiq.DrillScan;
 import org.eigenbase.rel.RelNode;
@@ -57,68 +45,60 @@ import org.eigenbase.reltype.RelDataTypeFactory;
 import org.eigenbase.sql.type.SqlTypeName;
 
 /** Optiq Table used by Drill. */
-public class DrillTable extends BaseQueryable<Object>
-    implements TranslatableTable<Object>
-{
+public class DrillTable extends BaseQueryable<Object> implements TranslatableTable<Object>{
+  
   private final Schema schema;
   private final String name;
   private final String storageEngineName;  
   private final RelDataType rowType;
   public final StorageEngineConfig storageEngineConfig;
-  public final Object selection;
-  private boolean useReferenceInterpreter;
-  // full engine connection information
-  public Drillbit bit;
+  private Object selection;
   public DrillClient client;
+  
 
   /** Creates a DrillTable. */
-  public DrillTable(Schema schema,
+  public DrillTable(DrillClient client, Schema schema,
       Type elementType,
       Expression expression,
       RelDataType rowType,
       String name,
-      StorageEngineConfig storageEngineConfig,
       Object selection,
-      String storageEngineName,
-      boolean useReferenceInterpreter     
+      StorageEngineConfig storageEngineConfig
       ) {
     super(schema.getQueryProvider(), elementType, expression);
     this.schema = schema;
     this.name = name;
     this.rowType = rowType;
-    this.storageEngineConfig = storageEngineConfig;
     this.selection = selection;
-    this.storageEngineName = storageEngineName;
-    this.useReferenceInterpreter = useReferenceInterpreter;
-
-    
-    DrillConfig config = DrillConfig.create();
-    RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
-    try {
-      bit = new Drillbit(config, serviceSet);
-      client = new DrillClient(config, serviceSet.getCoordinator());
-      bit.run();
-    } catch (IOException e) {
-      System.out.println("Error creating drill client or connecting to drillbit.");
-    } catch (Exception e) {
-      System.out.println("Error creating drill client or connecting to drillbit.");
-    }
+    this.storageEngineConfig = storageEngineConfig;
+    this.storageEngineName = schema.getName();
+  }
+  
+  public String getName() {
+    return name;
+  }
 
+  public StorageEngineConfig getStorageEngineConfig(){
+    return storageEngineConfig;
+  }
+  
+  public Object getSelection() {
+    return selection;
   }
 
-  private static DrillTable createTable(
+  public static DrillTable createTable(
+      DrillClient client,
       RelDataTypeFactory typeFactory,
-      MutableSchema schema,
+      Schema schema,
       String name,
       StorageEngineConfig storageEngineConfig,
-      Object selection,
-      String storageEngineName,
-      boolean useReferenceInterpreter
+      Object selection
       ) {
-    final MethodCallExpression call = Expressions.call(schema.getExpression(),
-        BuiltinMethod.DATA_CONTEXT_GET_TABLE.method,
-        Expressions.constant(name),
+    final MethodCallExpression call = Expressions.call(schema.getExpression(), //
+        BuiltinMethod.DATA_CONTEXT_GET_TABLE.method, //
+        Expressions.constant(name), //
         Expressions.constant(Object.class));
+    
     final RelDataType rowType =
         typeFactory.createStructType(
             Collections.singletonList(
@@ -126,9 +106,10 @@ public class DrillTable extends BaseQueryable<Object>
                     typeFactory.createSqlType(SqlTypeName.VARCHAR),
                     typeFactory.createSqlType(SqlTypeName.ANY))),
             Collections.singletonList("_MAP"));
-      return new DrillTable(schema, Object.class, call, rowType, name,
-          storageEngineConfig, selection, storageEngineName, useReferenceInterpreter);
+      return new DrillTable(client, schema, Object.class, call, rowType, name, selection, storageEngineConfig);
   }
+  
+  
 
   @Override
   public DataContext getDataContext() {
@@ -141,7 +122,7 @@ public class DrillTable extends BaseQueryable<Object>
   }
 
   public boolean useReferenceInterpreter() {
-    return useReferenceInterpreter;
+    return this.client == null;
   }
 
   @Override
@@ -165,71 +146,40 @@ public class DrillTable extends BaseQueryable<Object>
         table);
   }
 
-  /** Factory for custom tables in Optiq schema. */
-  @SuppressWarnings("UnusedDeclaration")
-  public static class Factory implements TableFactory<DrillTable> {
-
-    private static final List<String> DONUTS_TABLES = Arrays.asList(
-        "DONUTS");
-
-    private static final List<String> HR_TABLES = Arrays.asList(
-        "EMPLOYEES", "DEPARTMENTS");
-
-    private static final List<String> FOODMART_TABLES = Arrays.asList(
-        "ACCOUNT", "CATEGORY", "CURRENCY", "CUSTOMER", "DAYS", "DEPARTMENT",
-        "EMPLOYEE_CLOSURE", "EMPLOYEE", "EXPENSE_FACT", "INVENTORY_FACT_1997",
-        "INVENTORY_FACT_1998", "POSITION", "PRODUCT_CLASS", "PRODUCT",
-        "PROMOTION", "REGION", "RESERVE_EMPLOYEE", "SALARY", "SALES_FACT_1997",
-        "SALES_FACT_1998", "SALES_FACT_DEC_1998", "STORE", "STORE_RAGGED",
-        "TIME_BY_DAY", "WAREHOUSE", "WAREHOUSE_CLASS");
-
-//    public DrillTable create(
-//        JavaTypeFactory typeFactory,
-//        Schema schema,
-//        String name,
-//        Map<String, Object> operand, 
-//        RelDataType rowType) {
-//      final ClasspathRSE.ClasspathRSEConfig rseConfig = new ClasspathRSE.ClasspathRSEConfig();
-//      final ClasspathInputConfig inputConfig = new ClasspathInputConfig();
-//      assert DONUTS_TABLES.contains(name)
-//          || HR_TABLES.contains(name)
-//          || FOODMART_TABLES.contains(name)
-//          : name;
-//      inputConfig.path = "/" + name.toLowerCase() + ".json";
-//      inputConfig.type = DataWriter.ConverterType.JSON;
-//      boolean useReferenceInterpreter;
-//      if (operand.get("useReferenceInterpreter") != null){
-//        useReferenceInterpreter = operand.get("useReferenceInterpreter").equals("true") ? true : false;
-//      }
-//      else{
-//        useReferenceInterpreter = false;
-//      }      
-//      return createTable(typeFactory, (MutableSchema) schema, name, rseConfig,
-//          inputConfig, "donuts-json", useReferenceInterpreter);
-//    }
+  
+  
+//  private static final List<String> DONUTS_TABLES = Arrays.asList(
+//      "DONUTS");
 //
-    @Override
-    public DrillTable create(Schema schema, String name, Map<String, Object> operand, RelDataType rowType) {
-      
-      final ClasspathRSE.ClasspathRSEConfig rseConfig = new ClasspathRSE.ClasspathRSEConfig();
-      final ClasspathInputConfig inputConfig = new ClasspathInputConfig();
-      assert DONUTS_TABLES.contains(name)
-          || HR_TABLES.contains(name)
-          || FOODMART_TABLES.contains(name)
-          : name;
-      inputConfig.path = "/" + name.toLowerCase() + ".json";
-      inputConfig.type = DataWriter.ConverterType.JSON;
-      boolean useReferenceInterpreter;
-      if (operand.get("useReferenceInterpreter") != null){
-        useReferenceInterpreter = operand.get("useReferenceInterpreter").equals("true") ? true : false;
-      }
-      else{
-        useReferenceInterpreter = false;
-      }      
-      return createTable(schema.getTypeFactory(), (MutableSchema) schema, name, rseConfig,
-          inputConfig, "donuts-json", useReferenceInterpreter);
-    }
-  }
-}
-
-// End DrillTable.java
+//  private static final List<String> HR_TABLES = Arrays.asList(
+//      "EMPLOYEES", "DEPARTMENTS");
+//
+//  private static final List<String> FOODMART_TABLES = Arrays.asList(
+//      "ACCOUNT", "CATEGORY", "CURRENCY", "CUSTOMER", "DAYS", "DEPARTMENT",
+//      "EMPLOYEE_CLOSURE", "EMPLOYEE", "EXPENSE_FACT", "INVENTORY_FACT_1997",
+//      "INVENTORY_FACT_1998", "POSITION", "PRODUCT_CLASS", "PRODUCT",
+//      "PROMOTION", "REGION", "RESERVE_EMPLOYEE", "SALARY", "SALES_FACT_1997",
+//      "SALES_FACT_1998", "SALES_FACT_DEC_1998", "STORE", "STORE_RAGGED",
+//      "TIME_BY_DAY", "WAREHOUSE", "WAREHOUSE_CLASS");
+//  
+//  public static DrillTable create(Schema schema, String name, Map<String, Object> operand, RelDataType rowType) {
+//    
+//    final ClasspathRSE.ClasspathRSEConfig rseConfig = new ClasspathRSE.ClasspathRSEConfig();
+//    final ClasspathInputConfig inputConfig = new ClasspathInputConfig();
+//    assert DONUTS_TABLES.contains(name)
+//        || HR_TABLES.contains(name)
+//        || FOODMART_TABLES.contains(name)
+//        : name;
+//    inputConfig.path = "/" + name.toLowerCase() + ".json";
+//    inputConfig.type = DataWriter.ConverterType.JSON;
+//    boolean useReferenceInterpreter;
+//    if (operand.get("useReferenceInterpreter") != null){
+//      useReferenceInterpreter = operand.get("useReferenceInterpreter").equals("true") ? true : false;
+//    }else{
+//      useReferenceInterpreter = false;
+//    }      
+//    return createTable(schema.getTypeFactory(), (MutableSchema) schema, name, rseConfig,
+//        inputConfig, "donuts-json", useReferenceInterpreter);
+//  }
+  
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/Driver.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/Driver.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/Driver.java
index dba2162..7812c2c 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/Driver.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/Driver.java
@@ -17,12 +17,13 @@
  ******************************************************************************/
 package org.apache.drill.jdbc;
 
-import java.io.IOException;
-import java.sql.SQLException;
-
 import net.hydromatic.linq4j.function.Function0;
-import net.hydromatic.optiq.jdbc.*;
-import net.hydromatic.optiq.model.ModelHandler;
+import net.hydromatic.optiq.jdbc.DriverVersion;
+import net.hydromatic.optiq.jdbc.Handler;
+import net.hydromatic.optiq.jdbc.OptiqPrepare;
+import net.hydromatic.optiq.jdbc.UnregisteredDriver;
+
+import org.apache.drill.exec.client.DrillClient;
 import org.apache.drill.optiq.DrillPrepareImpl;
 
 /**
@@ -31,6 +32,8 @@ import org.apache.drill.optiq.DrillPrepareImpl;
 public class Driver extends UnregisteredDriver {
   public static final String CONNECT_STRING_PREFIX = "jdbc:drill:";
 
+  private volatile DrillHandler handler;
+  
   static {
     new Driver().register();
   }
@@ -46,56 +49,23 @@ public class Driver extends UnregisteredDriver {
   @Override
   protected Function0<OptiqPrepare> createPrepareFactory() {
     return new Function0<OptiqPrepare>() {
-        @Override
-        public OptiqPrepare apply() {
-            return new DrillPrepareImpl();
-        }
+      @Override
+      public OptiqPrepare apply() {
+        return new DrillPrepareImpl(Driver.this);
+      }
     };
   }
 
-    @Override
-  protected Handler createHandler() {
-    return new DrillHandler();
+  public DrillClient getClient(){
+    return handler.getClient();
   }
-
-  private static class DrillHandler extends HandlerImpl {
-    public void onConnectionInit(OptiqConnection connection)
-        throws SQLException
-    {
-      super.onConnectionInit(connection);
-
-      
-      final String model = connection.getProperties().getProperty("model");
-      if (model != null) {
-        try {
-          new ModelHandler(connection, model);
-        } catch (IOException e) {
-          throw new SQLException(e);
-        }
-      }
-
-      // The "schema" parameter currently gives a name to the schema. In future
-      // it will choose a schema that (presumably) already exists.
-      final String schemaName =
-          connection.getProperties().getProperty("schema");
-      if (schemaName != null) {
-        connection.setSchema(schemaName);
-      }
-    }
+  
+  @Override
+  protected Handler createHandler() {
+    this.handler = new DrillHandler();
+    return handler;
   }
-
-    /*
-
-optiq work
-==========
-
-1. todo: test can cast(<any> as varchar)  (or indeed to any type)
-
-2. We declare a variant record by adding a '_MAP any' field. It's nice that
-there can be some declared fields, and some undeclared. todo: Better syntactic
-sugar.
-
-*/
+  
 }
 
 // End Driver.java

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/StorageEngines.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/StorageEngines.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/StorageEngines.java
new file mode 100644
index 0000000..9bc83e7
--- /dev/null
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/StorageEngines.java
@@ -0,0 +1,57 @@
+package org.apache.drill.jdbc;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.logical.StorageEngineConfig;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Charsets;
+import com.google.common.io.Resources;
+
+public class StorageEngines implements Iterable<Map.Entry<String, StorageEngineConfig>>{
+  
+  private Map<String, StorageEngineConfig> storage;
+  
+  @JsonCreator
+  public StorageEngines(@JsonProperty("storage") Map<String, StorageEngineConfig> storage){
+    this.storage = storage;
+  }
+  
+  public static void main(String[] args) throws Exception{
+    DrillConfig config = DrillConfig.create();
+    String data = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8);
+    StorageEngines se = config.getMapper().readValue(data,  StorageEngines.class);
+    System.out.println(se);
+  }
+
+  @Override
+  public String toString() {
+    final int maxLen = 10;
+    return "StorageEngines [storage=" + (storage != null ? toString(storage.entrySet(), maxLen) : null) + "]";
+  }
+
+  @Override
+  public Iterator<Entry<String, StorageEngineConfig>> iterator() {
+    return storage.entrySet().iterator();
+  }
+
+  private String toString(Collection<?> collection, int maxLen) {
+    StringBuilder builder = new StringBuilder();
+    builder.append("[");
+    int i = 0;
+    for (Iterator<?> iterator = collection.iterator(); iterator.hasNext() && i < maxLen; i++) {
+      if (i > 0)
+        builder.append(", ");
+      builder.append(iterator.next());
+    }
+    builder.append("]");
+    return builder.toString();
+  }
+  
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java
index 57091c8..750fb18 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java
@@ -17,7 +17,11 @@
  ******************************************************************************/
 package org.apache.drill.optiq;
 
+import java.util.Set;
+
 import org.apache.drill.exec.ref.rse.QueueRSE.QueueOutputInfo;
+import org.apache.drill.jdbc.DrillTable;
+import org.eigenbase.rel.RelNode;
 
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.databind.JsonNode;
@@ -25,7 +29,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.eigenbase.rel.RelNode;
+import com.google.common.collect.Sets;
 
 /**
  * Context for converting a tree of {@link DrillRel} nodes into a Drill logical plan.
@@ -39,7 +43,9 @@ public class DrillImplementor {
   }
   private final ObjectNode rootNode = mapper.createObjectNode();
   private final ArrayNode operatorsNode;
-
+  private final ObjectNode sourcesNode;
+  private Set<DrillTable> tables = Sets.newHashSet();
+  
   public DrillImplementor() {
     final ObjectNode headNode = mapper.createObjectNode();
     rootNode.put("head", headNode);
@@ -53,26 +59,32 @@ public class DrillImplementor {
 
     // TODO: populate sources based on the sources of scans that occur in
     // the query
-    final ObjectNode sourcesNode = mapper.createObjectNode();
+    sourcesNode = mapper.createObjectNode();
     rootNode.put("storage", sourcesNode);
 
     // input file source
-    {
-      final ObjectNode sourceNode = mapper.createObjectNode();
-      sourceNode.put("type", "classpath");
-      sourcesNode.put("donuts-json", sourceNode);
-    }
-    {
-      final ObjectNode sourceNode = mapper.createObjectNode();
-      sourceNode.put("type", "queue");
-      sourcesNode.put("queue", sourceNode);
-    }
+//    {
+//      final ObjectNode sourceNode = mapper.createObjectNode();
+//      sourceNode.put("type", "classpath");
+//      sourcesNode.put("donuts-json", sourceNode);
+//    }
+//    {
+//      final ObjectNode sourceNode = mapper.createObjectNode();
+//      sourceNode.put("type", "queue");
+//      sourcesNode.put("queue", sourceNode);
+//    }
 
     final ArrayNode queryNode = mapper.createArrayNode();
     rootNode.put("query", queryNode);
 
     this.operatorsNode = queryNode;
   }
+  
+  public void registerSource(DrillTable table){
+    if(tables.add(table)){
+      sourcesNode.put(table.getStorageEngineName(), mapper.convertValue(table.getStorageEngineConfig(), JsonNode.class));  
+    }
+  }
 
   public int go(DrillRel root) {
     int inputId = root.implement(this);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRel.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRel.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRel.java
index 79e2da9..82a0c43 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRel.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRel.java
@@ -91,7 +91,7 @@ public class DrillJoinRel extends JoinRelBase implements DrillRel {
       final ObjectNode condition = implementor.mapper.createObjectNode();
       condition.put("relationship", "==");
       condition.put("left", leftFields.get(pair.left));
-      condition.put("right", rightFields.get(pair.left));
+      condition.put("right", rightFields.get(pair.right));
       conditions.add(condition);
     }
     return implementor.add(join);
@@ -123,7 +123,8 @@ public class DrillJoinRel extends JoinRelBase implements DrillRel {
       final ObjectNode objectNode = implementor.mapper.createObjectNode();
       transforms.add(objectNode);
       objectNode.put("expr", pair.left);
-      objectNode.put("ref", "output." + pair.right);
+//      objectNode.put("ref", "output." + pair.right);
+      objectNode.put("ref", pair.right);
     }
     return implementor.add(project);
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillOptiq.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillOptiq.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillOptiq.java
index 1a66c0c..e1c01fa 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillOptiq.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillOptiq.java
@@ -18,10 +18,23 @@
 package org.apache.drill.optiq;
 
 import net.hydromatic.linq4j.Ord;
+
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.work.FragmentRunnerListener;
 import org.eigenbase.rel.RelNode;
 import org.eigenbase.relopt.RelOptPlanner;
 import org.eigenbase.reltype.RelDataTypeField;
-import org.eigenbase.rex.*;
+import org.eigenbase.rex.RexCall;
+import org.eigenbase.rex.RexCorrelVariable;
+import org.eigenbase.rex.RexDynamicParam;
+import org.eigenbase.rex.RexFieldAccess;
+import org.eigenbase.rex.RexInputRef;
+import org.eigenbase.rex.RexLiteral;
+import org.eigenbase.rex.RexLocalRef;
+import org.eigenbase.rex.RexNode;
+import org.eigenbase.rex.RexOver;
+import org.eigenbase.rex.RexRangeRef;
+import org.eigenbase.rex.RexVisitorImpl;
 import org.eigenbase.sql.SqlSyntax;
 import org.eigenbase.sql.fun.SqlStdOperatorTable;
 
@@ -29,8 +42,10 @@ import org.eigenbase.sql.fun.SqlStdOperatorTable;
  * Utilities for Drill's planner.
  */
 public class DrillOptiq {
-  static void registerStandardPlannerRules(RelOptPlanner planner) {
-    planner.addRule(EnumerableDrillRule.ARRAY_INSTANCE);
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillOptiq.class);
+  
+  static void registerStandardPlannerRules(RelOptPlanner planner, DrillClient client) {
+    planner.addRule(EnumerableDrillRule.getInstance(client));
 
     // planner.addRule(DrillTableModificationConverterRule.INSTANCE);
     // planner.addRule(DrillCalcConverterRule.INSTANCE);
@@ -67,13 +82,16 @@ public class DrillOptiq {
 
     @Override
     public StringBuilder visitCall(RexCall call) {
+      logger.debug("RexCall {}, {}", call);
       final SqlSyntax syntax = call.getOperator().getSyntax();
       switch (syntax) {
       case Binary:
+        logger.debug("Binary");
         buf.append("(");
         call.getOperands().get(0).accept(this).append(" ").append(call.getOperator().getName()).append(" ");
         return call.getOperands().get(1).accept(this).append(")");
       case Function:
+        logger.debug("Function");
         buf.append(call.getOperator().getName().toLowerCase()).append("(");
         for (Ord<RexNode> operand : Ord.zip(call.getOperands())) {
           buf.append(operand.i > 0 ? ", " : "");
@@ -81,39 +99,81 @@ public class DrillOptiq {
         }
         return buf.append(")");
       case Special:
+        logger.debug("Special");
         switch (call.getKind()) {
         case Cast:
+          logger.debug("Cast {}", buf);
           // Ignore casts. Drill is type-less.
+          logger.debug("Ignoring cast {}, {}", call.getOperands().get(0), call.getOperands().get(0).getClass());
           return call.getOperands().get(0).accept(this);
         }
         if (call.getOperator() == SqlStdOperatorTable.itemOp) {
           final RexNode left = call.getOperands().get(0);
           final RexLiteral literal = (RexLiteral) call.getOperands().get(1);
           final String field = (String) literal.getValue2();
+//          buf.append('\'');
           final int length = buf.length();
           left.accept(this);
           if (buf.length() > length) {
             // check before generating empty LHS if inputName is null
             buf.append('.');
           }
-          return buf.append(field);
+          buf.append(field);
+//          buf.append('\'');
+          return buf;
         }
+        logger.debug("Not cast.");
         // fall through
       default:
         throw new AssertionError("todo: implement syntax " + syntax + "(" + call + ")");
       }
     }
 
+    private StringBuilder doUnknown(Object o){
+      logger.warn("Doesn't currently support consumption of {}.", o);
+      return buf;
+    }
+    @Override
+    public StringBuilder visitLocalRef(RexLocalRef localRef) {
+      return doUnknown(localRef);
+    }
+
+    @Override
+    public StringBuilder visitOver(RexOver over) {
+      return doUnknown(over);
+    }
+
+    @Override
+    public StringBuilder visitCorrelVariable(RexCorrelVariable correlVariable) {
+      return doUnknown(correlVariable);
+    }
+
+    @Override
+    public StringBuilder visitDynamicParam(RexDynamicParam dynamicParam) {
+      return doUnknown(dynamicParam);
+    }
+
+    @Override
+    public StringBuilder visitRangeRef(RexRangeRef rangeRef) {
+      return doUnknown(rangeRef);
+    }
+
+    @Override
+    public StringBuilder visitFieldAccess(RexFieldAccess fieldAccess) {
+      return super.visitFieldAccess(fieldAccess);
+    }
+
     @Override
     public StringBuilder visitInputRef(RexInputRef inputRef) {
       final int index = inputRef.getIndex();
       final RelDataTypeField field = input.getRowType().getFieldList().get(index);
-      return buf.append(field.getName());
+      buf.append(field.getName());
+      return buf;
     }
 
     @Override
     public StringBuilder visitLiteral(RexLiteral literal) {
-      return buf.append(literal);
+      return buf.append(literal.toString());
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillPrepareImpl.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillPrepareImpl.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillPrepareImpl.java
index f5dce0a..b80bc8c 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillPrepareImpl.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillPrepareImpl.java
@@ -19,20 +19,25 @@ package org.apache.drill.optiq;
 
 import net.hydromatic.optiq.prepare.OptiqPrepareImpl;
 import net.hydromatic.optiq.rules.java.JavaRules;
+
+import org.apache.drill.jdbc.DrillHandler;
+import org.apache.drill.jdbc.Driver;
 import org.eigenbase.relopt.RelOptPlanner;
 
 /**
  * Implementation of {@link net.hydromatic.optiq.jdbc.OptiqPrepare} for Drill.
  */
 public class DrillPrepareImpl extends OptiqPrepareImpl {
-  public DrillPrepareImpl() {
+  final Driver driver;
+  public DrillPrepareImpl(Driver driver) {
     super();
+    this.driver = driver;
   }
 
   @Override
   protected RelOptPlanner createPlanner() {
     final RelOptPlanner planner = super.createPlanner();
-    planner.addRule(EnumerableDrillRule.ARRAY_INSTANCE);
+    planner.addRule(EnumerableDrillRule.getInstance(driver.getClient()));
 
     // Enable when https://issues.apache.org/jira/browse/DRILL-57 fixed
     if (false) {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillProjectRel.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillProjectRel.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillProjectRel.java
index e2f1c28..0470f5e 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillProjectRel.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillProjectRel.java
@@ -74,7 +74,8 @@ public class DrillProjectRel extends ProjectRelBase implements DrillRel {
       transforms.add(objectNode);
       String expr = DrillOptiq.toDrill(getChild(), pair.left);
       objectNode.put("expr", expr);
-      String ref = "output." + pair.right;
+//      String ref = "output." + pair.right;
+      String ref = pair.right;
       objectNode.put("ref", ref);
     }
     return implementor.add(project);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillScan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillScan.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillScan.java
index 9f11f8c..6717ca4 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillScan.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillScan.java
@@ -27,7 +27,7 @@ public class DrillScan extends TableAccessRelBase implements DrillRel {
   @Override
   public void register(RelOptPlanner planner) {
     super.register(planner);
-    DrillOptiq.registerStandardPlannerRules(planner);
+    DrillOptiq.registerStandardPlannerRules(planner, drillTable.client);
   }
 
   public int implement(DrillImplementor implementor) {
@@ -36,7 +36,8 @@ public class DrillScan extends TableAccessRelBase implements DrillRel {
     node.put("memo", "initial_scan");
     node.put("ref", "_MAP"); // output is a record with a single field, '_MAP'
     node.put("storageengine", drillTable.getStorageEngineName());
-    node.put("selection", implementor.mapper.convertValue(drillTable.selection, JsonNode.class));
+    node.put("selection", implementor.mapper.convertValue(drillTable.getSelection(), JsonNode.class));
+    implementor.registerSource(drillTable);
     return implementor.add(node);
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrill.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrill.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrill.java
index c522b29..6b1dfce 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrill.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrill.java
@@ -18,8 +18,23 @@
 package org.apache.drill.optiq;
 
 import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 
 import net.hydromatic.linq4j.AbstractEnumerable;
 import net.hydromatic.linq4j.Enumerable;
@@ -32,6 +47,7 @@ import org.apache.drill.exec.ref.ReferenceInterpreter;
 import org.apache.drill.exec.ref.RunOutcome;
 import org.apache.drill.exec.ref.eval.BasicEvaluatorFactory;
 import org.apache.drill.exec.ref.rse.RSERegistry;
+import org.apache.drill.jdbc.DrillHandler;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -71,7 +87,7 @@ public class EnumerableDrill<E> extends AbstractEnumerable<E> implements Enumera
   /**
    * Creates a DrillEnumerable from a plan represented as a string. Each record returned is a {@link JsonNode}.
    */
-  public static <E> EnumerableDrill<E> of(String plan, final List<String> fieldNames, Class<E> clazz) {
+  public static <E> EnumerableDrill<E> of(DrillHandler handler, String plan, final List<String> fieldNames, Class<E> clazz) {
     DrillConfig config = DrillConfig.create();
     final LogicalPlan parse = LogicalPlan.parse(config, plan);
     return new EnumerableDrill<>(config, parse, clazz, fieldNames);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillFullEngine.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillFullEngine.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillFullEngine.java
index 7cd1082..9109b1c 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillFullEngine.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillFullEngine.java
@@ -27,7 +27,8 @@ import net.hydromatic.linq4j.Enumerator;
 import net.hydromatic.optiq.DataContext;
 
 import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.jdbc.DrillTable;
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.jdbc.DrillHandler.FakeSchema;
 import org.apache.drill.sql.client.full.DrillFullImpl;
 import org.apache.drill.sql.client.ref.DrillRefImpl;
 
@@ -41,8 +42,9 @@ public class EnumerableDrillFullEngine<E> extends AbstractEnumerable<E> implemen
   final BlockingQueue<Object> queue = new ArrayBlockingQueue<>(100);
   final DrillConfig config;
   private final List<String> fields;
-  private DataContext drillConnectionDataContext;
-
+  private DrillClient client;
+  private DataContext context;
+  
   /**
    * Creates a DrillEnumerable.
    * 
@@ -54,11 +56,12 @@ public class EnumerableDrillFullEngine<E> extends AbstractEnumerable<E> implemen
    *          Names of fields, or null to return the whole blob
    */
   public EnumerableDrillFullEngine(DrillConfig config, String plan, Class<E> clazz, List<String> fields,
-      DataContext drillConnectionDataContext) {
+      DrillClient client, DataContext context) {
     this.plan = plan;
     this.config = config;
     this.fields = fields;
-    this.drillConnectionDataContext = drillConnectionDataContext;
+    this.client = client;
+    this.context = context;
     config.setSinkQueues(0, queue);
   }
 
@@ -66,20 +69,20 @@ public class EnumerableDrillFullEngine<E> extends AbstractEnumerable<E> implemen
    * Creates a DrillEnumerable from a plan represented as a string. Each record returned is a {@link JsonNode}.
    */
   public static <E> EnumerableDrillFullEngine<E> of(String plan, final List<String> fieldNames, Class<E> clazz,
-      DataContext drillTable) {
+      DataContext context) {
     DrillConfig config = DrillConfig.create();
-    return new EnumerableDrillFullEngine<>(config, plan, clazz, fieldNames, drillTable);
+    return new EnumerableDrillFullEngine<>(config, plan, clazz, fieldNames, ((FakeSchema) context.getSubSchema("--FAKE--")).getClient(), context);
   }
 
   @Override
   public Enumerator<E> enumerator() {
-    DrillTable table = (DrillTable) drillConnectionDataContext.getSubSchema("DONUTS").getTable("DONUTS", Object.class);
-    if (table.useReferenceInterpreter()) {
+    
+    if (client == null) {
       DrillRefImpl<E> impl = new DrillRefImpl<E>(plan, config, fields, queue);
-      return impl.enumerator(table);
+      return impl.enumerator();
     } else {
-      DrillFullImpl<E> impl = new DrillFullImpl<E>(plan, config, fields);
-      return impl.enumerator(table);
+      DrillFullImpl<E> impl = new DrillFullImpl<E>(plan, config, fields, context);
+      return impl.enumerator(client);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRel.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRel.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRel.java
index 0a81f5e..b55a73c 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRel.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRel.java
@@ -17,24 +17,36 @@
  ******************************************************************************/
 package org.apache.drill.optiq;
 
-import net.hydromatic.linq4j.expressions.*;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.List;
+
+import net.hydromatic.linq4j.expressions.BlockBuilder;
+import net.hydromatic.linq4j.expressions.BlockStatement;
+import net.hydromatic.linq4j.expressions.Expression;
+import net.hydromatic.linq4j.expressions.Expressions;
 import net.hydromatic.linq4j.function.Function1;
 import net.hydromatic.linq4j.function.Functions;
 import net.hydromatic.optiq.DataContext;
 import net.hydromatic.optiq.impl.java.JavaTypeFactory;
-import net.hydromatic.optiq.rules.java.*;
+import net.hydromatic.optiq.rules.java.EnumerableConvention;
+import net.hydromatic.optiq.rules.java.EnumerableRel;
+import net.hydromatic.optiq.rules.java.EnumerableRelImplementor;
+import net.hydromatic.optiq.rules.java.JavaRowFormat;
+import net.hydromatic.optiq.rules.java.PhysType;
+import net.hydromatic.optiq.rules.java.PhysTypeImpl;
 
 import org.apache.drill.common.util.Hook;
-import org.apache.drill.optiq.EnumerableDrillFullEngine;
+import org.apache.drill.exec.client.DrillClient;
 import org.eigenbase.rel.RelNode;
 import org.eigenbase.rel.SingleRel;
-import org.eigenbase.relopt.*;
+import org.eigenbase.relopt.RelOptCluster;
+import org.eigenbase.relopt.RelOptCost;
+import org.eigenbase.relopt.RelOptPlanner;
+import org.eigenbase.relopt.RelTraitSet;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.lang.reflect.Method;
-import java.util.*;
-
 /**
  * Relational expression that converts from Drill to Enumerable. At runtime it executes a Drill query and returns the
  * results as an {@link net.hydromatic.linq4j.Enumerable}.
@@ -52,7 +64,8 @@ public class EnumerableDrillRel extends SingleRel implements EnumerableRel {
   private static final Method OF_METHOD;
 
   private PhysType physType;
-
+  private DrillClient client;
+  
   static {
     try {
       OF_METHOD = EnumerableDrillFullEngine.class.getMethod("of", String.class, List.class, Class.class,
@@ -63,10 +76,11 @@ public class EnumerableDrillRel extends SingleRel implements EnumerableRel {
     }
   }
 
-  public EnumerableDrillRel(RelOptCluster cluster, RelTraitSet traitSet, RelNode input) {
+  public EnumerableDrillRel(DrillClient client, RelOptCluster cluster, RelTraitSet traitSet, RelNode input) {
     super(cluster, traitSet, input);
     assert getConvention() instanceof EnumerableConvention;
     assert input.getConvention() == DrillRel.CONVENTION;
+    this.client = client;
     physType = PhysTypeImpl.of((JavaTypeFactory) cluster.getTypeFactory(), input.getRowType(), JavaRowFormat.ARRAY);
   }
 
@@ -81,7 +95,7 @@ public class EnumerableDrillRel extends SingleRel implements EnumerableRel {
 
   @Override
   public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
-    return new EnumerableDrillRel(getCluster(), traitSet, sole(inputs));
+    return new EnumerableDrillRel(client, getCluster(), traitSet, sole(inputs));
   }
 
   public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
@@ -101,10 +115,12 @@ public class EnumerableDrillRel extends SingleRel implements EnumerableRel {
     BlockStatement expr = new BlockBuilder().append(
         Expressions.call(
             OF_METHOD,
-            Expressions.constant(plan),
-            Expressions.call(Arrays.class, "asList",
-                Expressions.newArrayInit(String.class, Functions.apply(fieldNameList, TO_LITERAL))),
-            Expressions.constant(Object.class), Expressions.variable(DataContext.class, "root"))).toBlock();
+            Expressions.constant(plan), //
+            Expressions.call(Arrays.class, "asList", //
+                Expressions.newArrayInit(String.class, Functions.apply(fieldNameList, TO_LITERAL)) //
+                ),
+            Expressions.constant(Object.class), //
+            Expressions.variable(DataContext.class, "root"))).toBlock();
     final PhysType physType = PhysTypeImpl.of(implementor.getTypeFactory(), getRowType(),
         pref.prefer(JavaRowFormat.ARRAY));
     return new Result(expr, physType, JavaRowFormat.ARRAY);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRule.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRule.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRule.java
index a448171..2617749 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRule.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRule.java
@@ -18,6 +18,9 @@
 package org.apache.drill.optiq;
 
 import net.hydromatic.optiq.rules.java.EnumerableConvention;
+
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.jdbc.DrillHandler;
 import org.eigenbase.rel.RelNode;
 import org.eigenbase.rel.convert.ConverterRule;
 
@@ -25,10 +28,16 @@ import org.eigenbase.rel.convert.ConverterRule;
  * Rule that converts any Drill relational expression to enumerable format by adding a {@link EnumerableDrillRel}.
  */
 public class EnumerableDrillRule extends ConverterRule {
-  public static final EnumerableDrillRule ARRAY_INSTANCE = new EnumerableDrillRule(EnumerableConvention.INSTANCE);
 
-  private EnumerableDrillRule(EnumerableConvention outConvention) {
+  private final DrillClient client;
+  
+  public static EnumerableDrillRule getInstance(DrillClient client){
+    return new EnumerableDrillRule(EnumerableConvention.INSTANCE, client);
+  }
+  
+  private EnumerableDrillRule(EnumerableConvention outConvention, DrillClient client) {
     super(RelNode.class, DrillRel.CONVENTION, outConvention, "EnumerableDrillRule." + outConvention);
+    this.client = client;
   }
 
   @Override
@@ -39,6 +48,6 @@ public class EnumerableDrillRule extends ConverterRule {
   @Override
   public RelNode convert(RelNode rel) {
     assert rel.getTraitSet().contains(DrillRel.CONVENTION);
-    return new EnumerableDrillRel(rel.getCluster(), rel.getTraitSet().replace(getOutConvention()), rel);
+    return new EnumerableDrillRel(client, rel.getCluster(), rel.getTraitSet().replace(getOutConvention()), rel);
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java
index a60f92b..f58bfc8 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java
@@ -7,14 +7,17 @@ import java.util.Map;
 import java.util.Set;
 
 import jline.internal.Preconditions;
+import net.hydromatic.optiq.DataContext;
 
+import org.apache.drill.exec.client.DrillClient;
 import org.apache.drill.exec.exception.SchemaChangeException;
 import org.apache.drill.exec.record.RecordBatchLoader;
 import org.apache.drill.exec.record.VectorWrapper;
 import org.apache.drill.exec.rpc.RpcException;
 import org.apache.drill.exec.rpc.user.QueryResultBatch;
-import org.apache.drill.exec.server.DrillbitContext;
 import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VarBinaryVector;
+import org.apache.drill.exec.vector.VarCharVector;
 
 import com.fasterxml.jackson.core.JsonGenerationException;
 import com.fasterxml.jackson.core.JsonGenerator;
@@ -35,12 +38,16 @@ public class BatchLoaderMap implements Map<String, Object> {
   private final Map<String, ValueVector> fields = Maps.newHashMap();
   private int index;
   private Object[] objArr;
+  private JsonHelper helper;
+  private boolean loaded;
 
-  public BatchLoaderMap(List<String> requestedFields, BatchListener listener, DrillbitContext context) {
+  public BatchLoaderMap(List<String> requestedFields, BatchListener listener, DrillClient client, DataContext context) {
     this.listener = listener;
     this.requestedFields = requestedFields;
     this.objArr = new Object[requestedFields.size()];
-    this.loader = new RecordBatchLoader(context.getAllocator());
+    this.loader = new RecordBatchLoader(client.getAllocator());
+    this.helper = new JsonHelper(client.getConfig());
+    FileSystemSchema fsSchema = (FileSystemSchema) context;
   }
 
   private void load(QueryResultBatch batch) throws SchemaChangeException {
@@ -80,20 +87,39 @@ public class BatchLoaderMap implements Map<String, Object> {
     }
   }
 
-  public Object getCurrentAsObjectArray() {
-    
+
+  
+  public Object getCurrentObject() {
+    if(requestedFields.size() == 1){
+      if(requestedFields.iterator().next().equals("_MAP")){
+        return helper.get(fields, index); 
+      }else{
+        return getObj(fields.get(requestedFields.get(0)));
+      }
+      
+    }
+      
     for (int i = 0; i < requestedFields.size(); i++) {
       ValueVector vv = fields.get(requestedFields.get(i));
       if (vv == null) {
+        logger.debug("Failure while retrieving field of {}.  Didn't exist in returned set of {}", requestedFields.get(i), fields.keySet());
         objArr[i] = null;
       } else {
-        objArr[i] = vv.getAccessor().getObject(index);
+        objArr[i] = getObj(vv);
       }
     }
     return objArr;
 
   }
 
+  private Object getObj(ValueVector vv){
+    if(vv instanceof VarBinaryVector || vv instanceof VarCharVector){
+      return new String( (byte[]) vv.getAccessor().getObject(index));
+    }else{
+      return vv.getAccessor().getObject(index);  
+    }
+  }
+  
   @Override
   public int size() {
     return fields.size();

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillFullImpl.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillFullImpl.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillFullImpl.java
index 0974365..7f40c57 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillFullImpl.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillFullImpl.java
@@ -3,12 +3,11 @@ package org.apache.drill.sql.client.full;
 import java.util.List;
 
 import net.hydromatic.linq4j.Enumerator;
+import net.hydromatic.optiq.DataContext;
 
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.exec.client.DrillClient;
 import org.apache.drill.exec.proto.UserProtos;
-import org.apache.drill.exec.rpc.RpcException;
-import org.apache.drill.jdbc.DrillTable;
 
 public class DrillFullImpl<E>{
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillFullImpl.class);
@@ -16,24 +15,27 @@ public class DrillFullImpl<E>{
   private final String plan;
   final DrillConfig config;
   private final List<String> fields;
+  private DataContext context;
 
   
-  public DrillFullImpl(String plan, DrillConfig config, List<String> fields) {
+  public DrillFullImpl(String plan, DrillConfig config, List<String> fields, DataContext context) {
     super();
     this.plan = plan;
     this.config = config;
     this.fields = fields;
+    this.context = context;
   }
 
-  public Enumerator<E> enumerator(DrillTable table) {
+  @SuppressWarnings("unchecked")
+  public Enumerator<E> enumerator(DrillClient client) {
     
     BatchListener listener = new BatchListener();
 
     // TODO: use a completion service from the container
-    QueryRequestRunner runner = new QueryRequestRunner(plan, table.client, listener);
+    QueryRequestRunner runner = new QueryRequestRunner(plan, client, listener);
     runner.start();
     
-    return (Enumerator<E>) new ResultEnumerator(listener, table.bit.getContext(),fields);
+    return (Enumerator<E>) new ResultEnumerator(listener, client, fields, context);
     
   }
   
@@ -52,13 +54,7 @@ public class DrillFullImpl<E>{
 
     @Override
     public void run() {
-      try {
-        client.connect();
-        client.runQuery(UserProtos.QueryType.LOGICAL, plan, listener);
-      } catch (RpcException e) {
-        listener.submissionFailed(e);
-      }
-      
+      client.runQuery(UserProtos.QueryType.LOGICAL, plan, listener);
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillMutableSchema.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillMutableSchema.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillMutableSchema.java
new file mode 100644
index 0000000..b33363f
--- /dev/null
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillMutableSchema.java
@@ -0,0 +1,74 @@
+package org.apache.drill.sql.client.full;
+
+import java.util.Collection;
+import java.util.Map;
+
+import com.google.common.collect.Multimap;
+
+import net.hydromatic.linq4j.QueryProvider;
+import net.hydromatic.linq4j.expressions.Expression;
+import net.hydromatic.optiq.MutableSchema;
+import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.Table;
+import net.hydromatic.optiq.impl.java.JavaTypeFactory;
+
+public class DrillMutableSchema implements Schema {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillMutableSchema.class);
+
+  @Override
+  public Schema getParentSchema() {
+    return null;
+  }
+
+  @Override
+  public String getName() {
+    return null;
+  }
+
+  @Override
+  public Collection<TableFunctionInSchema> getTableFunctions(String name) {
+    return null;
+  }
+
+  @Override
+  public <E> Table<E> getTable(String name, Class<E> elementType) {
+    return null;
+  }
+
+  @Override
+  public Expression getExpression() {
+    return null;
+  }
+
+  @Override
+  public QueryProvider getQueryProvider() {
+    return null;
+  }
+
+  @Override
+  public Multimap<String, TableFunctionInSchema> getTableFunctions() {
+    return null;
+  }
+
+  @Override
+  public Collection<String> getSubSchemaNames() {
+    return null;
+  }
+
+  @Override
+  public Map<String, TableInSchema> getTables() {
+    return null;
+  }
+
+  @Override
+  public Schema getSubSchema(String name) {
+    return null;
+  }
+
+  @Override
+  public JavaTypeFactory getTypeFactory() {
+    return null;
+  }
+
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/FileSystemSchema.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/FileSystemSchema.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/FileSystemSchema.java
new file mode 100644
index 0000000..85a4125
--- /dev/null
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/FileSystemSchema.java
@@ -0,0 +1,137 @@
+package org.apache.drill.sql.client.full;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentMap;
+
+import net.hydromatic.linq4j.QueryProvider;
+import net.hydromatic.linq4j.expressions.Expression;
+import net.hydromatic.optiq.Schema;
+import net.hydromatic.optiq.Table;
+import net.hydromatic.optiq.impl.java.JavaTypeFactory;
+
+import org.apache.drill.common.logical.StorageEngineConfig;
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.store.SchemaProvider;
+import org.apache.drill.jdbc.DrillTable;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+
+public class FileSystemSchema implements Schema{
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemSchema.class);
+  
+  private ConcurrentMap<String, TableInSchema> tables = Maps.newConcurrentMap();
+
+  private final JavaTypeFactory typeFactory;
+  private final Schema parentSchema;
+  private final String name;
+  private final Expression expression;
+  private final QueryProvider queryProvider;
+  private final SchemaProvider schemaProvider;
+  private final DrillClient client;
+  private final StorageEngineConfig config;
+  
+  public FileSystemSchema(DrillClient client, StorageEngineConfig config, SchemaProvider schemaProvider, JavaTypeFactory typeFactory, Schema parentSchema, String name, Expression expression,
+      QueryProvider queryProvider) {
+    super();
+    this.client = client;
+    this.typeFactory = typeFactory;
+    this.parentSchema = parentSchema;
+    this.name = name;
+    this.expression = expression;
+    this.queryProvider = queryProvider;
+    this.schemaProvider = schemaProvider;
+    this.config = config;
+  }
+
+  @Override
+  public Schema getSubSchema(String name) {
+    return null;
+  }
+
+  @Override
+  public JavaTypeFactory getTypeFactory() {
+    return typeFactory;
+  }
+
+  @Override
+  public Schema getParentSchema() {
+    return parentSchema;
+  }
+
+  @Override
+  public String getName() {
+    return name;
+  }
+
+  @Override
+  public Expression getExpression() {
+    return expression;
+  }
+
+  @Override
+  public QueryProvider getQueryProvider() {    
+    return queryProvider;
+  }
+
+
+  @Override
+  public Collection<TableFunctionInSchema> getTableFunctions(String name) {
+    return Collections.emptyList();
+  }
+  
+  @Override
+  public Multimap<String, TableFunctionInSchema> getTableFunctions() {
+    return ArrayListMultimap.create();
+  }
+
+  @Override
+  public Collection<String> getSubSchemaNames() {
+    return Collections.EMPTY_LIST;
+  }
+  
+  @SuppressWarnings("unchecked")
+  @Override
+  public <E> Table<E> getTable(String name, Class<E> elementType) {
+    if( !elementType.isAssignableFrom(DrillTable.class)) throw new UnsupportedOperationException();
+    TableInfo info = (TableInfo) tables.get(name);
+    if(info != null) return (Table<E>) info.table;
+    Object selection = schemaProvider.getSelectionBaseOnName(name);
+    if(selection == null) return null;
+    
+    DrillTable table = DrillTable.createTable(client, typeFactory, this, name, config, selection);
+    info = new TableInfo(name, table);
+    TableInfo oldInfo = (TableInfo) tables.putIfAbsent(name, info);
+    if(oldInfo != null) return (Table<E>) oldInfo.table;
+    return (Table<E>) table;
+  }
+
+  @Override
+  public Map<String, TableInSchema> getTables() {
+    return this.tables;
+  }
+  
+  private class TableInfo extends TableInSchema{
+    
+    final DrillTable table;
+
+    public TableInfo(String name, DrillTable table) {
+      super(FileSystemSchema.this, name, TableType.TABLE);
+      this.table = table;
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public <E> Table<E> getTable(Class<E> elementType) {
+      if( !elementType.isAssignableFrom(DrillTable.class)) throw new UnsupportedOperationException();
+      return (Table<E>) table;
+    }
+    
+    
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/JsonHelper.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/JsonHelper.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/JsonHelper.java
new file mode 100644
index 0000000..74901ae
--- /dev/null
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/JsonHelper.java
@@ -0,0 +1,39 @@
+package org.apache.drill.sql.client.full;
+
+import java.util.Map;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.vector.ValueVector;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public class JsonHelper {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JsonHelper.class);
+  
+  private ObjectMapper mapper;
+  
+  JsonHelper(DrillConfig config){
+    mapper = config.getMapper();
+  }
+  
+  public JsonNode get(Map<String, ValueVector> vectors, int index){
+    final ObjectNode map = mapper.createObjectNode();
+    for(Map.Entry<String, ValueVector> e : vectors.entrySet()){
+      Object o = e.getValue().getAccessor().getObject(index);
+      if(o == null) continue;
+      String k = e.getKey().replace("_MAP.", "");
+      
+      if(o instanceof Integer) map.put(k, (Integer) o);
+      else if(o instanceof Long) map.put(k, (Long) o);
+      else if(o instanceof String) map.put(k, (String) o);
+      else if(o instanceof byte[]) map.put(k, new String((byte[]) o));
+      else if(o instanceof Float) map.put(k, (Float) o);
+      else if(o instanceof Double) map.put(k, (Double) o);
+      else map.put(k, "---Unknown Type---");
+      
+    }
+    return map; 
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/ResultEnumerator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/ResultEnumerator.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/ResultEnumerator.java
index 8f75e5f..f1f5bbd 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/ResultEnumerator.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/ResultEnumerator.java
@@ -1,13 +1,13 @@
 package org.apache.drill.sql.client.full;
 
 import java.util.List;
-import java.util.Map;
 
 import net.hydromatic.linq4j.Enumerator;
+import net.hydromatic.optiq.DataContext;
 
+import org.apache.drill.exec.client.DrillClient;
 import org.apache.drill.exec.exception.SchemaChangeException;
 import org.apache.drill.exec.rpc.RpcException;
-import org.apache.drill.exec.server.DrillbitContext;
 
 /**
    * Enumerator used for full execution engine.
@@ -17,8 +17,8 @@ import org.apache.drill.exec.server.DrillbitContext;
     private final BatchLoaderMap loaderMap;
     private Object current;
     
-    public ResultEnumerator(BatchListener listener, DrillbitContext context, List<String> fields) {
-      this.loaderMap = new BatchLoaderMap(fields, listener, context);
+    public ResultEnumerator(BatchListener listener, DrillClient client, List<String> fields, DataContext context) {
+      this.loaderMap = new BatchLoaderMap(fields, listener, client, context);
     }
 
     public Object current() {
@@ -30,7 +30,7 @@ import org.apache.drill.exec.server.DrillbitContext;
       try {
         boolean succ = loaderMap.next();
         if(succ){
-          current = loaderMap.getCurrentAsObjectArray();          
+          current = loaderMap.getCurrentObject();          
         }
         return succ;
         

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/ref/DrillRefImpl.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/ref/DrillRefImpl.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/ref/DrillRefImpl.java
index 0d01085..3792ddf 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/ref/DrillRefImpl.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/ref/DrillRefImpl.java
@@ -166,7 +166,7 @@ public class DrillRefImpl<E> {
 
   
   
-  public Enumerator<E> enumerator(DrillTable table) {
+  public Enumerator<E> enumerator() {
     // TODO: use a completion service from the container
     final ExecutorCompletionService<Collection<RunOutcome>> service = new ExecutorCompletionService<Collection<RunOutcome>>(
         new ThreadPoolExecutor(1, 1, 1, TimeUnit.SECONDS, new LinkedBlockingDeque<Runnable>(10)));

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/main/resources/storage-engines.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/resources/storage-engines.json b/sandbox/prototype/sqlparser/src/main/resources/storage-engines.json
new file mode 100644
index 0000000..1eead25
--- /dev/null
+++ b/sandbox/prototype/sqlparser/src/main/resources/storage-engines.json
@@ -0,0 +1,19 @@
+{
+  "storage":{
+    "parquet-local" :
+      {
+        "type":"parquet",
+        "dfsName" : "file:///"
+      },
+    "jsonl" :
+      {
+        "type":"json",
+        "dfsName" : "file:///"
+      },
+    "parquet" :
+      {
+        "type":"parquet",
+        "dfsName" : "maprfs://10.10.30.52/"
+      }
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
index 35a2414..72c19a6 100644
--- a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
+++ b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
@@ -13,7 +13,7 @@ import org.junit.rules.Timeout;
 import com.google.common.base.Charsets;
 import com.google.common.io.Resources;
 
-@Ignore
+
 public class FullEngineTest {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FullEngineTest.class);
 
@@ -34,16 +34,16 @@ public class FullEngineTest {
   public void fullSelectStarEngine() throws Exception {
     JdbcAssert.withModel(MODEL_FULL_ENGINE, "DONUTS")
     // .sql("select cast(_MAP['red'] as bigint) + 1 as red_inc from donuts ")
-        .sql("select * from donuts ").displayResults(50);
+        .sql("select _MAP['d'] as d, _MAP['b'] as b from \"parquet-local\".\"/tmp/parquet_test_file_many_types\" ").displayResults(50);
   }
 
-  @Test
-  public void fullEngine() throws Exception {
-    JdbcAssert
-        .withModel(MODEL_FULL_ENGINE, "DONUTS")
-        // .sql("select cast(_MAP['red'] as bigint) + 1 as red_inc from donuts ")
-        .sql(
-            "select cast(_MAP['RED'] as bigint)  as RED, cast(_MAP['GREEN'] as bigint)  as GREEN from donuts where cast(_MAP['red'] as BIGINT) > 1 ")
-        .displayResults(50);
-  }
+//  @Test
+//  public void fullEngine() throws Exception {
+//    JdbcAssert
+//        .withModel(MODEL_FULL_ENGINE, "DONUTS")
+//        // .sql("select cast(_MAP['red'] as bigint) + 1 as red_inc from donuts ")
+//        .sql(
+//            "select cast(_MAP['RED'] as bigint)  as RED, cast(_MAP['GREEN'] as bigint)  as GREEN from 'monkeys' where cast(_MAP['red'] as BIGINT) > 1 ")
+//        .displayResults(50);
+//  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
index 6768dda..52dbcfb 100644
--- a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
+++ b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
@@ -47,6 +47,7 @@ public class JdbcAssert {
     return new ModelAndSchema(info);
   }
 
+  
   static String toString(ResultSet resultSet, int expectedRecordCount) throws SQLException {
     StringBuilder buf = new StringBuilder();
     int total = 0, n;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/sqlparser/src/test/resources/logback.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/test/resources/logback.xml b/sandbox/prototype/sqlparser/src/test/resources/logback.xml
index 3b3289a..34d9356 100644
--- a/sandbox/prototype/sqlparser/src/test/resources/logback.xml
+++ b/sandbox/prototype/sqlparser/src/test/resources/logback.xml
@@ -19,13 +19,13 @@
   <logger name="org.apache.drill" additivity="false">
     <level value="debug" />
     <appender-ref ref="SOCKET" />
-    <appender-ref ref="STDOUT" />
+<!--     <appender-ref ref="STDOUT" /> -->
   </logger>
 
   <root>
     <level value="error" />
     <appender-ref ref="SOCKET" />
-    <appender-ref ref="STDOUT" />
+<!--     <appender-ref ref="STDOUT" /> -->
   </root>
 
 </configuration>
\ No newline at end of file


[03/18] git commit: DRILL-167, DRILL-80, DRILL-59 start make changes to support sqlline module and ensure no test jars on class path

Posted by ja...@apache.org.
DRILL-167, DRILL-80, DRILL-59 start
make changes to support sqlline module and ensure no test jars on class path


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/1a2352c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/1a2352c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/1a2352c0

Branch: refs/heads/master
Commit: 1a2352c017590fee07620e0f1735ae3ce32d71f3
Parents: 4515263
Author: Jacques Nadeau <ja...@apache.org>
Authored: Fri Aug 16 03:46:53 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Sep 1 15:16:12 2013 -0700

----------------------------------------------------------------------
 .../drill/common/expression/parser/ExprLexer.g  |   3 +-
 .../drill/common/expression/parser/ExprParser.g |   2 +-
 .../org/apache/drill/common/defs/OrderDef.java  |  11 +-
 .../drill/common/expression/FieldReference.java |   5 +
 .../drill/common/expression/PathSegment.java    |  22 +-
 .../drill/common/expression/SchemaPath.java     |  21 ++
 .../common/expression/fn/BooleanFunctions.java  |   2 +-
 .../drill/common/logical/data/Project.java      |   6 +-
 .../org/apache/drill/common/types/Types.java    |  12 +-
 .../common/expression/parser/TreeTest.java      |   5 +-
 sandbox/prototype/contrib/sqlline/pom.xml       |  38 +++
 sandbox/prototype/exec/java-exec/pom.xml        |  11 +
 .../templates/VariableLengthVectors.java        |  14 +-
 .../apache/drill/exec/client/DrillClient.java   |   7 +-
 .../drill/exec/coord/ZKClusterCoordinator.java  |  11 +-
 .../drill/exec/expr/EvaluationVisitor.java      |  24 +-
 .../exec/expr/fn/impl/ComparatorFunctions.java  |  64 +++++
 .../exec/expr/fn/impl/ComparisonFunctions.java  | 223 ++++++++++++++-
 .../drill/exec/expr/fn/impl/VarHelpers.java     | 148 ++++++++++
 .../org/apache/drill/exec/ops/QueryContext.java |  10 +-
 .../apache/drill/exec/opt/BasicOptimizer.java   | 129 +++++++--
 .../org/apache/drill/exec/opt/Optimizer.java    |   3 +-
 .../drill/exec/physical/ReadEntryWithPath.java  |   8 +
 .../drill/exec/physical/base/GroupScan.java     |   7 +-
 .../exec/physical/base/PhysicalOperator.java    |   3 +-
 .../exec/physical/config/MergeJoinPOP.java      |  14 +-
 .../physical/impl/filter/FilterTemplate.java    |   1 +
 .../impl/filter/ReturnValueExpression.java      |  10 +
 .../exec/physical/impl/join/MergeJoinBatch.java |  33 ++-
 .../exec/physical/impl/sort/SortBatch.java      |   2 +-
 .../impl/svremover/CopierTemplate2.java         |   2 +-
 .../exec/record/AbstractSingleRecordBatch.java  |   4 +-
 .../exec/record/selection/SelectionVector2.java |   3 +-
 .../org/apache/drill/exec/schema/Field.java     |   6 +-
 .../drill/exec/server/DrillbitContext.java      |   4 +-
 .../exec/store/DistributedStorageEngine.java    |   5 +
 .../apache/drill/exec/store/SchemaProvider.java |   8 +
 .../exec/store/SchemaProviderRegistry.java      |  82 ++++++
 .../apache/drill/exec/store/StorageEngine.java  |   4 +
 .../drill/exec/store/StorageEngineRegistry.java |   9 +-
 .../drill/exec/store/json/JSONGroupScan.java    |  61 ++---
 .../drill/exec/store/json/JSONRecordReader.java |  69 +++--
 .../exec/store/json/JSONScanBatchCreator.java   |   2 +-
 .../exec/store/json/JSONStorageEngine.java      |  31 ++-
 .../drill/exec/store/json/JSONSubScan.java      |  51 ++--
 .../exec/store/json/JsonSchemaProvider.java     |  43 +++
 .../exec/store/mock/MockStorageEngine.java      |   7 +-
 .../store/mock/MockStorageEngineConfig.java     |  59 ++++
 .../exec/store/parquet/ParquetGroupScan.java    |  49 +++-
 .../exec/store/parquet/ParquetRecordReader.java |  38 ++-
 .../exec/store/parquet/ParquetRowGroupScan.java |  50 ++--
 .../store/parquet/ParquetScanBatchCreator.java  |   3 +-
 .../store/parquet/ParquetSchemaProvider.java    |  44 +++
 .../store/parquet/ParquetStorageEngine.java     |  32 +--
 .../parquet/ParquetStorageEngineConfig.java     |  10 +-
 .../drill/exec/vector/ValueHolderHelper.java    |  21 ++
 .../drill/exec/work/FragmentRunnerListener.java |   2 -
 .../org/apache/drill/exec/work/WorkManager.java |   2 +-
 .../apache/drill/exec/work/foreman/Foreman.java |  41 +--
 .../src/main/resources/drill-module.conf        |  11 +-
 .../exec/physical/impl/TestOptiqPlans.java      | 273 +++++++++++++++++++
 .../store/parquet/ParquetRecordReaderTest.java  |   4 +-
 .../exec/store/parquet/TestFileGenerator.java   |  17 +-
 .../src/test/resources/logical_group.json       |  90 ++++++
 .../src/test/resources/logical_json_scan.json   |  36 +++
 .../src/test/resources/logical_order.json       |  57 ++++
 .../test/resources/logical_string_filter.json   |  52 ++++
 .../src/test/resources/physical_filter.json     |  44 +++
 .../src/test/resources/physical_join.json       | 117 ++++++++
 .../resources/physical_order_varbinary.json     |  48 ++++
 sandbox/prototype/exec/ref/pom.xml              |  11 +-
 sandbox/prototype/sqlline                       |   7 +-
 sandbox/prototype/sqlparser/pom.xml             |   3 +-
 .../org/apache/drill/jdbc/DrillHandler.java     | 147 ++++++++++
 .../java/org/apache/drill/jdbc/DrillTable.java  | 184 +++++--------
 .../main/java/org/apache/drill/jdbc/Driver.java |  70 ++---
 .../org/apache/drill/jdbc/StorageEngines.java   |  57 ++++
 .../apache/drill/optiq/DrillImplementor.java    |  38 ++-
 .../org/apache/drill/optiq/DrillJoinRel.java    |   5 +-
 .../java/org/apache/drill/optiq/DrillOptiq.java |  72 ++++-
 .../apache/drill/optiq/DrillPrepareImpl.java    |   9 +-
 .../org/apache/drill/optiq/DrillProjectRel.java |   3 +-
 .../java/org/apache/drill/optiq/DrillScan.java  |   5 +-
 .../org/apache/drill/optiq/EnumerableDrill.java |  22 +-
 .../drill/optiq/EnumerableDrillFullEngine.java  |  27 +-
 .../apache/drill/optiq/EnumerableDrillRel.java  |  44 ++-
 .../apache/drill/optiq/EnumerableDrillRule.java |  15 +-
 .../drill/sql/client/full/BatchLoaderMap.java   |  38 ++-
 .../drill/sql/client/full/DrillFullImpl.java    |  22 +-
 .../sql/client/full/DrillMutableSchema.java     |  74 +++++
 .../drill/sql/client/full/FileSystemSchema.java | 137 ++++++++++
 .../drill/sql/client/full/JsonHelper.java       |  39 +++
 .../drill/sql/client/full/ResultEnumerator.java |  10 +-
 .../drill/sql/client/ref/DrillRefImpl.java      |   2 +-
 .../src/main/resources/storage-engines.json     |  19 ++
 .../apache/drill/jdbc/test/FullEngineTest.java  |  22 +-
 .../org/apache/drill/jdbc/test/JdbcAssert.java  |   1 +
 .../sqlparser/src/test/resources/logback.xml    |   4 +-
 98 files changed, 2863 insertions(+), 564 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
----------------------------------------------------------------------
diff --git a/sandbox/prototype/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g b/sandbox/prototype/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
index 1500d07..efc2488 100644
--- a/sandbox/prototype/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
+++ b/sandbox/prototype/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
@@ -39,6 +39,7 @@ When     : 'when';
 Or       : '||';
 And      : '&&';
 Equals   : '==';
+Equals2   : '=';
 NEquals  : '<>';
 NEquals2 : '!=';
 GTEquals : '>=';
@@ -82,7 +83,7 @@ Number
 //  ;
   
 Identifier
-  :  ('a'..'z' | 'A'..'Z' | '_') ('a'..'z' | 'A'..'Z' | '_' | Digit)* ('.' ('a'..'z' | 'A'..'Z' | '_') ('a'..'z' | 'A'..'Z' | '_' | Digit)*)*
+  :  ('a'..'z' | 'A'..'Z' | '_' | '$') ('a'..'z' | 'A'..'Z' | '_' | '$' | Digit)* ('.' ('a'..'z' | 'A'..'Z' | '_' | '$' ) ('a'..'z' | 'A'..'Z' | '_' | '$' | Digit)*)*
   ;
 
 String

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
----------------------------------------------------------------------
diff --git a/sandbox/prototype/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g b/sandbox/prototype/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
index d2b46b8..da873ea 100644
--- a/sandbox/prototype/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
+++ b/sandbox/prototype/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
@@ -159,7 +159,7 @@ equExpr returns [LogicalExpression e]
 	  $e = registry.createByOp(exprs, p, cmps);
 	}
   :  r1=relExpr { exprs.add($r1.e); p = pos( $r1.start );
-    } ( cmpr= (Equals | NEquals | NEquals2 ) r2=relExpr {exprs.add($r2.e); cmps.add($cmpr.text); })*
+    } ( cmpr= (Equals2 | Equals | NEquals | NEquals2 ) r2=relExpr {exprs.add($r2.e); cmps.add($cmpr.text); })*
   ;
 
 relExpr returns [LogicalExpression e]

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/common/src/main/java/org/apache/drill/common/defs/OrderDef.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/defs/OrderDef.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/defs/OrderDef.java
index e873496..8e50058 100644
--- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/defs/OrderDef.java
+++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/defs/OrderDef.java
@@ -18,6 +18,8 @@
 package org.apache.drill.common.defs;
 
 import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.Order.Direction;
+import org.apache.drill.common.logical.data.Order.Ordering;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -49,12 +51,7 @@ public class OrderDef {
     return direction.description;
   }
 
-  public static enum Direction {
-    ASC("asc"), DESC("desc");
-    public final String description;
-
-    Direction(String d) {
-      description = d;
-    }
+  public static OrderDef create(Ordering o){
+    return new OrderDef(o.getDirection(), o.getExpr());
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/FieldReference.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
index f611614..2d62a41 100644
--- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
+++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
@@ -39,6 +39,11 @@ import com.fasterxml.jackson.databind.ser.std.StdSerializer;
 public class FieldReference extends SchemaPath {
   MajorType overrideType;
 
+  public FieldReference(SchemaPath sp){
+    super(sp);
+  }
+  
+  
   public FieldReference(String value, ExpressionPosition pos) {
     super(value, pos);
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/PathSegment.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
index 554c0ca..28e820b 100644
--- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
+++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
@@ -23,13 +23,15 @@ import org.apache.drill.common.expression.ValueExpressions.CollisionBehavior;
 
 public abstract class PathSegment{
 
-  private final ValueExpressions.CollisionBehavior collision;
-  private PathSegment child;
+  protected final ValueExpressions.CollisionBehavior collision;
+  protected PathSegment child;
 
   protected PathSegment(CollisionBehavior collision){
     this.collision = collision;
   }
   
+  public abstract PathSegment clone();
+  
   public static class ArraySegment extends PathSegment{
     private final int index;
     
@@ -61,8 +63,13 @@ public abstract class PathSegment{
       return "ArraySegment [index=" + index + ", getCollisionBehavior()=" + getCollisionBehavior() + ", getChild()="
           + getChild() + "]";
     }
-    
-    
+
+    @Override
+    public PathSegment clone() {
+      PathSegment seg = new ArraySegment(index, collision);
+      if(child != null) seg.setChild(child.clone());
+      return seg;
+    }
     
   }
   
@@ -123,6 +130,13 @@ public abstract class PathSegment{
         return false;
       return true;
     }
+
+    @Override
+    public PathSegment clone() {
+      PathSegment s = new NameSegment(this.path, this.collision);
+      if(child != null) s.setChild(child.clone());
+      return s;
+    }
     
     
     

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
index 5910c30..20d3b43 100644
--- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
+++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
@@ -27,6 +27,7 @@ import org.apache.drill.common.expression.visitors.ExprVisitor;
 import org.apache.drill.common.types.TypeProtos.MajorType;
 import org.apache.drill.common.types.Types;
 
+import com.google.common.base.Joiner;
 import com.google.common.collect.Iterators;
 import com.google.protobuf.DescriptorProtos.UninterpretedOption.NamePart;
 
@@ -57,6 +58,12 @@ public class SchemaPath extends LogicalExpressionBase {
   private final CharSequence originalPath;
   private final PathSegment rootSegment;
 
+  public SchemaPath(SchemaPath path){
+    super(path.getPosition());
+    this.originalPath = path.originalPath;
+    this.rootSegment = path.rootSegment;
+  }
+  
   public SchemaPath(CharSequence str, ExpressionPosition pos) {
     super(pos);
 
@@ -89,12 +96,26 @@ public class SchemaPath extends LogicalExpressionBase {
     rootSegment = r;
 
   }
+  
+  private SchemaPath(SchemaPath parent, String[] childPaths){
+    super(ExpressionPosition.UNKNOWN);
+    SchemaPath p = new SchemaPath(Joiner.on('.').join(childPaths), ExpressionPosition.UNKNOWN);
+    this.originalPath = parent.originalPath + "." + p.originalPath;
+    PathSegment seg = parent.getRootSegment().clone();
+    this.rootSegment = seg;
+    while(!seg.isLastPath()) seg = seg.getChild();
+    seg.setChild(p.getRootSegment());
+  }
 
   @Override
   public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
     return visitor.visitSchemaPath(this, value);
   }
 
+  public SchemaPath getChild(String[] childPaths){
+    return new SchemaPath(this, childPaths);
+  }
+  
   public PathSegment getRootSegment() {
     return rootSegment;
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/fn/BooleanFunctions.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/fn/BooleanFunctions.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/fn/BooleanFunctions.java
index 6b6a95f..f780f85 100644
--- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/fn/BooleanFunctions.java
+++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/fn/BooleanFunctions.java
@@ -36,7 +36,7 @@ public class BooleanFunctions implements CallProvider {
         FunctionDefinition.operator("and", new AllowedTypeList(2, Integer.MAX_VALUE, Types.REQUIRED_BIT), OutputTypeDeterminer.FIXED_BIT, "and", "&&"),
         FunctionDefinition.operator("greater than", new ComparableArguments(2), OutputTypeDeterminer.FIXED_BIT, ">"),
         FunctionDefinition.operator("less than", new ComparableArguments(2), OutputTypeDeterminer.FIXED_BIT, "<"),
-        FunctionDefinition.operator("equal", new ComparableArguments(2), OutputTypeDeterminer.FIXED_BIT, "=="),
+        FunctionDefinition.operator("equal", new ComparableArguments(2), OutputTypeDeterminer.FIXED_BIT, "==", "="),
         FunctionDefinition.operator("not equal", new ComparableArguments(2), OutputTypeDeterminer.FIXED_BIT, "<>", "!="),
         FunctionDefinition.operator("greater than or equal to", new ComparableArguments(2), OutputTypeDeterminer.FIXED_BIT, ">="),
         FunctionDefinition.operator("less than or equal to", new ComparableArguments(2), OutputTypeDeterminer.FIXED_BIT, "<="), };

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/Project.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/Project.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/Project.java
index 109d218..1fb5eb1 100644
--- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/Project.java
+++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/Project.java
@@ -40,9 +40,9 @@ public class Project extends SingleInputOperator {
     for (int i = 0; i < selections.length; i++) {
       PathSegment segment = selections[i].getRef().getRootSegment();
       CharSequence path = segment.getNameSegment().getPath();
-      if (!segment.isNamed() || !path.equals("output"))
-        throw new ExpressionParsingException(String.format(
-            "Outputs for projections always have to start with named path of output. First segment was named '%s' or was named [%s]", path, segment.isNamed()));
+//      if (!segment.isNamed() || !path.equals("output"))
+//        throw new ExpressionParsingException(String.format(
+//            "Outputs for projections always have to start with named path of output. First segment was named '%s' or was named [%s]", path, segment.isNamed()));
 
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/common/src/main/java/org/apache/drill/common/types/Types.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/types/Types.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/types/Types.java
index f07f726..ad7f771 100644
--- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/types/Types.java
+++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/types/Types.java
@@ -122,7 +122,17 @@ public class Types {
   
   
   public static boolean softEquals(MajorType a, MajorType b, boolean allowNullSwap){
-    if(a.getMinorType() != b.getMinorType()) return false;
+    if(a.getMinorType() != b.getMinorType()){
+      if(
+          (a.getMinorType() == MinorType.VARBINARY && b.getMinorType() == MinorType.VARCHAR) ||
+          (b.getMinorType() == MinorType.VARBINARY && a.getMinorType() == MinorType.VARCHAR) 
+          ){
+        // fall through;
+      }else{
+        return false;  
+      }
+      
+    }
     if(allowNullSwap){
       switch(a.getMode()){
       case OPTIONAL:

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java b/sandbox/prototype/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
index a297e13..1cab357 100644
--- a/sandbox/prototype/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
+++ b/sandbox/prototype/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
@@ -7,6 +7,8 @@ import org.antlr.runtime.CommonTokenStream;
 import org.antlr.runtime.tree.CommonTree;
 import org.antlr.runtime.tree.DOTTreeGenerator;
 import org.antlr.stringtemplate.StringTemplate;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.expression.FunctionRegistry;
 import org.apache.drill.common.expression.LogicalExpression;
 import org.apache.drill.common.expression.parser.ExprParser.parse_return;
 
@@ -20,7 +22,7 @@ public class TreeTest {
 
     ExprLexer lexer = new ExprLexer(
         new ANTLRStringStream(
-            "if ('blue.red') then case when 1 then 2 else 4 end else if(4==3) then 1 else if(x==3) then 7 else (if(2==1) then 6 else 4 end) end"));
+            "if ($F1) then case when (_MAP.R_NAME = 'AFRICA') then 2 else 4 end else if(4==3) then 1 else if(x==3) then 7 else (if(2==1) then 6 else 4 end) end"));
     // ExprLexer lexer = new ExprLexer(new
     // ANTLRStringStream("if ('blue.red') then 'orange' else if (false) then 1 else 0 end"));
     // ExprLexer lexer = new ExprLexer(new ANTLRStringStream("2+2"));
@@ -28,6 +30,7 @@ public class TreeTest {
     CommonTokenStream tokens = new CommonTokenStream(lexer);
 
     ExprParser parser = new ExprParser(tokens);
+    parser.setRegistry(new FunctionRegistry(DrillConfig.create()));
     parse_return ret = parser.parse();
     LogicalExpression e = ret.e;
     ObjectMapper mapper = new ObjectMapper();

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/contrib/sqlline/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/contrib/sqlline/pom.xml b/sandbox/prototype/contrib/sqlline/pom.xml
new file mode 100644
index 0000000..9710a49
--- /dev/null
+++ b/sandbox/prototype/contrib/sqlline/pom.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0"?>
+<project
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+  xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <artifactId>prototype-parent</artifactId>
+    <groupId>org.apache.drill</groupId>
+    <version>1.0-SNAPSHOT</version>
+  </parent>
+  <artifactId>sqlline</artifactId>
+  <name>sqlline</name>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.drill</groupId>
+      <artifactId>sqlparser</artifactId>
+      <version>1.0-SNAPSHOT</version>
+    </dependency>
+
+    <dependency>
+      <groupId>sqlline</groupId>
+      <artifactId>sqlline</artifactId>
+      <version>1.1.0</version>
+    </dependency>
+    <!-- Specify xalan and xerces versions to avoid setXIncludeAware error. -->
+    <dependency>
+      <groupId>xerces</groupId>
+      <artifactId>xercesImpl</artifactId>
+      <version>2.9.1</version>
+    </dependency>
+    <dependency>
+      <groupId>xalan</groupId>
+      <artifactId>xalan</artifactId>
+      <version>2.7.1</version>
+    </dependency>
+  </dependencies>
+</project>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/pom.xml b/sandbox/prototype/exec/java-exec/pom.xml
index a07e43a..03c1e52 100644
--- a/sandbox/prototype/exec/java-exec/pom.xml
+++ b/sandbox/prototype/exec/java-exec/pom.xml
@@ -38,6 +38,11 @@
       <version>2.6.1</version>
     </dependency>
     <dependency>
+      <groupId>net.hydromatic</groupId>
+      <artifactId>optiq</artifactId>
+      <version>0.4.10</version>
+    </dependency>    
+    <dependency>
       <groupId>org.freemarker</groupId>
       <artifactId>freemarker</artifactId>
       <version>2.3.19</version>
@@ -49,6 +54,11 @@
       <scope>test</scope>
     </dependency>
     <dependency>
+      <groupId>net.hydromatic</groupId>
+      <artifactId>optiq</artifactId>
+      <version>0.4.10</version>
+    </dependency>    
+    <dependency>
       <groupId>com.twitter</groupId>
       <artifactId>parquet-column</artifactId>
       <version>1.0.1</version>
@@ -93,6 +103,7 @@
       <artifactId>common</artifactId>
       <version>${project.version}</version>
       <classifier>tests</classifier>
+      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>com.beust</groupId>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
index a6a6aa4..a63552a 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
@@ -130,20 +130,20 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
   }
   
   public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
-    int start = from.offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(fromIndex);
-    int end =   from.offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(fromIndex+1);
+    int start = from.offsetVector.getAccessor().get(fromIndex);
+    int end =   from.offsetVector.getAccessor().get(fromIndex+1);
     int len = end - start;
     
     int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width});
-    from.data.getBytes(start, from.data, outputStart, len);
-    offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, len);
+    from.data.getBytes(start, data, outputStart, len);
+    offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, outputStart + len);
   }
   
   public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
     if(thisIndex >= getValueCapacity()) return false;
     
-    int start = from.offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(fromIndex);
-    int end =   from.offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(fromIndex+1);
+    int start = from.offsetVector.getAccessor().get(fromIndex);
+    int end =   from.offsetVector.getAccessor().get(fromIndex+1);
     int len = end - start;
     
     int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width});
@@ -151,7 +151,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
     if(data.capacity() < outputStart + len) return false;
     
     from.data.getBytes(start, data, outputStart, len);
-    offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, len);
+    offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, outputStart + len);
 
     return true;
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
index cf99abd..7f28dff 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
@@ -78,6 +78,9 @@ public class DrillClient implements Closeable{
     this.clusterCoordinator = coordinator;
   }
   
+  public DrillConfig getConfig(){
+    return config;
+  }
   
   /**
    * Connects the client to a Drillbit server
@@ -121,10 +124,8 @@ public class DrillClient implements Closeable{
 
   /**
    * Closes this client's connection to the server
-   *
-   * @throws IOException
    */
-  public void close() throws IOException {
+  public void close(){
     this.client.close();
     connected = false;
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ZKClusterCoordinator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ZKClusterCoordinator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ZKClusterCoordinator.java
index 75dce2c..b1d173d 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ZKClusterCoordinator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/coord/ZKClusterCoordinator.java
@@ -57,8 +57,13 @@ public class ZKClusterCoordinator extends ClusterCoordinator {
   private final String serviceName;
   private final CountDownLatch initialConnection = new CountDownLatch(1);
 
-  public ZKClusterCoordinator(DrillConfig config) throws IOException {
-
+  
+  public ZKClusterCoordinator(DrillConfig config) throws IOException{
+    this(config, null);
+  }
+  
+  public ZKClusterCoordinator(DrillConfig config, String connect) throws IOException {
+    connect = connect == null || connect.isEmpty() ? config.getString(ExecConstants.ZK_CONNECTION) : connect;
     this.basePath = config.getString(ExecConstants.ZK_ROOT);
     this.serviceName = config.getString(ExecConstants.SERVICE_NAME);
     RetryPolicy rp = new RetryNTimes(config.getInt(ExecConstants.ZK_RETRY_TIMES),
@@ -66,7 +71,7 @@ public class ZKClusterCoordinator extends ClusterCoordinator {
     curator = CuratorFrameworkFactory.builder()
       .connectionTimeoutMs(config.getInt(ExecConstants.ZK_TIMEOUT))
       .retryPolicy(rp)
-      .connectString(config.getString(ExecConstants.ZK_CONNECTION))
+      .connectString(connect)
       .build();
     curator.getConnectionStateListenable().addListener(new InitialConnectionListener());
     discovery = getDiscovery();

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
index fd454a7..8ed6551 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
@@ -16,17 +16,21 @@ import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
 import org.apache.drill.common.types.TypeProtos.MajorType;
 import org.apache.drill.common.types.TypeProtos.MinorType;
 import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.expr.CodeGenerator.BlockType;
 import org.apache.drill.exec.expr.CodeGenerator.HoldingContainer;
 import org.apache.drill.exec.expr.fn.DrillFuncHolder;
 import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
 import org.apache.drill.exec.physical.impl.filter.ReturnValueExpression;
 import org.apache.drill.exec.record.NullExpression;
+import org.apache.drill.exec.vector.ValueHolderHelper;
 
 import com.sun.codemodel.JBlock;
+import com.sun.codemodel.JClass;
 import com.sun.codemodel.JConditional;
 import com.sun.codemodel.JExpr;
 import com.sun.codemodel.JExpression;
 import com.sun.codemodel.JInvocation;
+import com.sun.codemodel.JType;
 import com.sun.codemodel.JVar;
 
 public class EvaluationVisitor {
@@ -239,18 +243,26 @@ public class EvaluationVisitor {
       LogicalExpression child = e.getChild();
       // Preconditions.checkArgument(child.getMajorType().equals(Types.REQUIRED_BOOLEAN));
       HoldingContainer hc = child.accept(this, generator);
-      generator.getEvalBlock()._return(hc.getValue().eq(JExpr.lit(1)));
+      if(e.isReturnTrueOnOne()){
+        generator.getEvalBlock()._return(hc.getValue().eq(JExpr.lit(1)));  
+      }else{
+        generator.getEvalBlock()._return(hc.getValue());
+      }
+      
       return null;
     }
 
     @Override
     public HoldingContainer visitQuotedStringConstant(QuotedString e, CodeGenerator<?> generator)
         throws RuntimeException {
-      throw new UnsupportedOperationException(
-          "We don't yet support string literals as we need to build the string value holders.");
-
-      // JExpr stringLiteral = JExpr.lit(e.value);
-      // CodeGenerator.block.decl(stringLiteral.invoke("getBytes").arg(JExpr.ref(Charsets.UTF_8));
+      MajorType majorType = Types.required(MinorType.VARCHAR);
+      JBlock setup = generator.getBlock(BlockType.SETUP);
+      JType holderType = generator.getHolderType(majorType);
+      JVar var = generator.declareClassField("string", holderType);
+      JExpression stringLiteral = JExpr.lit(e.value);
+      setup.assign(var, ((JClass)generator.getModel().ref(ValueHolderHelper.class)).staticInvoke("getVarCharHolder").arg(stringLiteral));
+      return new HoldingContainer(majorType, var, null, null);
+      
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java
index 1ad3c4d..1914c50 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java
@@ -13,6 +13,8 @@ import org.apache.drill.exec.expr.annotations.Param;
 import org.apache.drill.exec.record.RecordBatch;
 import org.apache.drill.exec.vector.BigIntHolder;
 import org.apache.drill.exec.vector.IntHolder;
+import org.apache.drill.exec.vector.VarBinaryHolder;
+import org.apache.drill.exec.vector.VarCharHolder;
 
 public class ComparatorFunctions {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComparatorFunctions.class);
@@ -32,6 +34,68 @@ public class ComparatorFunctions {
   }
   
   @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+  public static class VarBinaryComparator implements DrillSimpleFunc {
+
+      @Param VarBinaryHolder left;
+      @Param VarBinaryHolder right;
+      @Output IntHolder out;
+
+      public void setup(RecordBatch b) {}
+
+      public void eval() {
+        out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right);
+      }
+     
+  }
+  
+  
+  
+  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+  public static class VarCharComparator implements DrillSimpleFunc {
+
+      @Param VarCharHolder left;
+      @Param VarCharHolder right;
+      @Output IntHolder out;
+
+      public void setup(RecordBatch b) {}
+
+      public void eval() {
+        out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right);
+      }
+     
+  }  
+
+  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+  public static class VarBinaryCharComparator implements DrillSimpleFunc {
+
+      @Param VarBinaryHolder left;
+      @Param VarCharHolder right;
+      @Output IntHolder out;
+
+      public void setup(RecordBatch b) {}
+
+      public void eval() {
+        out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right);
+      }
+     
+  }    
+  
+  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+  public static class VarBinaryCharComparator2 implements DrillSimpleFunc {
+
+      @Param VarCharHolder left;
+      @Param VarBinaryHolder right;
+      @Output IntHolder out;
+
+      public void setup(RecordBatch b) {}
+
+      public void eval() {
+        out.value = -org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(right, left);
+      }
+     
+  }      
+  
+  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
   public static class Long implements DrillSimpleFunc {
 
       @Param BigIntHolder left;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctions.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctions.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctions.java
index cb47251..680ea66 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctions.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctions.java
@@ -1,19 +1,17 @@
 package org.apache.drill.exec.expr.fn.impl;
 
-import org.apache.drill.common.expression.ArgumentValidators;
-import org.apache.drill.common.expression.CallProvider;
-import org.apache.drill.common.expression.FunctionDefinition;
-import org.apache.drill.common.expression.OutputTypeDeterminer;
 import org.apache.drill.exec.expr.DrillSimpleFunc;
 import org.apache.drill.exec.expr.annotations.FunctionTemplate;
 import org.apache.drill.exec.expr.annotations.Output;
 import org.apache.drill.exec.expr.annotations.Param;
-import org.apache.drill.exec.vector.IntHolder;
+import org.apache.drill.exec.record.RecordBatch;
 import org.apache.drill.exec.vector.BigIntHolder;
+import org.apache.drill.exec.vector.BitHolder;
 import org.apache.drill.exec.vector.Float4Holder;
 import org.apache.drill.exec.vector.Float8Holder;
-import org.apache.drill.exec.vector.BitHolder;
-import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.vector.IntHolder;
+import org.apache.drill.exec.vector.VarBinaryHolder;
+import org.apache.drill.exec.vector.VarCharHolder;
 
 public class ComparisonFunctions {
     static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MathFunctions.class);
@@ -355,4 +353,215 @@ public class ComparisonFunctions {
             out.value = (left.value <= right.value) ? 1 : 0;
         }
     }
+    
+    @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+    public static class VarBinaryEqual implements DrillSimpleFunc {
+
+        @Param VarBinaryHolder left;
+        @Param VarBinaryHolder right;
+        @Output BitHolder out;
+
+        public void setup(RecordBatch b) {}
+
+        public void eval() {
+            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == 0 ? 1 : 0;
+        }
+    }
+    
+    @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+    public static class VarCharEqual implements DrillSimpleFunc {
+
+        @Param VarCharHolder left;
+        @Param VarCharHolder right;
+        @Output BitHolder out;
+
+        public void setup(RecordBatch b) {}
+
+        public void eval() {
+            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == 0 ? 1 : 0;
+        }
+    }
+    
+    @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+    public static class VarBinaryCharEqual implements DrillSimpleFunc {
+
+        @Param VarBinaryHolder left;
+        @Param VarCharHolder right;
+        @Output BitHolder out;
+
+        public void setup(RecordBatch b) {}
+
+        public void eval() {
+            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == 0 ? 1 : 0;
+        }
+    }
+    
+    @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+    public static class VarBinaryGTE implements DrillSimpleFunc {
+
+        @Param VarBinaryHolder left;
+        @Param VarBinaryHolder right;
+        @Output BitHolder out;
+
+        public void setup(RecordBatch b) {}
+
+        public void eval() {
+            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) > -1 ? 1 : 0;
+        }
+    }
+    
+    @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+    public static class VarCharGTE implements DrillSimpleFunc {
+
+        @Param VarCharHolder left;
+        @Param VarCharHolder right;
+        @Output BitHolder out;
+
+        public void setup(RecordBatch b) {}
+
+        public void eval() {
+            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) > -1 ? 1 : 0;
+        }
+    }
+    
+    @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+    public static class VarBinaryCharGTE implements DrillSimpleFunc {
+
+        @Param VarBinaryHolder left;
+        @Param VarCharHolder right;
+        @Output BitHolder out;
+
+        public void setup(RecordBatch b) {}
+
+        public void eval() {
+            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) > -1 ? 1 : 0;
+        }
+    }
+    
+    @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+    public static class VarBinaryGT implements DrillSimpleFunc {
+
+        @Param VarBinaryHolder left;
+        @Param VarBinaryHolder right;
+        @Output BitHolder out;
+
+        public void setup(RecordBatch b) {}
+
+        public void eval() {
+            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == 1 ? 1 : 0;
+        }
+    }
+    
+    @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+    public static class VarCharGT implements DrillSimpleFunc {
+
+        @Param VarCharHolder left;
+        @Param VarCharHolder right;
+        @Output BitHolder out;
+
+        public void setup(RecordBatch b) {}
+
+        public void eval() {
+            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == 1 ? 1 : 0;
+        }
+    }
+    
+    @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+    public static class VarBinaryCharGT implements DrillSimpleFunc {
+
+        @Param VarBinaryHolder left;
+        @Param VarCharHolder right;
+        @Output BitHolder out;
+
+        public void setup(RecordBatch b) {}
+
+        public void eval() {
+            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == 1? 1 : 0;
+        }
+    }
+    
+    @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+    public static class VarBinaryLTE implements DrillSimpleFunc {
+
+        @Param VarBinaryHolder left;
+        @Param VarBinaryHolder right;
+        @Output BitHolder out;
+
+        public void setup(RecordBatch b) {}
+
+        public void eval() {
+            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) < 1 ? 1 : 0;
+        }
+    }
+    
+    @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+    public static class VarCharLTE implements DrillSimpleFunc {
+
+        @Param VarCharHolder left;
+        @Param VarCharHolder right;
+        @Output BitHolder out;
+
+        public void setup(RecordBatch b) {}
+
+        public void eval() {
+            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) < 1 ? 1 : 0;
+        }
+    }
+    
+    @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+    public static class VarBinaryCharLTE implements DrillSimpleFunc {
+
+        @Param VarBinaryHolder left;
+        @Param VarCharHolder right;
+        @Output BitHolder out;
+
+        public void setup(RecordBatch b) {}
+
+        public void eval() {
+            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) < 1? 1 : 0;
+        }
+    }
+    
+    @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+    public static class VarBinaryLT implements DrillSimpleFunc {
+
+        @Param VarBinaryHolder left;
+        @Param VarBinaryHolder right;
+        @Output BitHolder out;
+
+        public void setup(RecordBatch b) {}
+
+        public void eval() {
+            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == -1 ? 1 : 0;
+        }
+    }
+    
+    @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+    public static class VarCharLT implements DrillSimpleFunc {
+
+        @Param VarCharHolder left;
+        @Param VarCharHolder right;
+        @Output BitHolder out;
+
+        public void setup(RecordBatch b) {}
+
+        public void eval() {
+            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == -1 ? 1 : 0;
+        }
+    }
+    
+    @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+    public static class VarBinaryCharLT implements DrillSimpleFunc {
+
+        @Param VarBinaryHolder left;
+        @Param VarCharHolder right;
+        @Output BitHolder out;
+
+        public void setup(RecordBatch b) {}
+
+        public void eval() {
+            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == -1 ? 1 : 0;
+        }
+    }
+    
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/VarHelpers.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/VarHelpers.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/VarHelpers.java
new file mode 100644
index 0000000..0ac99c2
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/VarHelpers.java
@@ -0,0 +1,148 @@
+package org.apache.drill.exec.expr.fn.impl;
+
+import org.apache.drill.exec.expr.DrillSimpleFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.vector.IntHolder;
+import org.apache.drill.exec.vector.VarBinaryHolder;
+import org.apache.drill.exec.vector.VarCharHolder;
+
+public class VarHelpers {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VarHelpers.class);
+  
+  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+  public static class VarBinaryComparator implements DrillSimpleFunc {
+
+      @Param VarBinaryHolder left;
+      @Param VarBinaryHolder right;
+      @Output IntHolder out;
+
+      public void setup(RecordBatch b) {}
+
+      public void eval() {
+        boolean doLengthEval = true;
+        int i =0;
+        for (int l = left.start, r = right.start; l < left.end && r < right.end; l++, r++, i++) {
+          byte leftByte = left.buffer.getByte(l);
+          byte rightByte = right.buffer.getByte(r);
+          if (leftByte != rightByte) {
+            out.value = ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
+            doLengthEval = false;
+            break;
+          }
+        }
+        if(doLengthEval){
+          int l = (left.end - left.start) - (right.end - right.start);
+          if(l > 0){
+            out.value = 1;
+          }else if(l == 0){
+            out.value = 0;
+          }else{
+            out.value = -1;
+          }
+        }
+
+      }
+     
+  }
+  
+  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+  public static class VarCharComparator implements DrillSimpleFunc {
+
+      @Param VarCharHolder left;
+      @Param VarCharHolder right;
+      @Output IntHolder out;
+
+      public void setup(RecordBatch b) {}
+
+      public void eval() {
+        boolean doLengthEval = true;
+        int i =0;
+        for (int l = left.start, r = right.start; l < left.end && r < right.end; l++, r++, i++) {
+          byte leftByte = left.buffer.getByte(l);
+          byte rightByte = right.buffer.getByte(r);
+          if (leftByte != rightByte) {
+            out.value = ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
+            doLengthEval = false;
+            break;
+          }
+        }
+        if(doLengthEval){
+          int l = (left.end - left.start) - (right.end - right.start);
+          if(l > 0){
+            out.value = 1;
+          }else if(l == 0){
+            out.value = 0;
+          }else{
+            out.value = -1;
+          }
+        }
+
+      }
+     
+  }  
+
+
+      public static final int compare(VarBinaryHolder left, VarCharHolder right) {
+        for (int l = left.start, r = right.start; l < left.end && r < right.end; l++, r++) {
+          byte leftByte = left.buffer.getByte(l);
+          byte rightByte = right.buffer.getByte(r);
+          if (leftByte != rightByte) {
+            return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
+          }
+        }
+        
+        int l = (left.end - left.start) - (right.end - right.start);
+        if(l > 0){
+          return 1;
+        }else if(l == 0){
+          return 0;
+        }else{
+          return -1;
+        }
+
+      }
+     
+      public static final int compare(VarCharHolder left, VarCharHolder right) {
+        for (int l = left.start, r = right.start; l < left.end && r < right.end; l++, r++) {
+          byte leftByte = left.buffer.getByte(l);
+          byte rightByte = right.buffer.getByte(r);
+          if (leftByte != rightByte) {
+            return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
+          }
+        }
+        
+        int l = (left.end - left.start) - (right.end - right.start);
+        if(l > 0){
+          return 1;
+        }else if(l == 0){
+          return 0;
+        }else{
+          return -1;
+        }
+
+      }
+      
+      public static final int compare(VarBinaryHolder left, VarBinaryHolder right) {
+        for (int l = left.start, r = right.start; l < left.end && r < right.end; l++, r++) {
+          byte leftByte = left.buffer.getByte(l);
+          byte rightByte = right.buffer.getByte(r);
+          if (leftByte != rightByte) {
+            return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
+          }
+        }
+        
+        int l = (left.end - left.start) - (right.end - right.start);
+        if(l > 0){
+          return 1;
+        }else if(l == 0){
+          return 0;
+        }else{
+          return -1;
+        }
+
+      }
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
index 44117ff..3d4930d 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
@@ -19,9 +19,10 @@ package org.apache.drill.exec.ops;
 
 import java.util.Collection;
 
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.logical.StorageEngineConfig;
 import org.apache.drill.exec.cache.DistributedCache;
-import org.apache.drill.exec.exception.SetupException;
 import org.apache.drill.exec.planner.PhysicalPlanReader;
 import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
 import org.apache.drill.exec.proto.UserBitShared.QueryId;
@@ -49,7 +50,7 @@ public class QueryContext {
     return queryId;
   }
 
-  public StorageEngine getStorageEngine(StorageEngineConfig config) throws SetupException {
+  public StorageEngine getStorageEngine(StorageEngineConfig config) throws ExecutionSetupException {
     return drillbitContext.getStorageEngine(config);
   }
 
@@ -68,4 +69,9 @@ public class QueryContext {
   public BitCom getBitCom(){
     return drillbitContext.getBitCom();
   }
+  
+  public DrillConfig getConfig(){
+    return drillbitContext.getConfig();
+  }
+  
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
index 2c2a342..5cd51a8 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
@@ -2,21 +2,30 @@ package org.apache.drill.exec.opt;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 
 import org.apache.drill.common.PlanProperties;
 import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.FunctionCall;
-import org.apache.drill.common.expression.FunctionDefinition;
-import org.apache.drill.common.expression.NoArgValidator;
-import org.apache.drill.common.expression.OutputTypeDeterminer;
+import org.apache.drill.common.defs.OrderDef;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.logical.LogicalPlan;
 import org.apache.drill.common.logical.StorageEngineConfig;
+import org.apache.drill.common.logical.data.CollapsingAggregate;
 import org.apache.drill.common.logical.data.Filter;
+import org.apache.drill.common.logical.data.Join;
+import org.apache.drill.common.logical.data.JoinCondition;
+import org.apache.drill.common.logical.data.NamedExpression;
+import org.apache.drill.common.logical.data.Order;
+import org.apache.drill.common.logical.data.Order.Direction;
+import org.apache.drill.common.logical.data.Order.Ordering;
 import org.apache.drill.common.logical.data.Project;
 import org.apache.drill.common.logical.data.Scan;
+import org.apache.drill.common.logical.data.Segment;
 import org.apache.drill.common.logical.data.SinkOperator;
 import org.apache.drill.common.logical.data.Store;
 import org.apache.drill.common.logical.data.visitors.AbstractLogicalVisitor;
@@ -24,16 +33,17 @@ import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.common.types.TypeProtos.DataMode;
 import org.apache.drill.common.types.TypeProtos.MinorType;
 import org.apache.drill.exec.exception.OptimizerException;
-import org.apache.drill.exec.exception.SetupException;
 import org.apache.drill.exec.ops.QueryContext;
 import org.apache.drill.exec.physical.PhysicalPlan;
 import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.config.MergeJoinPOP;
 import org.apache.drill.exec.physical.config.Screen;
 import org.apache.drill.exec.physical.config.SelectionVectorRemover;
+import org.apache.drill.exec.physical.config.Sort;
+import org.apache.drill.exec.physical.config.StreamingAggregate;
 import org.apache.drill.exec.store.StorageEngine;
-import org.apache.drill.exec.store.mock.MockGroupScanPOP;
 
-import com.fasterxml.jackson.core.type.TypeReference;
+import com.beust.jcommander.internal.Lists;
 
 public class BasicOptimizer extends Optimizer{
 
@@ -51,21 +61,15 @@ public class BasicOptimizer extends Optimizer{
   }
 
   @Override
-  public PhysicalPlan optimize(OptimizationContext context, LogicalPlan plan) {
+  public PhysicalPlan optimize(OptimizationContext context, LogicalPlan plan) throws OptimizerException{
     Object obj = new Object();
     Collection<SinkOperator> roots = plan.getGraph().getRoots();
     List<PhysicalOperator> physOps = new ArrayList<PhysicalOperator>(roots.size());
     LogicalConverter converter = new LogicalConverter(plan);
     for ( SinkOperator op : roots){
-      try {
-        PhysicalOperator pop  = op.accept(converter, obj);
-        System.out.println(pop);
-        physOps.add(pop);
-      } catch (OptimizerException e) {
-        e.printStackTrace();
-      } catch (Throwable throwable) {
-        throwable.printStackTrace();
-      }
+      PhysicalOperator pop  = op.accept(converter, obj);
+      System.out.println(pop);
+      physOps.add(pop);
     }
 
     PlanProperties props = new PlanProperties();
@@ -100,6 +104,81 @@ public class BasicOptimizer extends Optimizer{
       this.logicalPlan = logicalPlan;
     }
 
+    
+
+    @Override
+    public PhysicalOperator visitSegment(Segment segment, Object value) throws OptimizerException {
+      throw new OptimizerException("Segment operators aren't currently supported besides next to a collapsing aggregate operator.");
+    }
+
+
+
+    @Override
+    public PhysicalOperator visitOrder(Order order, Object value) throws OptimizerException {
+      PhysicalOperator input = order.getInput().accept(this, value);
+      List<OrderDef> ods = Lists.newArrayList();
+      for(Ordering o : order.getOrderings()){
+        ods.add(OrderDef.create(o));
+      }
+      return new SelectionVectorRemover(new Sort(input, ods, false));
+    }
+
+
+
+    @Override
+    public PhysicalOperator visitCollapsingAggregate(CollapsingAggregate agg, Object value)
+        throws OptimizerException {
+
+      if( !(agg.getInput() instanceof Segment) ){
+        throw new OptimizerException(String.format("Currently, Drill only supports CollapsingAggregate immediately preceded by a Segment.  The input of this operator is %s.", agg.getInput()));
+      }
+      Segment segment = (Segment) agg.getInput();
+
+      if(!agg.getWithin().equals(segment.getName())){
+        throw new OptimizerException(String.format("Currently, Drill only supports CollapsingAggregate immediately preceded by a Segment where the CollapsingAggregate works on the defined segments.  In this case, the segment has been defined based on the name %s but the collapsing aggregate is working within the field %s.", segment.getName(), agg.getWithin()));
+      }
+      
+      // a collapsing aggregate is a currently implemented as a sort followed by a streaming aggregate.
+      List<OrderDef> orderDefs = Lists.newArrayList();
+      
+      List<NamedExpression> keys = Lists.newArrayList();
+      for(LogicalExpression e : segment.getExprs()){
+        if( !(e instanceof SchemaPath)) throw new OptimizerException("The basic optimizer doesn't currently support collapsing aggregate where the segment value is something other than a SchemaPath.");
+        keys.add(new NamedExpression(e, new FieldReference((SchemaPath) e)));
+        orderDefs.add(new OrderDef(Direction.ASC, e));
+      }
+      Sort sort = new Sort(segment.getInput().accept(this, value), orderDefs, false);
+      
+      StreamingAggregate sa = new StreamingAggregate(sort, keys.toArray(new NamedExpression[keys.size()]), agg.getAggregations(), 1.0f);
+      SelectionVectorRemover svm = new SelectionVectorRemover(sa);
+      return svm;
+    }
+
+
+
+    @Override
+    public PhysicalOperator visitJoin(Join join, Object value) throws OptimizerException {
+      PhysicalOperator leftOp = join.getLeft().accept(this, value);
+      List<OrderDef> leftOrderDefs = Lists.newArrayList();
+      for(JoinCondition jc : join.getConditions()){
+        leftOrderDefs.add(new OrderDef(Direction.ASC, jc.getLeft()));
+      }
+      leftOp = new Sort(leftOp, leftOrderDefs, false);
+      leftOp = new SelectionVectorRemover(leftOp);
+      
+      PhysicalOperator rightOp = join.getRight().accept(this, value);
+      List<OrderDef> rightOrderDefs = Lists.newArrayList();
+      for(JoinCondition jc : join.getConditions()){
+        rightOrderDefs.add(new OrderDef(Direction.ASC, jc.getRight()));
+      }
+      rightOp = new Sort(rightOp, rightOrderDefs, false);
+      rightOp = new SelectionVectorRemover(rightOp);
+      
+      MergeJoinPOP mjp = new MergeJoinPOP(leftOp, rightOp, Arrays.asList(join.getConditions()));
+      return new SelectionVectorRemover(mjp);
+    }
+
+
 
     @Override
     public PhysicalOperator visitScan(Scan scan, Object obj) throws OptimizerException {
@@ -109,7 +188,7 @@ public class BasicOptimizer extends Optimizer{
       try {
         engine = context.getStorageEngine(config);
         return engine.getPhysicalScan(scan);
-      } catch (SetupException | IOException e) {
+      } catch (IOException | ExecutionSetupException e) {
         throw new OptimizerException("Failure while attempting to retrieve storage engine.", e);
       }
     }
@@ -124,9 +203,8 @@ public class BasicOptimizer extends Optimizer{
 
     @Override
     public PhysicalOperator visitProject(Project project, Object obj) throws OptimizerException {
-      return project.getInput().accept(this, obj);
-      // return new org.apache.drill.exec.physical.config.Project(
-      // Arrays.asList(project.getSelections()), project.iterator().next().accept(this, obj));
+//      return project.getInput().accept(this, obj);
+      return new org.apache.drill.exec.physical.config.Project(Arrays.asList(project.getSelections()), project.iterator().next().accept(this, obj));
     }
 
     @Override
@@ -134,11 +212,8 @@ public class BasicOptimizer extends Optimizer{
       TypeProtos.MajorType.Builder b = TypeProtos.MajorType.getDefaultInstance().newBuilderForType();
       b.setMode(DataMode.REQUIRED);
       b.setMinorType(MinorType.BIGINT);
-
-      return new SelectionVectorRemover(new org.apache.drill.exec.physical.config.Filter(filter.iterator().next()
-          .accept(this, obj), /* filter.getExpr() */
-      new FunctionCall(FunctionDefinition.simple("alternate", new NoArgValidator(), new OutputTypeDeterminer.FixedType(
-          b.build())), null, new ExpressionPosition("asdf", 1)), 1.0f));
+      PhysicalOperator child = filter.iterator().next().accept(this, obj);
+      return new SelectionVectorRemover(new org.apache.drill.exec.physical.config.Filter(child, filter.getExpr(), 1.0f));
     }
 
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/opt/Optimizer.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/opt/Optimizer.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/opt/Optimizer.java
index 9f506c1..e6e00e4 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/opt/Optimizer.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/opt/Optimizer.java
@@ -22,6 +22,7 @@ import java.io.Closeable;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.exceptions.DrillConfigurationException;
 import org.apache.drill.common.logical.LogicalPlan;
+import org.apache.drill.exec.exception.OptimizerException;
 import org.apache.drill.exec.physical.PhysicalPlan;
 
 public abstract class Optimizer implements Closeable{
@@ -30,7 +31,7 @@ public abstract class Optimizer implements Closeable{
   
   public abstract void init(DrillConfig config);
   
-  public abstract PhysicalPlan optimize(OptimizationContext context, LogicalPlan plan);
+  public abstract PhysicalPlan optimize(OptimizationContext context, LogicalPlan plan) throws OptimizerException;
   public abstract void close();
   
   public static Optimizer getOptimizer(DrillConfig config) throws DrillConfigurationException{

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryWithPath.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryWithPath.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryWithPath.java
index 57d1d0b..9d607d0 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryWithPath.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryWithPath.java
@@ -23,6 +23,14 @@ public class ReadEntryWithPath implements ReadEntry {
 
   protected String path;
 
+  
+  public ReadEntryWithPath(String path) {
+    super();
+    this.path = path;
+  }
+
+  public ReadEntryWithPath(){}
+  
   public String getPath(){
    return path;
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
index 870792a..fdd81d1 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
@@ -19,11 +19,10 @@ package org.apache.drill.exec.physical.base;
 
 import java.util.List;
 
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import org.apache.drill.exec.physical.ReadEntry;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
 
-import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 
 /**
  * A GroupScan operator represents all data which will be scanned by a given physical
@@ -33,7 +32,7 @@ public interface GroupScan extends Scan, HasAffinity{
 
   public abstract void applyAssignments(List<DrillbitEndpoint> endpoints);
 
-  public abstract SubScan getSpecificScan(int minorFragmentId);
+  public abstract SubScan getSpecificScan(int minorFragmentId) throws ExecutionSetupException;
 
   @JsonIgnore
   public int getMaxParallelizationWidth();

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalOperator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalOperator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalOperator.java
index c24836b..66841b8 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalOperator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalOperator.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.physical.base;
 
 import java.util.List;
 
+import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.graph.GraphValue;
 import org.apache.drill.exec.physical.OperatorCost;
 
@@ -75,6 +76,6 @@ public interface  PhysicalOperator extends GraphValue<PhysicalOperator> {
    * @return
    */
   @JsonIgnore
-  public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children);
+  public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) throws ExecutionSetupException;
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java
index 19351cc..9a1973e 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java
@@ -21,9 +21,9 @@ public class MergeJoinPOP extends AbstractBase{
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MergeJoinPOP.class);
 
   
-  private PhysicalOperator left;
-  private PhysicalOperator right;
-  private List<JoinCondition> conditions;
+  private final PhysicalOperator left;
+  private final PhysicalOperator right;
+  private final List<JoinCondition> conditions;
   
   @Override
   public OperatorCost getCost() {
@@ -62,6 +62,14 @@ public class MergeJoinPOP extends AbstractBase{
     return Iterators.forArray(left, right);
   }
 
+  public PhysicalOperator getLeft() {
+    return left;
+  }
+
+  public PhysicalOperator getRight() {
+    return right;
+  }
+
   public List<JoinCondition> getConditions() {
     return conditions;
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java
index a7e8f0c..5675475 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java
@@ -73,6 +73,7 @@ public abstract class FilterTemplate implements Filterer{
     for(char i =0; i < recordCount; i++){
       
       if(doEval(i, 0)){
+        logger.debug("Setting svIndex {} to {}", svIndex, (int) i);
         outgoingSelectionVector.setIndex(svIndex, i);
         svIndex++;
       }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ReturnValueExpression.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ReturnValueExpression.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ReturnValueExpression.java
index cfe520d..21373a1 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ReturnValueExpression.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/ReturnValueExpression.java
@@ -14,9 +14,15 @@ public class ReturnValueExpression implements LogicalExpression{
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ReturnValueExpression.class);
 
   private LogicalExpression child;
+  private boolean returnTrueOnOne;
   
   public ReturnValueExpression(LogicalExpression child) {
+    this(child, true);
+  }
+  
+  public ReturnValueExpression(LogicalExpression child, boolean returnTrueOnOne) {
     this.child = child;
+    this.returnTrueOnOne = returnTrueOnOne;
   }
 
   public LogicalExpression getChild() {
@@ -43,5 +49,9 @@ public class ReturnValueExpression implements LogicalExpression{
     return Iterators.singletonIterator(child);
   }
 
+  public boolean isReturnTrueOnOne() {
+    return returnTrueOnOne;
+  }
+
   
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
index af33ca4..02f72f5 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
@@ -4,10 +4,13 @@ import static org.apache.drill.exec.compile.sig.GeneratorMapping.GM;
 
 import java.io.IOException;
 
+import com.google.common.collect.ImmutableList;
 import com.sun.codemodel.*;
 
 import org.apache.drill.common.expression.ErrorCollector;
 import org.apache.drill.common.expression.ErrorCollectorImpl;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FunctionCall;
 import org.apache.drill.common.expression.LogicalExpression;
 import org.apache.drill.common.logical.data.JoinCondition;
 import org.apache.drill.exec.compile.sig.GeneratorMapping;
@@ -16,8 +19,11 @@ import org.apache.drill.exec.exception.ClassTransformationException;
 import org.apache.drill.exec.exception.SchemaChangeException;
 import org.apache.drill.exec.expr.CodeGenerator;
 import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
+import org.apache.drill.exec.expr.HoldingContainerExpression;
+import org.apache.drill.exec.expr.fn.impl.ComparatorFunctions;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.physical.config.MergeJoinPOP;
+import org.apache.drill.exec.physical.impl.filter.ReturnValueExpression;
 import org.apache.drill.exec.physical.impl.join.JoinWorker.JoinOutcome;
 import org.apache.drill.exec.record.*;
 import org.apache.drill.exec.vector.*;
@@ -256,16 +262,19 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
           ._return(JExpr.lit(1));
     }
 
-    // equality
-    cg.getEvalBlock()._if(compareLeftExprHolder.getValue().eq(compareRightExprHolder.getValue()))
-                     ._then()
-                       ._return(JExpr.lit(0));
-    // less than
-    cg.getEvalBlock()._if(compareLeftExprHolder.getValue().lt(compareRightExprHolder.getValue()))
-                     ._then()
-                       ._return(JExpr.lit(-1));
-    // greater than
-    cg.getEvalBlock()._return(JExpr.lit(1));
+    FunctionCall f = new FunctionCall(ComparatorFunctions.COMPARE_TO, ImmutableList.of((LogicalExpression) new HoldingContainerExpression(compareLeftExprHolder), (LogicalExpression)  new HoldingContainerExpression(compareRightExprHolder)), ExpressionPosition.UNKNOWN);
+    cg.addExpr(new ReturnValueExpression(f, false), false);
+//    
+//    // equality
+//    cg.getEvalBlock()._if(compareLeftExprHolder.getValue().eq(compareRightExprHolder.getValue()))
+//                     ._then()
+//                       ._return(JExpr.lit(0));
+//    // less than
+//    cg.getEvalBlock()._if(compareLeftExprHolder.getValue().lt(compareRightExprHolder.getValue()))
+//                     ._then()
+//                       ._return(JExpr.lit(-1));
+//    // greater than
+//    cg.getEvalBlock()._return(JExpr.lit(1));
 
 
     // generate compareNextLeftKey()
@@ -390,6 +399,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
     }
 
     public void alloc(int recordCount){
+      recordCount *=10;
       out.allocateNew(recordCount);
       out.getMutator().setValueCount(recordCount);
     }
@@ -406,7 +416,8 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
     }
 
     public void alloc(int recordCount){
-      out.allocateNew(in.getByteCapacity(), recordCount);
+      recordCount *= 10;
+      out.allocateNew(in.getByteCapacity()*10, recordCount);
       out.getMutator().setValueCount(recordCount);
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
index 8cffda3..dd09ed3 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
@@ -3,12 +3,12 @@ package org.apache.drill.exec.physical.impl.sort;
 import java.io.IOException;
 
 import org.apache.drill.common.defs.OrderDef;
-import org.apache.drill.common.defs.OrderDef.Direction;
 import org.apache.drill.common.expression.ErrorCollector;
 import org.apache.drill.common.expression.ErrorCollectorImpl;
 import org.apache.drill.common.expression.ExpressionPosition;
 import org.apache.drill.common.expression.FunctionCall;
 import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.Order.Direction;
 import org.apache.drill.exec.compile.sig.MappingSet;
 import org.apache.drill.exec.exception.ClassTransformationException;
 import org.apache.drill.exec.exception.SchemaChangeException;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/CopierTemplate2.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/CopierTemplate2.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/CopierTemplate2.java
index 1f99f04..563037f 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/CopierTemplate2.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/CopierTemplate2.java
@@ -34,7 +34,7 @@ public abstract class CopierTemplate2 implements Copier{
     int outgoingPosition = 0;
     
     for(int svIndex = 0; svIndex < recordCount; svIndex++, outgoingPosition++){
-      doEval(svIndex, outgoingPosition);
+      doEval(sv2.getIndex(svIndex), outgoingPosition);
     }
   }
   

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java
index 10cc7ab..fb2fc3a 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractSingleRecordBatch.java
@@ -8,6 +8,7 @@ public abstract class AbstractSingleRecordBatch<T extends PhysicalOperator> exte
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractSingleRecordBatch.class);
   
   protected final RecordBatch incoming;
+  private boolean first = true;
   
   public AbstractSingleRecordBatch(T popConfig, FragmentContext context, RecordBatch incoming) {
     super(popConfig, context);
@@ -22,7 +23,8 @@ public abstract class AbstractSingleRecordBatch<T extends PhysicalOperator> exte
   @Override
   public IterOutcome next() {
     IterOutcome upstream = incoming.next();
-    
+    if(first && upstream == IterOutcome.OK) upstream = IterOutcome.OK_NEW_SCHEMA;
+    first = false;
     switch(upstream){
     case NONE:
     case NOT_YET:

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java
index a38c8e5..7191f1c 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/selection/SelectionVector2.java
@@ -44,7 +44,8 @@ public class SelectionVector2 implements Closeable{
   }
 
   public char getIndex(int index){
-    return buffer.getChar(index);
+    
+    return buffer.getChar(index*2);
   }
 
   public void setIndex(int index, char value){

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/Field.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/Field.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/Field.java
index 080be92..c6dd622 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/Field.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/Field.java
@@ -19,8 +19,8 @@
 package org.apache.drill.exec.schema;
 
 import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FieldReference;
 import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.common.types.TypeProtos.MajorType;
 import org.apache.drill.exec.record.MaterializedField;
 
@@ -40,8 +40,8 @@ public abstract class Field {
     this.parentSchema = parentSchema;
   }
 
-  public MaterializedField getAsMaterializedField() {
-    return MaterializedField.create(new SchemaPath(getFieldName(), ExpressionPosition.UNKNOWN), fieldType);
+  public MaterializedField getAsMaterializedField(FieldReference ref) {
+    return MaterializedField.create(new SchemaPath(ref.getPath() + "." + getFieldName(), ExpressionPosition.UNKNOWN), fieldType);
   }
 
   public abstract String getFieldName();

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
index eb160be..a937ed7 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
@@ -22,6 +22,7 @@ import io.netty.channel.nio.NioEventLoopGroup;
 import java.util.Collection;
 
 import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.logical.StorageEngineConfig;
 import org.apache.drill.exec.ExecConstants;
 import org.apache.drill.exec.cache.DistributedCache;
@@ -36,6 +37,7 @@ import org.apache.drill.exec.store.StorageEngine;
 
 import com.google.common.base.Preconditions;
 import com.yammer.metrics.MetricRegistry;
+
 import org.apache.drill.exec.store.StorageEngineRegistry;
 
 public class DrillbitContext {
@@ -81,7 +83,7 @@ public class DrillbitContext {
     return context.getAllocator();
   }
   
-  public StorageEngine getStorageEngine(StorageEngineConfig config) throws SetupException {
+  public StorageEngine getStorageEngine(StorageEngineConfig config) throws ExecutionSetupException {
     return storageEngineRegistry.getEngine(config);
   }
   

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/DistributedStorageEngine.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/DistributedStorageEngine.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/DistributedStorageEngine.java
new file mode 100644
index 0000000..6283692
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/DistributedStorageEngine.java
@@ -0,0 +1,5 @@
+package org.apache.drill.exec.store;
+
+public interface DistributedStorageEngine {
+  public String getDfsName();
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProvider.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProvider.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProvider.java
new file mode 100644
index 0000000..e7d8044
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProvider.java
@@ -0,0 +1,8 @@
+package org.apache.drill.exec.store;
+
+public interface SchemaProvider {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SchemaProvider.class);
+  
+  public Object getSelectionBaseOnName(String tableName);
+
+}


[08/18] git commit: DRILL-167, DRILL-80, DRILL-59 continued

Posted by ja...@apache.org.
DRILL-167, DRILL-80, DRILL-59 continued


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/d5537492
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/d5537492
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/d5537492

Branch: refs/heads/master
Commit: d55374927814632796591268ffef48488fa903c8
Parents: 5ff952b
Author: Jacques Nadeau <ja...@apache.org>
Authored: Fri Aug 30 16:57:29 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Sep 1 15:16:21 2013 -0700

----------------------------------------------------------------------
 .../common/src/main/resources/drill-module.conf |   1 +
 .../drill/storage/MockStorageEngineConfig.java  |  59 -----
 sandbox/prototype/exec/java-exec/pom.xml        |  98 ++++---
 .../drill/exec/expr/EvaluationVisitor.java      |   7 +-
 .../exec/expr/fn/impl/ComparatorFunctions.java  |  71 ++++-
 .../drill/exec/expr/fn/impl/VarHelpers.java     | 265 ++++++++++---------
 .../apache/drill/exec/opt/BasicOptimizer.java   |   1 -
 .../exec/physical/impl/join/JoinTemplate.java   |   2 +-
 .../exec/physical/impl/join/MergeJoinBatch.java |  21 +-
 .../org/apache/drill/exec/schema/Field.java     |   7 +-
 .../drill/exec/store/json/JSONGroupScan.java    |   2 +-
 .../drill/exec/store/json/JSONRecordReader.java |   2 +-
 .../exec/physical/impl/TestOptiqPlans.java      |   3 +-
 .../drill/exec/store/JSONRecordReaderTest.java  |  12 +-
 sandbox/prototype/pom.xml                       |  10 +
 sandbox/prototype/sqlline                       |   2 +-
 .../apache/drill/jdbc/DrillDriverVersion.java   |   1 +
 .../apache/drill/optiq/DrillImplementor.java    |   5 +-
 .../java/org/apache/drill/optiq/DrillOptiq.java |   2 +-
 .../org/apache/drill/optiq/EnumerableDrill.java |   5 +
 .../drill/optiq/EnumerableDrillFullEngine.java  |   9 +-
 .../apache/drill/optiq/EnumerableDrillRel.java  |   5 +-
 .../drill/sql/client/full/BatchLoaderMap.java   |  16 +-
 .../drill/sql/client/full/DrillFullImpl.java    |   9 +-
 .../drill/sql/client/full/JsonHelper.java       |   1 +
 .../drill/sql/client/full/ResultEnumerator.java |  11 +-
 .../apache/drill/jdbc/test/FullEngineTest.java  |   2 +-
 .../org/apache/drill/jdbc/test/JdbcTest.java    |   5 +-
 .../sqlparser/src/test/resources/logback.xml    |   4 +-
 29 files changed, 347 insertions(+), 291 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/common/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/sandbox/prototype/common/src/main/resources/drill-module.conf b/sandbox/prototype/common/src/main/resources/drill-module.conf
index 28af168..2b265ea 100644
--- a/sandbox/prototype/common/src/main/resources/drill-module.conf
+++ b/sandbox/prototype/common/src/main/resources/drill-module.conf
@@ -1,3 +1,4 @@
 //  This file tells Drill to consider this module when class path scanning.  
 //  This file can also include any supplementary configuration information.  
 //  This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information.
+drill.exec.storage.packages += org.apache.drill.exec.store.mock
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/common/src/test/java/org/apache/drill/storage/MockStorageEngineConfig.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/common/src/test/java/org/apache/drill/storage/MockStorageEngineConfig.java b/sandbox/prototype/common/src/test/java/org/apache/drill/storage/MockStorageEngineConfig.java
deleted file mode 100644
index 5843c38..0000000
--- a/sandbox/prototype/common/src/test/java/org/apache/drill/storage/MockStorageEngineConfig.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- * http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.storage;
-
-import org.apache.drill.common.logical.StorageEngineConfigBase;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonTypeName;
-
-@JsonTypeName("mock")
-public class MockStorageEngineConfig extends StorageEngineConfigBase{
-
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockStorageEngineConfig.class);
-
-  private String url;
-  
-  @JsonCreator
-  public MockStorageEngineConfig(@JsonProperty("url") String url) {
-    this.url = url;
-  }
-
-  public String getUrl() {
-    return url;
-  }
-
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) return true;
-    if (o == null || getClass() != o.getClass()) return false;
-
-    MockStorageEngineConfig that = (MockStorageEngineConfig) o;
-
-    if (url != null ? !url.equals(that.url) : that.url != null) return false;
-
-    return true;
-  }
-
-  @Override
-  public int hashCode() {
-    return url != null ? url.hashCode() : 0;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/pom.xml b/sandbox/prototype/exec/java-exec/pom.xml
index d43ff23..781a5da 100644
--- a/sandbox/prototype/exec/java-exec/pom.xml
+++ b/sandbox/prototype/exec/java-exec/pom.xml
@@ -1,7 +1,7 @@
 <?xml version="1.0"?>
 <project
-    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
-    xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+  xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <artifactId>exec-parent</artifactId>
@@ -28,6 +28,16 @@
       <version>2.5.6</version>
     </dependency>
     <dependency>
+      <groupId>xerces</groupId>
+      <artifactId>xercesImpl</artifactId>
+      <version>2.9.1</version>
+    </dependency>
+    <dependency>
+      <groupId>xalan</groupId>
+      <artifactId>xalan</artifactId>
+      <version>2.7.1</version>
+    </dependency>
+    <dependency>
       <groupId>com.sun.codemodel</groupId>
       <artifactId>codemodel</artifactId>
       <version>2.6</version>
@@ -41,7 +51,7 @@
       <groupId>net.hydromatic</groupId>
       <artifactId>optiq</artifactId>
       <version>0.4.10</version>
-    </dependency>    
+    </dependency>
     <dependency>
       <groupId>org.freemarker</groupId>
       <artifactId>freemarker</artifactId>
@@ -57,7 +67,7 @@
       <groupId>net.hydromatic</groupId>
       <artifactId>optiq</artifactId>
       <version>0.4.10</version>
-    </dependency>    
+    </dependency>
     <dependency>
       <groupId>com.twitter</groupId>
       <artifactId>parquet-column</artifactId>
@@ -87,7 +97,7 @@
       <groupId>javax.inject</groupId>
       <artifactId>javax.inject</artifactId>
       <version>1</version>
-    </dependency>    
+    </dependency>
     <dependency>
       <groupId>org.apache.drill</groupId>
       <artifactId>common</artifactId>
@@ -261,13 +271,13 @@
                   </fileset>
                 </path>
                 <pathconvert pathsep=" " property="proto.files"
-                             refid="proto.path.files" />
+                  refid="proto.path.files" />
 
                 <exec executable="protoc">
                   <arg value="--java_out=${target.gen.source.path}" />
                   <arg value="--proto_path=${proto.cas.path}" />
                   <arg
-                      value="--proto_path=${project.basedir}/../../common/src/main/protobuf/" />
+                    value="--proto_path=${project.basedir}/../../common/src/main/protobuf/" />
                   <arg line="${proto.files}" />
                 </exec>
               </tasks>
@@ -298,43 +308,43 @@
           </execution>
         </executions>
       </plugin>
-        <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-shade-plugin</artifactId>
-                <version>2.1</version>
-                <executions>
-                  <execution>
-                    <phase>package</phase>
-                    <goals>
-                      <goal>shade</goal>
-                    </goals>
-                    <configuration>
-                      <artifactSet>
-                         <includes>
-                            <include>com.google.protobuf:protobuf-java</include>                     
-                         </includes>                 
-                      </artifactSet>
-                      <shadedArtifactAttached>true</shadedArtifactAttached>
-                      <shadedClassifierName>rebuffed</shadedClassifierName>
-                      <filters>
-                        <filter>
-                            <artifact>com.google.protobuf:protobuf-java</artifact>
-                            <excludes>
-                                <exclude>**</exclude>
-                            </excludes>
-                        </filter>
-                    </filters>
-                      <relocations>
-                        <relocation>
-                          <pattern>com.google.protobuf</pattern>
-                          <shadedPattern>250.com.google.protobuf</shadedPattern>
-                        </relocation>
-                      </relocations>
-                    </configuration>
-                  </execution>
-                </executions>
-              </plugin>
-        
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-shade-plugin</artifactId>
+        <version>2.1</version>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>shade</goal>
+            </goals>
+            <configuration>
+              <artifactSet>
+                <includes>
+                  <include>com.google.protobuf:protobuf-java</include>
+                </includes>
+              </artifactSet>
+              <shadedArtifactAttached>true</shadedArtifactAttached>
+              <shadedClassifierName>rebuffed</shadedClassifierName>
+              <filters>
+                <filter>
+                  <artifact>com.google.protobuf:protobuf-java</artifact>
+                  <excludes>
+                    <exclude>**</exclude>
+                  </excludes>
+                </filter>
+              </filters>
+              <relocations>
+                <relocation>
+                  <pattern>com.google.protobuf</pattern>
+                  <shadedPattern>250.com.google.protobuf</shadedPattern>
+                </relocation>
+              </relocations>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+
       <!-- <plugin> -->
       <!-- <groupId>com.github.igor-petruk.protobuf</groupId> -->
       <!-- <artifactId>protobuf-maven-plugin</artifactId> -->

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
index 8ed6551..a7e576f 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
@@ -210,10 +210,11 @@ public class EvaluationVisitor {
       JExpression indexVariable = generator.getMappingSet().getValueReadIndex();
 
       JInvocation getValueAccessor = vv1.invoke("getAccessor").invoke("get");
+      JInvocation getValueAccessor2 = vv1.invoke("getAccessor");
       if (e.isSuperReader()) {
 
-        getValueAccessor = ((JExpression) vv1.component(indexVariable.shrz(JExpr.lit(16)))).invoke("getAccessor")
-            .invoke("get");
+        getValueAccessor = ((JExpression) vv1.component(indexVariable.shrz(JExpr.lit(16)))).invoke("getAccessor").invoke("get");
+        getValueAccessor2 = ((JExpression) vv1.component(indexVariable.shrz(JExpr.lit(16)))).invoke("getAccessor");
         indexVariable = indexVariable.band(JExpr.lit((int) Character.MAX_VALUE));
       }
 
@@ -222,7 +223,7 @@ public class EvaluationVisitor {
 
       if (out.isOptional()) {
         JBlock blk = generator.getEvalBlock();
-        blk.assign(out.getIsSet(), vv1.invoke("getAccessor").invoke("isSet").arg(indexVariable));
+        blk.assign(out.getIsSet(), getValueAccessor2.invoke("isSet").arg(indexVariable));
         JConditional jc = blk._if(out.getIsSet().eq(JExpr.lit(1)));
         if (Types.usesHolderForGet(e.getMajorType())) {
           jc._then().add(getValueAccessor.arg(indexVariable).arg(out.getHolder()));

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java
index 1914c50..416ce29 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java
@@ -13,6 +13,9 @@ import org.apache.drill.exec.expr.annotations.Param;
 import org.apache.drill.exec.record.RecordBatch;
 import org.apache.drill.exec.vector.BigIntHolder;
 import org.apache.drill.exec.vector.IntHolder;
+import org.apache.drill.exec.vector.NullableBigIntHolder;
+import org.apache.drill.exec.vector.NullableIntHolder;
+import org.apache.drill.exec.vector.NullableVarCharHolder;
 import org.apache.drill.exec.vector.VarBinaryHolder;
 import org.apache.drill.exec.vector.VarCharHolder;
 
@@ -34,6 +37,58 @@ public class ComparatorFunctions {
   }
   
   @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+  public static class NullableIntComparator implements DrillSimpleFunc {
+
+      @Param NullableIntHolder left;
+      @Param NullableIntHolder right;
+      @Output IntHolder out;
+
+      public void setup(RecordBatch b) {}
+
+      public void eval() {
+        
+        if(left.isSet == 0){
+          if(right.isSet == 0){
+            out.value = 0; 
+          }else{
+            out.value = -1;
+          }
+        }else if(right.isSet == 0){
+          out.value = 1;
+        }else{
+          out.value = left.value < right.value ? -1 : ((left.value == right.value)? 0 : 1);  
+        }
+        
+      }
+  }
+  
+  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+  public static class NullableBigIntComparator implements DrillSimpleFunc {
+
+      @Param NullableBigIntHolder left;
+      @Param NullableBigIntHolder right;
+      @Output IntHolder out;
+
+      public void setup(RecordBatch b) {}
+
+      public void eval() {
+        
+        if(left.isSet == 0){
+          if(right.isSet == 0){
+            out.value = 0; 
+          }else{
+            out.value = -1;
+          }
+        }else if(right.isSet == 0){
+          out.value = 1;
+        }else{
+          out.value = left.value < right.value ? -1 : ((left.value == right.value)? 0 : 1);  
+        }
+        
+      }
+  }
+  
+  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
   public static class VarBinaryComparator implements DrillSimpleFunc {
 
       @Param VarBinaryHolder left;
@@ -48,7 +103,21 @@ public class ComparatorFunctions {
      
   }
   
-  
+  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+  public static class NullableVarCharComparator implements DrillSimpleFunc {
+
+      @Param NullableVarCharHolder left;
+      @Param NullableVarCharHolder right;
+      @Output IntHolder out;
+
+      public void setup(RecordBatch b) {}
+
+      public void eval() {
+        out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right);
+      }
+     
+  }
+
   
   @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
   public static class VarCharComparator implements DrillSimpleFunc {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/VarHelpers.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/VarHelpers.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/VarHelpers.java
index 0ac99c2..2942e05 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/VarHelpers.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/VarHelpers.java
@@ -1,148 +1,155 @@
 package org.apache.drill.exec.expr.fn.impl;
 
-import org.apache.drill.exec.expr.DrillSimpleFunc;
-import org.apache.drill.exec.expr.annotations.FunctionTemplate;
-import org.apache.drill.exec.expr.annotations.Output;
-import org.apache.drill.exec.expr.annotations.Param;
-import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.vector.IntHolder;
+import org.apache.drill.exec.vector.NullableVarBinaryHolder;
+import org.apache.drill.exec.vector.NullableVarCharHolder;
 import org.apache.drill.exec.vector.VarBinaryHolder;
 import org.apache.drill.exec.vector.VarCharHolder;
 
 public class VarHelpers {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VarHelpers.class);
-  
-  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-  public static class VarBinaryComparator implements DrillSimpleFunc {
-
-      @Param VarBinaryHolder left;
-      @Param VarBinaryHolder right;
-      @Output IntHolder out;
-
-      public void setup(RecordBatch b) {}
-
-      public void eval() {
-        boolean doLengthEval = true;
-        int i =0;
-        for (int l = left.start, r = right.start; l < left.end && r < right.end; l++, r++, i++) {
-          byte leftByte = left.buffer.getByte(l);
-          byte rightByte = right.buffer.getByte(r);
-          if (leftByte != rightByte) {
-            out.value = ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
-            doLengthEval = false;
-            break;
-          }
-        }
-        if(doLengthEval){
-          int l = (left.end - left.start) - (right.end - right.start);
-          if(l > 0){
-            out.value = 1;
-          }else if(l == 0){
-            out.value = 0;
-          }else{
-            out.value = -1;
-          }
-        }
 
+  public static final int compare(VarBinaryHolder left, VarCharHolder right) {
+    for (int l = left.start, r = right.start; l < left.end && r < right.end; l++, r++) {
+      byte leftByte = left.buffer.getByte(l);
+      byte rightByte = right.buffer.getByte(r);
+      if (leftByte != rightByte) {
+        return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
       }
-     
+    }
+
+    int l = (left.end - left.start) - (right.end - right.start);
+    if (l > 0) {
+      return 1;
+    } else if (l == 0) {
+      return 0;
+    } else {
+      return -1;
+    }
+
   }
-  
-  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-  public static class VarCharComparator implements DrillSimpleFunc {
-
-      @Param VarCharHolder left;
-      @Param VarCharHolder right;
-      @Output IntHolder out;
-
-      public void setup(RecordBatch b) {}
-
-      public void eval() {
-        boolean doLengthEval = true;
-        int i =0;
-        for (int l = left.start, r = right.start; l < left.end && r < right.end; l++, r++, i++) {
-          byte leftByte = left.buffer.getByte(l);
-          byte rightByte = right.buffer.getByte(r);
-          if (leftByte != rightByte) {
-            out.value = ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
-            doLengthEval = false;
-            break;
-          }
-        }
-        if(doLengthEval){
-          int l = (left.end - left.start) - (right.end - right.start);
-          if(l > 0){
-            out.value = 1;
-          }else if(l == 0){
-            out.value = 0;
-          }else{
-            out.value = -1;
-          }
-        }
 
+  public static final int compare(VarCharHolder left, VarCharHolder right) {
+    for (int l = left.start, r = right.start; l < left.end && r < right.end; l++, r++) {
+      byte leftByte = left.buffer.getByte(l);
+      byte rightByte = right.buffer.getByte(r);
+      if (leftByte != rightByte) {
+        return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
       }
-     
-  }  
-
-
-      public static final int compare(VarBinaryHolder left, VarCharHolder right) {
-        for (int l = left.start, r = right.start; l < left.end && r < right.end; l++, r++) {
-          byte leftByte = left.buffer.getByte(l);
-          byte rightByte = right.buffer.getByte(r);
-          if (leftByte != rightByte) {
-            return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
-          }
-        }
-        
-        int l = (left.end - left.start) - (right.end - right.start);
-        if(l > 0){
-          return 1;
-        }else if(l == 0){
-          return 0;
-        }else{
-          return -1;
-        }
+    }
+
+    int l = (left.end - left.start) - (right.end - right.start);
+    if (l > 0) {
+      return 1;
+    } else if (l == 0) {
+      return 0;
+    } else {
+      return -1;
+    }
+
+  }
+
+  public static final int compare(NullableVarBinaryHolder left, NullableVarBinaryHolder right) {
+    if (left.isSet == 0) {
+      if (right.isSet == 0)
+        return 0;
+      return -1;
+    } else if (right.isSet == 0) {
+      return 1;
+    }
 
+    for (int l = left.start, r = right.start; l < left.end && r < right.end; l++, r++) {
+      byte leftByte = left.buffer.getByte(l);
+      byte rightByte = right.buffer.getByte(r);
+      if (leftByte != rightByte) {
+        return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
       }
-     
-      public static final int compare(VarCharHolder left, VarCharHolder right) {
-        for (int l = left.start, r = right.start; l < left.end && r < right.end; l++, r++) {
-          byte leftByte = left.buffer.getByte(l);
-          byte rightByte = right.buffer.getByte(r);
-          if (leftByte != rightByte) {
-            return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
-          }
-        }
-        
-        int l = (left.end - left.start) - (right.end - right.start);
-        if(l > 0){
-          return 1;
-        }else if(l == 0){
-          return 0;
-        }else{
-          return -1;
-        }
+    }
 
+    int l = (left.end - left.start) - (right.end - right.start);
+    if (l > 0) {
+      return 1;
+    } else if (l == 0) {
+      return 0;
+    } else {
+      return -1;
+    }
+
+  }
+
+  public static final int compare(NullableVarBinaryHolder left, NullableVarCharHolder right) {
+    if (left.isSet == 0) {
+      if (right.isSet == 0)
+        return 0;
+      return -1;
+    } else if (right.isSet == 0) {
+      return 1;
+    }
+
+    for (int l = left.start, r = right.start; l < left.end && r < right.end; l++, r++) {
+      byte leftByte = left.buffer.getByte(l);
+      byte rightByte = right.buffer.getByte(r);
+      if (leftByte != rightByte) {
+        return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
       }
-      
-      public static final int compare(VarBinaryHolder left, VarBinaryHolder right) {
-        for (int l = left.start, r = right.start; l < left.end && r < right.end; l++, r++) {
-          byte leftByte = left.buffer.getByte(l);
-          byte rightByte = right.buffer.getByte(r);
-          if (leftByte != rightByte) {
-            return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
-          }
-        }
-        
-        int l = (left.end - left.start) - (right.end - right.start);
-        if(l > 0){
-          return 1;
-        }else if(l == 0){
-          return 0;
-        }else{
-          return -1;
-        }
+    }
+
+    int l = (left.end - left.start) - (right.end - right.start);
+    if (l > 0) {
+      return 1;
+    } else if (l == 0) {
+      return 0;
+    } else {
+      return -1;
+    }
 
+  }
+
+  public static final int compare(NullableVarCharHolder left, NullableVarCharHolder right) {
+    if (left.isSet == 0) {
+      if (right.isSet == 0)
+        return 0;
+      return -1;
+    } else if (right.isSet == 0) {
+      return 1;
+    }
+
+    for (int l = left.start, r = right.start; l < left.end && r < right.end; l++, r++) {
+      byte leftByte = left.buffer.getByte(l);
+      byte rightByte = right.buffer.getByte(r);
+      if (leftByte != rightByte) {
+        return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
       }
-  
+    }
+
+    int l = (left.end - left.start) - (right.end - right.start);
+    if (l > 0) {
+      return 1;
+    } else if (l == 0) {
+      return 0;
+    } else {
+      return -1;
+    }
+
+  }
+
+  public static final int compare(VarBinaryHolder left, VarBinaryHolder right) {
+    for (int l = left.start, r = right.start; l < left.end && r < right.end; l++, r++) {
+      byte leftByte = left.buffer.getByte(l);
+      byte rightByte = right.buffer.getByte(r);
+      if (leftByte != rightByte) {
+        return ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
+      }
+    }
+
+    int l = (left.end - left.start) - (right.end - right.start);
+    if (l > 0) {
+      return 1;
+    } else if (l == 0) {
+      return 0;
+    } else {
+      return -1;
+    }
+
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
index 5cd51a8..dfe6804 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
@@ -68,7 +68,6 @@ public class BasicOptimizer extends Optimizer{
     LogicalConverter converter = new LogicalConverter(plan);
     for ( SinkOperator op : roots){
       PhysicalOperator pop  = op.accept(converter, obj);
-      System.out.println(pop);
       physOps.add(pop);
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
index 5feb5ee..d6fc802 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
@@ -125,7 +125,7 @@ public abstract class JoinTemplate implements JoinWorker {
           }
           status.advanceRight();
 
-        } while (status.isRightPositionAllowed() && doCompare(status.getLeftPosition(), status.getRightPosition()) == 0);
+        } while (status.isRightPositionInCurrentBatch() && doCompare(status.getLeftPosition(), status.getRightPosition()) == 0);
 
         if (status.getRightPosition() > initialRightPosition && status.isLeftRepeating())
           // more than one matching result from right table; reset position in case of subsequent left match

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
index 02f72f5..02a1961 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
@@ -107,7 +107,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
     
     // we do this in the here instead of the constructor because don't necessary want to start consuming on construction.
     status.ensureInitial();
-    
+
     // loop so we can start over again if we find a new batch was created.
     while(true){
 
@@ -148,8 +148,8 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
         kill();
         return IterOutcome.STOP;
       case NO_MORE_DATA:
-        logger.debug("NO MORE DATA; returning {}", (status.getOutPosition() > 0 ? "OK" : "NONE"));
-        return status.getOutPosition() > 0 ? IterOutcome.OK: IterOutcome.NONE;
+        logger.debug("NO MORE DATA; returning {}", (status.getOutPosition() > 0 ? (first ? "OK_NEW_SCHEMA" : "OK") : "NONE"));
+        return status.getOutPosition() > 0 ? (first ? IterOutcome.OK_NEW_SCHEMA : IterOutcome.OK): IterOutcome.NONE;
       case SCHEMA_CHANGED:
         worker = null;
         if(status.getOutPosition() > 0){
@@ -327,8 +327,8 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
                                                       new TypedFieldId(vw.getField().getType(), vectorId));
       JVar vvOut = cg.declareVectorValueSetupAndMember("outgoing",
                                                        new TypedFieldId(vw.getField().getType(),vectorId));
-      // todo: check for room in vvOut
-      cg.getEvalBlock().add(vvOut.invoke("copyFrom")
+      // todo: check result of copyFromSafe and grow allocation
+      cg.getEvalBlock().add(vvOut.invoke("copyFromSafe")
                                    .arg(COPY_LEFT_MAPPING.getValueReadIndex())
                                    .arg(COPY_LEFT_MAPPING.getValueWriteIndex())
                                    .arg(vvIn));
@@ -346,7 +346,8 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
                                                       new TypedFieldId(vw.getField().getType(), vectorId - rightVectorBase));
       JVar vvOut = cg.declareVectorValueSetupAndMember("outgoing",
                                                        new TypedFieldId(vw.getField().getType(),vectorId));
-      cg.getEvalBlock().add(vvOut.invoke("copyFrom")
+      // todo: check result of copyFromSafe and grow allocation
+      cg.getEvalBlock().add(vvOut.invoke("copyFromSafe")
           .arg(COPY_RIGHT_MAPPING.getValueReadIndex())
           .arg(COPY_RIGHT_MAPPING.getValueWriteIndex())
           .arg(vvIn));
@@ -366,13 +367,13 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
     // add fields from both batches
     for (VectorWrapper<?> w : left) {
       ValueVector outgoingVector = TypeHelper.getNewVector(w.getField(), context.getAllocator());
-      getAllocator(w.getValueVector(), outgoingVector).alloc(left.getRecordCount() * 4);
+      getAllocator(w.getValueVector(), outgoingVector).alloc(left.getRecordCount() * 16);
       container.add(outgoingVector);
     }
 
     for (VectorWrapper<?> w : right) {
       ValueVector outgoingVector = TypeHelper.getNewVector(w.getField(), context.getAllocator());
-      getAllocator(w.getValueVector(), outgoingVector).alloc(right.getRecordCount() * 4);
+      getAllocator(w.getValueVector(), outgoingVector).alloc(right.getRecordCount() * 16);
       container.add(outgoingVector);
     }
 
@@ -399,7 +400,6 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
     }
 
     public void alloc(int recordCount){
-      recordCount *=10;
       out.allocateNew(recordCount);
       out.getMutator().setValueCount(recordCount);
     }
@@ -416,8 +416,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
     }
 
     public void alloc(int recordCount){
-      recordCount *= 10;
-      out.allocateNew(in.getByteCapacity()*10, recordCount);
+      out.allocateNew(in.getByteCapacity() * 4, recordCount);
       out.getMutator().setValueCount(recordCount);
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/Field.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/Field.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/Field.java
index c6dd622..434f85b 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/Field.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/Field.java
@@ -41,7 +41,12 @@ public abstract class Field {
   }
 
   public MaterializedField getAsMaterializedField(FieldReference ref) {
-    return MaterializedField.create(new SchemaPath(ref.getPath() + "." + getFieldName(), ExpressionPosition.UNKNOWN), fieldType);
+    if(ref != null){
+      return MaterializedField.create(new SchemaPath(ref.getPath() + "." + getFieldName(), ExpressionPosition.UNKNOWN), fieldType);  
+    }else{
+      return MaterializedField.create(new SchemaPath(getFieldName(), ExpressionPosition.UNKNOWN), fieldType);
+    }
+    
   }
 
   public abstract String getFieldName();

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java
index a28eacf..95716bb 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java
@@ -56,7 +56,7 @@ public class JSONGroupScan extends AbstractGroupScan {
   @JsonCreator
   public JSONGroupScan(@JsonProperty("entries") List<ScanEntry> entries,
                        @JsonProperty("storageengine") JSONStorageEngineConfig storageEngineConfig,
-                       @JacksonInject StorageEngineRegistry engineRegistry, FieldReference ref) throws ExecutionSetupException {
+                       @JacksonInject StorageEngineRegistry engineRegistry, @JsonProperty("ref") FieldReference ref) throws ExecutionSetupException {
     this(entries, (JSONStorageEngine) engineRegistry.getEngine(storageEngineConfig), ref);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java
index 0324260..526019e 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java
@@ -476,7 +476,7 @@ public class JSONRecordReader implements RecordReader {
   }
 
   private VectorHolder getOrCreateVectorHolder(Field field) throws SchemaChangeException {
-    String fullFieldName = ref.getPath() + "." + field.getFullFieldName();
+    String fullFieldName = ref != null ? ref.getPath() + "." + field.getFullFieldName() : field.getFullFieldName();
     VectorHolder holder = valueVectorMap.get(fullFieldName);
 
     if (holder == null) {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
index 9a71dde..32184ea 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
@@ -38,6 +38,7 @@ import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.VarBinaryHolder;
 import org.apache.drill.exec.vector.VarBinaryVector;
 import org.junit.AfterClass;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import com.google.common.base.Charsets;
@@ -45,7 +46,7 @@ import com.google.common.io.Files;
 import com.google.common.io.Resources;
 import com.yammer.metrics.MetricRegistry;
 
-
+@Ignore
 public class TestOptiqPlans {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestOptiqPlans.class);
   DrillConfig c = DrillConfig.create();

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
index 85c9e78..f0217be 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
@@ -119,7 +119,7 @@ public class JSONRecordReaderTest {
     };
     JSONRecordReader jr = new JSONRecordReader(context,
         FileUtils.getResourceAsFile("/scan_json_test_1.json").toURI().toString(),
-        FileSystem.getLocal(new Configuration()));
+        FileSystem.getLocal(new Configuration()), null);
 
     MockOutputMutator mutator = new MockOutputMutator();
     List<ValueVector> addFields = mutator.getAddFields();
@@ -149,7 +149,7 @@ public class JSONRecordReaderTest {
 
     JSONRecordReader jr = new JSONRecordReader(context,
         FileUtils.getResourceAsFile("/scan_json_test_2.json").toURI().toString(),
-        FileSystem.getLocal(new Configuration()));
+        FileSystem.getLocal(new Configuration()), null);
 
     MockOutputMutator mutator = new MockOutputMutator();
     List<ValueVector> addFields = mutator.getAddFields();
@@ -191,7 +191,7 @@ public class JSONRecordReaderTest {
     JSONRecordReader jr = new JSONRecordReader(context,
         FileUtils.getResourceAsFile("/scan_json_test_2.json").toURI().toString(),
         FileSystem.getLocal(new Configuration()),
-        64); // batch only fits 1 int
+        64, null); // batch only fits 1 int
     MockOutputMutator mutator = new MockOutputMutator();
     List<ValueVector> addFields = mutator.getAddFields();
     List<MaterializedField> removedFields = mutator.getRemovedFields();
@@ -249,7 +249,7 @@ public class JSONRecordReaderTest {
 
     JSONRecordReader jr = new JSONRecordReader(context,
         FileUtils.getResourceAsFile("/scan_json_test_3.json").toURI().toString(),
-        FileSystem.getLocal(new Configuration()));
+        FileSystem.getLocal(new Configuration()), null);
 
     MockOutputMutator mutator = new MockOutputMutator();
     List<ValueVector> addFields = mutator.getAddFields();
@@ -278,7 +278,7 @@ public class JSONRecordReaderTest {
 
     JSONRecordReader jr = new JSONRecordReader(context,
         FileUtils.getResourceAsFile("/scan_json_test_4.json").toURI().toString(),
-        FileSystem.getLocal(new Configuration()));
+        FileSystem.getLocal(new Configuration()), null);
 
     MockOutputMutator mutator = new MockOutputMutator();
     List<ValueVector> addFields = mutator.getAddFields();
@@ -311,7 +311,7 @@ public class JSONRecordReaderTest {
 
     JSONRecordReader jr = new JSONRecordReader(context,
         FileUtils.getResourceAsFile("/scan_json_test_5.json").toURI().toString(),
-        FileSystem.getLocal(new Configuration()));
+        FileSystem.getLocal(new Configuration()), null);
 
     MockOutputMutator mutator = new MockOutputMutator();
     List<ValueVector> addFields = mutator.getAddFields();

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/pom.xml b/sandbox/prototype/pom.xml
index 16fea84..2f2eae2 100644
--- a/sandbox/prototype/pom.xml
+++ b/sandbox/prototype/pom.xml
@@ -348,6 +348,16 @@
       <dependencyManagement>
         <dependencies>
           <dependency>
+              <groupId>xerces</groupId>
+              <artifactId>xercesImpl</artifactId>
+              <version>2.9.1</version>
+          </dependency>
+          <dependency>
+              <groupId>xalan</groupId>
+              <artifactId>xalan</artifactId>
+              <version>2.7.1</version>
+          </dependency>        
+          <dependency>
             <groupId>com.mapr.hadoop</groupId>
             <artifactId>maprfs</artifactId>
             <version>1.0.3-mapr-3.0.0</version>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/sqlline
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlline b/sandbox/prototype/sqlline
index 7cec974..0177e4f 100755
--- a/sandbox/prototype/sqlline
+++ b/sandbox/prototype/sqlline
@@ -11,7 +11,7 @@ if [ ! -f .classpath ]; then
 fi
 
 #exec java -Drebel.log=true -javaagent:/src/jrebel/jrebel.jar -Dlogback.configurationFile=/src/drill/sandbox/prototype/sqlparser/src/test/resources/logback.xml -cp ./:"$(cat .classpath)" sqlline.SqlLine "$@"
-exec java -Dlogback.configurationFile=/src/drill/sandbox/prototype/sqlparser/src/test/resources/logback.xml -cp ./:"$(cat .classpath)" sqlline.SqlLine "$@"
+exec java -Dlogback.configurationFile=/src/drill/sandbox/prototype/sqlparser/src/test/resources/logback.xml -cp ./:"$(cat .classpath)" sqlline.SqlLine --verbose=true "$@"
 
 
 # End sqlline

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillDriverVersion.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillDriverVersion.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillDriverVersion.java
index 26f5b01..1febcb9 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillDriverVersion.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillDriverVersion.java
@@ -38,4 +38,5 @@ class DrillDriverVersion extends DriverVersion {
   }
 }
 
+
 // End DrillDriverVersion.java

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java
index 750fb18..0d3ed68 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java
@@ -35,6 +35,9 @@ import com.google.common.collect.Sets;
  * Context for converting a tree of {@link DrillRel} nodes into a Drill logical plan.
  */
 public class DrillImplementor {
+  
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillImplementor.class);
+  
   final ObjectMapper mapper = new ObjectMapper();
   {
     mapper.enable(SerializationFeature.INDENT_OUTPUT);
@@ -111,7 +114,7 @@ public class DrillImplementor {
   /** Returns the generated plan. */
   public String getJsonString() {
     String s = rootNode.toString();
-    System.out.println(s);
+    if(logger.isDebugEnabled()) logger.debug("Optiq Generated Logical Plan: {}", s);
     return s;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillOptiq.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillOptiq.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillOptiq.java
index e1c01fa..e687435 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillOptiq.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillOptiq.java
@@ -173,7 +173,7 @@ public class DrillOptiq {
 
     @Override
     public StringBuilder visitLiteral(RexLiteral literal) {
-      return buf.append(literal.toString());
+      return buf.append(literal);
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrill.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrill.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrill.java
index 6b1dfce..a55429f 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrill.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrill.java
@@ -47,6 +47,7 @@ import org.apache.drill.exec.ref.ReferenceInterpreter;
 import org.apache.drill.exec.ref.RunOutcome;
 import org.apache.drill.exec.ref.eval.BasicEvaluatorFactory;
 import org.apache.drill.exec.ref.rse.RSERegistry;
+import org.apache.drill.exec.store.json.JSONRecordReader;
 import org.apache.drill.jdbc.DrillHandler;
 
 import com.fasterxml.jackson.databind.JsonNode;
@@ -58,6 +59,9 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
  * Runtime helper that executes a Drill query and converts it into an {@link Enumerable}.
  */
 public class EnumerableDrill<E> extends AbstractEnumerable<E> implements Enumerable<E> {
+  
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(EnumerableDrill.class);
+  
   private final LogicalPlan plan;
   final BlockingQueue<Object> queue = new ArrayBlockingQueue<>(100);
   final DrillConfig config;
@@ -103,6 +107,7 @@ public class EnumerableDrill<E> extends AbstractEnumerable<E> implements Enumera
     try {
       i.setup();
     } catch (IOException e) {
+      logger.error("IOException during query", e);
       throw new RuntimeException(e);
     }
     return service.submit(new Callable<Collection<RunOutcome>>() {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillFullEngine.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillFullEngine.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillFullEngine.java
index 9109b1c..c6cb707 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillFullEngine.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillFullEngine.java
@@ -43,7 +43,6 @@ public class EnumerableDrillFullEngine<E> extends AbstractEnumerable<E> implemen
   final DrillConfig config;
   private final List<String> fields;
   private DrillClient client;
-  private DataContext context;
   
   /**
    * Creates a DrillEnumerable.
@@ -56,12 +55,11 @@ public class EnumerableDrillFullEngine<E> extends AbstractEnumerable<E> implemen
    *          Names of fields, or null to return the whole blob
    */
   public EnumerableDrillFullEngine(DrillConfig config, String plan, Class<E> clazz, List<String> fields,
-      DrillClient client, DataContext context) {
+      DrillClient client) {
     this.plan = plan;
     this.config = config;
     this.fields = fields;
     this.client = client;
-    this.context = context;
     config.setSinkQueues(0, queue);
   }
 
@@ -71,17 +69,18 @@ public class EnumerableDrillFullEngine<E> extends AbstractEnumerable<E> implemen
   public static <E> EnumerableDrillFullEngine<E> of(String plan, final List<String> fieldNames, Class<E> clazz,
       DataContext context) {
     DrillConfig config = DrillConfig.create();
-    return new EnumerableDrillFullEngine<>(config, plan, clazz, fieldNames, ((FakeSchema) context.getSubSchema("--FAKE--")).getClient(), context);
+    return new EnumerableDrillFullEngine<>(config, plan, clazz, fieldNames, ((FakeSchema) context.getSubSchema("--FAKE--")).getClient());
   }
 
   @Override
   public Enumerator<E> enumerator() {
     
+    //DrillTable table = (DrillTable) drillConnectionDataContext.getSubSchema("DONUTS").getTable("DONUTS", Object.class);
     if (client == null) {
       DrillRefImpl<E> impl = new DrillRefImpl<E>(plan, config, fields, queue);
       return impl.enumerator();
     } else {
-      DrillFullImpl<E> impl = new DrillFullImpl<E>(plan, config, fields, context);
+      DrillFullImpl<E> impl = new DrillFullImpl<E>(plan, config, fields);
       return impl.enumerator(client);
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRel.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRel.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRel.java
index b55a73c..b280f8a 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRel.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRel.java
@@ -52,7 +52,7 @@ import org.slf4j.LoggerFactory;
  * results as an {@link net.hydromatic.linq4j.Enumerable}.
  */
 public class EnumerableDrillRel extends SingleRel implements EnumerableRel {
-  private static final Logger LOG = LoggerFactory.getLogger(EnumerableDrillRel.class);
+  private static final Logger logger = LoggerFactory.getLogger(EnumerableDrillRel.class);
 
   private static final Function1<String, Expression> TO_LITERAL = new Function1<String, Expression>() {
     @Override
@@ -72,6 +72,7 @@ public class EnumerableDrillRel extends SingleRel implements EnumerableRel {
           DataContext.class);
       // EnumerableDrillFullEngine.class.getMethod("of", String.class, List.class, Class.class);
     } catch (NoSuchMethodException e) {
+      logger.error("Exception during query", e);
       throw new RuntimeException(e);
     }
   }
@@ -99,7 +100,7 @@ public class EnumerableDrillRel extends SingleRel implements EnumerableRel {
   }
 
   public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
-    LOG.debug("implementing enumerable");
+    logger.debug("implementing enumerable");
 
     final DrillImplementor drillImplementor = new DrillImplementor();
     DrillRel input = (DrillRel) getChild();

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java
index f58bfc8..2df8f3e 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java
@@ -7,7 +7,6 @@ import java.util.Map;
 import java.util.Set;
 
 import jline.internal.Preconditions;
-import net.hydromatic.optiq.DataContext;
 
 import org.apache.drill.exec.client.DrillClient;
 import org.apache.drill.exec.exception.SchemaChangeException;
@@ -15,6 +14,8 @@ import org.apache.drill.exec.record.RecordBatchLoader;
 import org.apache.drill.exec.record.VectorWrapper;
 import org.apache.drill.exec.rpc.RpcException;
 import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.apache.drill.exec.vector.NullableVarBinaryVector;
+import org.apache.drill.exec.vector.NullableVarCharVector;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.VarBinaryVector;
 import org.apache.drill.exec.vector.VarCharVector;
@@ -41,13 +42,12 @@ public class BatchLoaderMap implements Map<String, Object> {
   private JsonHelper helper;
   private boolean loaded;
 
-  public BatchLoaderMap(List<String> requestedFields, BatchListener listener, DrillClient client, DataContext context) {
+  public BatchLoaderMap(List<String> requestedFields, BatchListener listener, DrillClient client) {
     this.listener = listener;
     this.requestedFields = requestedFields;
     this.objArr = new Object[requestedFields.size()];
     this.loader = new RecordBatchLoader(client.getAllocator());
     this.helper = new JsonHelper(client.getConfig());
-    FileSystemSchema fsSchema = (FileSystemSchema) context;
   }
 
   private void load(QueryResultBatch batch) throws SchemaChangeException {
@@ -113,10 +113,13 @@ public class BatchLoaderMap implements Map<String, Object> {
   }
 
   private Object getObj(ValueVector vv){
-    if(vv instanceof VarBinaryVector || vv instanceof VarCharVector){
-      return new String( (byte[]) vv.getAccessor().getObject(index));
+    Object val = vv.getAccessor().getObject(index);
+    if(val == null){
+      return null;
+    }else if(vv instanceof VarBinaryVector || vv instanceof VarCharVector || vv instanceof NullableVarBinaryVector || vv instanceof NullableVarCharVector){
+      return new String( (byte[]) val );
     }else{
-      return vv.getAccessor().getObject(index);  
+      return val;  
     }
   }
   
@@ -182,6 +185,7 @@ public class BatchLoaderMap implements Map<String, Object> {
     try {
       return new ObjectMapper().writeValueAsString(this);
     } catch (JsonProcessingException e) {
+      logger.error("Exception during query", e);
       throw new RuntimeException(e);
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillFullImpl.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillFullImpl.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillFullImpl.java
index 7f40c57..a243d11 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillFullImpl.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/DrillFullImpl.java
@@ -3,11 +3,12 @@ package org.apache.drill.sql.client.full;
 import java.util.List;
 
 import net.hydromatic.linq4j.Enumerator;
-import net.hydromatic.optiq.DataContext;
 
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.exec.client.DrillClient;
 import org.apache.drill.exec.proto.UserProtos;
+import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.jdbc.DrillTable;
 
 public class DrillFullImpl<E>{
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillFullImpl.class);
@@ -15,15 +16,13 @@ public class DrillFullImpl<E>{
   private final String plan;
   final DrillConfig config;
   private final List<String> fields;
-  private DataContext context;
 
   
-  public DrillFullImpl(String plan, DrillConfig config, List<String> fields, DataContext context) {
+  public DrillFullImpl(String plan, DrillConfig config, List<String> fields) {
     super();
     this.plan = plan;
     this.config = config;
     this.fields = fields;
-    this.context = context;
   }
 
   @SuppressWarnings("unchecked")
@@ -35,7 +34,7 @@ public class DrillFullImpl<E>{
     QueryRequestRunner runner = new QueryRequestRunner(plan, client, listener);
     runner.start();
     
-    return (Enumerator<E>) new ResultEnumerator(listener, client, fields, context);
+    return (Enumerator<E>) new ResultEnumerator(listener, client, fields);
     
   }
   

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/JsonHelper.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/JsonHelper.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/JsonHelper.java
index 74901ae..8390795 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/JsonHelper.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/JsonHelper.java
@@ -27,6 +27,7 @@ public class JsonHelper {
       
       if(o instanceof Integer) map.put(k, (Integer) o);
       else if(o instanceof Long) map.put(k, (Long) o);
+      else if(o instanceof Boolean) map.put(k, (Boolean) o);
       else if(o instanceof String) map.put(k, (String) o);
       else if(o instanceof byte[]) map.put(k, new String((byte[]) o));
       else if(o instanceof Float) map.put(k, (Float) o);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/ResultEnumerator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/ResultEnumerator.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/ResultEnumerator.java
index f1f5bbd..c6251e9 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/ResultEnumerator.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/ResultEnumerator.java
@@ -13,12 +13,13 @@ import org.apache.drill.exec.rpc.RpcException;
    * Enumerator used for full execution engine.
    */
   class ResultEnumerator implements Enumerator<Object> {
-
+    static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ResultEnumerator.class);
+    
     private final BatchLoaderMap loaderMap;
     private Object current;
     
-    public ResultEnumerator(BatchListener listener, DrillClient client, List<String> fields, DataContext context) {
-      this.loaderMap = new BatchLoaderMap(fields, listener, client, context);
+    public ResultEnumerator(BatchListener listener, DrillClient client, List<String> fields) {
+      this.loaderMap = new BatchLoaderMap(fields, listener, client);
     }
 
     public Object current() {
@@ -30,14 +31,16 @@ import org.apache.drill.exec.rpc.RpcException;
       try {
         boolean succ = loaderMap.next();
         if(succ){
-          current = loaderMap.getCurrentObject();          
+          current = loaderMap.getCurrentObject();         
         }
         return succ;
         
       } catch (InterruptedException e) {
         Thread.interrupted();
+        logger.error("Exception during query", e);
         throw new RuntimeException(e);
       } catch (RpcException | SchemaChangeException e) {
+        logger.error("Exception during query", e);
         throw new RuntimeException(e);
       }
     }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
index 72c19a6..c3bd51d 100644
--- a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
+++ b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
@@ -13,7 +13,7 @@ import org.junit.rules.Timeout;
 import com.google.common.base.Charsets;
 import com.google.common.io.Resources;
 
-
+@Ignore
 public class FullEngineTest {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FullEngineTest.class);
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
index 3bde4ed..ccab8fd 100644
--- a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
+++ b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
@@ -25,17 +25,14 @@ import java.sql.Statement;
 import org.apache.drill.exec.ref.ReferenceInterpreter;
 import org.junit.BeforeClass;
 import org.junit.Ignore;
-import org.junit.Rule;
 import org.junit.Test;
-import org.junit.rules.TestName;
-import org.junit.rules.TestRule;
-import org.junit.rules.Timeout;
 
 import com.google.common.base.Charsets;
 import com.google.common.base.Function;
 import com.google.common.io.Resources;
 
 /** Unit tests for Drill's JDBC driver. */
+@Ignore
 public class JdbcTest {
   private static String MODEL;
   private static String EXPECTED;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d5537492/sandbox/prototype/sqlparser/src/test/resources/logback.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/test/resources/logback.xml b/sandbox/prototype/sqlparser/src/test/resources/logback.xml
index 34d9356..0b3080a 100644
--- a/sandbox/prototype/sqlparser/src/test/resources/logback.xml
+++ b/sandbox/prototype/sqlparser/src/test/resources/logback.xml
@@ -23,9 +23,9 @@
   </logger>
 
   <root>
-    <level value="error" />
+    <level value="debug" />
     <appender-ref ref="SOCKET" />
 <!--     <appender-ref ref="STDOUT" /> -->
   </root>
 
-</configuration>
\ No newline at end of file
+</configuration>


[13/18] git commit: fix parquet query test

Posted by ja...@apache.org.
fix parquet query test


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/d91bfe61
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/d91bfe61
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/d91bfe61

Branch: refs/heads/master
Commit: d91bfe61e6edecf4430ddf0c613a14d1157752f2
Parents: 89eb945
Author: Ben Becker <be...@gmail.com>
Authored: Fri Aug 30 14:59:08 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Sep 1 15:16:22 2013 -0700

----------------------------------------------------------------------
 .../org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d91bfe61/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
index 9375263..000e9cc 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
@@ -16,7 +16,6 @@ import org.apache.drill.exec.exception.SchemaChangeException;
 import org.apache.drill.exec.expr.CodeGenerator;
 import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
 import org.apache.drill.exec.expr.HoldingContainerExpression;
-import org.apache.drill.exec.expr.TypeHelper;
 import org.apache.drill.exec.expr.fn.impl.ComparatorFunctions;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.physical.config.MergeJoinPOP;
@@ -28,6 +27,7 @@ import org.apache.drill.exec.record.RecordBatch;
 import org.apache.drill.exec.record.TypedFieldId;
 import org.apache.drill.exec.record.VectorContainer;
 import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.vector.TypeHelper;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.allocator.VectorAllocator;
 


[06/18] git commit: DRILL-191 ParquetGroupScan and AffinityCalculator do not properly handle directories or multiple files

Posted by ja...@apache.org.
DRILL-191 ParquetGroupScan and AffinityCalculator do not properly handle directories or multiple files


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/f77c5258
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/f77c5258
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/f77c5258

Branch: refs/heads/master
Commit: f77c5258fd58e1ecf6f8cb76d0ecc29b8529965e
Parents: 1a2352c
Author: Steven Phillips <sp...@maprtech.com>
Authored: Thu Aug 29 17:08:37 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Sep 1 15:16:21 2013 -0700

----------------------------------------------------------------------
 .../drill/exec/store/AffinityCalculator.java    | 24 ++++++++-----
 .../exec/store/parquet/ParquetGroupScan.java    | 38 ++++++++++----------
 .../exec/store/TestAffinityCalculator.java      |  6 ++--
 .../exec/store/TestParquetPhysicalPlan.java     | 13 +++++--
 4 files changed, 49 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f77c5258/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java
index b341ea4..6210059 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/AffinityCalculator.java
@@ -20,26 +20,25 @@ public class AffinityCalculator {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AffinityCalculator.class);
 
 
-  BlockLocation[] blocks;
-  ImmutableRangeMap<Long,BlockLocation> blockMap;
+  HashMap<String,ImmutableRangeMap<Long,BlockLocation>> blockMapMap = new HashMap<>();
   FileSystem fs;
   String fileName;
   Collection<DrillbitEndpoint> endpoints;
   HashMap<String,DrillbitEndpoint> endPointMap;
-  Stopwatch watch = new Stopwatch();
 
-  public AffinityCalculator(String fileName, FileSystem fs, Collection<DrillbitEndpoint> endpoints) {
+  public AffinityCalculator(FileSystem fs, Collection<DrillbitEndpoint> endpoints) {
     this.fs = fs;
-    this.fileName = fileName;
     this.endpoints = endpoints;
-    buildBlockMap();
     buildEndpointMap();
   }
 
   /**
    * Builds a mapping of block locations to file byte range
    */
-  private void buildBlockMap() {
+  private void buildBlockMap(String fileName) {
+    Stopwatch watch = new Stopwatch();
+    BlockLocation[] blocks;
+    ImmutableRangeMap<Long,BlockLocation> blockMap;
     try {
       watch.start();
       FileStatus file = fs.getFileStatus(new Path(fileName));
@@ -60,6 +59,7 @@ public class AffinityCalculator {
     blockMap = blockMapBuilder.build();
     watch.stop();
     logger.debug("Took {} ms to build block map", watch.elapsed(TimeUnit.MILLISECONDS));
+    blockMapMap.put(fileName, blockMap);
   }
   /**
    * For a given RowGroup, calculate how many bytes are available on each on drillbit endpoint
@@ -67,8 +67,14 @@ public class AffinityCalculator {
    * @param rowGroup the RowGroup to calculate endpoint bytes for
    */
   public void setEndpointBytes(ParquetGroupScan.RowGroupInfo rowGroup) {
-    watch.reset();
+    Stopwatch watch = new Stopwatch();
     watch.start();
+    String fileName = rowGroup.getPath();
+    if (!blockMapMap.containsKey(fileName)) {
+      buildBlockMap(fileName);
+    }
+
+    ImmutableRangeMap<Long,BlockLocation> blockMap = blockMapMap.get(fileName);
     HashMap<String,Long> hostMap = new HashMap<>();
     HashMap<DrillbitEndpoint,Long> endpointByteMap = new HashMap();
     long start = rowGroup.getStart();
@@ -116,7 +122,7 @@ public class AffinityCalculator {
    * Builds a mapping of drillbit endpoints to hostnames
    */
   private void buildEndpointMap() {
-    watch.reset();
+    Stopwatch watch = new Stopwatch();
     watch.start();
     endPointMap = new HashMap<String, DrillbitEndpoint>();
     for (DrillbitEndpoint d : endpoints) {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f77c5258/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
index 54618c1..2dc5102 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
@@ -46,6 +46,7 @@ import org.apache.drill.exec.store.StorageEngineRegistry;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
+import parquet.hadoop.Footer;
 import parquet.hadoop.ParquetFileReader;
 import parquet.hadoop.metadata.BlockMetaData;
 import parquet.hadoop.metadata.ColumnChunkMetaData;
@@ -87,7 +88,6 @@ public class ParquetGroupScan extends AbstractGroupScan {
   private StorageEngineRegistry engineRegistry;
   private ParquetStorageEngineConfig engineConfig;
   private FileSystem fs;
-  private String fileName;
   private final FieldReference ref;
   private List<EndpointAffinity> endpointAffinities;
 
@@ -106,7 +106,6 @@ public class ParquetGroupScan extends AbstractGroupScan {
     this.entries = entries;
     this.ref = ref;
     readFooter();
-    this.fileName = rowGroupInfos.get(0).getPath();
     calculateEndpointBytes();
   }
 
@@ -118,7 +117,6 @@ public class ParquetGroupScan extends AbstractGroupScan {
     this.entries = entries;
     this.ref = ref;
     readFooter();
-    this.fileName = rowGroupInfos.get(0).getPath();
     calculateEndpointBytes();
   }
 
@@ -130,23 +128,27 @@ public class ParquetGroupScan extends AbstractGroupScan {
     ColumnChunkMetaData columnChunkMetaData;
     for (ReadEntryWithPath readEntryWithPath : entries){
       Path path = new Path(readEntryWithPath.getPath());
-      ParquetMetadata footer = ParquetFileReader.readFooter(this.storageEngine.getHadoopConfig(), path);
+      List<Footer> footers = ParquetFileReader.readFooters(this.storageEngine.getHadoopConfig(), path);
       readEntryWithPath.getPath();
 
-      int i = 0;
-      for (BlockMetaData rowGroup : footer.getBlocks()){
-        // need to grab block information from HDFS
-        columnChunkMetaData = rowGroup.getColumns().iterator().next();
-        start = columnChunkMetaData.getFirstDataPageOffset();
-        // this field is not being populated correctly, but the column chunks know their sizes, just summing them for now
-        //end = start + rowGroup.getTotalByteSize();
-        length = 0;
-        for (ColumnChunkMetaData col : rowGroup.getColumns()){
-          length += col.getTotalSize();
+      for (Footer footer : footers) {
+        int index = 0;
+        ParquetMetadata metadata = footer.getParquetMetadata();
+        for (BlockMetaData rowGroup : metadata.getBlocks()){
+          // need to grab block information from HDFS
+          columnChunkMetaData = rowGroup.getColumns().iterator().next();
+          start = columnChunkMetaData.getFirstDataPageOffset();
+          // this field is not being populated correctly, but the column chunks know their sizes, just summing them for now
+          //end = start + rowGroup.getTotalByteSize();
+          length = 0;
+          for (ColumnChunkMetaData col : rowGroup.getColumns()){
+            length += col.getTotalSize();
+          }
+          String filePath = footer.getFile().toUri().getPath();
+          rowGroupInfos.add(new ParquetGroupScan.RowGroupInfo(filePath, start, length, index));
+          logger.debug("rowGroupInfo path: {} start: {} length {}", filePath, start, length);
+          index++;
         }
-        rowGroupInfos.add(new ParquetGroupScan.RowGroupInfo(readEntryWithPath.getPath(), start, length, i));
-        logger.debug("rowGroupInfo path: {} start: {} length {}", readEntryWithPath.getPath(), start, length);
-        i++;
       }
     }
     watch.stop();
@@ -156,7 +158,7 @@ public class ParquetGroupScan extends AbstractGroupScan {
   private void calculateEndpointBytes() {
     watch.reset();
     watch.start();
-    AffinityCalculator ac = new AffinityCalculator(fileName, fs, availableEndpoints);
+    AffinityCalculator ac = new AffinityCalculator(fs, availableEndpoints);
     for (RowGroupInfo e : rowGroupInfos) {
       ac.setEndpointBytes(e);
       totalBytes += e.getLength();

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f77c5258/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java
index cff7c86..b25c805 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestAffinityCalculator.java
@@ -103,7 +103,7 @@ public class TestAffinityCalculator {
     }};
 
 
-    AffinityCalculator ac = new AffinityCalculator(path, fs, endPoints);
+    AffinityCalculator ac = new AffinityCalculator(fs, endPoints);
     for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) {
       ac.setEndpointBytes(rowGroup);
     }
@@ -115,7 +115,7 @@ public class TestAffinityCalculator {
 
     buildRowGroups(rowGroups, numberOfBlocks, blockSize, 2);
 
-    ac = new AffinityCalculator(path, fs, endPoints);
+    ac = new AffinityCalculator(fs, endPoints);
     for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) {
       ac.setEndpointBytes(rowGroup);
     }
@@ -127,7 +127,7 @@ public class TestAffinityCalculator {
 
     buildRowGroups(rowGroups, numberOfBlocks, blockSize, 6);
 
-    ac = new AffinityCalculator(path, fs, endPoints);
+    ac = new AffinityCalculator(fs, endPoints);
     for (ParquetGroupScan.RowGroupInfo rowGroup : rowGroups) {
       ac.setEndpointBytes(rowGroup);
     }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f77c5258/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
index 18ac294..c58bc71 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
@@ -44,7 +44,7 @@ public class TestParquetPhysicalPlan {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestParquetPhysicalPlan.class);
 
   //public String fileName = "/physical_test2.json";
-  public String fileName = "parquet_scan_union_screen_physical.json";
+  public String fileName = "parquet/parquet_scan_union_screen_physical.json";
 //  public String fileName = "parquet-sample.json";
 
 
@@ -59,19 +59,28 @@ public class TestParquetPhysicalPlan {
       client.connect();
       List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, Resources.toString(Resources.getResource(fileName),Charsets.UTF_8));
       RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
+      int count = 0;
       for (QueryResultBatch b : results) {
         System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
+        count += b.getHeader().getRowCount();
         loader.load(b.getHeader().getDef(), b.getData());
         for (VectorWrapper vw : loader) {
+//          System.out.println(vw.getValueVector().getField().getName() + vw.getValueVector().getField().getType());
           System.out.println(vw.getValueVector().getField().getName());
           ValueVector vv = vw.getValueVector();
           for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
             Object o = vv.getAccessor().getObject(i);
-            System.out.println(vv.getAccessor().getObject(i));
+            if (o instanceof byte[]) {
+              System.out.println(new String((byte[]) o));
+            } else {
+              System.out.println(vv.getAccessor().getObject(i));
+            }
+            break;
           }
         }
       }
       client.close();
+      System.out.println(String.format("Got %d total results", count));
     }
   }
 


[11/18] Reorganize location of generated files to support other generated needs. (Clean up packages for outputs and change directory structure for inputs and inputs. Packages changes caused large set of changes to other files.)

Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
new file mode 100644
index 0000000..d86e449
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
@@ -0,0 +1,320 @@
+<@pp.dropOutputFile />
+<#list vv.types as type>
+<#list type.minor as minor>
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/Repeated${minor.class}Vector.java" />
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector;
+
+<#include "/@includes/vv_imports.ftl" />
+
+
+@SuppressWarnings("unused")
+/**
+ * Repeated${minor.class} implements a vector with multple values per row (e.g. JSON array or
+ * repeated protobuf field).  The implementation uses two additional value vectors; one to convert
+ * the index offset to the underlying element offset, and another to store the number of values
+ * in the vector.
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+
+ public final class Repeated${minor.class}Vector extends BaseValueVector implements Repeated<#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector {
+
+  private MaterializedField field;
+  
+  private int parentValueCount;
+  private int childValueCount;
+  
+  private final UInt4Vector offsets;   // offsets to start of each record
+  private final ${minor.class}Vector values;
+  private final Mutator mutator = new Mutator();
+  private final Accessor accessor = new Accessor();
+  
+  
+  public Repeated${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
+    super(field, allocator);
+    this.offsets = new UInt4Vector(null, allocator);
+    this.values = new ${minor.class}Vector(null, allocator);
+  }
+
+  public int getValueCapacity(){
+    return values.getValueCapacity();
+  }
+  
+  public int getBufferSize(){
+    return offsets.getBufferSize() + values.getBufferSize();
+  }
+  
+  public TransferPair getTransferPair(){
+    return new TransferImpl(getField());
+  }
+  public TransferPair getTransferPair(FieldReference ref){
+    return new TransferImpl(getField().clone(ref));
+  }
+  
+  public void transferTo(Repeated${minor.class}Vector target){
+    offsets.transferTo(target.offsets);
+    values.transferTo(target.values);
+    target.parentValueCount = parentValueCount;
+    target.childValueCount = childValueCount;
+    clear();
+  }
+  
+  private class TransferImpl implements TransferPair{
+    Repeated${minor.class}Vector to;
+    
+    public TransferImpl(MaterializedField field){
+      this.to = new Repeated${minor.class}Vector(field, allocator);
+    }
+    
+    public Repeated${minor.class}Vector getTo(){
+      return to;
+    }
+    
+    public void transfer(){
+      transferTo(to);
+    }
+    
+    @Override
+    public void copyValue(int fromIndex, int toIndex) {
+      to.copyFrom(fromIndex, toIndex, Repeated${minor.class}Vector.this);
+    }
+  }
+  
+  public void copyFrom(int inIndex, int outIndex, Repeated${minor.class}Vector v){
+    throw new UnsupportedOperationException();
+  }
+  
+  public boolean copyFromSafe(int inIndex, int outIndex, Repeated${minor.class}Vector v){
+    throw new UnsupportedOperationException();
+  }
+  
+  <#if type.major == "VarLen">
+  @Override
+  public FieldMetadata getMetadata() {
+    return FieldMetadata.newBuilder()
+             .setDef(getField().getDef())
+             .setGroupCount(this.parentValueCount)
+             .setValueCount(this.childValueCount)
+             .setVarByteLength(values.getVarByteLength())
+             .setBufferLength(getBufferSize())
+             .build();
+  }
+  
+  public void allocateNew(int totalBytes, int parentValueCount, int childValueCount) {
+    offsets.allocateNew(parentValueCount+1);
+    values.allocateNew(totalBytes, childValueCount);
+    mutator.reset();
+    accessor.reset();
+  }
+  
+  @Override
+  public int load(int dataBytes, int parentValueCount, int childValueCount, ByteBuf buf){
+    clear();
+    this.parentValueCount = parentValueCount;
+    this.childValueCount = childValueCount;
+    int loaded = 0;
+    loaded += offsets.load(parentValueCount+1, buf.slice(loaded, buf.capacity() - loaded));
+    loaded += values.load(dataBytes, childValueCount, buf.slice(loaded, buf.capacity() - loaded));
+    return loaded;
+  }
+  
+  @Override
+  public void load(FieldMetadata metadata, ByteBuf buffer) {
+    assert this.field.getDef().equals(metadata.getDef());
+    int loaded = load(metadata.getVarByteLength(), metadata.getGroupCount(), metadata.getValueCount(), buffer);
+    assert metadata.getBufferLength() == loaded;
+  }
+  
+  public int getByteCapacity(){
+    return values.getByteCapacity();
+  }
+
+  <#else>
+  
+  @Override
+  public FieldMetadata getMetadata() {
+    return FieldMetadata.newBuilder()
+             .setDef(getField().getDef())
+             .setGroupCount(this.parentValueCount)
+             .setValueCount(this.childValueCount)
+             .setBufferLength(getBufferSize())
+             .build();
+  }
+  
+  public void allocateNew(int parentValueCount, int childValueCount) {
+    clear();
+    offsets.allocateNew(parentValueCount+1);
+    values.allocateNew(childValueCount);
+    mutator.reset();
+    accessor.reset();
+  }
+  
+  public int load(int parentValueCount, int childValueCount, ByteBuf buf){
+    clear();
+    this.parentValueCount = parentValueCount;
+    this.childValueCount = childValueCount;
+    int loaded = 0;
+    loaded += offsets.load(parentValueCount+1, buf.slice(loaded, buf.capacity() - loaded));
+    loaded += values.load(childValueCount, buf.slice(loaded, buf.capacity() - loaded));
+    return loaded;
+  }
+  
+  @Override
+  public void load(FieldMetadata metadata, ByteBuf buffer) {
+    assert this.field.getDef().equals(metadata.getDef());
+    int loaded = load(metadata.getGroupCount(), metadata.getValueCount(), buffer);
+    assert metadata.getBufferLength() == loaded;
+  }
+  </#if>
+
+  @Override
+  public ByteBuf[] getBuffers() {
+    ByteBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(), values.getBuffers(), ByteBuf.class);
+    clear();
+    return buffers;
+  }
+
+  public void clear(){
+    offsets.clear();
+    values.clear();
+    parentValueCount = 0;
+    childValueCount = 0;
+  }
+
+  public Mutator getMutator(){
+    return mutator;
+  }
+  
+  public Accessor getAccessor(){
+    return accessor;
+  }
+  
+  public final class Accessor implements ValueVector.Accessor{
+    /**
+     * Get the elements at the given index.
+     */
+    public int getCount(int index) {
+      return offsets.getAccessor().get(index+1) - offsets.getAccessor().get(index);
+    }
+    
+    public Object getObject(int index) {
+      List<Object> vals = Lists.newArrayList();
+      int start = offsets.getAccessor().get(index);
+      int end = offsets.getAccessor().get(index+1);
+      for(int i = start; i < end; i++){
+        vals.add(values.getAccessor().getObject(i));
+      }
+      return vals;
+    }
+
+    /**
+     * Get a value for the given record.  Each element in the repeated field is accessed by
+     * the positionIndex param.
+     *
+     * @param  index           record containing the repeated field
+     * @param  positionIndex   position within the repeated field
+     * @return element at the given position in the given record
+     */
+    public <#if type.major == "VarLen">byte[]
+           <#else>${minor.javaType!type.javaType}
+           </#if> get(int index, int positionIndex) {
+      return values.getAccessor().get(offsets.getAccessor().get(index) + positionIndex);
+    }
+           
+    public void get(int index, Repeated${minor.class}Holder holder){
+      holder.start = offsets.getAccessor().get(index);
+      holder.end =  offsets.getAccessor().get(index+1);
+      holder.vector = values;
+    }
+
+    public MaterializedField getField() {
+      return field;
+    }
+    
+    public int getGroupCount(){
+      return parentValueCount;
+    }
+    
+    public int getValueCount(){
+      return childValueCount;
+    }
+    
+    public void reset(){
+      
+    }
+  }
+  
+  public final class Mutator implements RepeatedMutator {
+
+    
+    private Mutator(){
+    }
+
+    public void startNewGroup(int index) {
+      offsets.getMutator().set(index+1, offsets.getAccessor().get(index));
+    }
+
+    /**
+     * Add an element to the given record index.  This is similar to the set() method in other
+     * value vectors, except that it permits setting multiple values for a single record.
+     *
+     * @param index   record of the element to add
+     * @param value   value to add to the given row
+     */
+    public void add(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
+      int nextOffset = offsets.getAccessor().get(index+1);
+      values.getMutator().set(nextOffset, value);
+      offsets.getMutator().set(index+1, nextOffset+1);
+    }
+
+    public void add(int index, ${minor.class}Holder holder){
+      int nextOffset = offsets.getAccessor().get(index+1);
+      values.getMutator().set(nextOffset, holder);
+      offsets.getMutator().set(index+1, nextOffset+1);
+    }
+    
+    public void add(int index, Repeated${minor.class}Holder holder){
+      
+      ${minor.class}Vector.Accessor accessor = holder.vector.getAccessor();
+      ${minor.class}Holder innerHolder = new ${minor.class}Holder();
+      
+      for(int i = holder.start; i < holder.end; i++){
+        accessor.get(i, innerHolder);
+        add(index, innerHolder);
+      }
+    }
+    
+    /**
+     * Set the number of value groups in this repeated field.
+     * @param groupCount Count of Value Groups.
+     */
+    public void setValueCount(int groupCount) {
+      parentValueCount = groupCount;
+      childValueCount = offsets.getAccessor().get(groupCount);
+      offsets.getMutator().setValueCount(groupCount+1);
+      values.getMutator().setValueCount(childValueCount);
+    }
+    
+    public void generateTestData(){
+      setValueCount(offsets.getAccessor().getValueCount() - 1);
+      int valCount = offsets.getValueCapacity();
+      int[] sizes = {1,2,0,6};
+      int size = 0;
+      int runningOffset = 0;
+      for(int i =1; i < valCount; i++, size++){
+        runningOffset += sizes[size % sizes.length];
+        offsets.getMutator().set(i, runningOffset);  
+      }
+      values.getMutator().generateTestData();
+    }
+    
+    public void reset(){
+      
+    }
+    
+  }
+}
+</#list>
+</#list>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/TypeHelper.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/TypeHelper.java b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/TypeHelper.java
new file mode 100644
index 0000000..51521bb
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/TypeHelper.java
@@ -0,0 +1,99 @@
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/TypeHelper.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.expr;
+
+<#include "/@includes/vv_imports.ftl" />
+
+
+public class TypeHelper {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TypeHelper.class);
+
+  private static final int WIDTH_ESTIMATE = 50;
+
+  public static int getSize(MajorType major) {
+    switch (major.getMinorType()) {
+<#list vv.types as type>
+  <#list type.minor as minor>
+    case ${minor.class?upper_case}:
+      return ${type.width}<#if minor.class?substring(0, 3) == "Var" ||
+                               minor.class?substring(0, 3) == "PRO" ||
+                               minor.class?substring(0, 3) == "MSG"> + WIDTH_ESTIMATE</#if>;
+  </#list>
+</#list>
+      case FIXEDCHAR: return major.getWidth();
+      case FIXED16CHAR: return major.getWidth();
+      case FIXEDBINARY: return major.getWidth();
+    }
+    throw new UnsupportedOperationException();
+  }
+
+  public static Class<?> getValueVectorClass(MinorType type, DataMode mode){
+    switch (type) {
+<#list vv.types as type>
+  <#list type.minor as minor>
+      case ${minor.class?upper_case}:
+        switch (mode) {
+          case REQUIRED:
+            return ${minor.class}Vector.class;
+          case OPTIONAL:
+            return Nullable${minor.class}Vector.class;
+          case REPEATED:
+            return Repeated${minor.class}Vector.class;
+        }
+  </#list>
+</#list>
+    default:
+      break;
+    }
+    throw new UnsupportedOperationException();
+  }
+
+  public static JType getHolderType(JCodeModel model, MinorType type, DataMode mode){
+    switch (type) {
+<#list vv.types as type>
+  <#list type.minor as minor>
+      case ${minor.class?upper_case}:
+        switch (mode) {
+          case REQUIRED:
+            return model._ref(${minor.class}Holder.class);
+          case OPTIONAL:
+            return model._ref(Nullable${minor.class}Holder.class);
+          case REPEATED:
+            return model._ref(Repeated${minor.class}Holder.class);
+        }
+  </#list>
+</#list>
+      default:
+        break;
+      }
+      throw new UnsupportedOperationException();
+  }
+
+  public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator){
+    MajorType type = field.getType();
+
+    switch (type.getMinorType()) {
+<#list vv.  types as type>
+  <#list type.minor as minor>
+    case ${minor.class?upper_case}:
+      switch (type.getMode()) {
+        case REQUIRED:
+          return new ${minor.class}Vector(field, allocator);
+        case OPTIONAL:
+          return new Nullable${minor.class}Vector(field, allocator);
+        case REPEATED:
+          return new Repeated${minor.class}Vector(field, allocator);
+      }
+  </#list>
+</#list>
+    default:
+      break;
+    }
+    // All ValueVector types have been handled.
+    throw new UnsupportedOperationException(type.getMinorType() + " type is not supported. Mode: " + type.getMode());
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/ValueHolders.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/ValueHolders.java b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/ValueHolders.java
new file mode 100644
index 0000000..f0983b7
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/ValueHolders.java
@@ -0,0 +1,75 @@
+<@pp.dropOutputFile />
+<#list vv.modes as mode>
+<#list vv.types as type>
+<#list type.minor as minor>
+
+<#assign className="${mode.prefix}${minor.class}Holder" />
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/holders/${className}.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.expr.holders;
+
+<#include "/@includes/vv_imports.ftl" />
+
+public final class ${className} implements ValueHolder{
+  
+  public static final MajorType TYPE = Types.${mode.name?lower_case}(MinorType.${minor.class?upper_case});
+
+    <#if mode.name != "Repeated">
+      
+    public static final int WIDTH = ${type.width};
+      <#if mode.name == "Optional">
+      /** Whether the given holder holds a valid value.  1 means non-null.  0 means null. **/
+      public int isSet;
+      </#if>
+      
+      <#if type.major != "VarLen">
+      
+      <#if (type.width > 8)>
+      public int start;
+      public ByteBuf buffer;
+      <#else>
+        public ${minor.javaType!type.javaType} value;
+      
+      </#if>
+      <#else>
+      /** The first offset (inclusive) into the buffer. **/
+      public int start;
+      
+      /** The last offset (exclusive) into the buffer. **/
+      public int end;
+      
+      /** The buffer holding actual values. **/
+      public ByteBuf buffer;
+
+      public String toString() {
+      <#if mode.name == "Optional">
+        if (isSet == 0)
+          return "<NULL>";
+      </#if>
+        byte[] buf = new byte[end-start];
+        buffer.getBytes(start, buf, 0, end-start);
+        return new String(buf);
+      }
+
+      </#if>
+
+    <#else> 
+    
+      /** The first index (inclusive) into the Vector. **/
+      public int start;
+      
+      /** The last index (exclusive) into the Vector. **/
+      public int end;
+      
+      /** The Vector holding the actual values. **/
+      public ${minor.class}Vector vector;
+    </#if>
+  
+    
+}
+
+</#list>
+</#list>
+</#list>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
new file mode 100644
index 0000000..3e5464c
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
@@ -0,0 +1,325 @@
+<@pp.dropOutputFile />
+<#list vv.types as type>
+<#list type.minor as minor>
+
+<#if type.major == "VarLen">
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/${minor.class}Vector.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/**
+ * ${minor.class}Vector implements a vector of variable width values.  Elements in the vector
+ * are accessed by position from the logical start of the vector.  A fixed width offsetVector
+ * is used to convert an element's position to it's offset from the start of the (0-based)
+ * ByteBuf.  Size is inferred by adjacent elements.
+ *   The width of each element is ${type.width} byte(s)
+ *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+@SuppressWarnings("unused")
+public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector{
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
+
+  private final UInt${type.width}Vector offsetVector;
+  private final Accessor accessor = new Accessor();
+  private final Mutator mutator = new Mutator();
+  
+  public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
+    super(field, allocator);
+    this.offsetVector = new UInt${type.width}Vector(null, allocator);
+  }
+
+  public int getBufferSize(){
+    return offsetVector.getBufferSize() + data.writerIndex();
+  }
+  
+  int getSizeFromCount(int valueCount) {
+    return valueCount * ${type.width};
+  }
+  
+  public int getValueCapacity(){
+    return offsetVector.getValueCapacity();
+  }
+  
+  public int getByteCapacity(){
+    return data.capacity(); 
+  }
+  
+  /**
+   * Return the number of bytes contained in the current var len byte vector.
+   * @return
+   */
+  public int getVarByteLength(){
+    return offsetVector.getAccessor().get(valueCount); 
+  }
+  
+  @Override
+  public FieldMetadata getMetadata() {
+    int len = (valueCount + 1) * ${type.width} + getVarByteLength();
+    return FieldMetadata.newBuilder()
+             .setDef(getField().getDef())
+             .setValueCount(valueCount)
+             .setVarByteLength(getVarByteLength())
+             .setBufferLength(len)
+             .build();
+  }
+
+  public int load(int dataBytes, int valueCount, ByteBuf buf){
+    this.valueCount = valueCount;
+    int loaded = offsetVector.load(valueCount+1, buf);
+    data = buf.slice(loaded, dataBytes);
+    data.retain();
+    return loaded + dataBytes;
+  }
+  
+  @Override
+  public void load(FieldMetadata metadata, ByteBuf buffer) {
+    assert this.field.getDef().equals(metadata.getDef());
+    int loaded = load(metadata.getVarByteLength(), metadata.getValueCount(), buffer);
+    assert metadata.getBufferLength() == loaded;
+  }
+  
+  @Override
+  public void clear() {
+    super.clear();
+    offsetVector.clear();
+  }
+
+  
+  @Override
+  public ByteBuf[] getBuffers() {
+    ByteBuf[] buffers = ObjectArrays.concat(offsetVector.getBuffers(), super.getBuffers(), ByteBuf.class);
+    clear();
+    return buffers;
+  }
+  
+  public TransferPair getTransferPair(){
+    return new TransferImpl(getField());
+  }
+  public TransferPair getTransferPair(FieldReference ref){
+    return new TransferImpl(getField().clone(ref));
+  }
+  
+  public void transferTo(${minor.class}Vector target){
+    this.offsetVector.transferTo(target.offsetVector);
+    target.data = data;
+    target.data.retain();
+    target.valueCount = valueCount;
+    clear();
+  }
+  
+  public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
+    int start = from.offsetVector.getAccessor().get(fromIndex);
+    int end =   from.offsetVector.getAccessor().get(fromIndex+1);
+    int len = end - start;
+    
+    int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width});
+    from.data.getBytes(start, data, outputStart, len);
+    offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, outputStart + len);
+  }
+  
+  public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
+    if(thisIndex >= getValueCapacity()) return false;
+    
+    int start = from.offsetVector.getAccessor().get(fromIndex);
+    int end =   from.offsetVector.getAccessor().get(fromIndex+1);
+    int len = end - start;
+    
+    int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width});
+    
+    if(data.capacity() < outputStart + len) return false;
+    
+    from.data.getBytes(start, data, outputStart, len);
+    offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, outputStart + len);
+
+    return true;
+  }
+
+  
+  private class TransferImpl implements TransferPair{
+    ${minor.class}Vector to;
+    
+    public TransferImpl(MaterializedField field){
+      this.to = new ${minor.class}Vector(field, allocator);
+    }
+    
+    public ${minor.class}Vector getTo(){
+      return to;
+    }
+    
+    public void transfer(){
+      transferTo(to);
+    }
+    
+    @Override
+    public void copyValue(int fromIndex, int toIndex) {
+      to.copyFrom(fromIndex, toIndex, ${minor.class}Vector.this);
+    }
+  }
+  
+  public void allocateNew(int totalBytes, int valueCount) {
+    clear();
+    assert totalBytes >= 0;
+    data = allocator.buffer(totalBytes);
+    data.readerIndex(0);
+    offsetVector.allocateNew(valueCount+1);
+  }
+
+  public Accessor getAccessor(){
+    return accessor;
+  }
+  
+  public Mutator getMutator() {
+    return mutator;
+  }
+  
+  public final class Accessor extends BaseValueVector.BaseAccessor{
+    
+    public byte[] get(int index) {
+      assert index >= 0;
+      int startIdx = offsetVector.getAccessor().get(index);
+      int length = offsetVector.getAccessor().get(index + 1) - startIdx;
+      assert length >= 0;
+      byte[] dst = new byte[length];
+      data.getBytes(startIdx, dst, 0, length);
+      return dst;
+    }
+    
+    public void get(int index, ${minor.class}Holder holder){
+      holder.start = offsetVector.getAccessor().get(index);
+      holder.end = offsetVector.getAccessor().get(index + 1);
+      holder.buffer = data;
+    }
+    
+    void get(int index, Nullable${minor.class}Holder holder){
+      holder.start = offsetVector.getAccessor().get(index);
+      holder.end = offsetVector.getAccessor().get(index + 1);
+      holder.buffer = data;
+    }
+    
+    public Object getObject(int index) {
+      return get(index);
+    }
+    
+    public int getValueCount() {
+      return valueCount;
+    }
+
+    public UInt${type.width}Vector getOffsetVector(){
+      return offsetVector;
+    }
+  }
+  
+  /**
+   * Mutable${minor.class} implements a vector of variable width values.  Elements in the vector
+   * are accessed by position from the logical start of the vector.  A fixed width offsetVector
+   * is used to convert an element's position to it's offset from the start of the (0-based)
+   * ByteBuf.  Size is inferred by adjacent elements.
+   *   The width of each element is ${type.width} byte(s)
+   *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
+   *
+   * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+   */
+  public final class Mutator extends BaseValueVector.BaseMutator{
+
+    /**
+     * Set the variable length element at the specified index to the supplied byte array.
+     *
+     * @param index   position of the bit to set
+     * @param bytes   array of bytes to write
+     */
+    public void set(int index, byte[] bytes) {
+      assert index >= 0;
+      int currentOffset = offsetVector.getAccessor().get(index);
+      offsetVector.getMutator().set(index + 1, currentOffset + bytes.length);
+      data.setBytes(currentOffset, bytes);
+    }
+
+   
+    public boolean setSafe(int index, Nullable${minor.class}Holder holder){
+      assert holder.isSet == 1;
+      if(index >= getValueCapacity()) return false;
+      
+      int start = holder.start;
+      int end =   holder.end;
+      int len = end - start;
+      
+      int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+      
+      if(data.capacity() < outputStart + len) return false;
+      
+      holder.buffer.getBytes(start, data, outputStart, len);
+      offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (index+1) * ${type.width}, len);
+
+      return true;
+    }
+    
+    public boolean setSafe(int index, ${minor.class}Holder holder){
+      if(index >= getValueCapacity()) return false;
+      
+      int start = holder.start;
+      int end =   holder.end;
+      int len = end - start;
+      
+      int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+      
+      if(data.capacity() < outputStart + len) return false;
+      
+      holder.buffer.getBytes(start, data, outputStart, len);
+      offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (index+1) * ${type.width}, len);
+
+      return true;
+    }
+    
+    public void set(int index, int start, int length, ByteBuf buffer){
+      assert index >= 0;
+      int currentOffset = offsetVector.getAccessor().get(index);
+      offsetVector.getMutator().set(index + 1, currentOffset + length);
+      ByteBuf bb = buffer.slice(start, length);
+      data.setBytes(currentOffset, bb);
+    }
+
+    void set(int index, Nullable${minor.class}Holder holder){
+      int length = holder.end - holder.start;
+      int currentOffset = offsetVector.getAccessor().get(index);
+      offsetVector.getMutator().set(index + 1, currentOffset + length);
+      data.setBytes(currentOffset, holder.buffer, holder.start, length);
+    }
+    
+    public void set(int index, ${minor.class}Holder holder){
+      int length = holder.end - holder.start;
+      int currentOffset = offsetVector.getAccessor().get(index);
+      offsetVector.getMutator().set(index + 1, currentOffset + length);
+      data.setBytes(currentOffset, holder.buffer, holder.start, length);
+    }
+    
+    public void setValueCount(int valueCount) {
+      ${minor.class}Vector.this.valueCount = valueCount;
+      data.writerIndex(offsetVector.getAccessor().get(valueCount));
+      offsetVector.getMutator().setValueCount(valueCount+1);
+    }
+
+    @Override
+    public void generateTestData(){
+      boolean even = true;
+      for(int i =0; i < valueCount; i++, even = !even){
+        if(even){
+          set(i, new String("aaaaa").getBytes(Charsets.UTF_8));
+        }else{
+          set(i, new String("bbbbbbbbbb").getBytes(Charsets.UTF_8));
+        }
+      }
+    }
+  }
+  
+}
+
+
+</#if> <#-- type.major -->
+</#list>
+</#list>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
index 0c9ca7e..01929b9 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CodeGenerator.java
@@ -20,7 +20,6 @@ import org.apache.drill.exec.compile.sig.SignatureHolder;
 import org.apache.drill.exec.exception.SchemaChangeException;
 import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
 import org.apache.drill.exec.record.TypedFieldId;
-import org.apache.drill.exec.vector.TypeHelper;
 
 import com.beust.jcommander.internal.Lists;
 import com.beust.jcommander.internal.Maps;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/agg/impl/CountFunctions.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/agg/impl/CountFunctions.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/agg/impl/CountFunctions.java
index 08fc941..b8c4dde 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/agg/impl/CountFunctions.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/agg/impl/CountFunctions.java
@@ -6,9 +6,9 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
 import org.apache.drill.exec.expr.annotations.Output;
 import org.apache.drill.exec.expr.annotations.Param;
 import org.apache.drill.exec.expr.annotations.Workspace;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.IntHolder;
 import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.vector.BigIntHolder;
-import org.apache.drill.exec.vector.IntHolder;
 
 public class CountFunctions {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CountFunctions.class);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/agg/impl/SumFunctions.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/agg/impl/SumFunctions.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/agg/impl/SumFunctions.java
index e42636d..6a771f1 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/agg/impl/SumFunctions.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/agg/impl/SumFunctions.java
@@ -6,9 +6,9 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
 import org.apache.drill.exec.expr.annotations.Output;
 import org.apache.drill.exec.expr.annotations.Param;
 import org.apache.drill.exec.expr.annotations.Workspace;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.IntHolder;
 import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.vector.BigIntHolder;
-import org.apache.drill.exec.vector.IntHolder;
 
 public class SumFunctions {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SumFunctions.class);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Alternator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Alternator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Alternator.java
index d79120d..073a065 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Alternator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Alternator.java
@@ -11,8 +11,8 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate;
 import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
 import org.apache.drill.exec.expr.annotations.Output;
 import org.apache.drill.exec.expr.annotations.Workspace;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
 import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.vector.BigIntHolder;
 
 
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteSubstring.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteSubstring.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteSubstring.java
index e06fee2..eea1dd8 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteSubstring.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteSubstring.java
@@ -18,16 +18,20 @@
 
 package org.apache.drill.exec.expr.fn.impl;
 
-import org.apache.drill.common.expression.*;
+import org.apache.drill.common.expression.Arg;
+import org.apache.drill.common.expression.BasicArgumentValidator;
+import org.apache.drill.common.expression.CallProvider;
+import org.apache.drill.common.expression.FunctionDefinition;
+import org.apache.drill.common.expression.OutputTypeDeterminer;
 import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.types.TypeProtos.MajorType;
 import org.apache.drill.common.types.Types;
 import org.apache.drill.exec.expr.DrillSimpleFunc;
 import org.apache.drill.exec.expr.annotations.FunctionTemplate;
 import org.apache.drill.exec.expr.annotations.Output;
 import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.VarBinaryHolder;
 import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.vector.*;
 
 // TODO: implement optional length parameter
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CharSubstring.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CharSubstring.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CharSubstring.java
index b60299c..4794ae2 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CharSubstring.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CharSubstring.java
@@ -18,15 +18,20 @@
 
 package org.apache.drill.exec.expr.fn.impl;
 
-import org.apache.drill.common.expression.*;
+import org.apache.drill.common.expression.Arg;
+import org.apache.drill.common.expression.BasicArgumentValidator;
+import org.apache.drill.common.expression.CallProvider;
+import org.apache.drill.common.expression.FunctionDefinition;
+import org.apache.drill.common.expression.OutputTypeDeterminer;
 import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.common.types.Types;
 import org.apache.drill.exec.expr.DrillSimpleFunc;
 import org.apache.drill.exec.expr.annotations.FunctionTemplate;
 import org.apache.drill.exec.expr.annotations.Output;
 import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
 import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.vector.*;
 
 /**
  * Evaluate a substring expression for a given UTF-8 value; specifying the start

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java
index 416ce29..ddd2752 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java
@@ -10,14 +10,14 @@ import org.apache.drill.exec.expr.DrillSimpleFunc;
 import org.apache.drill.exec.expr.annotations.FunctionTemplate;
 import org.apache.drill.exec.expr.annotations.Output;
 import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.IntHolder;
+import org.apache.drill.exec.expr.holders.NullableBigIntHolder;
+import org.apache.drill.exec.expr.holders.NullableIntHolder;
+import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
+import org.apache.drill.exec.expr.holders.VarBinaryHolder;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
 import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.vector.BigIntHolder;
-import org.apache.drill.exec.vector.IntHolder;
-import org.apache.drill.exec.vector.NullableBigIntHolder;
-import org.apache.drill.exec.vector.NullableIntHolder;
-import org.apache.drill.exec.vector.NullableVarCharHolder;
-import org.apache.drill.exec.vector.VarBinaryHolder;
-import org.apache.drill.exec.vector.VarCharHolder;
 
 public class ComparatorFunctions {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComparatorFunctions.class);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctions.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctions.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctions.java
index 680ea66..6bc6313 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctions.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctions.java
@@ -4,14 +4,14 @@ import org.apache.drill.exec.expr.DrillSimpleFunc;
 import org.apache.drill.exec.expr.annotations.FunctionTemplate;
 import org.apache.drill.exec.expr.annotations.Output;
 import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.BitHolder;
+import org.apache.drill.exec.expr.holders.Float4Holder;
+import org.apache.drill.exec.expr.holders.Float8Holder;
+import org.apache.drill.exec.expr.holders.IntHolder;
+import org.apache.drill.exec.expr.holders.VarBinaryHolder;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
 import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.vector.BigIntHolder;
-import org.apache.drill.exec.vector.BitHolder;
-import org.apache.drill.exec.vector.Float4Holder;
-import org.apache.drill.exec.vector.Float8Holder;
-import org.apache.drill.exec.vector.IntHolder;
-import org.apache.drill.exec.vector.VarBinaryHolder;
-import org.apache.drill.exec.vector.VarCharHolder;
 
 public class ComparisonFunctions {
     static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MathFunctions.class);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctionsNullable.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctionsNullable.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctionsNullable.java
index aa6f57e..976b759 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctionsNullable.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctionsNullable.java
@@ -4,10 +4,10 @@ import org.apache.drill.exec.expr.DrillSimpleFunc;
 import org.apache.drill.exec.expr.annotations.FunctionTemplate;
 import org.apache.drill.exec.expr.annotations.Output;
 import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.BitHolder;
+import org.apache.drill.exec.expr.holders.NullableBigIntHolder;
+import org.apache.drill.exec.expr.holders.NullableIntHolder;
 import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.vector.NullableBigIntHolder;
-import org.apache.drill.exec.vector.BitHolder;
-import org.apache.drill.exec.vector.NullableIntHolder;
 
 public class ComparisonFunctionsNullable {
     static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComparisonFunctionsNullable.class);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Hash.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Hash.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Hash.java
index bc89b5f..f58c400 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Hash.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Hash.java
@@ -18,15 +18,18 @@
 
 package org.apache.drill.exec.expr.fn.impl;
 
-import org.apache.drill.common.expression.*;
+import org.apache.drill.common.expression.ArgumentValidators;
+import org.apache.drill.common.expression.CallProvider;
+import org.apache.drill.common.expression.FunctionDefinition;
+import org.apache.drill.common.expression.OutputTypeDeterminer;
 import org.apache.drill.exec.expr.DrillSimpleFunc;
 import org.apache.drill.exec.expr.annotations.FunctionTemplate;
 import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
 import org.apache.drill.exec.expr.annotations.Output;
 import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.IntHolder;
 import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.vector.BigIntHolder;
-import org.apache.drill.exec.vector.IntHolder;
 
 @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
 public class Hash implements DrillSimpleFunc {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNotNull.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNotNull.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNotNull.java
index a33dca3..9b6bdf7 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNotNull.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNotNull.java
@@ -26,9 +26,9 @@ import org.apache.drill.exec.expr.DrillSimpleFunc;
 import org.apache.drill.exec.expr.annotations.FunctionTemplate;
 import org.apache.drill.exec.expr.annotations.Output;
 import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.BitHolder;
+import org.apache.drill.exec.expr.holders.NullableFloat8Holder;
 import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.vector.BitHolder;
-import org.apache.drill.exec.vector.NullableFloat8Holder;
 
 @FunctionTemplate(name = "isNotNull", scope = FunctionTemplate.FunctionScope.SIMPLE)
 public class IsNotNull implements DrillSimpleFunc {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNull.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNull.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNull.java
index c8630b7..31f26c4 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNull.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNull.java
@@ -26,9 +26,9 @@ import org.apache.drill.exec.expr.DrillSimpleFunc;
 import org.apache.drill.exec.expr.annotations.FunctionTemplate;
 import org.apache.drill.exec.expr.annotations.Output;
 import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.BitHolder;
+import org.apache.drill.exec.expr.holders.NullableFloat8Holder;
 import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.vector.BitHolder;
-import org.apache.drill.exec.vector.NullableFloat8Holder;
 
 @FunctionTemplate(name = "isNull", scope = FunctionTemplate.FunctionScope.SIMPLE)
 public class IsNull implements DrillSimpleFunc {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
index ccec5e3..3f4b976 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
@@ -6,9 +6,9 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
 import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
 import org.apache.drill.exec.expr.annotations.Output;
 import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.IntHolder;
 import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.vector.BigIntHolder;
-import org.apache.drill.exec.vector.IntHolder;
 
 public class MathFunctions{
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MathFunctions.class);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SimpleRepeatedFunctions.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SimpleRepeatedFunctions.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SimpleRepeatedFunctions.java
index 87c2fd8..972d9ea 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SimpleRepeatedFunctions.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SimpleRepeatedFunctions.java
@@ -1,7 +1,6 @@
 package org.apache.drill.exec.expr.fn.impl;
 
 import org.apache.drill.common.expression.Arg;
-import org.apache.drill.common.expression.ArgumentValidators;
 import org.apache.drill.common.expression.BasicArgumentValidator;
 import org.apache.drill.common.expression.CallProvider;
 import org.apache.drill.common.expression.FunctionDefinition;
@@ -12,12 +11,12 @@ import org.apache.drill.exec.expr.DrillSimpleFunc;
 import org.apache.drill.exec.expr.annotations.FunctionTemplate;
 import org.apache.drill.exec.expr.annotations.Output;
 import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.BitHolder;
+import org.apache.drill.exec.expr.holders.IntHolder;
+import org.apache.drill.exec.expr.holders.RepeatedBigIntHolder;
+import org.apache.drill.exec.expr.holders.RepeatedIntHolder;
 import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.vector.BigIntHolder;
-import org.apache.drill.exec.vector.BitHolder;
-import org.apache.drill.exec.vector.IntHolder;
-import org.apache.drill.exec.vector.RepeatedBigIntHolder;
-import org.apache.drill.exec.vector.RepeatedIntHolder;
 
 public class SimpleRepeatedFunctions {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MathFunctions.class);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/VarHelpers.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/VarHelpers.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/VarHelpers.java
index 2942e05..5a7f911 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/VarHelpers.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/VarHelpers.java
@@ -1,9 +1,9 @@
 package org.apache.drill.exec.expr.fn.impl;
 
-import org.apache.drill.exec.vector.NullableVarBinaryHolder;
-import org.apache.drill.exec.vector.NullableVarCharHolder;
-import org.apache.drill.exec.vector.VarBinaryHolder;
-import org.apache.drill.exec.vector.VarCharHolder;
+import org.apache.drill.exec.expr.holders.NullableVarBinaryHolder;
+import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
+import org.apache.drill.exec.expr.holders.VarBinaryHolder;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
 
 public class VarHelpers {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VarHelpers.class);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggBatch.java
index c352b16..97e66f9 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggBatch.java
@@ -18,18 +18,18 @@ import org.apache.drill.exec.expr.CodeGenerator.BlockType;
 import org.apache.drill.exec.expr.CodeGenerator.HoldingContainer;
 import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
 import org.apache.drill.exec.expr.HoldingContainerExpression;
+import org.apache.drill.exec.expr.TypeHelper;
 import org.apache.drill.exec.expr.ValueVectorWriteExpression;
 import org.apache.drill.exec.expr.fn.impl.ComparatorFunctions;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.physical.config.StreamingAggregate;
 import org.apache.drill.exec.record.AbstractRecordBatch;
+import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
 import org.apache.drill.exec.record.MaterializedField;
 import org.apache.drill.exec.record.RecordBatch;
 import org.apache.drill.exec.record.TypedFieldId;
-import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
 import org.apache.drill.exec.record.selection.SelectionVector2;
 import org.apache.drill.exec.record.selection.SelectionVector4;
-import org.apache.drill.exec.vector.TypeHelper;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.allocator.VectorAllocator;
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
index 5a62c1f..9375263 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
@@ -4,31 +4,40 @@ import static org.apache.drill.exec.compile.sig.GeneratorMapping.GM;
 
 import java.io.IOException;
 
-import com.google.common.collect.ImmutableList;
-import com.sun.codemodel.*;
-
 import org.apache.drill.common.expression.ErrorCollector;
 import org.apache.drill.common.expression.ErrorCollectorImpl;
 import org.apache.drill.common.expression.ExpressionPosition;
 import org.apache.drill.common.expression.FunctionCall;
 import org.apache.drill.common.expression.LogicalExpression;
 import org.apache.drill.common.logical.data.JoinCondition;
-import org.apache.drill.exec.compile.sig.GeneratorMapping;
 import org.apache.drill.exec.compile.sig.MappingSet;
 import org.apache.drill.exec.exception.ClassTransformationException;
 import org.apache.drill.exec.exception.SchemaChangeException;
 import org.apache.drill.exec.expr.CodeGenerator;
 import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
 import org.apache.drill.exec.expr.HoldingContainerExpression;
+import org.apache.drill.exec.expr.TypeHelper;
 import org.apache.drill.exec.expr.fn.impl.ComparatorFunctions;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.physical.config.MergeJoinPOP;
 import org.apache.drill.exec.physical.impl.filter.ReturnValueExpression;
 import org.apache.drill.exec.physical.impl.join.JoinWorker.JoinOutcome;
-import org.apache.drill.exec.record.*;
-import org.apache.drill.exec.vector.*;
+import org.apache.drill.exec.record.AbstractRecordBatch;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.allocator.VectorAllocator;
 
+import com.google.common.collect.ImmutableList;
+import com.sun.codemodel.JClass;
+import com.sun.codemodel.JExpr;
+import com.sun.codemodel.JMod;
+import com.sun.codemodel.JType;
+import com.sun.codemodel.JVar;
+
 /**
  * A merge join combining to incoming in-order batches.
  */

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/OutgoingRecordBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/OutgoingRecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/OutgoingRecordBatch.java
index e429402..979ff27 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/OutgoingRecordBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/OutgoingRecordBatch.java
@@ -22,6 +22,7 @@ import java.util.Iterator;
 
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.expr.TypeHelper;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.physical.config.HashPartitionSender;
 import org.apache.drill.exec.proto.ExecProtos;
@@ -39,7 +40,6 @@ import org.apache.drill.exec.record.selection.SelectionVector4;
 import org.apache.drill.exec.rpc.BaseRpcOutcomeListener;
 import org.apache.drill.exec.rpc.RpcException;
 import org.apache.drill.exec.rpc.bit.BitTunnel;
-import org.apache.drill.exec.vector.TypeHelper;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.allocator.VectorAllocator;
 import org.apache.drill.exec.work.foreman.ErrorHelper;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
index f96e3cf..219f520 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/partitionsender/PartitionSenderRootExec.java
@@ -18,25 +18,33 @@
 
 package org.apache.drill.exec.physical.impl.partitionsender;
 
-import com.google.common.collect.Lists;
-import com.sun.codemodel.*;
-import org.apache.drill.common.expression.*;
+import java.io.IOException;
+
+import org.apache.drill.common.expression.ErrorCollector;
+import org.apache.drill.common.expression.ErrorCollectorImpl;
+import org.apache.drill.common.expression.LogicalExpression;
 import org.apache.drill.exec.exception.ClassTransformationException;
 import org.apache.drill.exec.exception.SchemaChangeException;
 import org.apache.drill.exec.expr.CodeGenerator;
 import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
+import org.apache.drill.exec.expr.TypeHelper;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.physical.config.HashPartitionSender;
 import org.apache.drill.exec.physical.impl.RootExec;
-import org.apache.drill.exec.physical.impl.filter.Filterer;
 import org.apache.drill.exec.physical.impl.filter.ReturnValueExpression;
 import org.apache.drill.exec.proto.CoordinationProtos;
-import org.apache.drill.exec.record.*;
-import org.apache.drill.exec.vector.TypeHelper;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.record.VectorWrapper;
 import org.apache.drill.exec.vector.ValueVector;
 
-import java.io.IOException;
-import java.util.List;
+import com.sun.codemodel.JArray;
+import com.sun.codemodel.JClass;
+import com.sun.codemodel.JExpr;
+import com.sun.codemodel.JExpression;
+import com.sun.codemodel.JMod;
+import com.sun.codemodel.JType;
+import com.sun.codemodel.JVar;
 
 class PartitionSenderRootExec implements RootExec {
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
index c44d988..b7874bd 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
@@ -11,6 +11,7 @@ import org.apache.drill.exec.exception.ClassTransformationException;
 import org.apache.drill.exec.exception.SchemaChangeException;
 import org.apache.drill.exec.expr.CodeGenerator;
 import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
+import org.apache.drill.exec.expr.TypeHelper;
 import org.apache.drill.exec.expr.ValueVectorReadExpression;
 import org.apache.drill.exec.expr.ValueVectorWriteExpression;
 import org.apache.drill.exec.ops.FragmentContext;
@@ -23,7 +24,6 @@ import org.apache.drill.exec.record.TransferPair;
 import org.apache.drill.exec.record.TypedFieldId;
 import org.apache.drill.exec.record.VectorWrapper;
 import org.apache.drill.exec.vector.AllocationHelper;
-import org.apache.drill.exec.vector.TypeHelper;
 import org.apache.drill.exec.vector.ValueVector;
 
 import com.google.common.base.Preconditions;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
index 521061c..0b8e157 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/svremover/RemovingRecordBatch.java
@@ -6,6 +6,7 @@ import java.util.List;
 import org.apache.drill.exec.exception.ClassTransformationException;
 import org.apache.drill.exec.exception.SchemaChangeException;
 import org.apache.drill.exec.expr.CodeGenerator;
+import org.apache.drill.exec.expr.TypeHelper;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.physical.config.SelectionVectorRemover;
 import org.apache.drill.exec.record.AbstractSingleRecordBatch;
@@ -16,7 +17,6 @@ import org.apache.drill.exec.record.TypedFieldId;
 import org.apache.drill.exec.record.VectorWrapper;
 import org.apache.drill.exec.record.WritableBatch;
 import org.apache.drill.exec.vector.FixedWidthVector;
-import org.apache.drill.exec.vector.TypeHelper;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.VariableWidthVector;
 import org.apache.drill.exec.vector.allocator.FixedVectorAllocator;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
index d974441..6a65ce8 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
@@ -18,20 +18,16 @@
 package org.apache.drill.exec.record;
 
 import java.util.Iterator;
-import java.util.List;
 
 import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.LogicalExpression;
 import org.apache.drill.common.expression.PathSegment;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.types.TypeProtos.DataMode;
 import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.expr.TypeHelper;
 import org.apache.drill.exec.proto.SchemaDefProtos.FieldDef;
 import org.apache.drill.exec.proto.SchemaDefProtos.NamePart;
 import org.apache.drill.exec.proto.SchemaDefProtos.NamePart.Type;
-import org.apache.drill.exec.vector.TypeHelper;
-
-import com.google.common.collect.Lists;
 
 public class MaterializedField{
   private final FieldDef def;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
index 9ac53f0..31497d1 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
@@ -25,15 +25,13 @@ import java.util.Map;
 
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.expr.TypeHelper;
 import org.apache.drill.exec.memory.BufferAllocator;
 import org.apache.drill.exec.proto.SchemaDefProtos.FieldDef;
 import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
 import org.apache.drill.exec.proto.UserBitShared.RecordBatchDef;
-import org.apache.drill.exec.vector.TypeHelper;
 import org.apache.drill.exec.vector.ValueVector;
 
-import com.beust.jcommander.internal.Lists;
-import com.google.common.base.Preconditions;
 import com.google.common.collect.Maps;
 
 public class RecordBatchLoader implements Iterable<VectorWrapper<?>>{

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java
index 526019e..df44fe0 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java
@@ -18,6 +18,10 @@ import org.apache.drill.common.types.TypeProtos.MajorType;
 import org.apache.drill.common.types.TypeProtos.MinorType;
 import org.apache.drill.common.types.Types;
 import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.expr.holders.NullableBitHolder;
+import org.apache.drill.exec.expr.holders.NullableFloat4Holder;
+import org.apache.drill.exec.expr.holders.NullableIntHolder;
 import org.apache.drill.exec.memory.BufferAllocator;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.physical.impl.OutputMutator;
@@ -32,18 +36,14 @@ import org.apache.drill.exec.schema.json.jackson.JacksonHelper;
 import org.apache.drill.exec.store.RecordReader;
 import org.apache.drill.exec.store.VectorHolder;
 import org.apache.drill.exec.vector.AllocationHelper;
-import org.apache.drill.exec.vector.NullableBitHolder;
 import org.apache.drill.exec.vector.NullableBitVector;
-import org.apache.drill.exec.vector.NullableFloat4Holder;
 import org.apache.drill.exec.vector.NullableFloat4Vector;
-import org.apache.drill.exec.vector.NullableIntHolder;
 import org.apache.drill.exec.vector.NullableIntVector;
 import org.apache.drill.exec.vector.NullableVarCharVector;
 import org.apache.drill.exec.vector.RepeatedBitVector;
 import org.apache.drill.exec.vector.RepeatedFloat4Vector;
 import org.apache.drill.exec.vector.RepeatedIntVector;
 import org.apache.drill.exec.vector.RepeatedVarCharVector;
-import org.apache.drill.exec.vector.TypeHelper;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java
index d5f1d8f..8febb06 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockGroupScanPOP.java
@@ -24,14 +24,15 @@ import java.util.List;
 import org.apache.drill.common.types.TypeProtos.DataMode;
 import org.apache.drill.common.types.TypeProtos.MajorType;
 import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.expr.TypeHelper;
 import org.apache.drill.exec.physical.EndpointAffinity;
 import org.apache.drill.exec.physical.OperatorCost;
 import org.apache.drill.exec.physical.ReadEntry;
-import org.apache.drill.exec.physical.base.*;
 import org.apache.drill.exec.physical.base.AbstractGroupScan;
-import org.apache.drill.exec.physical.base.GroupScan;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.physical.base.SubScan;
 import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-import org.apache.drill.exec.vector.TypeHelper;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java
index c5375b2..ca45755 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockRecordReader.java
@@ -22,6 +22,7 @@ import org.apache.drill.common.expression.ExpressionPosition;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.types.TypeProtos.MajorType;
 import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.expr.TypeHelper;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.physical.impl.OutputMutator;
 import org.apache.drill.exec.record.MaterializedField;
@@ -29,7 +30,6 @@ import org.apache.drill.exec.store.RecordReader;
 import org.apache.drill.exec.store.mock.MockGroupScanPOP.MockColumn;
 import org.apache.drill.exec.store.mock.MockGroupScanPOP.MockScanEntry;
 import org.apache.drill.exec.vector.AllocationHelper;
-import org.apache.drill.exec.vector.TypeHelper;
 import org.apache.drill.exec.vector.ValueVector;
 
 public class MockRecordReader implements RecordReader {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockSubScanPOP.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockSubScanPOP.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockSubScanPOP.java
index 38bf337..e128c74 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockSubScanPOP.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockSubScanPOP.java
@@ -17,32 +17,23 @@
  ******************************************************************************/
 package org.apache.drill.exec.store.mock;
 
-import java.util.Collections;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 
-import com.google.common.collect.Iterators;
-import org.apache.drill.common.graph.GraphVisitor;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.exec.physical.EndpointAffinity;
 import org.apache.drill.exec.physical.OperatorCost;
-import org.apache.drill.exec.physical.ReadEntry;
-import org.apache.drill.exec.physical.base.*;
-import org.apache.drill.exec.physical.base.AbstractGroupScan;
-import org.apache.drill.exec.physical.base.GroupScan;
-import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
-import org.apache.drill.exec.vector.TypeHelper;
+import org.apache.drill.exec.physical.base.AbstractBase;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.base.PhysicalVisitor;
+import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.physical.base.SubScan;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonInclude.Include;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterators;
 
 @JsonTypeName("mock-sub-scan")
 public class MockSubScanPOP extends AbstractBase implements SubScan {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
index 0a0470d..b62ba27 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
@@ -32,12 +32,12 @@ import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.common.types.Types;
 import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.expr.TypeHelper;
 import org.apache.drill.exec.memory.BufferAllocator;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.physical.impl.OutputMutator;
 import org.apache.drill.exec.record.MaterializedField;
 import org.apache.drill.exec.store.RecordReader;
-import org.apache.drill.exec.vector.TypeHelper;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
index 8d21298..d16b998 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
@@ -3,6 +3,8 @@ package org.apache.drill.exec.vector;
 import io.netty.buffer.ByteBuf;
 
 import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.exec.expr.holders.BitHolder;
+import org.apache.drill.exec.expr.holders.NullableBitHolder;
 import org.apache.drill.exec.memory.BufferAllocator;
 import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
 import org.apache.drill.exec.record.MaterializedField;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
index ed9b3b1..f81be83 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
@@ -2,6 +2,8 @@ package org.apache.drill.exec.vector;
 
 import io.netty.buffer.UnpooledByteBufAllocator;
 
+import org.apache.drill.exec.expr.holders.VarCharHolder;
+
 import com.google.common.base.Charsets;
 
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
index 32184ea..1dace1d 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
@@ -12,6 +12,7 @@ import org.apache.drill.exec.cache.DistributedCache;
 import org.apache.drill.exec.client.DrillClient;
 import org.apache.drill.exec.coord.ClusterCoordinator;
 import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.expr.holders.VarBinaryHolder;
 import org.apache.drill.exec.memory.BufferAllocator;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.ops.QueryContext;
@@ -35,7 +36,6 @@ import org.apache.drill.exec.server.DrillbitContext;
 import org.apache.drill.exec.server.RemoteServiceSet;
 import org.apache.drill.exec.store.StorageEngineRegistry;
 import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.VarBinaryHolder;
 import org.apache.drill.exec.vector.VarBinaryVector;
 import org.junit.AfterClass;
 import org.junit.Ignore;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
index ce5dccd..2209e58 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
@@ -18,16 +18,18 @@
 
 package org.apache.drill.exec.physical.impl;
 
-import com.google.common.base.Charsets;
-import com.google.common.io.Files;
-import com.yammer.metrics.MetricRegistry;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import mockit.Injectable;
 import mockit.NonStrictExpectations;
+
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.expression.ExpressionPosition;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.util.FileUtils;
 import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.expr.holders.NullableVarBinaryHolder;
+import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
 import org.apache.drill.exec.memory.BufferAllocator;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.physical.PhysicalPlan;
@@ -37,15 +39,14 @@ import org.apache.drill.exec.proto.CoordinationProtos;
 import org.apache.drill.exec.proto.ExecProtos;
 import org.apache.drill.exec.rpc.user.UserServer;
 import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.vector.NullableVarBinaryHolder;
 import org.apache.drill.exec.vector.NullableVarBinaryVector;
-import org.apache.drill.exec.vector.NullableVarCharHolder;
 import org.apache.drill.exec.vector.NullableVarCharVector;
 import org.junit.After;
 import org.junit.Test;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import com.yammer.metrics.MetricRegistry;
 
 public class TestSimpleFunctions {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleFunctions.class);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
index 08f118c..b4412c0 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
@@ -7,6 +7,7 @@ import java.nio.charset.Charset;
 import org.apache.drill.common.types.TypeProtos.DataMode;
 import org.apache.drill.common.types.TypeProtos.MajorType;
 import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.expr.TypeHelper;
 import org.apache.drill.exec.memory.DirectBufferAllocator;
 import org.apache.drill.exec.proto.SchemaDefProtos.FieldDef;
 import org.apache.drill.exec.record.MaterializedField;
@@ -14,7 +15,6 @@ import org.apache.drill.exec.vector.BitVector;
 import org.apache.drill.exec.vector.NullableFloat4Vector;
 import org.apache.drill.exec.vector.NullableUInt4Vector;
 import org.apache.drill.exec.vector.NullableVarCharVector;
-import org.apache.drill.exec.vector.TypeHelper;
 import org.apache.drill.exec.vector.UInt4Vector;
 import org.junit.Test;
 


[09/18] git commit: fix jackson conflict for mapr build

Posted by ja...@apache.org.
fix jackson conflict for mapr build


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/5ff952b9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/5ff952b9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/5ff952b9

Branch: refs/heads/master
Commit: 5ff952b9ca3c645e8de12f7734892193bf2bf6c8
Parents: 35f43f5
Author: Steven Phillips <sp...@maprtech.com>
Authored: Fri Aug 30 11:53:49 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Sep 1 15:16:21 2013 -0700

----------------------------------------------------------------------
 sandbox/prototype/pom.xml | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5ff952b9/sandbox/prototype/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/pom.xml b/sandbox/prototype/pom.xml
index 5a714df..16fea84 100644
--- a/sandbox/prototype/pom.xml
+++ b/sandbox/prototype/pom.xml
@@ -358,7 +358,28 @@
             <version>1.0.3-mapr-3.0.0</version>
             <exclusions>
               <exclusion>
-                <artifactId>jets3t</artifactId>
+                <groupId>org.codehaus.jackson</groupId>
+                <artifactId>jackson-core-asl</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.codehaus.jackson</groupId>
+                <artifactId>jackson-mapper-asl</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.codehaus.jackson</groupId>
+                <artifactId>jackson-xc</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>org.codehaus.jackson</groupId>
+                <artifactId>jackson-jaxrs</artifactId>
+              </exclusion>
+              <exclusion>
+                <artifactId>mockito-all</artifactId>
+                <groupId>org.codehaus</groupId>
+              </exclusion>
+              <exclusion>
+
+              <artifactId>jets3t</artifactId>
                 <groupId>net.java.dev.jets3t</groupId>
               </exclusion>
               <exclusion>


[17/18] git commit: some comparison function fixes and updates to support ref engine sql testing

Posted by ja...@apache.org.
some comparison function fixes and updates to support ref engine sql testing


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/d3b2f9f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/d3b2f9f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/d3b2f9f6

Branch: refs/heads/master
Commit: d3b2f9f61e859df36681a3d27f58da0a8426ccb0
Parents: 4a603de
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sat Aug 31 22:46:12 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Sep 1 15:16:22 2013 -0700

----------------------------------------------------------------------
 .../drill/common/expression/FieldReference.java |   2 +-
 .../drill/common/logical/data/Project.java      |   6 +-
 .../codegen/templates/ComparisonFunctions.java  | 233 ++++++++++++++-----
 .../exec/expr/fn/DrillSimpleFuncHolder.java     |   8 +
 .../expr/fn/FunctionImplementationRegistry.java |  21 +-
 .../physical/impl/filter/FilterTemplate.java    |   1 -
 .../impl/project/ProjectRecordBatch.java        |  16 +-
 .../org/apache/drill/exec/ref/ROPConverter.java |   2 +-
 .../org/apache/drill/jdbc/DrillHandler.java     | 117 +++++-----
 .../java/org/apache/drill/jdbc/DrillTable.java  | 108 ++++++---
 .../apache/drill/optiq/DrillImplementor.java    |  44 ++--
 .../org/apache/drill/optiq/DrillJoinRel.java    |   4 +-
 .../org/apache/drill/optiq/DrillProjectRel.java |   4 +-
 .../drill/optiq/EnumerableDrillFullEngine.java  |   5 +-
 .../apache/drill/optiq/EnumerableDrillRel.java  |   2 +-
 .../drill/sql/client/full/FileSystemSchema.java |   2 +-
 .../drill/sql/client/ref/DrillRefImpl.java      | 145 ++++++------
 .../org/apache/drill/jdbc/test/JdbcAssert.java  |   2 +-
 .../org/apache/drill/jdbc/test/JdbcTest.java    |   4 +-
 19 files changed, 477 insertions(+), 249 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d3b2f9f6/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/FieldReference.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
index 2d62a41..cb3f0b4 100644
--- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
+++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
@@ -44,7 +44,7 @@ public class FieldReference extends SchemaPath {
   }
   
   
-  public FieldReference(String value, ExpressionPosition pos) {
+  public FieldReference(CharSequence value, ExpressionPosition pos) {
     super(value, pos);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d3b2f9f6/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/Project.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/Project.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/Project.java
index 1fb5eb1..109d218 100644
--- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/Project.java
+++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/Project.java
@@ -40,9 +40,9 @@ public class Project extends SingleInputOperator {
     for (int i = 0; i < selections.length; i++) {
       PathSegment segment = selections[i].getRef().getRootSegment();
       CharSequence path = segment.getNameSegment().getPath();
-//      if (!segment.isNamed() || !path.equals("output"))
-//        throw new ExpressionParsingException(String.format(
-//            "Outputs for projections always have to start with named path of output. First segment was named '%s' or was named [%s]", path, segment.isNamed()));
+      if (!segment.isNamed() || !path.equals("output"))
+        throw new ExpressionParsingException(String.format(
+            "Outputs for projections always have to start with named path of output. First segment was named '%s' or was named [%s]", path, segment.isNamed()));
 
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d3b2f9f6/sandbox/prototype/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
index a5b560d..d30783a 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
@@ -1,62 +1,70 @@
 <@pp.dropOutputFile />
 
-<#macro compareBlock mode left right output>
+<#macro compareBlock mode left right output nullCompare>
 
 outside:{
-  <#if left?starts_with("Nullable")>
-  <#if right?starts_with("Nullable")>
-  <#-- Both are nullable. -->
-  if(left.isSet == 0){
-    if(right.isSet == 0){
-      ${output} = 0;
-      break outside;
-    }else{
-      ${output} = 1;
-    }
-  }else if(right.isSet == 0){
-    ${output} = -1;
-    break outside;
-  }
-  <#else>
-  <#-- Left is nullable but right is not. -->
-  if(left.isSet == 0){
-    ${output} = 1;
-    break outside;
-  }
-  </#if>
-<#elseif right?starts_with("Nullable")>
-  if(right.isSet == 0){
-    ${output} = -1;
-    break outside;
-  }
-  </#if>
-  
   
-<#if mode == "var">
-      
-      for (int l = left.start, r = right.start; l < left.end && r < right.end; l++, r++) {
-        byte leftByte = left.buffer.getByte(l);
-        byte rightByte = right.buffer.getByte(r);
-        if (leftByte != rightByte) {
-          ${output} = ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
+  <#if nullCompare>
+      <#if left?starts_with("Nullable")>
+        <#if right?starts_with("Nullable")>
+        <#-- Both are nullable. -->
+        if(left.isSet == 0){
+          if(right.isSet == 0){
+            ${output} = 0;
+            break outside;
+          }else{
+            ${output} = 1;
+            break outside;
+          }
+        }else if(right.isSet == 0){
+          ${output} = -1;
           break outside;
         }
-      }
-      
-      int l = (left.end - left.start) - (right.end - right.start);
-      if (l > 0) {
-        ${output} = 1;
-        break outside;
-      } else if (l == 0) {
-        ${output} = 0;
-        break outside;
-      } else {
+        <#else>
+        <#-- Left is nullable but right is not. -->
+        if(left.isSet == 0){
+          ${output} = 1;
+          break outside;
+        }
+        </#if>
+    <#elseif right?starts_with("Nullable")>
+      if(right.isSet == 0){
         ${output} = -1;
         break outside;
       }
-<#elseif mode == "fixed">
-    ${output} = left.value < right.value ? -1 : ((left.value == right.value)? 0 : 1);
-</#if>
+      </#if>
+    </#if>
+    
+    <#if mode == "var">
+    
+    for (int l = left.start, r = right.start; l < left.end && r < right.end; l++, r++) {
+      byte leftByte = left.buffer.getByte(l);
+      byte rightByte = right.buffer.getByte(r);
+      if (leftByte != rightByte) {
+        ${output} = ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
+        break outside;
+      }
+    }
+    
+    int l = (left.end - left.start) - (right.end - right.start);
+    if (l > 0) {
+      ${output} = 1;
+      break outside;
+    } else if (l == 0) {
+      ${output} = 0;
+      break outside;
+    } else {
+      ${output} = -1;
+      break outside;
+    }
+    <#elseif mode == "fixed">
+      ${output} = left.value < right.value ? -1 : ((left.value == right.value)? 0 : 1);
+    </#if>    
+  
+
+  
+  
+
 }
 </#macro>
 
@@ -89,7 +97,7 @@ public class GCompare${left}${right}{
       public void setup(RecordBatch b) {}
 
       public void eval() {
-        <@compareBlock mode=type.mode left=left right=right output="out.value" />
+        <@compareBlock mode=type.mode left=left right=right output="out.value" nullCompare=true />
       }
   }
   
@@ -103,9 +111,24 @@ public class GCompare${left}${right}{
       public void setup(RecordBatch b) {}
 
       public void eval() {
+        sout: {
+        <#if left?starts_with("Nullable")>
+        if(left.isSet ==0){
+          out.value = 0;
+          break sout;
+        }
+        </#if>
+        <#if right?starts_with("Nullable")>
+        if(right.isSet ==0){
+          out.value = 0;
+          break sout;
+        }
+        </#if>
+        
         int cmp;
-        <@compareBlock mode=type.mode left=left right=right output="cmp" />
+        <@compareBlock mode=type.mode left=left right=right output="cmp" nullCompare=false/>
         out.value = cmp == -1 ? 1 : 0;
+        }
       }
   }
   
@@ -119,10 +142,25 @@ public class GCompare${left}${right}{
       public void setup(RecordBatch b) {}
 
       public void eval() {
+        sout: {
+        <#if left?starts_with("Nullable")>
+        if(left.isSet ==0){
+          out.value = 0;
+          break sout;
+        }
+        </#if>
+        <#if right?starts_with("Nullable")>
+        if(right.isSet ==0){
+          out.value = 0;
+          break sout;
+        }
+        </#if>
+        
         int cmp;
-        <@compareBlock mode=type.mode left=left right=right output="cmp" />
+        <@compareBlock mode=type.mode left=left right=right output="cmp" nullCompare=false/>
         out.value = cmp < 1 ? 1 : 0;
-      }
+        }
+    }
   }
   
   @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
@@ -135,10 +173,25 @@ public class GCompare${left}${right}{
       public void setup(RecordBatch b) {}
 
       public void eval() {
+        sout: {
+        <#if left?starts_with("Nullable")>
+        if(left.isSet ==0){
+          out.value = 0;
+          break sout;
+        }
+        </#if>
+        <#if right?starts_with("Nullable")>
+        if(right.isSet ==0){
+          out.value = 0;
+          break sout;
+        }
+        </#if>
+        
         int cmp;
-        <@compareBlock mode=type.mode left=left right=right output="cmp" />
+        <@compareBlock mode=type.mode left=left right=right output="cmp" nullCompare=false/>
         out.value = cmp == 1 ? 1 : 0;
-      }
+        }
+    }
   }
   
   @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
@@ -151,13 +204,28 @@ public class GCompare${left}${right}{
       public void setup(RecordBatch b) {}
 
       public void eval() {
+        sout: {
+        <#if left?starts_with("Nullable")>
+        if(left.isSet ==0){
+          out.value = 0;
+          break sout;
+        }
+        </#if>
+        <#if right?starts_with("Nullable")>
+        if(right.isSet ==0){
+          out.value = 0;
+          break sout;
+        }
+        </#if>
+        
         int cmp;
-        <@compareBlock mode=type.mode left=left right=right output="cmp" />
+        <@compareBlock mode=type.mode left=left right=right output="cmp" nullCompare=false/>
         out.value = cmp > -1 ? 1 : 0;
+        }
       }
   }
   
-  @FunctionTemplate(name = "equals", scope = FunctionTemplate.FunctionScope.SIMPLE)
+  @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
   public static class Equals${left}${right} implements DrillSimpleFunc {
 
       @Param ${left}Holder left;
@@ -167,9 +235,56 @@ public class GCompare${left}${right}{
       public void setup(RecordBatch b) {}
 
       public void eval() {
+        sout: {
+          <#if left?starts_with("Nullable")>
+          if(left.isSet ==0){
+            out.value = 0;
+            break sout;
+          }
+          </#if>
+          <#if right?starts_with("Nullable")>
+          if(right.isSet ==0){
+            out.value = 0;
+            break sout;
+          }
+          </#if>
+          
+          int cmp;
+          <@compareBlock mode=type.mode left=left right=right output="cmp" nullCompare=false/>
+          out.value = cmp == 0 ? 1 : 0;
+        }
+      }
+  }
+  
+  @FunctionTemplate(name = "not equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+  public static class NotEquals${left}${right} implements DrillSimpleFunc {
+
+      @Param ${left}Holder left;
+      @Param ${right}Holder right;
+      @Output BitHolder out;
+
+      public void setup(RecordBatch b) {}
+
+      public void eval() {
+        sout: {
+        <#if left?starts_with("Nullable")>
+        if(left.isSet ==0){
+          out.value = 0;
+          break sout;
+        }
+        </#if>
+        <#if right?starts_with("Nullable")>
+        if(right.isSet ==0){
+          out.value = 0;
+          break sout;
+        }
+        </#if>
+        
         int cmp;
-        <@compareBlock mode=type.mode left=left right=right output="cmp" />
-        out.value = cmp == 0 ? 1 : 0;
+        <@compareBlock mode=type.mode left=left right=right output="cmp" nullCompare=false/>
+        out.value = cmp == 0 ? 0 : 1;
+        }
+        
       }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d3b2f9f6/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java
index 0a8e35f..f92c82f 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java
@@ -1,5 +1,6 @@
 package org.apache.drill.exec.expr.fn;
 
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 
@@ -99,5 +100,12 @@ class DrillSimpleFuncHolder extends DrillFuncHolder{
     if (sub != topSub) sub.assign(internalOutput.ref("isSet"),JExpr.lit(1));// Assign null if NULL_IF_NULL mode
     return out;
   }
+
+@Override
+public String toString() {
+  final int maxLen = 10;
+  return "DrillSimpleFuncHolder [, functionName=" + functionName + ", nullHandling=" + nullHandling + "parameters="
+      + (parameters != null ? Arrays.asList(parameters).subList(0, Math.min(parameters.length, maxLen)) : null) + "]";
+}
   
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d3b2f9f6/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
index 3b45446..b492610 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
@@ -43,8 +43,25 @@ public class FunctionImplementationRegistry {
     for(LogicalExpression e : call.args){
       types.add(e.getMajorType());
     }
-    
-    throw new UnsupportedOperationException(String.format("Unable to find matching function implementation for call %s with args %s", call.getDefinition().getName(), types));
+    StringBuilder sb = new StringBuilder();
+    sb.append("Missing function implementation: ");
+    sb.append("[");
+    sb.append(call.getDefinition().getName());
+    sb.append("(");
+    boolean first = true;
+    for(MajorType mt : types){
+      if(first){
+        first = false;
+      }else{
+        sb.append(", ");
+      }
+      sb.append(mt.getMinorType().name());
+      sb.append("-");
+      sb.append(mt.getMode().name());
+    }
+    sb.append(")");
+    sb.append("]");
+    throw new UnsupportedOperationException(sb.toString());
   }
 
   

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d3b2f9f6/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java
index 5675475..a7e8f0c 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate.java
@@ -73,7 +73,6 @@ public abstract class FilterTemplate implements Filterer{
     for(char i =0; i < recordCount; i++){
       
       if(doEval(i, 0)){
-        logger.debug("Setting svIndex {} to {}", svIndex, (int) i);
         outgoingSelectionVector.setIndex(svIndex, i);
         svIndex++;
       }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d3b2f9f6/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
index b7874bd..14a4905 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
@@ -5,7 +5,9 @@ import java.util.List;
 
 import org.apache.drill.common.expression.ErrorCollector;
 import org.apache.drill.common.expression.ErrorCollectorImpl;
+import org.apache.drill.common.expression.FieldReference;
 import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.PathSegment;
 import org.apache.drill.common.logical.data.NamedExpression;
 import org.apache.drill.exec.exception.ClassTransformationException;
 import org.apache.drill.exec.exception.SchemaChangeException;
@@ -57,6 +59,16 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
     }
   }
 
+  /** hack to make ref and full work together... need to figure out if this is still necessary. **/
+  private FieldReference getRef(NamedExpression e){
+    FieldReference ref = e.getRef();
+    PathSegment seg = ref.getRootSegment();
+    if(seg.isNamed() && "output".contentEquals(seg.getNameSegment().getPath())){
+      return new FieldReference(ref.getPath().toString().subSequence(7, ref.getPath().length()), ref.getPosition());
+    }
+    return ref;
+  }
+  
   @Override
   protected void setupNewSchema() throws SchemaChangeException{
     this.allocationVectors = Lists.newArrayList();
@@ -70,7 +82,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
     for(int i =0; i < exprs.size(); i++){
       final NamedExpression namedExpression = exprs.get(i);
       final LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector);
-      final MaterializedField outputField = MaterializedField.create(namedExpression.getRef(), expr.getMajorType());
+      final MaterializedField outputField = MaterializedField.create(getRef(namedExpression), expr.getMajorType());
       if(collector.hasErrors()){
         throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema.  Errors:\n %s.", collector.toErrorString()));
       }
@@ -81,7 +93,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
         ValueVector vvIn = incoming.getValueAccessorById(vectorRead.getFieldId().getFieldId(), TypeHelper.getValueVectorClass(vectorRead.getMajorType().getMinorType(), vectorRead.getMajorType().getMode())).getValueVector();
         Preconditions.checkNotNull(incoming);
 
-        TransferPair tp = vvIn.getTransferPair(namedExpression.getRef());
+        TransferPair tp = vvIn.getTransferPair(getRef(namedExpression));
         transfers.add(tp);
         container.add(tp.getTo());
         logger.debug("Added transfer.");

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d3b2f9f6/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/ROPConverter.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/ROPConverter.java b/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/ROPConverter.java
index 384af2d..fb21a7f 100644
--- a/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/ROPConverter.java
+++ b/sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/ROPConverter.java
@@ -106,7 +106,7 @@ class ROPConverter {
 
   private ReferenceStorageEngine getEngine(String name){
     StorageEngineConfig config = plan.getStorageEngineConfig(name);
-    if(config == null) throw new SetupException(String.format("Unable to find define logical plan of name [%s].", name));
+    if(config == null) throw new SetupException(String.format("Unable to find define storage engine of name [%s].", name));
     ReferenceStorageEngine engine = engineRegistry.getEngine(config);
     return engine;
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d3b2f9f6/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
index f2265f8..6cc25e0 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
@@ -13,6 +13,7 @@ import net.hydromatic.optiq.impl.java.JavaTypeFactory;
 import net.hydromatic.optiq.impl.java.MapSchema;
 import net.hydromatic.optiq.jdbc.HandlerImpl;
 import net.hydromatic.optiq.jdbc.OptiqConnection;
+import net.hydromatic.optiq.model.ModelHandler;
 
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.logical.StorageEngineConfig;
@@ -40,58 +41,68 @@ public class DrillHandler extends HandlerImpl {
   public void onConnectionInit(OptiqConnection connection) throws SQLException {
     super.onConnectionInit(connection);
 
-    registry = new SchemaProviderRegistry(config);
     final Properties p = connection.getProperties();
-    Preconditions.checkArgument(bit == null);
-    Preconditions.checkArgument(client == null);
-    Preconditions.checkArgument(coordinator == null);
-    // final String model = p.getProperty("model");
-    // if (model != null) {
-    // if (model != null) {
-    // try {
-    // new ModelHandler(connection, model);
-    // } catch (IOException e) {
-    // throw new SQLException(e);
-    // }
-    // }
-    // }
-
-    final String zk = connection.getProperties().getProperty("zk");
-
-    try {
-      String enginesData = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8);
-
-      StorageEngines engines = config.getMapper().readValue(enginesData, StorageEngines.class);
-      MutableSchema rootSchema = connection.getRootSchema();
-
-      for (Map.Entry<String, StorageEngineConfig> entry : engines) {
-        SchemaProvider provider = registry.getSchemaProvider(entry.getValue());
-        FileSystemSchema schema = new FileSystemSchema(client, entry.getValue(), provider, rootSchema.getTypeFactory(),
-            rootSchema, entry.getKey(), rootSchema.getExpression(), rootSchema.getQueryProvider());
-        rootSchema.addSchema(entry.getKey(), schema);
+    
+    if (p.getProperty("ref") != null) {
+      final String model = p.getProperty("model");
+      if (model != null) {
+        if (model != null) {
+          try {
+            new ModelHandler(connection, model);
+          } catch (IOException e) {
+            throw new SQLException(e);
+          }
+        }
       }
-
-      rootSchema.addSchema("--FAKE--", new FakeSchema(rootSchema, rootSchema.getQueryProvider(), rootSchema.getTypeFactory(), "fake", rootSchema.getExpression()));
+    } else {
+      
+      registry = new SchemaProviderRegistry(config);
       
-      if (zk != null) {
-        coordinator = new ZKClusterCoordinator(config, zk);
-        coordinator.start(10000);
-        DrillClient cl = new DrillClient(config, coordinator);
-        cl.connect();
-        client = cl;
-      } else {
-
-        RemoteServiceSet local = RemoteServiceSet.getLocalServiceSet();
-        this.coordinator = local.getCoordinator();
-        bit = new Drillbit(config, local);
-        bit.run();
-        
-        DrillClient cl = new DrillClient(config, coordinator);
-        cl.connect();
-        client = cl;
+      Preconditions.checkArgument(bit == null);
+      Preconditions.checkArgument(client == null);
+      Preconditions.checkArgument(coordinator == null);
+
+      final String zk = connection.getProperties().getProperty("zk");
+
+      try {
+        String enginesData = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8);
+
+        StorageEngines engines = config.getMapper().readValue(enginesData, StorageEngines.class);
+        MutableSchema rootSchema = connection.getRootSchema();
+
+        for (Map.Entry<String, StorageEngineConfig> entry : engines) {
+          SchemaProvider provider = registry.getSchemaProvider(entry.getValue());
+          FileSystemSchema schema = new FileSystemSchema(client, entry.getValue(), provider,
+              rootSchema.getTypeFactory(), rootSchema, entry.getKey(), rootSchema.getExpression(),
+              rootSchema.getQueryProvider());
+          rootSchema.addSchema(entry.getKey(), schema);
+        }
+
+        rootSchema.addSchema(
+            "--FAKE--",
+            new FakeSchema(rootSchema, rootSchema.getQueryProvider(), rootSchema.getTypeFactory(), "fake", rootSchema
+                .getExpression()));
+
+        if (zk != null) {
+          coordinator = new ZKClusterCoordinator(config, zk);
+          coordinator.start(10000);
+          DrillClient cl = new DrillClient(config, coordinator);
+          cl.connect();
+          client = cl;
+        } else {
+
+          RemoteServiceSet local = RemoteServiceSet.getLocalServiceSet();
+          this.coordinator = local.getCoordinator();
+          bit = new Drillbit(config, local);
+          bit.run();
+
+          DrillClient cl = new DrillClient(config, coordinator);
+          cl.connect();
+          client = cl;
+        }
+      } catch (Exception ex) {
+        throw new SQLException("Failure trying to connect to Drill.", ex);
       }
-    } catch (Exception ex) {
-      throw new SQLException("Failure trying to connect to Drill.", ex);
     }
 
     // The "schema" parameter currently gives a name to the schema. In future
@@ -102,24 +113,24 @@ public class DrillHandler extends HandlerImpl {
     }
 
   }
-  
-  public class FakeSchema extends MapSchema{
+
+  public class FakeSchema extends MapSchema {
 
     public FakeSchema(Schema parentSchema, QueryProvider queryProvider, JavaTypeFactory typeFactory, String name,
         Expression expression) {
       super(parentSchema, queryProvider, typeFactory, name, expression);
-      
+
     }
+
     public DrillClient getClient() {
       return client;
     }
   }
-  
+
   public DrillClient getClient() {
     return client;
   }
 
-
   @Override
   public void onConnectionClose(OptiqConnection connection) throws RuntimeException {
     super.onConnectionClose(connection);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d3b2f9f6/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillTable.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillTable.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillTable.java
index 6dfea48..da20756 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillTable.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillTable.java
@@ -18,7 +18,10 @@
 package org.apache.drill.jdbc;
 
 import java.lang.reflect.Type;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
+import java.util.Map;
 
 import net.hydromatic.linq4j.BaseQueryable;
 import net.hydromatic.linq4j.Enumerator;
@@ -32,10 +35,14 @@ import net.hydromatic.optiq.MutableSchema;
 import net.hydromatic.optiq.Schema;
 import net.hydromatic.optiq.Statistic;
 import net.hydromatic.optiq.Statistics;
+import net.hydromatic.optiq.TableFactory;
 import net.hydromatic.optiq.TranslatableTable;
 
 import org.apache.drill.common.logical.StorageEngineConfig;
 import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.ref.rops.DataWriter;
+import org.apache.drill.exec.ref.rse.ClasspathRSE;
+import org.apache.drill.exec.ref.rse.ClasspathRSE.ClasspathInputConfig;
 import org.apache.drill.optiq.DrillRel;
 import org.apache.drill.optiq.DrillScan;
 import org.eigenbase.rel.RelNode;
@@ -56,22 +63,28 @@ public class DrillTable extends BaseQueryable<Object> implements TranslatableTab
   public DrillClient client;
   
 
+  public boolean isRefEngine(){
+    return client == null;
+  }
+  
   /** Creates a DrillTable. */
   public DrillTable(DrillClient client, Schema schema,
       Type elementType,
       Expression expression,
       RelDataType rowType,
       String name,
+      String storageEngineName,
       Object selection,
       StorageEngineConfig storageEngineConfig
       ) {
     super(schema.getQueryProvider(), elementType, expression);
+    this.client = client;
     this.schema = schema;
     this.name = name;
     this.rowType = rowType;
     this.selection = selection;
     this.storageEngineConfig = storageEngineConfig;
-    this.storageEngineName = schema.getName();
+    this.storageEngineName = client == null ? storageEngineName : schema.getName();
   }
   
   public String getName() {
@@ -91,6 +104,7 @@ public class DrillTable extends BaseQueryable<Object> implements TranslatableTab
       RelDataTypeFactory typeFactory,
       Schema schema,
       String name,
+      String storageEngineName,
       StorageEngineConfig storageEngineConfig,
       Object selection
       ) {
@@ -106,7 +120,7 @@ public class DrillTable extends BaseQueryable<Object> implements TranslatableTab
                     typeFactory.createSqlType(SqlTypeName.VARCHAR),
                     typeFactory.createSqlType(SqlTypeName.ANY))),
             Collections.singletonList("_MAP"));
-      return new DrillTable(client, schema, Object.class, call, rowType, name, selection, storageEngineConfig);
+      return new DrillTable(client, schema, Object.class, call, rowType, name, storageEngineName, selection, storageEngineConfig);
   }
   
   
@@ -148,38 +162,64 @@ public class DrillTable extends BaseQueryable<Object> implements TranslatableTab
 
   
   
-//  private static final List<String> DONUTS_TABLES = Arrays.asList(
-//      "DONUTS");
-//
-//  private static final List<String> HR_TABLES = Arrays.asList(
-//      "EMPLOYEES", "DEPARTMENTS");
+  /** Factory for custom tables in Optiq schema. */
+  @SuppressWarnings("UnusedDeclaration")
+  public static class Factory implements TableFactory<DrillTable> {
+
+    private static final List<String> DONUTS_TABLES = Arrays.asList(
+        "DONUTS");
+
+    private static final List<String> HR_TABLES = Arrays.asList(
+        "EMPLOYEES", "DEPARTMENTS");
+
+    private static final List<String> FOODMART_TABLES = Arrays.asList(
+        "ACCOUNT", "CATEGORY", "CURRENCY", "CUSTOMER", "DAYS", "DEPARTMENT",
+        "EMPLOYEE_CLOSURE", "EMPLOYEE", "EXPENSE_FACT", "INVENTORY_FACT_1997",
+        "INVENTORY_FACT_1998", "POSITION", "PRODUCT_CLASS", "PRODUCT",
+        "PROMOTION", "REGION", "RESERVE_EMPLOYEE", "SALARY", "SALES_FACT_1997",
+        "SALES_FACT_1998", "SALES_FACT_DEC_1998", "STORE", "STORE_RAGGED",
+        "TIME_BY_DAY", "WAREHOUSE", "WAREHOUSE_CLASS");
+
+//    public DrillTable create(
+//        JavaTypeFactory typeFactory,
+//        Schema schema,
+//        String name,
+//        Map<String, Object> operand, 
+//        RelDataType rowType) {
+//      final ClasspathRSE.ClasspathRSEConfig rseConfig = new ClasspathRSE.ClasspathRSEConfig();
+//      final ClasspathInputConfig inputConfig = new ClasspathInputConfig();
+//      assert DONUTS_TABLES.contains(name)
+//          || HR_TABLES.contains(name)
+//          || FOODMART_TABLES.contains(name)
+//          : name;
+//      inputConfig.path = "/" + name.toLowerCase() + ".json";
+//      inputConfig.type = DataWriter.ConverterType.JSON;
+//      boolean useReferenceInterpreter;
+//      if (operand.get("useReferenceInterpreter") != null){
+//        useReferenceInterpreter = operand.get("useReferenceInterpreter").equals("true") ? true : false;
+//      }
+//      else{
+//        useReferenceInterpreter = false;
+//      }      
+//      return createTable(typeFactory, (MutableSchema) schema, name, rseConfig,
+//          inputConfig, "donuts-json", useReferenceInterpreter);
+//    }
 //
-//  private static final List<String> FOODMART_TABLES = Arrays.asList(
-//      "ACCOUNT", "CATEGORY", "CURRENCY", "CUSTOMER", "DAYS", "DEPARTMENT",
-//      "EMPLOYEE_CLOSURE", "EMPLOYEE", "EXPENSE_FACT", "INVENTORY_FACT_1997",
-//      "INVENTORY_FACT_1998", "POSITION", "PRODUCT_CLASS", "PRODUCT",
-//      "PROMOTION", "REGION", "RESERVE_EMPLOYEE", "SALARY", "SALES_FACT_1997",
-//      "SALES_FACT_1998", "SALES_FACT_DEC_1998", "STORE", "STORE_RAGGED",
-//      "TIME_BY_DAY", "WAREHOUSE", "WAREHOUSE_CLASS");
-//  
-//  public static DrillTable create(Schema schema, String name, Map<String, Object> operand, RelDataType rowType) {
-//    
-//    final ClasspathRSE.ClasspathRSEConfig rseConfig = new ClasspathRSE.ClasspathRSEConfig();
-//    final ClasspathInputConfig inputConfig = new ClasspathInputConfig();
-//    assert DONUTS_TABLES.contains(name)
-//        || HR_TABLES.contains(name)
-//        || FOODMART_TABLES.contains(name)
-//        : name;
-//    inputConfig.path = "/" + name.toLowerCase() + ".json";
-//    inputConfig.type = DataWriter.ConverterType.JSON;
-//    boolean useReferenceInterpreter;
-//    if (operand.get("useReferenceInterpreter") != null){
-//      useReferenceInterpreter = operand.get("useReferenceInterpreter").equals("true") ? true : false;
-//    }else{
-//      useReferenceInterpreter = false;
-//    }      
-//    return createTable(schema.getTypeFactory(), (MutableSchema) schema, name, rseConfig,
-//        inputConfig, "donuts-json", useReferenceInterpreter);
-//  }
+    @Override
+    public DrillTable create(Schema schema, String name, Map<String, Object> operand, RelDataType rowType) {
+      
+      final ClasspathRSE.ClasspathRSEConfig rseConfig = new ClasspathRSE.ClasspathRSEConfig();
+      final ClasspathInputConfig inputConfig = new ClasspathInputConfig();
+      assert DONUTS_TABLES.contains(name)
+          || HR_TABLES.contains(name)
+          || FOODMART_TABLES.contains(name)
+          : name;
+      inputConfig.path = "/" + name.toLowerCase() + ".json";
+      inputConfig.type = DataWriter.ConverterType.JSON;
+      return createTable(null, schema.getTypeFactory(), (MutableSchema) schema, name, "donuts-json", rseConfig, inputConfig);
+    }
+  }
+
+  
   
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d3b2f9f6/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java
index 0d3ed68..27b9630 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java
@@ -17,13 +17,16 @@
  ******************************************************************************/
 package org.apache.drill.optiq;
 
+import java.io.IOException;
 import java.util.Set;
 
 import org.apache.drill.exec.ref.rse.QueueRSE.QueueOutputInfo;
 import org.apache.drill.jdbc.DrillTable;
 import org.eigenbase.rel.RelNode;
 
+import com.fasterxml.jackson.core.JsonParseException;
 import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
@@ -48,8 +51,10 @@ public class DrillImplementor {
   private final ArrayNode operatorsNode;
   private final ObjectNode sourcesNode;
   private Set<DrillTable> tables = Sets.newHashSet();
+  private final boolean isRef;
   
-  public DrillImplementor() {
+  public DrillImplementor(boolean isRef) {
+    this.isRef = isRef;
     final ObjectNode headNode = mapper.createObjectNode();
     rootNode.put("head", headNode);
     headNode.put("type", "APACHE_DRILL_LOGICAL");
@@ -65,17 +70,19 @@ public class DrillImplementor {
     sourcesNode = mapper.createObjectNode();
     rootNode.put("storage", sourcesNode);
 
-    // input file source
-//    {
-//      final ObjectNode sourceNode = mapper.createObjectNode();
-//      sourceNode.put("type", "classpath");
-//      sourcesNode.put("donuts-json", sourceNode);
-//    }
-//    {
-//      final ObjectNode sourceNode = mapper.createObjectNode();
-//      sourceNode.put("type", "queue");
-//      sourcesNode.put("queue", sourceNode);
-//    }
+    if(isRef){
+      {
+        final ObjectNode sourceNode = mapper.createObjectNode();
+        sourceNode.put("type", "classpath");
+        sourcesNode.put("donuts-json", sourceNode);
+      }
+      {
+        final ObjectNode sourceNode = mapper.createObjectNode();
+        sourceNode.put("type", "queue");
+        sourcesNode.put("queue", sourceNode);
+      }
+    }
+
 
     final ArrayNode queryNode = mapper.createArrayNode();
     rootNode.put("query", queryNode);
@@ -114,7 +121,18 @@ public class DrillImplementor {
   /** Returns the generated plan. */
   public String getJsonString() {
     String s = rootNode.toString();
-    if(logger.isDebugEnabled()) logger.debug("Optiq Generated Logical Plan: {}", s);
+    
+    if(logger.isDebugEnabled()){
+      JsonNode node;
+      try {
+        ObjectMapper mapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
+        node = mapper.readValue(s, JsonNode.class);
+        logger.debug("Optiq Generated Logical Plan: {}", mapper.writeValueAsString(node));
+      } catch (IOException e) {
+        logger.error("Failure while trying to parse logical plan string of {}", s, e);
+      }
+      
+    }
     return s;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d3b2f9f6/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRel.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRel.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRel.java
index 82a0c43..f367e6d 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRel.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRel.java
@@ -123,8 +123,8 @@ public class DrillJoinRel extends JoinRelBase implements DrillRel {
       final ObjectNode objectNode = implementor.mapper.createObjectNode();
       transforms.add(objectNode);
       objectNode.put("expr", pair.left);
-//      objectNode.put("ref", "output." + pair.right);
-      objectNode.put("ref", pair.right);
+      objectNode.put("ref", "output." + pair.right);
+//      objectNode.put("ref", pair.right);
     }
     return implementor.add(project);
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d3b2f9f6/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillProjectRel.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillProjectRel.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillProjectRel.java
index 0470f5e..1c0b00b 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillProjectRel.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillProjectRel.java
@@ -74,8 +74,8 @@ public class DrillProjectRel extends ProjectRelBase implements DrillRel {
       transforms.add(objectNode);
       String expr = DrillOptiq.toDrill(getChild(), pair.left);
       objectNode.put("expr", expr);
-//      String ref = "output." + pair.right;
-      String ref = pair.right;
+      String ref = "output." + pair.right;
+//      String ref = pair.right;
       objectNode.put("ref", ref);
     }
     return implementor.add(project);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d3b2f9f6/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillFullEngine.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillFullEngine.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillFullEngine.java
index c6cb707..a71ab8c 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillFullEngine.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillFullEngine.java
@@ -69,13 +69,12 @@ public class EnumerableDrillFullEngine<E> extends AbstractEnumerable<E> implemen
   public static <E> EnumerableDrillFullEngine<E> of(String plan, final List<String> fieldNames, Class<E> clazz,
       DataContext context) {
     DrillConfig config = DrillConfig.create();
-    return new EnumerableDrillFullEngine<>(config, plan, clazz, fieldNames, ((FakeSchema) context.getSubSchema("--FAKE--")).getClient());
+    FakeSchema s = (FakeSchema) context.getSubSchema("--FAKE--");
+    return new EnumerableDrillFullEngine<>(config, plan, clazz, fieldNames, s == null ? null : s.getClient());
   }
 
   @Override
   public Enumerator<E> enumerator() {
-    
-    //DrillTable table = (DrillTable) drillConnectionDataContext.getSubSchema("DONUTS").getTable("DONUTS", Object.class);
     if (client == null) {
       DrillRefImpl<E> impl = new DrillRefImpl<E>(plan, config, fields, queue);
       return impl.enumerator();

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d3b2f9f6/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRel.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRel.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRel.java
index b280f8a..2f9cd9b 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRel.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/EnumerableDrillRel.java
@@ -102,7 +102,7 @@ public class EnumerableDrillRel extends SingleRel implements EnumerableRel {
   public Result implement(EnumerableRelImplementor implementor, Prefer pref) {
     logger.debug("implementing enumerable");
 
-    final DrillImplementor drillImplementor = new DrillImplementor();
+    final DrillImplementor drillImplementor = new DrillImplementor(this.client == null);
     DrillRel input = (DrillRel) getChild();
 
     drillImplementor.go(input);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d3b2f9f6/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/FileSystemSchema.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/FileSystemSchema.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/FileSystemSchema.java
index 85a4125..f506ebd 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/FileSystemSchema.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/full/FileSystemSchema.java
@@ -103,7 +103,7 @@ public class FileSystemSchema implements Schema{
     Object selection = schemaProvider.getSelectionBaseOnName(name);
     if(selection == null) return null;
     
-    DrillTable table = DrillTable.createTable(client, typeFactory, this, name, config, selection);
+    DrillTable table = DrillTable.createTable(client, typeFactory, this, name, null, config, selection);
     info = new TableInfo(name, table);
     TableInfo oldInfo = (TableInfo) tables.putIfAbsent(name, info);
     if(oldInfo != null) return (Table<E>) oldInfo.table;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d3b2f9f6/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/ref/DrillRefImpl.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/ref/DrillRefImpl.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/ref/DrillRefImpl.java
index 3792ddf..a4256a1 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/ref/DrillRefImpl.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/sql/client/ref/DrillRefImpl.java
@@ -9,7 +9,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.SortedMap;
 import java.util.TreeMap;
-import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CompletionService;
@@ -28,24 +27,23 @@ import org.apache.drill.exec.ref.ReferenceInterpreter;
 import org.apache.drill.exec.ref.RunOutcome;
 import org.apache.drill.exec.ref.eval.BasicEvaluatorFactory;
 import org.apache.drill.exec.ref.rse.RSERegistry;
-import org.apache.drill.jdbc.DrillTable;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.Lists;
 
 public class DrillRefImpl<E> {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillRefImpl.class);
 
   private static final ObjectMapper mapper = createMapper();
-  
+
   private final String plan;
   final BlockingQueue<Object> queue;
   final DrillConfig config;
   private final List<String> fields;
 
-  
   public DrillRefImpl(String plan, DrillConfig config, List<String> fields, BlockingQueue<Object> queue) {
     super();
     this.plan = plan;
@@ -53,8 +51,7 @@ public class DrillRefImpl<E> {
     this.fields = fields;
     this.queue = queue;
   }
-  
-  
+
   private static ObjectMapper createMapper() {
     return new ObjectMapper();
   }
@@ -67,17 +64,20 @@ public class DrillRefImpl<E> {
     private final String holder;
     private final List<String> fields;
     private Object current;
+    private Future<Collection<RunOutcome>> futureOutcomes;
 
-    public JsonEnumerator(BlockingQueue<Object> queue, List<String> fields) {
+    public JsonEnumerator(Future<Collection<RunOutcome>> futureOutcomes, BlockingQueue<Object> queue,
+        List<String> fields) {
+      this.futureOutcomes = futureOutcomes;
       this.queue = queue;
       this.holder = null;
       this.fields = fields;
     }
 
-    public void close(){
-      
+    public void close() {
+
     }
-    
+
     public Object current() {
       return current;
     }
@@ -87,13 +87,32 @@ public class DrillRefImpl<E> {
         Object o = queue.take();
         if (o instanceof RunOutcome.OutcomeType) {
           switch ((RunOutcome.OutcomeType) o) {
-            case SUCCESS:
-              return false; // end of data
-            case CANCELED:
-              throw new RuntimeException("canceled");
-            case FAILED:
-            default:
-              throw new RuntimeException("failed");
+          case SUCCESS:
+            return false; // end of data
+          case CANCELED:
+            throw new RuntimeException("canceled");
+          case FAILED:
+          default:
+            try {
+              Collection<RunOutcome> outcomes = this.futureOutcomes.get();
+              List<RunOutcome> l = Lists.newArrayList(outcomes);
+              for (int i = 1; i < outcomes.size(); i++) {
+                RunOutcome out = l.get(i);
+                logger.error("Failure while running query", out, out.exception);
+              }
+              if (!outcomes.isEmpty()) {
+                RunOutcome out = outcomes.iterator().next();
+                if (out.exception != null) {
+                  throw new RuntimeException("Query Failed while running.", out.exception);
+                } else {
+                  throw new RuntimeException("Query Failed while running. " + o);
+                }
+              }
+            } catch (Exception e) {
+              throw new RuntimeException("failed", e);
+            }
+            
+            throw new RuntimeException("failed");
           }
         } else {
           Object o1 = parseJson((byte[]) o);
@@ -127,45 +146,38 @@ public class DrillRefImpl<E> {
     }
   }
 
-
   /**
    * Runs the plan as a background task.
    */
-  Future<Collection<RunOutcome>> runRefInterpreterPlan(
-      CompletionService<Collection<RunOutcome>> service) {
+  Future<Collection<RunOutcome>> runRefInterpreterPlan(CompletionService<Collection<RunOutcome>> service) {
     LogicalPlan parsedPlan = LogicalPlan.parse(DrillConfig.create(), plan);
     IteratorRegistry ir = new IteratorRegistry();
     DrillConfig config = DrillConfig.create();
     config.setSinkQueues(0, queue);
-    final ReferenceInterpreter i =
-        new ReferenceInterpreter(parsedPlan, ir, new BasicEvaluatorFactory(ir),
-            new RSERegistry(config));
+    final ReferenceInterpreter i = new ReferenceInterpreter(parsedPlan, ir, new BasicEvaluatorFactory(ir),
+        new RSERegistry(config));
     try {
       i.setup();
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
-    return service.submit(
-        new Callable<Collection<RunOutcome>>() {
-          @Override
-          public Collection<RunOutcome> call() throws Exception {
-            Collection<RunOutcome> outcomes = i.run();
-
-            for (RunOutcome outcome : outcomes) {
-              System.out.println("============");
-              System.out.println(outcome);
-              if (outcome.outcome == RunOutcome.OutcomeType.FAILED
-                  && outcome.exception != null) {
-                outcome.exception.printStackTrace();
-              }
-            }
-            return outcomes;
+    return service.submit(new Callable<Collection<RunOutcome>>() {
+      @Override
+      public Collection<RunOutcome> call() throws Exception {
+        Collection<RunOutcome> outcomes = i.run();
+
+        for (RunOutcome outcome : outcomes) {
+          System.out.println("============");
+          System.out.println(outcome);
+          if (outcome.outcome == RunOutcome.OutcomeType.FAILED && outcome.exception != null) {
+            outcome.exception.printStackTrace();
           }
-        });
+        }
+        return outcomes;
+      }
+    });
   }
 
-  
-  
   public Enumerator<E> enumerator() {
     // TODO: use a completion service from the container
     final ExecutorCompletionService<Collection<RunOutcome>> service = new ExecutorCompletionService<Collection<RunOutcome>>(
@@ -177,13 +189,13 @@ public class DrillRefImpl<E> {
     // TODO: use the result of task, and check for exceptions
     final Future<Collection<RunOutcome>> task = runRefInterpreterPlan(service);
 
-    return new JsonEnumerator(queue, fields);
+    return new JsonEnumerator(task, queue, fields);
 
   }
-  
+
   /**
-   * Converts a JSON document, represented as an array of bytes, into a Java
-   * object (consisting of Map, List, String, Integer, Double, Boolean).
+   * Converts a JSON document, represented as an array of bytes, into a Java object (consisting of Map, List, String,
+   * Integer, Double, Boolean).
    */
   static Object parseJson(byte[] bytes) {
     try {
@@ -192,33 +204,31 @@ public class DrillRefImpl<E> {
       throw new RuntimeException(e);
     }
   }
-  
-
 
   /**
-   * Converts a JSON node to Java objects ({@link List}, {@link Map},
-   * {@link String}, {@link Integer}, {@link Double}, {@link Boolean}.
+   * Converts a JSON node to Java objects ({@link List}, {@link Map}, {@link String}, {@link Integer}, {@link Double},
+   * {@link Boolean}.
    */
   static Object wrapper(JsonNode node) {
     switch (node.asToken()) {
-      case START_OBJECT:
-        return map((ObjectNode) node);
-      case START_ARRAY:
-        return array((ArrayNode) node);
-      case VALUE_STRING:
-        return node.asText();
-      case VALUE_NUMBER_INT:
-        return node.asInt();
-      case VALUE_NUMBER_FLOAT:
-        return node.asDouble();
-      case VALUE_TRUE:
-        return Boolean.TRUE;
-      case VALUE_FALSE:
-        return Boolean.FALSE;
-      case VALUE_NULL:
-        return null;
-      default:
-        throw new AssertionError("unexpected: " + node + ": " + node.asToken());
+    case START_OBJECT:
+      return map((ObjectNode) node);
+    case START_ARRAY:
+      return array((ArrayNode) node);
+    case VALUE_STRING:
+      return node.asText();
+    case VALUE_NUMBER_INT:
+      return node.asInt();
+    case VALUE_NUMBER_FLOAT:
+      return node.asDouble();
+    case VALUE_TRUE:
+      return Boolean.TRUE;
+    case VALUE_FALSE:
+      return Boolean.FALSE;
+    case VALUE_NULL:
+      return null;
+    default:
+      throw new AssertionError("unexpected: " + node + ": " + node.asToken());
     }
   }
 
@@ -241,5 +251,4 @@ public class DrillRefImpl<E> {
     return Collections.unmodifiableSortedMap(map);
   }
 
-  
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d3b2f9f6/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
index 52dbcfb..d3927f4 100644
--- a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
+++ b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
@@ -113,7 +113,7 @@ public class JdbcAssert {
       this.connectionFactory = new ConnectionFactory() {
         public Connection createConnection() throws Exception {
           Class.forName("org.apache.drill.jdbc.Driver");
-          return DriverManager.getConnection("jdbc:drill:", ModelAndSchema.this.info);
+          return DriverManager.getConnection("jdbc:drill:ref=true", ModelAndSchema.this.info);
         }
       };
     }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/d3b2f9f6/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
index ccab8fd..d9dd11a 100644
--- a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
+++ b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
@@ -32,7 +32,7 @@ import com.google.common.base.Function;
 import com.google.common.io.Resources;
 
 /** Unit tests for Drill's JDBC driver. */
-@Ignore
+
 public class JdbcTest {
   private static String MODEL;
   private static String EXPECTED;
@@ -65,7 +65,7 @@ public class JdbcTest {
   @Test
   public void testConnect() throws Exception {
     Class.forName("org.apache.drill.jdbc.Driver");
-    final Connection connection = DriverManager.getConnection("jdbc:drill:schema=DONUTS");
+    final Connection connection = DriverManager.getConnection("jdbc:drill:ref=true&schema=DONUTS");
     connection.close();
   }
 


[10/18] git commit: Tool for submitting physical or logical plan

Posted by ja...@apache.org.
Tool for submitting physical or logical plan


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/35f43f56
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/35f43f56
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/35f43f56

Branch: refs/heads/master
Commit: 35f43f5628d9fd6119684ef6fa183d2faf0f01c0
Parents: b03740e
Author: Steven Phillips <sp...@maprtech.com>
Authored: Fri Aug 30 02:27:28 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Sep 1 15:16:21 2013 -0700

----------------------------------------------------------------------
 .../drill/exec/client/QuerySubmitter.java       | 116 +++++++++++++++++++
 .../exec/store/TestParquetPhysicalPlan.java     |   6 +-
 2 files changed, 121 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/35f43f56/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
new file mode 100644
index 0000000..a9cad92
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
@@ -0,0 +1,116 @@
+package org.apache.drill.exec.client;
+
+import com.beust.jcommander.internal.Lists;
+import com.google.common.base.Charsets;
+import com.google.common.base.Stopwatch;
+import com.google.common.io.Resources;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.coord.ZKClusterCoordinator;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.proto.UserProtos;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.rpc.RemoteRpcException;
+import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.apache.drill.exec.rpc.user.UserResultsListener;
+import org.apache.drill.exec.server.BootStrapContext;
+import org.apache.drill.exec.vector.ValueVector;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.CharsetDecoder;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class QuerySubmitter {
+
+  public static void main(String args[]) throws Exception {
+    QuerySubmitter submitter = new QuerySubmitter();
+    System.exit(submitter.submitQuery(args[0], args[1], args[2]));
+  }
+
+  public int submitQuery(String planLocation, String type, String zkQuorum) throws Exception {
+    DrillConfig config = DrillConfig.create();
+    ZKClusterCoordinator clusterCoordinator = new ZKClusterCoordinator(config, zkQuorum);
+    clusterCoordinator.start(10000);
+    DrillClient client = new DrillClient(config, clusterCoordinator);
+    client.connect();
+    QueryResultsListener listener = new QueryResultsListener();
+    String plan = Charsets.UTF_8.decode(ByteBuffer.wrap(Files.readAllBytes(Paths.get(planLocation)))).toString();
+    UserProtos.QueryType queryType;
+    type = type.toLowerCase();
+    switch(type) {
+      case "logical":
+        queryType = UserProtos.QueryType.LOGICAL;
+        break;
+      case "physical":
+        queryType = UserProtos.QueryType.PHYSICAL;
+        break;
+      default:
+        System.out.println("Invalid query type: " + type);
+        return -1;
+    }
+    Stopwatch watch = new Stopwatch();
+    watch.start();
+    client.runQuery(queryType, plan, listener);
+    System.out.println(String.format("Got %d total records in %d seconds", listener.await(), watch.elapsed(TimeUnit.SECONDS)));
+    return 0;
+  }
+
+  private class QueryResultsListener implements UserResultsListener {
+    AtomicInteger count = new AtomicInteger();
+    private CountDownLatch latch = new CountDownLatch(1);
+    @Override
+    public void submissionFailed(RpcException ex) {
+      System.out.println(String.format("Query failed: %s", ex));
+      latch.countDown();
+    }
+
+    @Override
+    public void resultArrived(QueryResultBatch result) {
+      int rows = result.getHeader().getRowCount();
+      RecordBatchLoader loader = new RecordBatchLoader(new BootStrapContext(DrillConfig.create()).getAllocator());
+      try {
+        loader.load(result.getHeader().getDef(), result.getData());
+      } catch (SchemaChangeException e) {
+        submissionFailed(new RpcException(e));
+      }
+      List<String> columns = Lists.newArrayList();
+      for (VectorWrapper vw : loader) {
+        columns.add(vw.getValueVector().getField().getName());
+      }
+      for (int row = 0; row < rows; row++) {
+        if (row%50 == 0) {
+          for (String column : columns) {
+            System.out.printf("| %-15s", column.length() <= 15 ? column : column.substring(0, 14));
+          }
+          System.out.printf("|\n");
+        }
+        for (VectorWrapper vw : loader) {
+          Object o = vw.getValueVector().getAccessor().getObject(row);
+          if (o instanceof byte[]) {
+            String value = new String((byte[]) o);
+            System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0, 14));
+          } else {
+            String value = o.toString();
+            System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0,14));
+          }
+        }
+        System.out.printf("|\n");
+      }
+      count.addAndGet(rows);
+      if (result.getHeader().getIsLastChunk()) latch.countDown();
+      result.release();
+    }
+
+    public int await() throws Exception {
+      latch.await();
+      return count.get();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/35f43f56/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
index e1344f4..79ba5c9 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
@@ -2,6 +2,7 @@ package org.apache.drill.exec.store;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Charsets;
+import com.google.common.base.Stopwatch;
 import com.google.common.io.Files;
 import com.google.common.io.Resources;
 import mockit.Injectable;
@@ -36,6 +37,7 @@ import java.nio.charset.Charset;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static junit.framework.Assert.assertNull;
@@ -116,8 +118,10 @@ public class TestParquetPhysicalPlan {
     try(DrillClient client = new DrillClient(config);){
       client.connect();
       ParquetResultsListener listener = new ParquetResultsListener();
+      Stopwatch watch = new Stopwatch();
+      watch.start();
       client.runQuery(UserProtos.QueryType.PHYSICAL, Resources.toString(Resources.getResource(fileName),Charsets.UTF_8), listener);
-      System.out.println(String.format("Got %d total records.", listener.await()));
+      System.out.println(String.format("Got %d total records in %d seconds", listener.await(), watch.elapsed(TimeUnit.SECONDS)));
       client.close();
     }
   }


[12/18] git commit: Reorganize location of generated files to support other generated needs. (Clean up packages for outputs and change directory structure for inputs and inputs. Packages changes caused large set of changes to other files.)

Posted by ja...@apache.org.
Reorganize location of generated files to support other generated needs.  (Clean up packages for outputs and change directory structure for inputs and inputs.  Packages changes caused large set of changes to other files.)


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/e897fe3a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/e897fe3a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/e897fe3a

Branch: refs/heads/master
Commit: e897fe3acd4f5b9acbc283a16ac58fd6642967ce
Parents: cc329aa
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sat Aug 31 12:33:33 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Sep 1 15:16:22 2013 -0700

----------------------------------------------------------------------
 sandbox/prototype/exec/java-exec/pom.xml        |   6 +-
 .../src/main/codegen/ValueVectors/config.fmpp   |   3 -
 .../ValueVectors/data/ValueVectorTypes.tdd      | 107 ------
 .../templates/FixedValueVectors.java            | 300 ----------------
 .../templates/NullableValueVectors.java         | 340 ------------------
 .../templates/RepeatedValueVectors.java         | 341 -------------------
 .../ValueVectors/templates/TypeHelper.java      | 117 -------
 .../ValueVectors/templates/ValueHolders.java    |  76 -----
 .../templates/VariableLengthVectors.java        | 341 -------------------
 .../exec/java-exec/src/main/codegen/config.fmpp |   8 +
 .../src/main/codegen/data/ValueVectorTypes.tdd  | 107 ++++++
 .../src/main/codegen/includes/license.ftl       |  18 +
 .../src/main/codegen/includes/vv_imports.ftl    |  36 ++
 .../codegen/templates/FixedValueVectors.java    | 288 ++++++++++++++++
 .../codegen/templates/NullableValueVectors.java | 317 +++++++++++++++++
 .../codegen/templates/RepeatedValueVectors.java | 320 +++++++++++++++++
 .../src/main/codegen/templates/TypeHelper.java  |  99 ++++++
 .../main/codegen/templates/ValueHolders.java    |  75 ++++
 .../templates/VariableLengthVectors.java        | 325 ++++++++++++++++++
 .../apache/drill/exec/expr/CodeGenerator.java   |   1 -
 .../exec/expr/fn/agg/impl/CountFunctions.java   |   4 +-
 .../exec/expr/fn/agg/impl/SumFunctions.java     |   4 +-
 .../drill/exec/expr/fn/impl/Alternator.java     |   2 +-
 .../drill/exec/expr/fn/impl/ByteSubstring.java  |  10 +-
 .../drill/exec/expr/fn/impl/CharSubstring.java  |   9 +-
 .../exec/expr/fn/impl/ComparatorFunctions.java  |  14 +-
 .../exec/expr/fn/impl/ComparisonFunctions.java  |  14 +-
 .../fn/impl/ComparisonFunctionsNullable.java    |   6 +-
 .../apache/drill/exec/expr/fn/impl/Hash.java    |   9 +-
 .../drill/exec/expr/fn/impl/IsNotNull.java      |   4 +-
 .../apache/drill/exec/expr/fn/impl/IsNull.java  |   4 +-
 .../drill/exec/expr/fn/impl/MathFunctions.java  |   4 +-
 .../expr/fn/impl/SimpleRepeatedFunctions.java   |  11 +-
 .../drill/exec/expr/fn/impl/VarHelpers.java     |   8 +-
 .../exec/physical/impl/aggregate/AggBatch.java  |   4 +-
 .../exec/physical/impl/join/MergeJoinBatch.java |  21 +-
 .../partitionsender/OutgoingRecordBatch.java    |   2 +-
 .../PartitionSenderRootExec.java                |  24 +-
 .../impl/project/ProjectRecordBatch.java        |   2 +-
 .../impl/svremover/RemovingRecordBatch.java     |   2 +-
 .../drill/exec/record/MaterializedField.java    |   6 +-
 .../drill/exec/record/RecordBatchLoader.java    |   4 +-
 .../drill/exec/store/json/JSONRecordReader.java |   8 +-
 .../drill/exec/store/mock/MockGroupScanPOP.java |   7 +-
 .../drill/exec/store/mock/MockRecordReader.java |   2 +-
 .../drill/exec/store/mock/MockSubScanPOP.java   |  21 +-
 .../exec/store/parquet/ParquetRecordReader.java |   2 +-
 .../org/apache/drill/exec/vector/BitVector.java |   2 +
 .../drill/exec/vector/ValueHolderHelper.java    |   2 +
 .../exec/physical/impl/TestOptiqPlans.java      |   2 +-
 .../exec/physical/impl/TestSimpleFunctions.java |  15 +-
 .../exec/record/vector/TestValueVector.java     |   2 +-
 52 files changed, 1721 insertions(+), 1735 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/pom.xml b/sandbox/prototype/exec/java-exec/pom.xml
index 781a5da..bd29966 100644
--- a/sandbox/prototype/exec/java-exec/pom.xml
+++ b/sandbox/prototype/exec/java-exec/pom.xml
@@ -294,9 +294,9 @@
         <artifactId>fmpp-maven-plugin</artifactId>
         <version>1.0</version>
         <configuration>
-          <cfgFile>src/main/codegen/ValueVectors/config.fmpp</cfgFile>
-          <outputDirectory>target/generated-sources/org/apache/drill/exec/vector</outputDirectory>
-          <templateDirectory>src/main/codegen/ValueVectors/templates</templateDirectory>
+          <cfgFile>src/main/codegen/config.fmpp</cfgFile>
+          <outputDirectory>target/generated-sources</outputDirectory>
+          <templateDirectory>src/main/codegen/templates</templateDirectory>
         </configuration>
         <executions>
           <execution>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/config.fmpp
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/config.fmpp b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/config.fmpp
deleted file mode 100644
index da05e2d..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/config.fmpp
+++ /dev/null
@@ -1,3 +0,0 @@
-data: {
-    tdd(../data/ValueVectorTypes.tdd)
-}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/data/ValueVectorTypes.tdd
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/data/ValueVectorTypes.tdd b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/data/ValueVectorTypes.tdd
deleted file mode 100644
index d201671..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/data/ValueVectorTypes.tdd
+++ /dev/null
@@ -1,107 +0,0 @@
-{
-  modes: [
-    {name: "Optional", prefix: "Nullable"},
-    {name: "Required", prefix: ""},
-    {name: "Repeated", prefix: "Repeated"}
-    ],
-  types: [
-    {
-      major: "Fixed",
-      width: 1,
-      javaType: "byte",
-      boxedType: "Byte",
-      minor: [
-        { class: "TinyInt", valueHolder: "IntHolder"},
-        { class: "UInt1", valueHolder: "UInt1Holder"}
-      ]
-    },
-    {
-      major: "Fixed",
-      width: 2,
-      javaType: "char",
-      boxedType: "Character",
-      minor: [
-        { class: "UInt2", valueHolder: "UInt2Holder"}
-      ]
-    },    {
-      major: "Fixed",
-      width: 2,
-      javaType: "short",
-      boxedType: "Short",
-      minor: [
-        { class: "SmallInt", valueHolder: "Int2Holder"},
-      ]
-    },
-    {
-      major: "Fixed",
-      width: 4,
-      javaType: "int",
-      boxedType: "Integer",
-      minor: [
-        { class: "Int", valueHolder: "IntHolder" },
-        { class: "UInt4", valueHolder: "UInt4Holder" },
-        { class: "Float4", javaType: "float" },
-        <#-- 
-        { class: "Decimal4", maxPrecisionDigits: 8, scale: 4, javaType: "float"},
-        { class: "Date" }
-        -->
-      ]
-    },
-    {
-      major: "Fixed",
-      width: 8,
-      javaType: "long",
-      boxedType: "Long",
-      minor: [
-        { class: "BigInt" },
-        { class: "UInt8" },
-        { class: "Float8", javaType: "double" , boxedType: "Double"},
-        <#-- 
-        { class: "Decimal8", maxPrecisionDigits: 18, scale: 4, javaType: "double", boxedType: "Double"},
-        { class: "Money", maxPrecisionDigits: 2, scale: 1, }, 
-        { class: "Time" },
-        { class: "TimeStamp" },
-        { class: "DateTime" }
-        -->
-      ]
-    },
-    <#--
-    {
-      major: "Fixed",
-      width: 12,
-      javaType: "ByteBuf",
-      minor: [
-        { class: "Decimal12", maxPrecisionDigits: 28, scale: 5},
-        { class: "TimeTZ" },
-        { class: "Interval" }
-      ]
-    },
-    {
-      major: "Fixed",
-      width: 16,
-      javaType: "ByteBuf",
-      minor: [
-        { class: "Decimal16", maxPrecisionDigits: 37, scale: 6}
-      ]
-    },
-    -->
-    {
-      major: "VarLen",
-      width: 4,
-      javaType: "int",
-      minor: [
-        { class: "VarBinary" },
-        { class: "VarChar" }
-        { class: "Var16Char" }
-      ]
-    },
-    {
-      major: "Bit",
-      width: 1,
-      javaType: "int",
-      minor: [
-        { class: "Bit" }
-      ]
-    }
-  ]
-}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/FixedValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/FixedValueVectors.java
deleted file mode 100644
index 2eeb9ed..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/FixedValueVectors.java
+++ /dev/null
@@ -1,300 +0,0 @@
-<@pp.dropOutputFile />
-<#list types as type>
-<#list type.minor as minor>
-
-<#if type.major == "Fixed">
-<@pp.changeOutputFile name="${minor.class}Vector.java" />
-package org.apache.drill.exec.vector;
-
-import io.netty.buffer.ByteBuf;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.TransferPair;
-import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
-import org.apache.drill.exec.record.DeadBuf;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.vector.BaseValueVector;
-import org.apache.drill.exec.vector.BitVector;
-import org.apache.drill.common.expression.FieldReference;
-
-import java.io.ObjectInputStream.GetField;
-import java.util.Arrays;
-import java.util.Random;
-
-
-/**
- * ${minor.class} implements a vector of fixed width values.  Elements in the vector are accessed
- * by position, starting from the logical start of the vector.  Values should be pushed onto the
- * vector sequentially, but may be randomly accessed.
- *   The width of each element is ${type.width} byte(s)
- *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
- *
- * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
- */
-@SuppressWarnings("unused")
-public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector{
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
-
- 
-  private final Accessor accessor = new Accessor();
-  private final Mutator mutator = new Mutator();
-  
-  public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
-    super(field, allocator);
-  }
-
-  public int getValueCapacity(){
-    return (int) (data.capacity() *1.0 / ${type.width});
-  }
-
-  public Accessor getAccessor(){
-    return accessor;
-  }
-  
-  public Mutator getMutator(){
-    return mutator;
-  }
-  
-  
-
-  /**
-   * Allocate a new buffer that supports setting at least the provided number of values.  May actually be sized bigger depending on underlying buffer rounding size. Must be called prior to using the ValueVector.
-   * @param valueCount
-   */
-  public void allocateNew(int valueCount) {
-    clear();
-    this.data = allocator.buffer(valueCount * ${type.width});
-    this.data.readerIndex(0);
-  }
-  
-  @Override
-  public FieldMetadata getMetadata() {
-    return FieldMetadata.newBuilder()
-             .setDef(getField().getDef())
-             .setValueCount(valueCount)
-             .setBufferLength(valueCount * ${type.width})
-             .build();
-  }
-
-  @Override
-  public int load(int valueCount, ByteBuf buf){
-    clear();
-    this.valueCount = valueCount;
-    int len = valueCount * ${type.width};
-    data = buf.slice(0, len);
-    data.retain();
-    return len;
-  }
-  
-  @Override
-  public void load(FieldMetadata metadata, ByteBuf buffer) {
-    assert this.field.getDef().equals(metadata.getDef());
-    int loaded = load(metadata.getValueCount(), buffer);
-    assert metadata.getBufferLength() == loaded;
-  }
-  
-  public TransferPair getTransferPair(){
-    return new TransferImpl(getField());
-  }
-  public TransferPair getTransferPair(FieldReference ref){
-    return new TransferImpl(getField().clone(ref));
-  }
-  
-  public void transferTo(${minor.class}Vector target){
-    target.data = data;
-    target.data.retain();
-    target.valueCount = valueCount;
-    clear();
-  }
-  
-  private class TransferImpl implements TransferPair{
-    ${minor.class}Vector to;
-    
-    public TransferImpl(MaterializedField field){
-      this.to = new ${minor.class}Vector(field, allocator);
-    }
-    
-    public ${minor.class}Vector getTo(){
-      return to;
-    }
-    
-    public void transfer(){
-      transferTo(to);
-    }
-    
-    @Override
-    public void copyValue(int fromIndex, int toIndex) {
-      to.copyFrom(fromIndex, toIndex, ${minor.class}Vector.this);
-    }
-  }
-  
-  public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
-    <#if (type.width > 8)>
-    data.getBytes(fromIndex * ${type.width}, from.data, thisIndex * ${type.width}, ${type.width});
-    <#else> <#-- type.width <= 8 -->
-    data.set${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width}, 
-        from.data.get${(minor.javaType!type.javaType)?cap_first}(fromIndex * ${type.width})
-    );
-    </#if> <#-- type.width -->
-  }
-  
-  public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
-    if(thisIndex >= getValueCapacity()) return false;
-    copyFrom(fromIndex, thisIndex, from);
-    return true;
-  }
-
-  public final class Accessor extends BaseValueVector.BaseAccessor{
-
-    public int getValueCount() {
-      return valueCount;
-    }
-    
-    <#if (type.width > 8)>
-
-    public ${minor.javaType!type.javaType} get(int index) {
-      ByteBuf dst = allocator.buffer(${type.width});
-      data.getBytes(index * ${type.width}, dst, 0, ${type.width});
-      return dst;
-    }
-
-    public void get(int index, ${minor.class}Holder holder){
-      holder.buffer = data;
-      holder.start = index * ${type.width};
-    }
-    
-    void get(int index, Nullable${minor.class}Holder holder){
-      holder.buffer = data;
-      holder.start = index * ${type.width};
-    }
-
-    @Override
-    public Object getObject(int index) {
-      ByteBuf dst = allocator.buffer(${type.width});
-      data.getBytes(index, dst, 0, ${type.width});
-      return dst;
-    }
-
-    <#else> <#-- type.width <= 8 -->
-
-    public ${minor.javaType!type.javaType} get(int index) {
-      return data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
-    }
-
-    public Object getObject(int index) {
-      return get(index);
-    }
-    
-    public void get(int index, ${minor.class}Holder holder){
-      holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
-    }
-
-    void get(int index, Nullable${minor.class}Holder holder){
-      holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
-    }
-
-
-   </#if> <#-- type.width -->
- }
- 
- /**
-  * ${minor.class}.Mutator implements a mutable vector of fixed width values.  Elements in the
-  * vector are accessed by position from the logical start of the vector.  Values should be pushed
-  * onto the vector sequentially, but may be randomly accessed.
-  *   The width of each element is ${type.width} byte(s)
-  *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
-  *
-  * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
-  */
-  public final class Mutator extends BaseValueVector.BaseMutator{
-
-    private Mutator(){};
-   /**
-    * Set the element at the given index to the given value.  Note that widths smaller than
-    * 32 bits are handled by the ByteBuf interface.
-    *
-    * @param index   position of the bit to set
-    * @param value   value to set
-    */
-  <#if (type.width > 8)>
-   public void set(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
-     data.setBytes(index * ${type.width}, value);
-   }
-   
-   public void set(int index, ${minor.class}Holder holder){
-     data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width});
-   }
-   
-   public boolean setSafe(int index, ${minor.class}Holder holder){
-     if(index >= getValueCapacity()) return false;
-     set(index, holder);
-     return true;
-   }
-
-   void set(int index, Nullable${minor.class}Holder holder){
-     data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width});
-   }
-   
-   @Override
-   public void generateTestData() {
-     setValueCount(getValueCapacity());
-     boolean even = true;
-     for(int i =0; i < valueCount; i++, even = !even){
-       byte b = even ? Byte.MIN_VALUE : Byte.MAX_VALUE;
-       for(int w = 0; w < ${type.width}; w++){
-         data.setByte(i + w, b);
-       }
-     }
-   }
-   
-  <#else> <#-- type.width <= 8 -->
-   public void set(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
-     data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, value);
-   }
-   
-   public boolean setSafe(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
-     if(index >= getValueCapacity()) return false;
-     set(index, value);
-     return true;
-   }
-
-   public void set(int index, ${minor.class}Holder holder){
-     data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value);
-   }
-
-   void set(int index, Nullable${minor.class}Holder holder){
-     data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value);
-   }
-
-   @Override
-   public void generateTestData() {
-     setValueCount(getValueCapacity());
-     boolean even = true;
-     for(int i =0; i < valueCount; i++, even = !even){
-       if(even){
-         set(i, ${minor.boxedType!type.boxedType}.MIN_VALUE);
-       }else{
-         set(i, ${minor.boxedType!type.boxedType}.MAX_VALUE);
-       }
-     }
-   }
-
-  </#if> <#-- type.width -->
-  
-
-  
-   public void setValueCount(int valueCount) {
-     ${minor.class}Vector.this.valueCount = valueCount;
-     data.writerIndex(${type.width} * valueCount);
-   }
-
-
-
-
-  
- }
-}
-
-</#if> <#-- type.major -->
-</#list>
-</#list>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
deleted file mode 100644
index a543131..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
+++ /dev/null
@@ -1,340 +0,0 @@
-<@pp.dropOutputFile />
-<#list types as type>
-<#list type.minor as minor>
-
-<#assign className = "Nullable${minor.class}Vector" />
-<#assign valuesName = "${minor.class}Vector" />
-<@pp.changeOutputFile name="${className}.java" />
-
-package org.apache.drill.exec.vector;
-
-
-
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-import io.netty.buffer.ByteBuf;
-
-import java.io.Closeable;
-import java.util.List;
-
-import org.apache.commons.lang3.ArrayUtils;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.proto.SchemaDefProtos;
-import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
-import org.apache.drill.exec.record.DeadBuf;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.TransferPair;
-import org.apache.drill.exec.vector.BaseValueVector;
-import org.apache.drill.exec.vector.BitVector;
-import org.apache.drill.exec.vector.UInt2Vector;
-import org.apache.drill.exec.vector.UInt4Vector;
-import org.apache.drill.common.expression.FieldReference;
-
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.ObjectArrays;
-
-/**
- * Nullable${minor.class} implements a vector of values which could be null.  Elements in the vector
- * are first checked against a fixed length vector of boolean values.  Then the element is retrieved
- * from the base class (if not null).
- *
- * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
- */
-@SuppressWarnings("unused")
-public final class ${className} extends BaseValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector{
-
-  private int valueCount;
-  final BitVector bits;
-  final ${valuesName} values;
-  private final Accessor accessor = new Accessor();
-  private final Mutator mutator = new Mutator();
-
-  public ${className}(MaterializedField field, BufferAllocator allocator) {
-    super(field, allocator);
-    this.bits = new BitVector(null, allocator);
-    this.values = new ${minor.class}Vector(null, allocator);
-  }
-  
-  public int getValueCapacity(){
-    return bits.getValueCapacity();
-  }
-  
-  @Override
-  public ByteBuf[] getBuffers() {
-    ByteBuf[] buffers = ObjectArrays.concat(bits.getBuffers(), values.getBuffers(), ByteBuf.class);
-    clear();
-    return buffers;
-  }
-  
-  @Override
-  public void clear() {
-    valueCount = 0;
-    bits.clear();
-    values.clear();
-  }
-  
-  public int getBufferSize(){
-    return values.getBufferSize() + bits.getBufferSize();
-  }
-
-  <#if type.major == "VarLen">
-  @Override
-  public FieldMetadata getMetadata() {
-    return FieldMetadata.newBuilder()
-             .setDef(getField().getDef())
-             .setValueCount(valueCount)
-             .setVarByteLength(values.getVarByteLength())
-             .setBufferLength(getBufferSize())
-             .build();
-  }
-
-  @Override
-  public void allocateNew(int totalBytes, int valueCount) {
-    values.allocateNew(totalBytes, valueCount);
-    bits.allocateNew(valueCount);
-    mutator.reset();
-    accessor.reset();
-  }
-
-  @Override
-  public int load(int dataBytes, int valueCount, ByteBuf buf){
-    clear();
-    this.valueCount = valueCount;
-    int loaded = bits.load(valueCount, buf);
-    
-    // remove bits part of buffer.
-    buf = buf.slice(loaded, buf.capacity() - loaded);
-    loaded += values.load(dataBytes, valueCount, buf);
-    return loaded;
-  }
-  
-  @Override
-  public void load(FieldMetadata metadata, ByteBuf buffer) {
-    assert this.field.getDef().equals(metadata.getDef());
-    int loaded = load(metadata.getVarByteLength(), metadata.getValueCount(), buffer);
-    assert metadata.getBufferLength() == loaded;
-  }
-  
-  @Override
-  public int getByteCapacity(){
-    return values.getByteCapacity();
-  }
-
-  <#else>
-  @Override
-  public FieldMetadata getMetadata() {
-    return FieldMetadata.newBuilder()
-             .setDef(getField().getDef())
-             .setValueCount(valueCount)
-             .setBufferLength(getBufferSize())
-             .build();
-  }
-  
-  @Override
-  public void allocateNew(int valueCount) {
-    values.allocateNew(valueCount);
-    bits.allocateNew(valueCount);
-    mutator.reset();
-    accessor.reset();
-  }
-  
-  @Override
-  public int load(int valueCount, ByteBuf buf){
-    clear();
-    this.valueCount = valueCount;
-    int loaded = bits.load(valueCount, buf);
-    
-    // remove bits part of buffer.
-    buf = buf.slice(loaded, buf.capacity() - loaded);
-    loaded += values.load(valueCount, buf);
-    return loaded;
-  }
-  
-  @Override
-  public void load(FieldMetadata metadata, ByteBuf buffer) {
-    assert this.field.getDef().equals(metadata.getDef());
-    int loaded = load(metadata.getValueCount(), buffer);
-    assert metadata.getBufferLength() == loaded;
-  }
-  
-  </#if>
-  
-  public TransferPair getTransferPair(){
-    return new TransferImpl(getField());
-  }
-  public TransferPair getTransferPair(FieldReference ref){
-    return new TransferImpl(getField().clone(ref));
-  }
-
-  
-  public void transferTo(Nullable${minor.class}Vector target){
-    bits.transferTo(target.bits);
-    values.transferTo(target.values);
-    target.valueCount = valueCount;
-    clear();
-  }
-  
-  private class TransferImpl implements TransferPair{
-    Nullable${minor.class}Vector to;
-    
-    public TransferImpl(MaterializedField field){
-      this.to = new Nullable${minor.class}Vector(field, allocator);
-    }
-    
-    public Nullable${minor.class}Vector getTo(){
-      return to;
-    }
-    
-    public void transfer(){
-      transferTo(to);
-    }
-    
-    @Override
-    public void copyValue(int fromIndex, int toIndex) {
-      to.copyFrom(fromIndex, toIndex, Nullable${minor.class}Vector.this);
-    }
-  }
-  
-  public Accessor getAccessor(){
-    return accessor;
-  }
-  
-  public Mutator getMutator(){
-    return mutator;
-  }
-  
-  public ${minor.class}Vector convertToRequiredVector(){
-    ${minor.class}Vector v = new ${minor.class}Vector(getField().getOtherNullableVersion(), allocator);
-    v.data = values.data;
-    v.valueCount = this.valueCount;
-    v.data.retain();
-    clear();
-    return v;
-  }
-
-  
-  public void copyFrom(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){
-    bits.copyFrom(fromIndex, thisIndex, from.bits);
-    values.copyFrom(fromIndex, thisIndex, from.values);
-  }
-  
-  public boolean copyFromSafe(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){
-    return bits.copyFromSafe(fromIndex, thisIndex, from.bits) && values.copyFromSafe(fromIndex, thisIndex, from.values);
-  }
-
-  
-  public final class Accessor implements ValueVector.Accessor{
-
-    /**
-     * Get the element at the specified position.
-     *
-     * @param   index   position of the value
-     * @return  value of the element, if not null
-     * @throws  NullValueException if the value is null
-     */
-    public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
-      assert !isNull(index);
-      return values.getAccessor().get(index);
-    }
-
-    public boolean isNull(int index) {
-      return isSet(index) == 0;
-    }
-
-    public int isSet(int index){
-      return bits.getAccessor().get(index);
-    }
-    
-    public void get(int index, Nullable${minor.class}Holder holder){
-      holder.isSet = bits.getAccessor().get(index);
-      values.getAccessor().get(index, holder);
-    }
-    
-    @Override
-    public Object getObject(int index) {
-      return isNull(index) ? null : values.getAccessor().getObject(index);
-    }
-    
-    public int getValueCount(){
-      return valueCount;
-    }
-    
-    public void reset(){}
-  }
-  
-  public final class Mutator implements ValueVector.Mutator{
-    
-    private int setCount;
-    
-    private Mutator(){
-    }
-
-    /**
-     * Set the variable length element at the specified index to the supplied byte array.
-     *
-     * @param index   position of the bit to set
-     * @param bytes   array of bytes to write
-     */
-    public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
-      setCount++;
-      bits.getMutator().set(index, 1);
-      values.getMutator().set(index, value);
-    }
-    
-    public void setSkipNull(int index, ${minor.class}Holder holder){
-      values.getMutator().set(index, holder);
-    }
-
-    public void setSkipNull(int index, Nullable${minor.class}Holder holder){
-      values.getMutator().set(index, holder);
-    }
-    
-    public void set(int index, Nullable${minor.class}Holder holder){
-      bits.getMutator().set(index, holder.isSet);
-      values.getMutator().set(index, holder);
-    }
-
-    public void set(int index, ${minor.class}Holder holder){
-      bits.getMutator().set(index, 1);
-      values.getMutator().set(index, holder);
-    }
-    
-    public boolean setSafe(int index, <#if type.major == "VarLen">Nullable${minor.class}Holder <#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value){
-      boolean b1 = bits.getMutator().setSafe(index, 1);
-      boolean b2 = values.getMutator().setSafe(index, value);
-      if(b1 && b2){
-        setCount++;
-        return true;
-      }else{
-        return false;
-      }
-
-    }
-
-    
-    public void setValueCount(int valueCount) {
-      assert valueCount >= 0;
-      Nullable${minor.class}Vector.this.valueCount = valueCount;
-      values.getMutator().setValueCount(valueCount);
-      bits.getMutator().setValueCount(valueCount);
-    }
-    
-    public boolean noNulls(){
-      return valueCount == setCount;
-    }
-    
-    public void generateTestData(){
-      bits.getMutator().generateTestData();
-      values.getMutator().generateTestData();
-    }
-    
-    public void reset(){
-      setCount = 0;
-    }
-    
-  }
-}
-</#list>
-</#list>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
deleted file mode 100644
index 907a3b1..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
+++ /dev/null
@@ -1,341 +0,0 @@
-<@pp.dropOutputFile />
-<#list types as type>
-<#list type.minor as minor>
-<@pp.changeOutputFile name="Repeated${minor.class}Vector.java" />
-package org.apache.drill.exec.vector;
-
-import org.apache.drill.exec.vector.UInt2Vector;
-import org.apache.drill.exec.vector.UInt4Vector;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-import io.netty.buffer.ByteBuf;
-
-import java.io.Closeable;
-import java.util.Random;
-import java.util.Vector;
-
-import org.apache.commons.lang3.ArrayUtils;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.proto.SchemaDefProtos;
-import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
-import org.apache.drill.exec.record.DeadBuf;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.TransferPair;
-import org.apache.drill.common.expression.FieldReference;
-
-
-import java.util.List;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.ObjectArrays;
-
-@SuppressWarnings("unused")
-/**
- * Repeated${minor.class} implements a vector with multple values per row (e.g. JSON array or
- * repeated protobuf field).  The implementation uses two additional value vectors; one to convert
- * the index offset to the underlying element offset, and another to store the number of values
- * in the vector.
- *
- * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
- */
-
- public final class Repeated${minor.class}Vector extends BaseValueVector implements Repeated<#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector {
-
-  private MaterializedField field;
-  
-  private int parentValueCount;
-  private int childValueCount;
-  
-  private final UInt4Vector offsets;   // offsets to start of each record
-  private final ${minor.class}Vector values;
-  private final Mutator mutator = new Mutator();
-  private final Accessor accessor = new Accessor();
-  
-  
-  public Repeated${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
-    super(field, allocator);
-    this.offsets = new UInt4Vector(null, allocator);
-    this.values = new ${minor.class}Vector(null, allocator);
-  }
-
-  public int getValueCapacity(){
-    return values.getValueCapacity();
-  }
-  
-  public int getBufferSize(){
-    return offsets.getBufferSize() + values.getBufferSize();
-  }
-  
-  public TransferPair getTransferPair(){
-    return new TransferImpl(getField());
-  }
-  public TransferPair getTransferPair(FieldReference ref){
-    return new TransferImpl(getField().clone(ref));
-  }
-  
-  public void transferTo(Repeated${minor.class}Vector target){
-    offsets.transferTo(target.offsets);
-    values.transferTo(target.values);
-    target.parentValueCount = parentValueCount;
-    target.childValueCount = childValueCount;
-    clear();
-  }
-  
-  private class TransferImpl implements TransferPair{
-    Repeated${minor.class}Vector to;
-    
-    public TransferImpl(MaterializedField field){
-      this.to = new Repeated${minor.class}Vector(field, allocator);
-    }
-    
-    public Repeated${minor.class}Vector getTo(){
-      return to;
-    }
-    
-    public void transfer(){
-      transferTo(to);
-    }
-    
-    @Override
-    public void copyValue(int fromIndex, int toIndex) {
-      to.copyFrom(fromIndex, toIndex, Repeated${minor.class}Vector.this);
-    }
-  }
-  
-  public void copyFrom(int inIndex, int outIndex, Repeated${minor.class}Vector v){
-    throw new UnsupportedOperationException();
-  }
-  
-  public boolean copyFromSafe(int inIndex, int outIndex, Repeated${minor.class}Vector v){
-    throw new UnsupportedOperationException();
-  }
-  
-  <#if type.major == "VarLen">
-  @Override
-  public FieldMetadata getMetadata() {
-    return FieldMetadata.newBuilder()
-             .setDef(getField().getDef())
-             .setGroupCount(this.parentValueCount)
-             .setValueCount(this.childValueCount)
-             .setVarByteLength(values.getVarByteLength())
-             .setBufferLength(getBufferSize())
-             .build();
-  }
-  
-  public void allocateNew(int totalBytes, int parentValueCount, int childValueCount) {
-    offsets.allocateNew(parentValueCount+1);
-    values.allocateNew(totalBytes, childValueCount);
-    mutator.reset();
-    accessor.reset();
-  }
-  
-  @Override
-  public int load(int dataBytes, int parentValueCount, int childValueCount, ByteBuf buf){
-    clear();
-    this.parentValueCount = parentValueCount;
-    this.childValueCount = childValueCount;
-    int loaded = 0;
-    loaded += offsets.load(parentValueCount+1, buf.slice(loaded, buf.capacity() - loaded));
-    loaded += values.load(dataBytes, childValueCount, buf.slice(loaded, buf.capacity() - loaded));
-    return loaded;
-  }
-  
-  @Override
-  public void load(FieldMetadata metadata, ByteBuf buffer) {
-    assert this.field.getDef().equals(metadata.getDef());
-    int loaded = load(metadata.getVarByteLength(), metadata.getGroupCount(), metadata.getValueCount(), buffer);
-    assert metadata.getBufferLength() == loaded;
-  }
-  
-  public int getByteCapacity(){
-    return values.getByteCapacity();
-  }
-
-  <#else>
-  
-  @Override
-  public FieldMetadata getMetadata() {
-    return FieldMetadata.newBuilder()
-             .setDef(getField().getDef())
-             .setGroupCount(this.parentValueCount)
-             .setValueCount(this.childValueCount)
-             .setBufferLength(getBufferSize())
-             .build();
-  }
-  
-  public void allocateNew(int parentValueCount, int childValueCount) {
-    clear();
-    offsets.allocateNew(parentValueCount+1);
-    values.allocateNew(childValueCount);
-    mutator.reset();
-    accessor.reset();
-  }
-  
-  public int load(int parentValueCount, int childValueCount, ByteBuf buf){
-    clear();
-    this.parentValueCount = parentValueCount;
-    this.childValueCount = childValueCount;
-    int loaded = 0;
-    loaded += offsets.load(parentValueCount+1, buf.slice(loaded, buf.capacity() - loaded));
-    loaded += values.load(childValueCount, buf.slice(loaded, buf.capacity() - loaded));
-    return loaded;
-  }
-  
-  @Override
-  public void load(FieldMetadata metadata, ByteBuf buffer) {
-    assert this.field.getDef().equals(metadata.getDef());
-    int loaded = load(metadata.getGroupCount(), metadata.getValueCount(), buffer);
-    assert metadata.getBufferLength() == loaded;
-  }
-  </#if>
-
-  @Override
-  public ByteBuf[] getBuffers() {
-    ByteBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(), values.getBuffers(), ByteBuf.class);
-    clear();
-    return buffers;
-  }
-
-  public void clear(){
-    offsets.clear();
-    values.clear();
-    parentValueCount = 0;
-    childValueCount = 0;
-  }
-
-  public Mutator getMutator(){
-    return mutator;
-  }
-  
-  public Accessor getAccessor(){
-    return accessor;
-  }
-  
-  public final class Accessor implements ValueVector.Accessor{
-    /**
-     * Get the elements at the given index.
-     */
-    public int getCount(int index) {
-      return offsets.getAccessor().get(index+1) - offsets.getAccessor().get(index);
-    }
-    
-    public Object getObject(int index) {
-      List<Object> vals = Lists.newArrayList();
-      int start = offsets.getAccessor().get(index);
-      int end = offsets.getAccessor().get(index+1);
-      for(int i = start; i < end; i++){
-        vals.add(values.getAccessor().getObject(i));
-      }
-      return vals;
-    }
-
-    /**
-     * Get a value for the given record.  Each element in the repeated field is accessed by
-     * the positionIndex param.
-     *
-     * @param  index           record containing the repeated field
-     * @param  positionIndex   position within the repeated field
-     * @return element at the given position in the given record
-     */
-    public <#if type.major == "VarLen">byte[]
-           <#else>${minor.javaType!type.javaType}
-           </#if> get(int index, int positionIndex) {
-      return values.getAccessor().get(offsets.getAccessor().get(index) + positionIndex);
-    }
-           
-    public void get(int index, Repeated${minor.class}Holder holder){
-      holder.start = offsets.getAccessor().get(index);
-      holder.end =  offsets.getAccessor().get(index+1);
-      holder.vector = values;
-    }
-
-    public MaterializedField getField() {
-      return field;
-    }
-    
-    public int getGroupCount(){
-      return parentValueCount;
-    }
-    
-    public int getValueCount(){
-      return childValueCount;
-    }
-    
-    public void reset(){
-      
-    }
-  }
-  
-  public final class Mutator implements RepeatedMutator {
-
-    
-    private Mutator(){
-    }
-
-    public void startNewGroup(int index) {
-      offsets.getMutator().set(index+1, offsets.getAccessor().get(index));
-    }
-
-    /**
-     * Add an element to the given record index.  This is similar to the set() method in other
-     * value vectors, except that it permits setting multiple values for a single record.
-     *
-     * @param index   record of the element to add
-     * @param value   value to add to the given row
-     */
-    public void add(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
-      int nextOffset = offsets.getAccessor().get(index+1);
-      values.getMutator().set(nextOffset, value);
-      offsets.getMutator().set(index+1, nextOffset+1);
-    }
-
-    public void add(int index, ${minor.class}Holder holder){
-      int nextOffset = offsets.getAccessor().get(index+1);
-      values.getMutator().set(nextOffset, holder);
-      offsets.getMutator().set(index+1, nextOffset+1);
-    }
-    
-    public void add(int index, Repeated${minor.class}Holder holder){
-      
-      ${minor.class}Vector.Accessor accessor = holder.vector.getAccessor();
-      ${minor.class}Holder innerHolder = new ${minor.class}Holder();
-      
-      for(int i = holder.start; i < holder.end; i++){
-        accessor.get(i, innerHolder);
-        add(index, innerHolder);
-      }
-    }
-    
-    /**
-     * Set the number of value groups in this repeated field.
-     * @param groupCount Count of Value Groups.
-     */
-    public void setValueCount(int groupCount) {
-      parentValueCount = groupCount;
-      childValueCount = offsets.getAccessor().get(groupCount);
-      offsets.getMutator().setValueCount(groupCount+1);
-      values.getMutator().setValueCount(childValueCount);
-    }
-    
-    public void generateTestData(){
-      setValueCount(offsets.getAccessor().getValueCount() - 1);
-      int valCount = offsets.getValueCapacity();
-      int[] sizes = {1,2,0,6};
-      int size = 0;
-      int runningOffset = 0;
-      for(int i =1; i < valCount; i++, size++){
-        runningOffset += sizes[size % sizes.length];
-        offsets.getMutator().set(i, runningOffset);  
-      }
-      values.getMutator().generateTestData();
-    }
-    
-    public void reset(){
-      
-    }
-    
-  }
-}
-</#list>
-</#list>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/TypeHelper.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/TypeHelper.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/TypeHelper.java
deleted file mode 100644
index ecb5551..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/TypeHelper.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.vector;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.exec.record.MaterializedField;
-
-import com.sun.codemodel.JType;
-import com.sun.codemodel.JCodeModel;
-
-public class TypeHelper {
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TypeHelper.class);
-
-  private static final int WIDTH_ESTIMATE = 50;
-
-  public static int getSize(MajorType major) {
-    switch (major.getMinorType()) {
-<#list types as type>
-  <#list type.minor as minor>
-    case ${minor.class?upper_case}:
-      return ${type.width}<#if minor.class?substring(0, 3) == "Var" ||
-                               minor.class?substring(0, 3) == "PRO" ||
-                               minor.class?substring(0, 3) == "MSG"> + WIDTH_ESTIMATE</#if>;
-  </#list>
-</#list>
-      case FIXEDCHAR: return major.getWidth();
-      case FIXED16CHAR: return major.getWidth();
-      case FIXEDBINARY: return major.getWidth();
-    }
-    throw new UnsupportedOperationException();
-  }
-
-  public static Class<?> getValueVectorClass(MinorType type, DataMode mode){
-    switch (type) {
-<#list types as type>
-  <#list type.minor as minor>
-      case ${minor.class?upper_case}:
-        switch (mode) {
-          case REQUIRED:
-            return ${minor.class}Vector.class;
-          case OPTIONAL:
-            return Nullable${minor.class}Vector.class;
-          case REPEATED:
-            return Repeated${minor.class}Vector.class;
-        }
-  </#list>
-</#list>
-    default:
-      break;
-    }
-    throw new UnsupportedOperationException();
-  }
-
-  public static JType getHolderType(JCodeModel model, MinorType type, DataMode mode){
-    switch (type) {
-<#list types as type>
-  <#list type.minor as minor>
-      case ${minor.class?upper_case}:
-        switch (mode) {
-          case REQUIRED:
-            return model._ref(${minor.class}Holder.class);
-          case OPTIONAL:
-            return model._ref(Nullable${minor.class}Holder.class);
-          case REPEATED:
-            return model._ref(Repeated${minor.class}Holder.class);
-        }
-  </#list>
-</#list>
-      default:
-        break;
-      }
-      throw new UnsupportedOperationException();
-  }
-
-  public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator){
-    MajorType type = field.getType();
-
-    switch (type.getMinorType()) {
-<#list types as type>
-  <#list type.minor as minor>
-    case ${minor.class?upper_case}:
-      switch (type.getMode()) {
-        case REQUIRED:
-          return new ${minor.class}Vector(field, allocator);
-        case OPTIONAL:
-          return new Nullable${minor.class}Vector(field, allocator);
-        case REPEATED:
-          return new Repeated${minor.class}Vector(field, allocator);
-      }
-  </#list>
-</#list>
-    default:
-      break;
-    }
-    // All ValueVector types have been handled.
-    throw new UnsupportedOperationException(type.getMinorType() + " type is not supported. Mode: " + type.getMode());
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/ValueHolders.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/ValueHolders.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/ValueHolders.java
deleted file mode 100644
index 18be0f3..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/ValueHolders.java
+++ /dev/null
@@ -1,76 +0,0 @@
-<@pp.dropOutputFile />
-<#list modes as mode>
-<#list types as type>
-<#list type.minor as minor>
-
-<#assign className="${mode.prefix}${minor.class}Holder" />
-<@pp.changeOutputFile name="${className}.java" />
-package org.apache.drill.exec.vector;
-
-import org.apache.drill.common.types.Types;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.exec.expr.holders.ValueHolder;
-import io.netty.buffer.ByteBuf;
-
-public final class ${className} implements ValueHolder{
-  
-  public static final MajorType TYPE = Types.${mode.name?lower_case}(MinorType.${minor.class?upper_case});
-
-    <#if mode.name != "Repeated">
-      
-    public static final int WIDTH = ${type.width};
-      <#if mode.name == "Optional">
-      /** Whether the given holder holds a valid value.  1 means non-null.  0 means null. **/
-      public int isSet;
-      </#if>
-      
-      <#if type.major != "VarLen">
-      
-      <#if (type.width > 8)>
-      public int start;
-      public ByteBuf buffer;
-      <#else>
-        public ${minor.javaType!type.javaType} value;
-      
-      </#if>
-      <#else>
-      /** The first offset (inclusive) into the buffer. **/
-      public int start;
-      
-      /** The last offset (exclusive) into the buffer. **/
-      public int end;
-      
-      /** The buffer holding actual values. **/
-      public ByteBuf buffer;
-
-      public String toString() {
-      <#if mode.name == "Optional">
-        if (isSet == 0)
-          return "<NULL>";
-      </#if>
-        byte[] buf = new byte[end-start];
-        buffer.getBytes(start, buf, 0, end-start);
-        return new String(buf);
-      }
-
-      </#if>
-
-    <#else> 
-    
-      /** The first index (inclusive) into the Vector. **/
-      public int start;
-      
-      /** The last index (exclusive) into the Vector. **/
-      public int end;
-      
-      /** The Vector holding the actual values. **/
-      public ${minor.class}Vector vector;
-    </#if>
-  
-    
-}
-
-</#list>
-</#list>
-</#list>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
deleted file mode 100644
index a63552a..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
+++ /dev/null
@@ -1,341 +0,0 @@
-<@pp.dropOutputFile />
-<#list types as type>
-<#list type.minor as minor>
-
-<#if type.major == "VarLen">
-<@pp.changeOutputFile name="${minor.class}Vector.java" />
-package org.apache.drill.exec.vector;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-import io.netty.buffer.ByteBuf;
-
-import java.io.Closeable;
-import java.nio.ByteBuffer;
-import java.util.Random;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.proto.SchemaDefProtos;
-import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
-import org.apache.drill.exec.record.DeadBuf;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.TransferPair;
-import org.apache.drill.common.expression.FieldReference;
-
-
-import com.google.common.base.Charsets;
-import com.google.common.collect.ObjectArrays;
-
-
-/**
- * ${minor.class}Vector implements a vector of variable width values.  Elements in the vector
- * are accessed by position from the logical start of the vector.  A fixed width offsetVector
- * is used to convert an element's position to it's offset from the start of the (0-based)
- * ByteBuf.  Size is inferred by adjacent elements.
- *   The width of each element is ${type.width} byte(s)
- *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
- *
- * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
- */
-@SuppressWarnings("unused")
-public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector{
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
-
-  private final UInt${type.width}Vector offsetVector;
-  private final Accessor accessor = new Accessor();
-  private final Mutator mutator = new Mutator();
-  
-  public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
-    super(field, allocator);
-    this.offsetVector = new UInt${type.width}Vector(null, allocator);
-  }
-
-  public int getBufferSize(){
-    return offsetVector.getBufferSize() + data.writerIndex();
-  }
-  
-  int getSizeFromCount(int valueCount) {
-    return valueCount * ${type.width};
-  }
-  
-  public int getValueCapacity(){
-    return offsetVector.getValueCapacity();
-  }
-  
-  public int getByteCapacity(){
-    return data.capacity(); 
-  }
-  
-  /**
-   * Return the number of bytes contained in the current var len byte vector.
-   * @return
-   */
-  public int getVarByteLength(){
-    return offsetVector.getAccessor().get(valueCount); 
-  }
-  
-  @Override
-  public FieldMetadata getMetadata() {
-    int len = (valueCount + 1) * ${type.width} + getVarByteLength();
-    return FieldMetadata.newBuilder()
-             .setDef(getField().getDef())
-             .setValueCount(valueCount)
-             .setVarByteLength(getVarByteLength())
-             .setBufferLength(len)
-             .build();
-  }
-
-  public int load(int dataBytes, int valueCount, ByteBuf buf){
-    this.valueCount = valueCount;
-    int loaded = offsetVector.load(valueCount+1, buf);
-    data = buf.slice(loaded, dataBytes);
-    data.retain();
-    return loaded + dataBytes;
-  }
-  
-  @Override
-  public void load(FieldMetadata metadata, ByteBuf buffer) {
-    assert this.field.getDef().equals(metadata.getDef());
-    int loaded = load(metadata.getVarByteLength(), metadata.getValueCount(), buffer);
-    assert metadata.getBufferLength() == loaded;
-  }
-  
-  @Override
-  public void clear() {
-    super.clear();
-    offsetVector.clear();
-  }
-
-  
-  @Override
-  public ByteBuf[] getBuffers() {
-    ByteBuf[] buffers = ObjectArrays.concat(offsetVector.getBuffers(), super.getBuffers(), ByteBuf.class);
-    clear();
-    return buffers;
-  }
-  
-  public TransferPair getTransferPair(){
-    return new TransferImpl(getField());
-  }
-  public TransferPair getTransferPair(FieldReference ref){
-    return new TransferImpl(getField().clone(ref));
-  }
-  
-  public void transferTo(${minor.class}Vector target){
-    this.offsetVector.transferTo(target.offsetVector);
-    target.data = data;
-    target.data.retain();
-    target.valueCount = valueCount;
-    clear();
-  }
-  
-  public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
-    int start = from.offsetVector.getAccessor().get(fromIndex);
-    int end =   from.offsetVector.getAccessor().get(fromIndex+1);
-    int len = end - start;
-    
-    int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width});
-    from.data.getBytes(start, data, outputStart, len);
-    offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, outputStart + len);
-  }
-  
-  public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
-    if(thisIndex >= getValueCapacity()) return false;
-    
-    int start = from.offsetVector.getAccessor().get(fromIndex);
-    int end =   from.offsetVector.getAccessor().get(fromIndex+1);
-    int len = end - start;
-    
-    int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width});
-    
-    if(data.capacity() < outputStart + len) return false;
-    
-    from.data.getBytes(start, data, outputStart, len);
-    offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, outputStart + len);
-
-    return true;
-  }
-
-  
-  private class TransferImpl implements TransferPair{
-    ${minor.class}Vector to;
-    
-    public TransferImpl(MaterializedField field){
-      this.to = new ${minor.class}Vector(field, allocator);
-    }
-    
-    public ${minor.class}Vector getTo(){
-      return to;
-    }
-    
-    public void transfer(){
-      transferTo(to);
-    }
-    
-    @Override
-    public void copyValue(int fromIndex, int toIndex) {
-      to.copyFrom(fromIndex, toIndex, ${minor.class}Vector.this);
-    }
-  }
-  
-  public void allocateNew(int totalBytes, int valueCount) {
-    clear();
-    assert totalBytes >= 0;
-    data = allocator.buffer(totalBytes);
-    data.readerIndex(0);
-    offsetVector.allocateNew(valueCount+1);
-  }
-
-  public Accessor getAccessor(){
-    return accessor;
-  }
-  
-  public Mutator getMutator() {
-    return mutator;
-  }
-  
-  public final class Accessor extends BaseValueVector.BaseAccessor{
-    
-    public byte[] get(int index) {
-      assert index >= 0;
-      int startIdx = offsetVector.getAccessor().get(index);
-      int length = offsetVector.getAccessor().get(index + 1) - startIdx;
-      assert length >= 0;
-      byte[] dst = new byte[length];
-      data.getBytes(startIdx, dst, 0, length);
-      return dst;
-    }
-    
-    public void get(int index, ${minor.class}Holder holder){
-      holder.start = offsetVector.getAccessor().get(index);
-      holder.end = offsetVector.getAccessor().get(index + 1);
-      holder.buffer = data;
-    }
-    
-    void get(int index, Nullable${minor.class}Holder holder){
-      holder.start = offsetVector.getAccessor().get(index);
-      holder.end = offsetVector.getAccessor().get(index + 1);
-      holder.buffer = data;
-    }
-    
-    public Object getObject(int index) {
-      return get(index);
-    }
-    
-    public int getValueCount() {
-      return valueCount;
-    }
-
-    public UInt${type.width}Vector getOffsetVector(){
-      return offsetVector;
-    }
-  }
-  
-  /**
-   * Mutable${minor.class} implements a vector of variable width values.  Elements in the vector
-   * are accessed by position from the logical start of the vector.  A fixed width offsetVector
-   * is used to convert an element's position to it's offset from the start of the (0-based)
-   * ByteBuf.  Size is inferred by adjacent elements.
-   *   The width of each element is ${type.width} byte(s)
-   *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
-   *
-   * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
-   */
-  public final class Mutator extends BaseValueVector.BaseMutator{
-
-    /**
-     * Set the variable length element at the specified index to the supplied byte array.
-     *
-     * @param index   position of the bit to set
-     * @param bytes   array of bytes to write
-     */
-    public void set(int index, byte[] bytes) {
-      assert index >= 0;
-      int currentOffset = offsetVector.getAccessor().get(index);
-      offsetVector.getMutator().set(index + 1, currentOffset + bytes.length);
-      data.setBytes(currentOffset, bytes);
-    }
-
-   
-    public boolean setSafe(int index, Nullable${minor.class}Holder holder){
-      assert holder.isSet == 1;
-      if(index >= getValueCapacity()) return false;
-      
-      int start = holder.start;
-      int end =   holder.end;
-      int len = end - start;
-      
-      int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
-      
-      if(data.capacity() < outputStart + len) return false;
-      
-      holder.buffer.getBytes(start, data, outputStart, len);
-      offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (index+1) * ${type.width}, len);
-
-      return true;
-    }
-    
-    public boolean setSafe(int index, ${minor.class}Holder holder){
-      if(index >= getValueCapacity()) return false;
-      
-      int start = holder.start;
-      int end =   holder.end;
-      int len = end - start;
-      
-      int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
-      
-      if(data.capacity() < outputStart + len) return false;
-      
-      holder.buffer.getBytes(start, data, outputStart, len);
-      offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (index+1) * ${type.width}, len);
-
-      return true;
-    }
-    
-    public void set(int index, int start, int length, ByteBuf buffer){
-      assert index >= 0;
-      int currentOffset = offsetVector.getAccessor().get(index);
-      offsetVector.getMutator().set(index + 1, currentOffset + length);
-      ByteBuf bb = buffer.slice(start, length);
-      data.setBytes(currentOffset, bb);
-    }
-
-    void set(int index, Nullable${minor.class}Holder holder){
-      int length = holder.end - holder.start;
-      int currentOffset = offsetVector.getAccessor().get(index);
-      offsetVector.getMutator().set(index + 1, currentOffset + length);
-      data.setBytes(currentOffset, holder.buffer, holder.start, length);
-    }
-    
-    public void set(int index, ${minor.class}Holder holder){
-      int length = holder.end - holder.start;
-      int currentOffset = offsetVector.getAccessor().get(index);
-      offsetVector.getMutator().set(index + 1, currentOffset + length);
-      data.setBytes(currentOffset, holder.buffer, holder.start, length);
-    }
-    
-    public void setValueCount(int valueCount) {
-      ${minor.class}Vector.this.valueCount = valueCount;
-      data.writerIndex(offsetVector.getAccessor().get(valueCount));
-      offsetVector.getMutator().setValueCount(valueCount+1);
-    }
-
-    @Override
-    public void generateTestData(){
-      boolean even = true;
-      for(int i =0; i < valueCount; i++, even = !even){
-        if(even){
-          set(i, new String("aaaaa").getBytes(Charsets.UTF_8));
-        }else{
-          set(i, new String("bbbbbbbbbb").getBytes(Charsets.UTF_8));
-        }
-      }
-    }
-  }
-  
-}
-
-
-</#if> <#-- type.major -->
-</#list>
-</#list>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/codegen/config.fmpp
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/config.fmpp b/sandbox/prototype/exec/java-exec/src/main/codegen/config.fmpp
new file mode 100644
index 0000000..fc37ea1
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/config.fmpp
@@ -0,0 +1,8 @@
+data: {
+    vv: tdd(../data/ValueVectorTypes.tdd)
+}
+freemarkerLinks: {
+    includes: includes/
+} 
+
+    

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd b/sandbox/prototype/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd
new file mode 100644
index 0000000..d201671
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/data/ValueVectorTypes.tdd
@@ -0,0 +1,107 @@
+{
+  modes: [
+    {name: "Optional", prefix: "Nullable"},
+    {name: "Required", prefix: ""},
+    {name: "Repeated", prefix: "Repeated"}
+    ],
+  types: [
+    {
+      major: "Fixed",
+      width: 1,
+      javaType: "byte",
+      boxedType: "Byte",
+      minor: [
+        { class: "TinyInt", valueHolder: "IntHolder"},
+        { class: "UInt1", valueHolder: "UInt1Holder"}
+      ]
+    },
+    {
+      major: "Fixed",
+      width: 2,
+      javaType: "char",
+      boxedType: "Character",
+      minor: [
+        { class: "UInt2", valueHolder: "UInt2Holder"}
+      ]
+    },    {
+      major: "Fixed",
+      width: 2,
+      javaType: "short",
+      boxedType: "Short",
+      minor: [
+        { class: "SmallInt", valueHolder: "Int2Holder"},
+      ]
+    },
+    {
+      major: "Fixed",
+      width: 4,
+      javaType: "int",
+      boxedType: "Integer",
+      minor: [
+        { class: "Int", valueHolder: "IntHolder" },
+        { class: "UInt4", valueHolder: "UInt4Holder" },
+        { class: "Float4", javaType: "float" },
+        <#-- 
+        { class: "Decimal4", maxPrecisionDigits: 8, scale: 4, javaType: "float"},
+        { class: "Date" }
+        -->
+      ]
+    },
+    {
+      major: "Fixed",
+      width: 8,
+      javaType: "long",
+      boxedType: "Long",
+      minor: [
+        { class: "BigInt" },
+        { class: "UInt8" },
+        { class: "Float8", javaType: "double" , boxedType: "Double"},
+        <#-- 
+        { class: "Decimal8", maxPrecisionDigits: 18, scale: 4, javaType: "double", boxedType: "Double"},
+        { class: "Money", maxPrecisionDigits: 2, scale: 1, }, 
+        { class: "Time" },
+        { class: "TimeStamp" },
+        { class: "DateTime" }
+        -->
+      ]
+    },
+    <#--
+    {
+      major: "Fixed",
+      width: 12,
+      javaType: "ByteBuf",
+      minor: [
+        { class: "Decimal12", maxPrecisionDigits: 28, scale: 5},
+        { class: "TimeTZ" },
+        { class: "Interval" }
+      ]
+    },
+    {
+      major: "Fixed",
+      width: 16,
+      javaType: "ByteBuf",
+      minor: [
+        { class: "Decimal16", maxPrecisionDigits: 37, scale: 6}
+      ]
+    },
+    -->
+    {
+      major: "VarLen",
+      width: 4,
+      javaType: "int",
+      minor: [
+        { class: "VarBinary" },
+        { class: "VarChar" }
+        { class: "Var16Char" }
+      ]
+    },
+    {
+      major: "Bit",
+      width: 1,
+      javaType: "int",
+      minor: [
+        { class: "Bit" }
+      ]
+    }
+  ]
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/codegen/includes/license.ftl
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/includes/license.ftl b/sandbox/prototype/exec/java-exec/src/main/codegen/includes/license.ftl
new file mode 100644
index 0000000..0455fd8
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/includes/license.ftl
@@ -0,0 +1,18 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/includes/vv_imports.ftl b/sandbox/prototype/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
new file mode 100644
index 0000000..2029280
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
@@ -0,0 +1,36 @@
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.ObjectArrays;
+import com.google.common.base.Charsets;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ByteBuf;
+
+import org.apache.commons.lang3.ArrayUtils;
+
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.proto.SchemaDefProtos;
+import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
+import org.apache.drill.exec.record.*;
+import org.apache.drill.exec.vector.*;
+import org.apache.drill.exec.expr.holders.*;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.types.TypeProtos.*;
+import org.apache.drill.common.types.Types;
+
+import com.sun.codemodel.JType;
+import com.sun.codemodel.JCodeModel;
+
+import java.util.Arrays;
+import java.util.Random;
+import java.util.List;
+
+import java.io.Closeable;
+
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
new file mode 100644
index 0000000..9227532
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
@@ -0,0 +1,288 @@
+<@pp.dropOutputFile />
+<#list vv.types as type>
+<#list type.minor as minor>
+
+<#if type.major == "Fixed">
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/${minor.class}Vector.java" />
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/**
+ * ${minor.class} implements a vector of fixed width values.  Elements in the vector are accessed
+ * by position, starting from the logical start of the vector.  Values should be pushed onto the
+ * vector sequentially, but may be randomly accessed.
+ *   The width of each element is ${type.width} byte(s)
+ *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+@SuppressWarnings("unused")
+public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector{
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
+
+ 
+  private final Accessor accessor = new Accessor();
+  private final Mutator mutator = new Mutator();
+  
+  public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
+    super(field, allocator);
+  }
+
+  public int getValueCapacity(){
+    return (int) (data.capacity() *1.0 / ${type.width});
+  }
+
+  public Accessor getAccessor(){
+    return accessor;
+  }
+  
+  public Mutator getMutator(){
+    return mutator;
+  }
+  
+  
+
+  /**
+   * Allocate a new buffer that supports setting at least the provided number of values.  May actually be sized bigger depending on underlying buffer rounding size. Must be called prior to using the ValueVector.
+   * @param valueCount
+   */
+  public void allocateNew(int valueCount) {
+    clear();
+    this.data = allocator.buffer(valueCount * ${type.width});
+    this.data.readerIndex(0);
+  }
+  
+  @Override
+  public FieldMetadata getMetadata() {
+    return FieldMetadata.newBuilder()
+             .setDef(getField().getDef())
+             .setValueCount(valueCount)
+             .setBufferLength(valueCount * ${type.width})
+             .build();
+  }
+
+  @Override
+  public int load(int valueCount, ByteBuf buf){
+    clear();
+    this.valueCount = valueCount;
+    int len = valueCount * ${type.width};
+    data = buf.slice(0, len);
+    data.retain();
+    return len;
+  }
+  
+  @Override
+  public void load(FieldMetadata metadata, ByteBuf buffer) {
+    assert this.field.getDef().equals(metadata.getDef());
+    int loaded = load(metadata.getValueCount(), buffer);
+    assert metadata.getBufferLength() == loaded;
+  }
+  
+  public TransferPair getTransferPair(){
+    return new TransferImpl(getField());
+  }
+  public TransferPair getTransferPair(FieldReference ref){
+    return new TransferImpl(getField().clone(ref));
+  }
+  
+  public void transferTo(${minor.class}Vector target){
+    target.data = data;
+    target.data.retain();
+    target.valueCount = valueCount;
+    clear();
+  }
+  
+  private class TransferImpl implements TransferPair{
+    ${minor.class}Vector to;
+    
+    public TransferImpl(MaterializedField field){
+      this.to = new ${minor.class}Vector(field, allocator);
+    }
+    
+    public ${minor.class}Vector getTo(){
+      return to;
+    }
+    
+    public void transfer(){
+      transferTo(to);
+    }
+    
+    @Override
+    public void copyValue(int fromIndex, int toIndex) {
+      to.copyFrom(fromIndex, toIndex, ${minor.class}Vector.this);
+    }
+  }
+  
+  public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
+    <#if (type.width > 8)>
+    data.getBytes(fromIndex * ${type.width}, from.data, thisIndex * ${type.width}, ${type.width});
+    <#else> <#-- type.width <= 8 -->
+    data.set${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width}, 
+        from.data.get${(minor.javaType!type.javaType)?cap_first}(fromIndex * ${type.width})
+    );
+    </#if> <#-- type.width -->
+  }
+  
+  public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
+    if(thisIndex >= getValueCapacity()) return false;
+    copyFrom(fromIndex, thisIndex, from);
+    return true;
+  }
+
+  public final class Accessor extends BaseValueVector.BaseAccessor{
+
+    public int getValueCount() {
+      return valueCount;
+    }
+    
+    <#if (type.width > 8)>
+
+    public ${minor.javaType!type.javaType} get(int index) {
+      ByteBuf dst = allocator.buffer(${type.width});
+      data.getBytes(index * ${type.width}, dst, 0, ${type.width});
+      return dst;
+    }
+
+    public void get(int index, ${minor.class}Holder holder){
+      holder.buffer = data;
+      holder.start = index * ${type.width};
+    }
+    
+    void get(int index, Nullable${minor.class}Holder holder){
+      holder.buffer = data;
+      holder.start = index * ${type.width};
+    }
+
+    @Override
+    public Object getObject(int index) {
+      ByteBuf dst = allocator.buffer(${type.width});
+      data.getBytes(index, dst, 0, ${type.width});
+      return dst;
+    }
+
+    <#else> <#-- type.width <= 8 -->
+
+    public ${minor.javaType!type.javaType} get(int index) {
+      return data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+    }
+
+    public Object getObject(int index) {
+      return get(index);
+    }
+    
+    public void get(int index, ${minor.class}Holder holder){
+      holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+    }
+
+    void get(int index, Nullable${minor.class}Holder holder){
+      holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+    }
+
+
+   </#if> <#-- type.width -->
+ }
+ 
+ /**
+  * ${minor.class}.Mutator implements a mutable vector of fixed width values.  Elements in the
+  * vector are accessed by position from the logical start of the vector.  Values should be pushed
+  * onto the vector sequentially, but may be randomly accessed.
+  *   The width of each element is ${type.width} byte(s)
+  *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
+  *
+  * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+  */
+  public final class Mutator extends BaseValueVector.BaseMutator{
+
+    private Mutator(){};
+   /**
+    * Set the element at the given index to the given value.  Note that widths smaller than
+    * 32 bits are handled by the ByteBuf interface.
+    *
+    * @param index   position of the bit to set
+    * @param value   value to set
+    */
+  <#if (type.width > 8)>
+   public void set(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
+     data.setBytes(index * ${type.width}, value);
+   }
+   
+   public void set(int index, ${minor.class}Holder holder){
+     data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width});
+   }
+   
+   public boolean setSafe(int index, ${minor.class}Holder holder){
+     if(index >= getValueCapacity()) return false;
+     set(index, holder);
+     return true;
+   }
+
+   void set(int index, Nullable${minor.class}Holder holder){
+     data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width});
+   }
+   
+   @Override
+   public void generateTestData() {
+     setValueCount(getValueCapacity());
+     boolean even = true;
+     for(int i =0; i < valueCount; i++, even = !even){
+       byte b = even ? Byte.MIN_VALUE : Byte.MAX_VALUE;
+       for(int w = 0; w < ${type.width}; w++){
+         data.setByte(i + w, b);
+       }
+     }
+   }
+   
+  <#else> <#-- type.width <= 8 -->
+   public void set(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
+     data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, value);
+   }
+   
+   public boolean setSafe(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
+     if(index >= getValueCapacity()) return false;
+     set(index, value);
+     return true;
+   }
+
+   public void set(int index, ${minor.class}Holder holder){
+     data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value);
+   }
+
+   void set(int index, Nullable${minor.class}Holder holder){
+     data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value);
+   }
+
+   @Override
+   public void generateTestData() {
+     setValueCount(getValueCapacity());
+     boolean even = true;
+     for(int i =0; i < valueCount; i++, even = !even){
+       if(even){
+         set(i, ${minor.boxedType!type.boxedType}.MIN_VALUE);
+       }else{
+         set(i, ${minor.boxedType!type.boxedType}.MAX_VALUE);
+       }
+     }
+   }
+
+  </#if> <#-- type.width -->
+  
+
+  
+   public void setValueCount(int valueCount) {
+     ${minor.class}Vector.this.valueCount = valueCount;
+     data.writerIndex(${type.width} * valueCount);
+   }
+
+
+
+
+  
+ }
+}
+
+</#if> <#-- type.major -->
+</#list>
+</#list>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e897fe3a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
new file mode 100644
index 0000000..bf28ecc
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
@@ -0,0 +1,317 @@
+<@pp.dropOutputFile />
+<#list vv.types as type>
+<#list type.minor as minor>
+
+<#assign className = "Nullable${minor.class}Vector" />
+<#assign valuesName = "${minor.class}Vector" />
+<@pp.changeOutputFile name="/org/apache/drill/exec/vector/${className}.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.vector;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/**
+ * Nullable${minor.class} implements a vector of values which could be null.  Elements in the vector
+ * are first checked against a fixed length vector of boolean values.  Then the element is retrieved
+ * from the base class (if not null).
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+@SuppressWarnings("unused")
+public final class ${className} extends BaseValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector{
+
+  private int valueCount;
+  final BitVector bits;
+  final ${valuesName} values;
+  private final Accessor accessor = new Accessor();
+  private final Mutator mutator = new Mutator();
+
+  public ${className}(MaterializedField field, BufferAllocator allocator) {
+    super(field, allocator);
+    this.bits = new BitVector(null, allocator);
+    this.values = new ${minor.class}Vector(null, allocator);
+  }
+  
+  public int getValueCapacity(){
+    return bits.getValueCapacity();
+  }
+  
+  @Override
+  public ByteBuf[] getBuffers() {
+    ByteBuf[] buffers = ObjectArrays.concat(bits.getBuffers(), values.getBuffers(), ByteBuf.class);
+    clear();
+    return buffers;
+  }
+  
+  @Override
+  public void clear() {
+    valueCount = 0;
+    bits.clear();
+    values.clear();
+  }
+  
+  public int getBufferSize(){
+    return values.getBufferSize() + bits.getBufferSize();
+  }
+
+  <#if type.major == "VarLen">
+  @Override
+  public FieldMetadata getMetadata() {
+    return FieldMetadata.newBuilder()
+             .setDef(getField().getDef())
+             .setValueCount(valueCount)
+             .setVarByteLength(values.getVarByteLength())
+             .setBufferLength(getBufferSize())
+             .build();
+  }
+
+  @Override
+  public void allocateNew(int totalBytes, int valueCount) {
+    values.allocateNew(totalBytes, valueCount);
+    bits.allocateNew(valueCount);
+    mutator.reset();
+    accessor.reset();
+  }
+
+  @Override
+  public int load(int dataBytes, int valueCount, ByteBuf buf){
+    clear();
+    this.valueCount = valueCount;
+    int loaded = bits.load(valueCount, buf);
+    
+    // remove bits part of buffer.
+    buf = buf.slice(loaded, buf.capacity() - loaded);
+    loaded += values.load(dataBytes, valueCount, buf);
+    return loaded;
+  }
+  
+  @Override
+  public void load(FieldMetadata metadata, ByteBuf buffer) {
+    assert this.field.getDef().equals(metadata.getDef());
+    int loaded = load(metadata.getVarByteLength(), metadata.getValueCount(), buffer);
+    assert metadata.getBufferLength() == loaded;
+  }
+  
+  @Override
+  public int getByteCapacity(){
+    return values.getByteCapacity();
+  }
+
+  <#else>
+  @Override
+  public FieldMetadata getMetadata() {
+    return FieldMetadata.newBuilder()
+             .setDef(getField().getDef())
+             .setValueCount(valueCount)
+             .setBufferLength(getBufferSize())
+             .build();
+  }
+  
+  @Override
+  public void allocateNew(int valueCount) {
+    values.allocateNew(valueCount);
+    bits.allocateNew(valueCount);
+    mutator.reset();
+    accessor.reset();
+  }
+  
+  @Override
+  public int load(int valueCount, ByteBuf buf){
+    clear();
+    this.valueCount = valueCount;
+    int loaded = bits.load(valueCount, buf);
+    
+    // remove bits part of buffer.
+    buf = buf.slice(loaded, buf.capacity() - loaded);
+    loaded += values.load(valueCount, buf);
+    return loaded;
+  }
+  
+  @Override
+  public void load(FieldMetadata metadata, ByteBuf buffer) {
+    assert this.field.getDef().equals(metadata.getDef());
+    int loaded = load(metadata.getValueCount(), buffer);
+    assert metadata.getBufferLength() == loaded;
+  }
+  
+  </#if>
+  
+  public TransferPair getTransferPair(){
+    return new TransferImpl(getField());
+  }
+  public TransferPair getTransferPair(FieldReference ref){
+    return new TransferImpl(getField().clone(ref));
+  }
+
+  
+  public void transferTo(Nullable${minor.class}Vector target){
+    bits.transferTo(target.bits);
+    values.transferTo(target.values);
+    target.valueCount = valueCount;
+    clear();
+  }
+  
+  private class TransferImpl implements TransferPair{
+    Nullable${minor.class}Vector to;
+    
+    public TransferImpl(MaterializedField field){
+      this.to = new Nullable${minor.class}Vector(field, allocator);
+    }
+    
+    public Nullable${minor.class}Vector getTo(){
+      return to;
+    }
+    
+    public void transfer(){
+      transferTo(to);
+    }
+    
+    @Override
+    public void copyValue(int fromIndex, int toIndex) {
+      to.copyFrom(fromIndex, toIndex, Nullable${minor.class}Vector.this);
+    }
+  }
+  
+  public Accessor getAccessor(){
+    return accessor;
+  }
+  
+  public Mutator getMutator(){
+    return mutator;
+  }
+  
+  public ${minor.class}Vector convertToRequiredVector(){
+    ${minor.class}Vector v = new ${minor.class}Vector(getField().getOtherNullableVersion(), allocator);
+    v.data = values.data;
+    v.valueCount = this.valueCount;
+    v.data.retain();
+    clear();
+    return v;
+  }
+
+  
+  public void copyFrom(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){
+    bits.copyFrom(fromIndex, thisIndex, from.bits);
+    values.copyFrom(fromIndex, thisIndex, from.values);
+  }
+  
+  public boolean copyFromSafe(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){
+    return bits.copyFromSafe(fromIndex, thisIndex, from.bits) && values.copyFromSafe(fromIndex, thisIndex, from.values);
+  }
+
+  
+  public final class Accessor implements ValueVector.Accessor{
+
+    /**
+     * Get the element at the specified position.
+     *
+     * @param   index   position of the value
+     * @return  value of the element, if not null
+     * @throws  NullValueException if the value is null
+     */
+    public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
+      assert !isNull(index);
+      return values.getAccessor().get(index);
+    }
+
+    public boolean isNull(int index) {
+      return isSet(index) == 0;
+    }
+
+    public int isSet(int index){
+      return bits.getAccessor().get(index);
+    }
+    
+    public void get(int index, Nullable${minor.class}Holder holder){
+      holder.isSet = bits.getAccessor().get(index);
+      values.getAccessor().get(index, holder);
+    }
+    
+    @Override
+    public Object getObject(int index) {
+      return isNull(index) ? null : values.getAccessor().getObject(index);
+    }
+    
+    public int getValueCount(){
+      return valueCount;
+    }
+    
+    public void reset(){}
+  }
+  
+  public final class Mutator implements ValueVector.Mutator{
+    
+    private int setCount;
+    
+    private Mutator(){
+    }
+
+    /**
+     * Set the variable length element at the specified index to the supplied byte array.
+     *
+     * @param index   position of the bit to set
+     * @param bytes   array of bytes to write
+     */
+    public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
+      setCount++;
+      bits.getMutator().set(index, 1);
+      values.getMutator().set(index, value);
+    }
+    
+    public void setSkipNull(int index, ${minor.class}Holder holder){
+      values.getMutator().set(index, holder);
+    }
+
+    public void setSkipNull(int index, Nullable${minor.class}Holder holder){
+      values.getMutator().set(index, holder);
+    }
+    
+    public void set(int index, Nullable${minor.class}Holder holder){
+      bits.getMutator().set(index, holder.isSet);
+      values.getMutator().set(index, holder);
+    }
+
+    public void set(int index, ${minor.class}Holder holder){
+      bits.getMutator().set(index, 1);
+      values.getMutator().set(index, holder);
+    }
+    
+    public boolean setSafe(int index, <#if type.major == "VarLen">Nullable${minor.class}Holder <#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value){
+      boolean b1 = bits.getMutator().setSafe(index, 1);
+      boolean b2 = values.getMutator().setSafe(index, value);
+      if(b1 && b2){
+        setCount++;
+        return true;
+      }else{
+        return false;
+      }
+
+    }
+
+    
+    public void setValueCount(int valueCount) {
+      assert valueCount >= 0;
+      Nullable${minor.class}Vector.this.valueCount = valueCount;
+      values.getMutator().setValueCount(valueCount);
+      bits.getMutator().setValueCount(valueCount);
+    }
+    
+    public boolean noNulls(){
+      return valueCount == setCount;
+    }
+    
+    public void generateTestData(){
+      bits.getMutator().generateTestData();
+      values.getMutator().generateTestData();
+    }
+    
+    public void reset(){
+      setCount = 0;
+    }
+    
+  }
+}
+</#list>
+</#list>
\ No newline at end of file


[05/18] git commit: Limit width to 5x the number of endpoints

Posted by ja...@apache.org.
Limit width to 5x the number of endpoints


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/b03740e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/b03740e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/b03740e8

Branch: refs/heads/master
Commit: b03740e897e17638ad49ff72a55931d777cfe7dd
Parents: 78a6197
Author: Steven Phillips <sp...@maprtech.com>
Authored: Thu Aug 29 19:34:13 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Sep 1 15:16:21 2013 -0700

----------------------------------------------------------------------
 .../org/apache/drill/exec/planner/fragment/Wrapper.java |  1 +
 .../drill/exec/store/TestParquetPhysicalPlan.java       | 12 +++++++++---
 2 files changed, 10 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b03740e8/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Wrapper.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Wrapper.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Wrapper.java
index 8c4b0b4..7e2497e 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Wrapper.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Wrapper.java
@@ -152,6 +152,7 @@ public class Wrapper {
         endpoints.add(all.get(i % div));
       }
     } else {
+      width = Math.min(width, values.size()*5);
       // get nodes with highest affinity.
       Collections.sort(values);
       values = Lists.reverse(values);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b03740e8/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
index c58bc71..e1344f4 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestParquetPhysicalPlan.java
@@ -36,6 +36,7 @@ import java.nio.charset.Charset;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import static junit.framework.Assert.assertNull;
 import static org.junit.Assert.assertEquals;
@@ -85,6 +86,7 @@ public class TestParquetPhysicalPlan {
   }
 
   private class ParquetResultsListener implements UserResultsListener {
+    AtomicInteger count = new AtomicInteger();
     private CountDownLatch latch = new CountDownLatch(1);
     @Override
     public void submissionFailed(RpcException ex) {
@@ -94,12 +96,16 @@ public class TestParquetPhysicalPlan {
 
     @Override
     public void resultArrived(QueryResultBatch result) {
-      System.out.printf("Result batch arrived. Number of records: %d", result.getHeader().getRowCount());
+      int rows = result.getHeader().getRowCount();
+      System.out.println(String.format("Result batch arrived. Number of records: %d", rows));
+      count.addAndGet(rows);
       if (result.getHeader().getIsLastChunk()) latch.countDown();
+      result.release();
     }
 
-    public void await() throws Exception {
+    public int await() throws Exception {
       latch.await();
+      return count.get();
     }
   }
   @Test
@@ -111,7 +117,7 @@ public class TestParquetPhysicalPlan {
       client.connect();
       ParquetResultsListener listener = new ParquetResultsListener();
       client.runQuery(UserProtos.QueryType.PHYSICAL, Resources.toString(Resources.getResource(fileName),Charsets.UTF_8), listener);
-      listener.await();
+      System.out.println(String.format("Got %d total records.", listener.await()));
       client.close();
     }
   }


[07/18] git commit: fix hadoop dependency problem in ref

Posted by ja...@apache.org.
fix hadoop dependency problem in ref


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/78a6197f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/78a6197f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/78a6197f

Branch: refs/heads/master
Commit: 78a6197f6c14ed65dc74b8f1233bd3bd94342fa9
Parents: f77c525
Author: Steven Phillips <sp...@maprtech.com>
Authored: Fri Aug 30 11:18:10 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Sep 1 15:16:21 2013 -0700

----------------------------------------------------------------------
 sandbox/prototype/exec/java-exec/pom.xml | 5 +++++
 sandbox/prototype/exec/ref/pom.xml       | 5 +++++
 2 files changed, 10 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/78a6197f/sandbox/prototype/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/pom.xml b/sandbox/prototype/exec/java-exec/pom.xml
index 03c1e52..d43ff23 100644
--- a/sandbox/prototype/exec/java-exec/pom.xml
+++ b/sandbox/prototype/exec/java-exec/pom.xml
@@ -169,6 +169,11 @@
   <profiles>
     <profile>
       <id>default-hadoop</id>
+      <activation>
+        <property>
+          <name>!alt-hadoop</name>
+        </property>
+      </activation>
       <dependencies>
         <dependency>
           <groupId>org.apache.hadoop</groupId>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/78a6197f/sandbox/prototype/exec/ref/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/ref/pom.xml b/sandbox/prototype/exec/ref/pom.xml
index ec664d5..a8970ba 100644
--- a/sandbox/prototype/exec/ref/pom.xml
+++ b/sandbox/prototype/exec/ref/pom.xml
@@ -37,6 +37,11 @@
   <profiles>
     <profile>
       <id>default-hadoop</id>
+      <activation>
+        <property>
+          <name>!alt-hadoop</name>
+        </property>
+      </activation>
       <dependencies>
         <dependency>
           <groupId>org.apache.hadoop</groupId>


[16/18] git commit: implemented left and inner join types

Posted by ja...@apache.org.
implemented left and inner join types


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/6f300a3f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/6f300a3f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/6f300a3f

Branch: refs/heads/master
Commit: 6f300a3fc217a72f53819ea8d49faad18078daf4
Parents: d91bfe6
Author: Ben Becker <be...@gmail.com>
Authored: Sat Aug 31 23:02:27 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Sep 1 15:16:22 2013 -0700

----------------------------------------------------------------------
 .../apache/drill/exec/opt/BasicOptimizer.java   |  2 +-
 .../exec/physical/config/MergeJoinPOP.java      | 16 ++++--
 .../exec/physical/impl/join/JoinTemplate.java   | 15 ++++--
 .../exec/physical/impl/join/MergeJoinBatch.java |  7 ++-
 .../drill/exec/record/AbstractRecordBatch.java  |  4 ++
 .../exec/physical/impl/join/TestMergeJoin.java  | 55 +++++++++++++++++++-
 .../join/merge_inner_single_batch.json          | 46 ++++++++++++++++
 .../test/resources/join/merge_multi_batch.json  |  1 +
 .../test/resources/join/merge_single_batch.json |  1 +
 9 files changed, 133 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f300a3f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
index dfe6804..9f54dd0 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
@@ -173,7 +173,7 @@ public class BasicOptimizer extends Optimizer{
       rightOp = new Sort(rightOp, rightOrderDefs, false);
       rightOp = new SelectionVectorRemover(rightOp);
       
-      MergeJoinPOP mjp = new MergeJoinPOP(leftOp, rightOp, Arrays.asList(join.getConditions()));
+      MergeJoinPOP mjp = new MergeJoinPOP(leftOp, rightOp, Arrays.asList(join.getConditions()), join.getJointType());
       return new SelectionVectorRemover(mjp);
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f300a3f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java
index 9a1973e..0ffd218 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java
@@ -3,6 +3,7 @@ package org.apache.drill.exec.physical.config;
 import java.util.Iterator;
 import java.util.List;
 
+import org.apache.drill.common.logical.data.Join;
 import org.apache.drill.common.logical.data.JoinCondition;
 import org.apache.drill.exec.physical.OperatorCost;
 import org.apache.drill.exec.physical.base.AbstractBase;
@@ -24,7 +25,8 @@ public class MergeJoinPOP extends AbstractBase{
   private final PhysicalOperator left;
   private final PhysicalOperator right;
   private final List<JoinCondition> conditions;
-  
+  private final Join.JoinType joinType;
+
   @Override
   public OperatorCost getCost() {
     return new OperatorCost(0,0,0,0);
@@ -34,11 +36,13 @@ public class MergeJoinPOP extends AbstractBase{
   public MergeJoinPOP(
       @JsonProperty("left") PhysicalOperator left, 
       @JsonProperty("right") PhysicalOperator right,
-      @JsonProperty("join-conditions") List<JoinCondition> conditions 
-      ) {
+      @JsonProperty("join-conditions") List<JoinCondition> conditions,
+      @JsonProperty("join-type") Join.JoinType joinType
+  ) {
     this.left = left;
     this.right = right;
     this.conditions = conditions;
+    this.joinType = joinType;
   }
   
   @Override
@@ -54,7 +58,7 @@ public class MergeJoinPOP extends AbstractBase{
   @Override
   public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) {
     Preconditions.checkArgument(children.size() == 2);
-    return new MergeJoinPOP(children.get(0), children.get(1), conditions);
+    return new MergeJoinPOP(children.get(0), children.get(1), conditions, joinType);
   }
 
   @Override
@@ -70,6 +74,10 @@ public class MergeJoinPOP extends AbstractBase{
     return right;
   }
 
+  public Join.JoinType getJoinType() {
+    return joinType;
+  }
+
   public List<JoinCondition> getConditions() {
     return conditions;
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f300a3f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
index 3d6dfce..9a3042b 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
@@ -2,8 +2,10 @@ package org.apache.drill.exec.physical.impl.join;
 
 import javax.inject.Named;
 
+import org.apache.drill.common.logical.data.Join;
 import org.apache.drill.exec.exception.SchemaChangeException;
 import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.config.MergeJoinPOP;
 import org.apache.drill.exec.record.VectorContainer;
 
 /**
@@ -71,10 +73,12 @@ public abstract class JoinTemplate implements JoinWorker {
 
       // validate input iterators (advancing to the next record batch if necessary)
       if (!status.isRightPositionAllowed()) {
-        // we've hit the end of the right record batch; copy any remaining values from the left batch
-        while (status.isLeftPositionAllowed()) {
-          doCopyLeft(status.getLeftPosition(), status.fetchAndIncOutputPos());
-          status.advanceLeft();
+        if (((MergeJoinPOP)status.outputBatch.getPopConfig()).getJoinType() == Join.JoinType.LEFT) {
+          // we've hit the end of the right record batch; copy any remaining values from the left batch
+          while (status.isLeftPositionAllowed()) {
+            doCopyLeft(status.getLeftPosition(), status.fetchAndIncOutputPos());
+            status.advanceLeft();
+          }
         }
         return;
       }
@@ -86,7 +90,8 @@ public abstract class JoinTemplate implements JoinWorker {
 
       case -1:
         // left key < right key
-        doCopyLeft(status.getLeftPosition(), status.fetchAndIncOutputPos());
+        if (((MergeJoinPOP)status.outputBatch.getPopConfig()).getJoinType() == Join.JoinType.LEFT)
+          doCopyLeft(status.getLeftPosition(), status.fetchAndIncOutputPos());
         status.advanceLeft();
         continue;
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f300a3f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
index 000e9cc..a8fd276 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
@@ -9,6 +9,7 @@ import org.apache.drill.common.expression.ErrorCollectorImpl;
 import org.apache.drill.common.expression.ExpressionPosition;
 import org.apache.drill.common.expression.FunctionCall;
 import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.Join;
 import org.apache.drill.common.logical.data.JoinCondition;
 import org.apache.drill.exec.compile.sig.MappingSet;
 import org.apache.drill.exec.exception.ClassTransformationException;
@@ -16,6 +17,7 @@ import org.apache.drill.exec.exception.SchemaChangeException;
 import org.apache.drill.exec.expr.CodeGenerator;
 import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
 import org.apache.drill.exec.expr.HoldingContainerExpression;
+import org.apache.drill.exec.expr.TypeHelper;
 import org.apache.drill.exec.expr.fn.impl.ComparatorFunctions;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.physical.config.MergeJoinPOP;
@@ -27,7 +29,6 @@ import org.apache.drill.exec.record.RecordBatch;
 import org.apache.drill.exec.record.TypedFieldId;
 import org.apache.drill.exec.record.VectorContainer;
 import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.vector.TypeHelper;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.allocator.VectorAllocator;
 
@@ -92,8 +93,9 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
   private final RecordBatch left;
   private final RecordBatch right;
   private final JoinStatus status;
+  private final JoinCondition condition;
+  private final Join.JoinType joinType;
   private JoinWorker worker;
-  private JoinCondition condition;
   public MergeJoinBatchBuilder batchBuilder;
   
   protected MergeJoinBatch(MergeJoinPOP popConfig, FragmentContext context, RecordBatch left, RecordBatch right) {
@@ -102,6 +104,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
     this.right = right;
     this.status = new JoinStatus(left, right, this);
     this.batchBuilder = new MergeJoinBatchBuilder(context, status);
+    this.joinType = popConfig.getJoinType();
     this.condition = popConfig.getConditions().get(0);
     // currently only one join condition is supported
     assert popConfig.getConditions().size() == 1;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f300a3f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
index ccd2468..0d44368 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/AbstractRecordBatch.java
@@ -31,6 +31,10 @@ public abstract class AbstractRecordBatch<T extends PhysicalOperator> implements
     return context;
   }
 
+  public PhysicalOperator getPopConfig() {
+    return popConfig;
+  }
+
   @Override
   public BatchSchema getSchema() {
     return container.getSchema();

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f300a3f/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
index 6aa651b..a8b48a7 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java
@@ -106,8 +106,8 @@ public class TestMergeJoin {
   }
 
   @Test
-  public void orderedEqualityJoin(@Injectable final DrillbitContext bitContext,
-                                  @Injectable UserServer.UserClientConnection connection) throws Throwable {
+  public void orderedEqualityLeftJoin(@Injectable final DrillbitContext bitContext,
+                                      @Injectable UserServer.UserClientConnection connection) throws Throwable {
 
     new NonStrictExpectations(){{
       bitContext.getMetrics(); result = new MetricRegistry("test");
@@ -157,6 +157,57 @@ public class TestMergeJoin {
 
   }
 
+  @Test
+  public void orderedEqualityInnerJoin(@Injectable final DrillbitContext bitContext,
+                                       @Injectable UserServer.UserClientConnection connection) throws Throwable {
+
+    new NonStrictExpectations(){{
+      bitContext.getMetrics(); result = new MetricRegistry("test");
+      bitContext.getAllocator(); result = BufferAllocator.getAllocator(c);
+      bitContext.getConfig(); result = c;
+    }};
+
+    PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(),CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), new StorageEngineRegistry(bitContext));
+    PhysicalPlan plan = reader.readPhysicalPlan(
+        Files.toString(
+            FileUtils.getResourceAsFile("/join/merge_inner_single_batch.json"), Charsets.UTF_8)
+            .replace("#{LEFT_FILE}", FileUtils.getResourceAsFile("/join/merge_single_batch.left.json").toURI().toString())
+            .replace("#{RIGHT_FILE}", FileUtils.getResourceAsFile("/join/merge_single_batch.right.json").toURI().toString()));
+    FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
+    FragmentContext context = new FragmentContext(bitContext, ExecProtos.FragmentHandle.getDefaultInstance(), connection, null, registry);
+    SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
+
+    int totalRecordCount = 0;
+    while (exec.next()) {
+      totalRecordCount += exec.getRecordCount();
+      System.out.println("got next with record count: " + exec.getRecordCount() + " (total: " + totalRecordCount + "):");
+      System.out.println("       t1                 t2");
+
+      for (int valueIdx = 0; valueIdx < exec.getRecordCount(); valueIdx++) {
+        List<Object> row = Lists.newArrayList();
+        for (ValueVector v : exec)
+          row.add(v.getField().getName() + ":" + v.getAccessor().getObject(valueIdx));
+        for (Object cell : row) {
+          if (cell == null) {
+            System.out.print("<null>    ");
+            continue;
+          }
+          int len = cell.toString().length();
+          System.out.print(cell + " ");
+          for (int i = 0; i < (10 - len); ++i)
+            System.out.print(" ");
+        }
+        System.out.println();
+      }
+    }
+    System.out.println("Total Record Count: " + totalRecordCount);
+    assertEquals(23, totalRecordCount);
+
+    if (context.getFailureCause() != null)
+      throw context.getFailureCause();
+    assertTrue(!context.isFailed());
+
+  }
 
   @Test
   public void orderedEqualityMultiBatchJoin(@Injectable final DrillbitContext bitContext,

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f300a3f/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_inner_single_batch.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_inner_single_batch.json b/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_inner_single_batch.json
new file mode 100644
index 0000000..5c7b8e3
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_inner_single_batch.json
@@ -0,0 +1,46 @@
+{
+  head:{
+    type:"APACHE_DRILL_PHYSICAL",
+    version:"1",
+    generator:{
+      type:"manual"
+    }
+  },
+  graph:[
+    {
+      @id:1,
+      pop:"json-sub-scan",
+      readEntries:[
+        {path: "#{LEFT_FILE}"}
+      ],
+      engineConfig:{
+         "type":"json",
+         "dfsName" : "file:///"
+      }      
+    },
+    {
+      @id:2,
+      pop:"json-sub-scan",
+      readEntries:[
+        {path: "#{RIGHT_FILE}"}
+      ],
+      engineConfig:{
+         "type":"json",
+         "dfsName" : "file:///"
+      }      
+    },
+    {
+      @id: 3,
+      left: 1,
+      right: 2,
+      pop: "merge-join",
+      join-type: "INNER",
+      join-conditions: [ { relationship: "==", left: "a", right: "aa" } ]
+    },
+    {
+      @id: 4,
+      child: 3,
+      pop: "screen"
+    }
+  ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f300a3f/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_multi_batch.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_multi_batch.json b/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_multi_batch.json
index ebdfd38..7abd19f 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_multi_batch.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_multi_batch.json
@@ -36,6 +36,7 @@
       left: 1,
       right: 2,
       pop: "merge-join",
+      join-type: "LEFT",
       join-conditions: [ { relationship: "==", left: "a", right: "aa" } ]
     },
     {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f300a3f/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_single_batch.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_single_batch.json b/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_single_batch.json
index ad33d26..1f7e0b5 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_single_batch.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/join/merge_single_batch.json
@@ -34,6 +34,7 @@
       left: 1,
       right: 2,
       pop: "merge-join",
+      join-type: "LEFT",
       join-conditions: [ { relationship: "==", left: "a", right: "aa" } ]
     },
     {


[15/18] git commit: Updates to fix SQL Handler issues. Add simple full query jdbc test execution (no results checking)

Posted by ja...@apache.org.
Updates to fix SQL Handler issues.  Add simple full query jdbc test execution (no results checking)


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/89eb9457
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/89eb9457
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/89eb9457

Branch: refs/heads/master
Commit: 89eb9457420ef444aafa867d77453bb293ae6ec8
Parents: d3b2f9f
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sun Sep 1 08:38:41 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Sep 1 15:16:22 2013 -0700

----------------------------------------------------------------------
 .../org/apache/drill/jdbc/DrillHandler.java     | 30 +++++++++++---------
 .../apache/drill/optiq/DrillImplementor.java    |  2 +-
 .../apache/drill/jdbc/test/FullEngineTest.java  | 16 ++++++-----
 .../org/apache/drill/jdbc/test/JdbcAssert.java  | 13 ++++++++-
 4 files changed, 38 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/89eb9457/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
index 6cc25e0..380993b 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
@@ -68,20 +68,6 @@ public class DrillHandler extends HandlerImpl {
         String enginesData = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8);
 
         StorageEngines engines = config.getMapper().readValue(enginesData, StorageEngines.class);
-        MutableSchema rootSchema = connection.getRootSchema();
-
-        for (Map.Entry<String, StorageEngineConfig> entry : engines) {
-          SchemaProvider provider = registry.getSchemaProvider(entry.getValue());
-          FileSystemSchema schema = new FileSystemSchema(client, entry.getValue(), provider,
-              rootSchema.getTypeFactory(), rootSchema, entry.getKey(), rootSchema.getExpression(),
-              rootSchema.getQueryProvider());
-          rootSchema.addSchema(entry.getKey(), schema);
-        }
-
-        rootSchema.addSchema(
-            "--FAKE--",
-            new FakeSchema(rootSchema, rootSchema.getQueryProvider(), rootSchema.getTypeFactory(), "fake", rootSchema
-                .getExpression()));
 
         if (zk != null) {
           coordinator = new ZKClusterCoordinator(config, zk);
@@ -100,6 +86,22 @@ public class DrillHandler extends HandlerImpl {
           cl.connect();
           client = cl;
         }
+        
+        MutableSchema rootSchema = connection.getRootSchema();
+
+        for (Map.Entry<String, StorageEngineConfig> entry : engines) {
+          SchemaProvider provider = registry.getSchemaProvider(entry.getValue());
+          FileSystemSchema schema = new FileSystemSchema(client, entry.getValue(), provider,
+              rootSchema.getTypeFactory(), rootSchema, entry.getKey(), rootSchema.getExpression(),
+              rootSchema.getQueryProvider());
+          rootSchema.addSchema(entry.getKey(), schema);
+        }
+
+        rootSchema.addSchema(
+            "--FAKE--",
+            new FakeSchema(rootSchema, rootSchema.getQueryProvider(), rootSchema.getTypeFactory(), "fake", rootSchema
+                .getExpression()));
+        
       } catch (Exception ex) {
         throw new SQLException("Failure trying to connect to Drill.", ex);
       }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/89eb9457/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java
index 27b9630..0a9e28c 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillImplementor.java
@@ -91,7 +91,7 @@ public class DrillImplementor {
   }
   
   public void registerSource(DrillTable table){
-    if(tables.add(table)){
+    if(!table.isRefEngine() && tables.add(table)){
       sourcesNode.put(table.getStorageEngineName(), mapper.convertValue(table.getStorageEngineConfig(), JsonNode.class));  
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/89eb9457/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
index c3bd51d..e88dc0d 100644
--- a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
+++ b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
@@ -13,7 +13,7 @@ import org.junit.rules.Timeout;
 import com.google.common.base.Charsets;
 import com.google.common.io.Resources;
 
-@Ignore
+
 public class FullEngineTest {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FullEngineTest.class);
 
@@ -25,16 +25,18 @@ public class FullEngineTest {
   // Set a timeout unless we're debugging.
   @Rule public TestRule globalTimeout = IS_DEBUG ? new TestName() : new Timeout(10000);
   
-  @BeforeClass
-  public static void setupFixtures() throws IOException {
-    MODEL_FULL_ENGINE = Resources.toString(Resources.getResource("full-model.json"), Charsets.UTF_8);
-  }
+//  @BeforeClass
+//  public static void setupFixtures() throws IOException {
+//    MODEL_FULL_ENGINE = Resources.toString(Resources.getResource("full-model.json"), Charsets.UTF_8);
+//  }
+  
+  
   
   @Test
   public void fullSelectStarEngine() throws Exception {
-    JdbcAssert.withModel(MODEL_FULL_ENGINE, "DONUTS")
+    JdbcAssert.withFull("parquet-local")
     // .sql("select cast(_MAP['red'] as bigint) + 1 as red_inc from donuts ")
-        .sql("select _MAP['d'] as d, _MAP['b'] as b from \"parquet-local\".\"/tmp/parquet_test_file_many_types\" ").displayResults(50);
+        .sql("select _MAP['d'] as d, _MAP['b'] as b from \"/tmp/parquet_test_file_many_types\" ").displayResults(50);
   }
 
 //  @Test

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/89eb9457/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
index d3927f4..54af383 100644
--- a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
+++ b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
@@ -47,6 +47,12 @@ public class JdbcAssert {
     return new ModelAndSchema(info);
   }
 
+  public static ModelAndSchema withFull(String schema) {
+    final Properties info = new Properties();
+    info.setProperty("schema", schema);
+    return new ModelAndSchema(info, false);
+  }
+
   
   static String toString(ResultSet resultSet, int expectedRecordCount) throws SQLException {
     StringBuilder buf = new StringBuilder();
@@ -109,11 +115,16 @@ public class JdbcAssert {
     private final ConnectionFactory connectionFactory;
 
     public ModelAndSchema(Properties info) {
+      this(info, true);
+    }
+    
+    public ModelAndSchema(Properties info, final boolean ref) {
       this.info = info;
       this.connectionFactory = new ConnectionFactory() {
         public Connection createConnection() throws Exception {
           Class.forName("org.apache.drill.jdbc.Driver");
-          return DriverManager.getConnection("jdbc:drill:ref=true", ModelAndSchema.this.info);
+          String connect = ref ? "jdbc:drill:ref=true" : "jdbc:drill:";
+          return DriverManager.getConnection(connect, ModelAndSchema.this.info);  
         }
       };
     }


[02/18] DRILL-167, DRILL-80, DRILL-59 start make changes to support sqlline module and ensure no test jars on class path

Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProviderRegistry.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProviderRegistry.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProviderRegistry.java
new file mode 100644
index 0000000..4e96dd8
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaProviderRegistry.java
@@ -0,0 +1,82 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.store;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.logical.StorageEngineConfig;
+import org.apache.drill.common.util.PathScanner;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.exception.SetupException;
+
+public class SchemaProviderRegistry {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SchemaProviderRegistry.class);
+  
+  private Map<Object, Constructor<? extends SchemaProvider>> allProviders = new HashMap<Object, Constructor<? extends SchemaProvider>>();
+  private Map<StorageEngineConfig, SchemaProvider> activeEngines = new HashMap<StorageEngineConfig, SchemaProvider>();
+
+  private final DrillConfig config;
+  
+  public SchemaProviderRegistry(DrillConfig config){
+    init(config);
+    this.config = config;
+  }
+  
+  @SuppressWarnings("unchecked")
+  public void init(DrillConfig config){
+    Collection<Class<? extends SchemaProvider>> providers = PathScanner.scanForImplementations(SchemaProvider.class, config.getStringList(ExecConstants.STORAGE_ENGINE_SCAN_PACKAGES));
+    logger.debug("Loading schema providers {}", providers);
+    for(Class<? extends SchemaProvider> schema: providers){
+      int i =0;
+      for(Constructor<?> c : schema.getConstructors()){
+        Class<?>[] params = c.getParameterTypes();
+        if(params.length != 2 || params[1] != DrillConfig.class || !StorageEngineConfig.class.isAssignableFrom(params[0])){
+          logger.info("Skipping SchemaProvider constructor {} for provider class {} since it doesn't implement a [constructor(StorageEngineConfig, DrillConfig)]", c, schema);
+          continue;
+        }
+        allProviders.put(params[0], (Constructor<? extends SchemaProvider>) c);
+        i++;
+      }
+      if(i == 0){
+        logger.debug("Skipping registration of StorageSchemaProvider {} as it doesn't have a constructor with the parameters of (StorangeEngineConfig, Config)", schema.getName());
+      }
+    }
+  }
+  
+  public SchemaProvider getSchemaProvider(StorageEngineConfig engineConfig) throws SetupException{
+    SchemaProvider engine = activeEngines.get(engineConfig);
+    if(engine != null) return engine;
+    Constructor<? extends SchemaProvider> c = allProviders.get(engineConfig.getClass());
+    if(c == null) throw new SetupException(String.format("Failure finding StorageSchemaProvider constructor for config %s", engineConfig));
+    try {
+      return c.newInstance(engineConfig, config);
+    } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
+      Throwable t = e instanceof InvocationTargetException ? ((InvocationTargetException)e).getTargetException() : e;
+      if(t instanceof SetupException) throw ((SetupException) t);
+      throw new SetupException(String.format("Failure setting up new storage engine configuration for config %s", engineConfig), t);
+    }
+  }
+  
+
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngine.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngine.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngine.java
index b24521d..107d1ae 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngine.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngine.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.util.Collection;
 import java.util.List;
 
+import org.apache.drill.common.logical.StorageEngineConfig;
 import org.apache.drill.common.logical.data.Scan;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.physical.ReadEntry;
@@ -52,6 +53,9 @@ public interface StorageEngine {
    */
   public AbstractGroupScan getPhysicalScan(Scan scan) throws IOException;
 
+  public SchemaProvider getSchemaProvider();
+  
+  
   /**
    * Get the set of Drillbit endpoints that are available for each read entry. Note that it is possible for a read entry
    * to have no Drillbit locations. In that case, the multimap will contain no values for that read entry.

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngineRegistry.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngineRegistry.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngineRegistry.java
index 4551c1f..810bbc9 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngineRegistry.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/StorageEngineRegistry.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
 import java.util.Map;
 
 import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.logical.StorageEngineConfig;
 import org.apache.drill.common.util.PathScanner;
 import org.apache.drill.exec.ExecConstants;
@@ -63,17 +64,17 @@ public class StorageEngineRegistry {
     }
   }
   
-  public StorageEngine getEngine(StorageEngineConfig engineConfig) throws SetupException{
+  public StorageEngine getEngine(StorageEngineConfig engineConfig) throws ExecutionSetupException{
     StorageEngine engine = activeEngines.get(engineConfig);
     if(engine != null) return engine;
     Constructor<? extends StorageEngine> c = availableEngines.get(engineConfig.getClass());
-    if(c == null) throw new SetupException(String.format("Failure finding StorageEngine constructor for config %s", engineConfig));
+    if(c == null) throw new ExecutionSetupException(String.format("Failure finding StorageEngine constructor for config %s", engineConfig));
     try {
       return c.newInstance(engineConfig, context);
     } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
       Throwable t = e instanceof InvocationTargetException ? ((InvocationTargetException)e).getTargetException() : e;
-      if(t instanceof SetupException) throw ((SetupException) t);
-      throw new SetupException(String.format("Failure setting up new storage engine configuration for config %s", engineConfig), t);
+      if(t instanceof ExecutionSetupException) throw ((ExecutionSetupException) t);
+      throw new ExecutionSetupException(String.format("Failure setting up new storage engine configuration for config %s", engineConfig), t);
     }
   }
   

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java
index b44565b..a28eacf 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONGroupScan.java
@@ -18,57 +18,62 @@
 
 package org.apache.drill.exec.store.json;
 
-import com.fasterxml.jackson.annotation.*;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.exec.exception.SetupException;
+import static com.google.common.base.Preconditions.checkArgument;
+
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.FieldReference;
 import org.apache.drill.exec.physical.EndpointAffinity;
 import org.apache.drill.exec.physical.OperatorCost;
 import org.apache.drill.exec.physical.ReadEntry;
-import org.apache.drill.exec.physical.ReadEntryWithPath;
 import org.apache.drill.exec.physical.base.AbstractGroupScan;
 import org.apache.drill.exec.physical.base.PhysicalOperator;
 import org.apache.drill.exec.physical.base.Size;
 import org.apache.drill.exec.physical.base.SubScan;
 import org.apache.drill.exec.proto.CoordinationProtos;
 import org.apache.drill.exec.store.StorageEngineRegistry;
-import org.apache.drill.exec.store.parquet.ParquetStorageEngineConfig;
-
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
 
-import static com.google.common.base.Preconditions.checkArgument;
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
 
 @JsonTypeName("json-scan")
 public class JSONGroupScan extends AbstractGroupScan {
   private static int ESTIMATED_RECORD_SIZE = 1024; // 1kb
-  private final StorageEngineRegistry registry;
-  private final StorageEngineConfig engineConfig;
+  private final JSONStorageEngine engine;
 
   private LinkedList<JSONGroupScan.ScanEntry>[] mappings;
   private final List<JSONGroupScan.ScanEntry> readEntries;
   private final OperatorCost cost;
   private final Size size;
+  private final FieldReference ref;
 
   @JsonCreator
   public JSONGroupScan(@JsonProperty("entries") List<ScanEntry> entries,
                        @JsonProperty("storageengine") JSONStorageEngineConfig storageEngineConfig,
-                       @JacksonInject StorageEngineRegistry engineRegistry) throws SetupException {
-    engineRegistry.init(DrillConfig.create());
-    this.registry = engineRegistry;
-    this.engineConfig = storageEngineConfig;
+                       @JacksonInject StorageEngineRegistry engineRegistry, FieldReference ref) throws ExecutionSetupException {
+    this(entries, (JSONStorageEngine) engineRegistry.getEngine(storageEngineConfig), ref);
+  }
+
+  public JSONGroupScan(List<ScanEntry> entries, JSONStorageEngine engine, FieldReference ref) {
+    this.engine = engine;
     this.readEntries = entries;
     OperatorCost cost = new OperatorCost(0, 0, 0, 0);
     Size size = new Size(0, 0);
     for (JSONGroupScan.ScanEntry r : readEntries) {
-      cost = cost.add(r.getCost());
-      size = size.add(r.getSize());
+    cost = cost.add(r.getCost());
+    size = size.add(r.getSize());
     }
     this.cost = cost;
     this.size = size;
+    this.ref = ref;
   }
-
+  
   @SuppressWarnings("unchecked")
   @Override
   public void applyAssignments(List<CoordinationProtos.DrillbitEndpoint> endpoints) {
@@ -91,14 +96,9 @@ public class JSONGroupScan extends AbstractGroupScan {
 
   @SuppressWarnings("unchecked")
   @Override
-  public SubScan getSpecificScan(int minorFragmentId) {
+  public SubScan getSpecificScan(int minorFragmentId) throws ExecutionSetupException{
     checkArgument(minorFragmentId < mappings.length, "Mappings length [%s] should be longer than minor fragment id [%s] but it isn't.", mappings.length, minorFragmentId);
-    try {
-      return new JSONSubScan(registry, engineConfig, mappings[minorFragmentId]);
-    } catch (SetupException e) {
-      e.printStackTrace();
-    }
-    return null;
+    return new JSONSubScan(mappings[minorFragmentId], engine, ref);
   }
 
   @Override
@@ -109,12 +109,7 @@ public class JSONGroupScan extends AbstractGroupScan {
   @Override
   @JsonIgnore
   public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) {
-    try {
-      return new JSONGroupScan(readEntries, (JSONStorageEngineConfig) engineConfig, registry);
-    } catch (SetupException e) {
-      e.printStackTrace();
-    }
-    return null;
+    return new JSONGroupScan(readEntries, engine, ref);
   }
 
   public static class ScanEntry implements ReadEntry {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java
index f2c7f96..0324260 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONRecordReader.java
@@ -1,19 +1,18 @@
 package org.apache.drill.exec.store.json;
 
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonToken;
-import com.google.common.base.Charsets;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.io.Files;
-import com.google.common.io.InputSupplier;
-import com.google.common.io.Resources;
+import static com.fasterxml.jackson.core.JsonToken.END_ARRAY;
+import static com.fasterxml.jackson.core.JsonToken.END_OBJECT;
+import static com.fasterxml.jackson.core.JsonToken.FIELD_NAME;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
 
 import org.apache.drill.common.exceptions.DrillRuntimeException;
 import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FieldReference;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.types.TypeProtos.MajorType;
 import org.apache.drill.common.types.TypeProtos.MinorType;
@@ -23,24 +22,38 @@ import org.apache.drill.exec.memory.BufferAllocator;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.physical.impl.OutputMutator;
 import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.schema.*;
+import org.apache.drill.exec.schema.DiffSchema;
+import org.apache.drill.exec.schema.Field;
+import org.apache.drill.exec.schema.NamedField;
+import org.apache.drill.exec.schema.ObjectSchema;
+import org.apache.drill.exec.schema.RecordSchema;
+import org.apache.drill.exec.schema.SchemaIdGenerator;
 import org.apache.drill.exec.schema.json.jackson.JacksonHelper;
 import org.apache.drill.exec.store.RecordReader;
 import org.apache.drill.exec.store.VectorHolder;
-import org.apache.drill.exec.vector.*;
-import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.drill.exec.vector.AllocationHelper;
+import org.apache.drill.exec.vector.NullableBitHolder;
+import org.apache.drill.exec.vector.NullableBitVector;
+import org.apache.drill.exec.vector.NullableFloat4Holder;
+import org.apache.drill.exec.vector.NullableFloat4Vector;
+import org.apache.drill.exec.vector.NullableIntHolder;
+import org.apache.drill.exec.vector.NullableIntVector;
+import org.apache.drill.exec.vector.NullableVarCharVector;
+import org.apache.drill.exec.vector.RepeatedBitVector;
+import org.apache.drill.exec.vector.RepeatedFloat4Vector;
+import org.apache.drill.exec.vector.RepeatedIntVector;
+import org.apache.drill.exec.vector.RepeatedVarCharVector;
+import org.apache.drill.exec.vector.TypeHelper;
+import org.apache.drill.exec.vector.ValueVector;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.URI;
-import java.nio.charset.Charset;
-import java.util.List;
-import java.util.Map;
-
-import static com.fasterxml.jackson.core.JsonToken.*;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 
 public class JSONRecordReader implements RecordReader {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JSONRecordReader.class);
@@ -59,17 +72,19 @@ public class JSONRecordReader implements RecordReader {
   private OutputMutator outputMutator;
   private BufferAllocator allocator;
   private int batchSize;
+  private final FieldReference ref;
 
-  public JSONRecordReader(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem, int batchSize) {
+  public JSONRecordReader(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem, int batchSize, FieldReference ref) {
     this.hadoopPath = new Path(inputPath);
     this.fileSystem = fileSystem;
     this.allocator = fragmentContext.getAllocator();
     this.batchSize = batchSize;
     valueVectorMap = Maps.newHashMap();
+    this.ref = ref;
   }
 
-  public JSONRecordReader(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem) {
-    this(fragmentContext, inputPath, fileSystem, DEFAULT_LENGTH);
+  public JSONRecordReader(FragmentContext fragmentContext, String inputPath, FileSystem fileSystem, FieldReference ref) {
+    this(fragmentContext, inputPath, fileSystem, DEFAULT_LENGTH, ref);
   }
 
   private JsonParser getParser() {
@@ -122,7 +137,7 @@ public class JSONRecordReader implements RecordReader {
       // Garbage collect fields never referenced in this batch
       for (Field field : Iterables.concat(currentSchema.removeUnreadFields(), removedFields)) {
         diffSchema.addRemovedField(field);
-        outputMutator.removeField(field.getAsMaterializedField());
+        outputMutator.removeField(field.getAsMaterializedField(ref));
       }
 
       if (diffSchema.isChanged()) {
@@ -461,7 +476,7 @@ public class JSONRecordReader implements RecordReader {
   }
 
   private VectorHolder getOrCreateVectorHolder(Field field) throws SchemaChangeException {
-    String fullFieldName = field.getFullFieldName();
+    String fullFieldName = ref.getPath() + "." + field.getFullFieldName();
     VectorHolder holder = valueVectorMap.get(fullFieldName);
 
     if (holder == null) {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONScanBatchCreator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONScanBatchCreator.java
index a79fa81..d5291b4 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONScanBatchCreator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONScanBatchCreator.java
@@ -38,7 +38,7 @@ public class JSONScanBatchCreator implements BatchCreator<JSONSubScan> {
         List<JSONGroupScan.ScanEntry> entries = config.getReadEntries();
         List<RecordReader> readers = Lists.newArrayList();
         for (JSONGroupScan.ScanEntry e : entries) {
-            readers.add(new JSONRecordReader(context, e.getPath(), config.getStorageEngine().getFileSystem()));
+            readers.add(new JSONRecordReader(context, e.getPath(), config.getStorageEngine().getFileSystem(), config.getRef()));
         }
 
         return new ScanBatch(context, readers.iterator());

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java
index 532a8b9..b0a2e3b 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java
@@ -17,21 +17,31 @@
  ******************************************************************************/
 package org.apache.drill.exec.store.json;
 
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.apache.drill.common.logical.data.Scan;
 import org.apache.drill.exec.server.DrillbitContext;
 import org.apache.drill.exec.store.AbstractStorageEngine;
+import org.apache.drill.exec.store.SchemaProvider;
+import org.apache.drill.exec.store.json.JSONGroupScan.ScanEntry;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 
-import java.io.IOException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
 
 public class JSONStorageEngine extends AbstractStorageEngine {
   private final JSONStorageEngineConfig config;
   private final Configuration conf;
   private FileSystem fileSystem;
   public static final String HADOOP_DEFAULT_NAME = "fs.default.name";
-
+  private final JsonSchemaProvider schemaProvider;
+ 
   public JSONStorageEngine(JSONStorageEngineConfig config, DrillbitContext context) {
     this.config = config;
+    this.schemaProvider = new JsonSchemaProvider(config, context.getConfig());
+    
     try {
       this.conf = new Configuration();
       this.conf.set(HADOOP_DEFAULT_NAME, config.getDfsName());
@@ -45,4 +55,21 @@ public class JSONStorageEngine extends AbstractStorageEngine {
   public FileSystem getFileSystem() {
     return fileSystem;
   }
+
+  public JSONStorageEngineConfig getConfig(){
+    return config;
+  }
+  
+  @Override
+  public JSONGroupScan getPhysicalScan(Scan scan) throws IOException {
+    ArrayList<ScanEntry> readEntries = scan.getSelection().getListWith(new ObjectMapper(), new TypeReference<ArrayList<ScanEntry>>() {});
+    return new JSONGroupScan(readEntries, this, scan.getOutputReference());
+  }
+
+  @Override
+  public SchemaProvider getSchemaProvider() {
+    return schemaProvider;
+  }
+  
+  
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONSubScan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONSubScan.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONSubScan.java
index d3a7fbc..602e543 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONSubScan.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONSubScan.java
@@ -18,16 +18,26 @@
 
 package org.apache.drill.exec.store.json;
 
-import com.fasterxml.jackson.annotation.*;
-import com.google.common.collect.Iterators;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.FieldReference;
 import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.exec.exception.SetupException;
 import org.apache.drill.exec.physical.OperatorCost;
-import org.apache.drill.exec.physical.base.*;
+import org.apache.drill.exec.physical.base.AbstractBase;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.base.PhysicalVisitor;
+import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.physical.base.SubScan;
 import org.apache.drill.exec.store.StorageEngineRegistry;
 
-import java.util.Iterator;
-import java.util.List;
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.collect.Iterators;
 
 @JsonTypeName("json-sub-scan")
 public class JSONSubScan extends AbstractBase implements SubScan {
@@ -35,18 +45,19 @@ public class JSONSubScan extends AbstractBase implements SubScan {
   protected final List<JSONGroupScan.ScanEntry> readEntries;
   private final OperatorCost cost;
   private final Size size;
-  private final StorageEngineRegistry registry;
-  private final JSONStorageEngineConfig engineConfig;
   private final JSONStorageEngine storageEngine;
+  private final FieldReference ref;
 
   @JsonCreator
   public JSONSubScan(@JacksonInject StorageEngineRegistry registry,
                      @JsonProperty("engineConfig") StorageEngineConfig engineConfig,
-                     @JsonProperty("readEntries") List<JSONGroupScan.ScanEntry> readEntries) throws SetupException {
+                     @JsonProperty("readEntries") List<JSONGroupScan.ScanEntry> readEntries, @JsonProperty("ref") FieldReference ref) throws ExecutionSetupException {
+    this(readEntries, (JSONStorageEngine) registry.getEngine(engineConfig), ref);
+  }
+  
+  JSONSubScan(List<JSONGroupScan.ScanEntry> readEntries, JSONStorageEngine engine, FieldReference ref){
     this.readEntries = readEntries;
-    this.registry = registry;
-    this.engineConfig = (JSONStorageEngineConfig) engineConfig;
-    this.storageEngine = (JSONStorageEngine) registry.getEngine(engineConfig);
+    this.storageEngine = engine;
     OperatorCost cost = new OperatorCost(0, 0, 0, 0);
     Size size = new Size(0, 0);
     for (JSONGroupScan.ScanEntry r : readEntries) {
@@ -55,6 +66,11 @@ public class JSONSubScan extends AbstractBase implements SubScan {
     }
     this.cost = cost;
     this.size = size;
+    this.ref = ref;
+  }
+  
+  public FieldReference getRef() {
+    return ref;
   }
 
   public List<JSONGroupScan.ScanEntry> getReadEntries() {
@@ -62,7 +78,7 @@ public class JSONSubScan extends AbstractBase implements SubScan {
   }
 
   public StorageEngineConfig getEngineConfig() {
-    return engineConfig;
+    return storageEngine.getConfig();
   }
 
   @JsonIgnore
@@ -71,13 +87,8 @@ public class JSONSubScan extends AbstractBase implements SubScan {
   }
 
   @Override
-  public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) {
-    try {
-      return new JSONSubScan(registry, (StorageEngineConfig) engineConfig, readEntries);
-    } catch (SetupException e) {
-      e.printStackTrace();
-    }
-    return null;
+  public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) throws ExecutionSetupException{
+    throw new UnsupportedOperationException();
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JsonSchemaProvider.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JsonSchemaProvider.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JsonSchemaProvider.java
new file mode 100644
index 0000000..065652a
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JsonSchemaProvider.java
@@ -0,0 +1,43 @@
+package org.apache.drill.exec.store.json;
+
+import java.io.IOException;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.physical.ReadEntryWithPath;
+import org.apache.drill.exec.store.SchemaProvider;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+
+import com.beust.jcommander.internal.Lists;
+
+public class JsonSchemaProvider implements SchemaProvider{
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JsonSchemaProvider.class);
+
+  public static final String HADOOP_DEFAULT_NAME = "fs.default.name";
+  final JSONStorageEngineConfig configuration;
+  final FileSystem fs;
+  final Configuration conf;
+
+  public JsonSchemaProvider(JSONStorageEngineConfig configuration, DrillConfig config){
+    this.configuration = configuration;
+    try {
+      this.conf = new Configuration();
+      this.conf.set(HADOOP_DEFAULT_NAME, "file:///");
+      this.fs = FileSystem.get(conf);
+    } catch (IOException ie) {
+      throw new RuntimeException("Error setting up filesystem");
+    }
+  }
+
+  @Override
+  public Object getSelectionBaseOnName(String tableName) {
+    try{
+//      if(!fs.exists(new Path(tableName))) return null;
+      ReadEntryWithPath re = new ReadEntryWithPath(tableName);
+      return Lists.newArrayList(re);
+    }catch(Exception e){
+      logger.warn(String.format("Failure while checking table name %s.", tableName), e);
+      return null;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java
index 1ea6958..5f45a6d 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java
@@ -24,8 +24,8 @@ import org.apache.drill.common.logical.data.Scan;
 import org.apache.drill.exec.physical.base.AbstractGroupScan;
 import org.apache.drill.exec.server.DrillbitContext;
 import org.apache.drill.exec.store.AbstractStorageEngine;
+import org.apache.drill.exec.store.SchemaProvider;
 import org.apache.drill.exec.store.mock.MockGroupScanPOP.MockScanEntry;
-import org.apache.drill.storage.MockStorageEngineConfig;
 
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -47,4 +47,9 @@ public class MockStorageEngine extends AbstractStorageEngine {
     return new MockGroupScanPOP(null, readEntries);
   }
 
+  @Override
+  public SchemaProvider getSchemaProvider() {
+    throw new UnsupportedOperationException();
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngineConfig.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngineConfig.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngineConfig.java
new file mode 100644
index 0000000..343a563
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngineConfig.java
@@ -0,0 +1,59 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.store.mock;
+
+import org.apache.drill.common.logical.StorageEngineConfigBase;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+@JsonTypeName("mock")
+public class MockStorageEngineConfig extends StorageEngineConfigBase{
+
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockStorageEngineConfig.class);
+
+  private String url;
+  
+  @JsonCreator
+  public MockStorageEngineConfig(@JsonProperty("url") String url) {
+    this.url = url;
+  }
+
+  public String getUrl() {
+    return url;
+  }
+
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+
+    MockStorageEngineConfig that = (MockStorageEngineConfig) o;
+
+    if (url != null ? !url.equals(that.url) : that.url != null) return false;
+
+    return true;
+  }
+
+  @Override
+  public int hashCode() {
+    return url != null ? url.hashCode() : 0;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
index 64ced87..54618c1 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetGroupScan.java
@@ -18,15 +18,21 @@
 package org.apache.drill.exec.store.parquet;
 
 import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
-import com.google.common.base.Stopwatch;
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Multimap;
 import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.exec.exception.SetupException;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.logical.StorageEngineConfig;
 import org.apache.drill.exec.physical.EndpointAffinity;
 import org.apache.drill.exec.physical.OperatorCost;
 import org.apache.drill.exec.physical.ReadEntryFromHDFS;
@@ -37,7 +43,6 @@ import org.apache.drill.exec.physical.base.Size;
 import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
 import org.apache.drill.exec.store.AffinityCalculator;
 import org.apache.drill.exec.store.StorageEngineRegistry;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
@@ -52,6 +57,10 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
 
 
 @JsonTypeName("parquet-scan")
@@ -79,13 +88,15 @@ public class ParquetGroupScan extends AbstractGroupScan {
   private ParquetStorageEngineConfig engineConfig;
   private FileSystem fs;
   private String fileName;
+  private final FieldReference ref;
   private List<EndpointAffinity> endpointAffinities;
 
   @JsonCreator
   public ParquetGroupScan(@JsonProperty("entries") List<ReadEntryWithPath> entries,
                           @JsonProperty("storageengine") ParquetStorageEngineConfig storageEngineConfig,
-                          @JacksonInject StorageEngineRegistry engineRegistry
-                           )throws SetupException,IOException {
+                          @JacksonInject StorageEngineRegistry engineRegistry,
+                          @JsonProperty("ref") FieldReference ref
+                           )throws IOException, ExecutionSetupException {
     engineRegistry.init(DrillConfig.create());
     this.storageEngine = (ParquetStorageEngine) engineRegistry.getEngine(storageEngineConfig);
     this.availableEndpoints = storageEngine.getContext().getBits();
@@ -93,17 +104,19 @@ public class ParquetGroupScan extends AbstractGroupScan {
     this.engineConfig = storageEngineConfig;
     this.engineRegistry = engineRegistry;
     this.entries = entries;
+    this.ref = ref;
     readFooter();
     this.fileName = rowGroupInfos.get(0).getPath();
     calculateEndpointBytes();
   }
 
   public ParquetGroupScan(ArrayList<ReadEntryWithPath> entries,
-                          ParquetStorageEngine storageEngine) throws IOException {
+                          ParquetStorageEngine storageEngine, FieldReference ref) throws IOException {
     this.storageEngine = storageEngine;
     this.availableEndpoints = storageEngine.getContext().getBits();
     this.fs = storageEngine.getFileSystem();
     this.entries = entries;
+    this.ref = ref;
     readFooter();
     this.fileName = rowGroupInfos.get(0).getPath();
     calculateEndpointBytes();
@@ -322,11 +335,12 @@ public class ParquetGroupScan extends AbstractGroupScan {
       logger.debug("minorFragmentId: {} Path: {} RowGroupIndex: {}",minorFragmentId, rg.getPath(),rg.getRowGroupIndex());
     }
     Preconditions.checkArgument(!mappings.get(minorFragmentId).isEmpty(), String.format("MinorFragmentId %d has no read entries assigned", minorFragmentId));
-    try {
-      return new ParquetRowGroupScan(storageEngine, engineConfig, mappings.get(minorFragmentId));
-    } catch (SetupException e) {
-      throw new RuntimeException("Error setting up ParquetRowGroupScan", e);
-    }
+    return new ParquetRowGroupScan(storageEngine, engineConfig, mappings.get(minorFragmentId), ref);
+  }
+
+  
+  public FieldReference getRef() {
+    return ref;
   }
 
   @Override
@@ -346,6 +360,11 @@ public class ParquetGroupScan extends AbstractGroupScan {
     return new Size(10,10);
   }
 
+  @JsonProperty("storageengine")
+  public StorageEngineConfig getStorageEngineConfig(){
+    return this.engineConfig;
+  }
+  
   @Override
   @JsonIgnore
   public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
index 3aaa987..0a0470d 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRecordReader.java
@@ -17,10 +17,17 @@
  ******************************************************************************/
 package org.apache.drill.exec.store.parquet;
 
+import static com.google.common.base.Preconditions.checkArgument;
 import io.netty.buffer.ByteBuf;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.drill.common.exceptions.DrillRuntimeException;
 import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FieldReference;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.common.types.Types;
@@ -35,6 +42,7 @@ import org.apache.drill.exec.vector.ValueVector;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+
 import parquet.column.ColumnDescriptor;
 import parquet.hadoop.CodecFactoryExposer;
 import parquet.hadoop.metadata.ColumnChunkMetaData;
@@ -42,14 +50,7 @@ import parquet.hadoop.metadata.ParquetMetadata;
 import parquet.schema.MessageType;
 import parquet.schema.PrimitiveType;
 
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
+import com.google.common.base.Joiner;
 
 public class ParquetRecordReader implements RecordReader {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRecordReader.class);
@@ -74,6 +75,7 @@ public class ParquetRecordReader implements RecordReader {
   private boolean allFieldsFixedLength;
   private int recordsPerBatch;
   private ByteBuf bufferWithAllData;
+  private final FieldReference ref;
   long totalRecords;
   long rowGroupOffset;
 
@@ -94,18 +96,19 @@ public class ParquetRecordReader implements RecordReader {
 
   public ParquetRecordReader(FragmentContext fragmentContext,
                              String path, int rowGroupIndex, FileSystem fs,
-                             CodecFactoryExposer codecFactoryExposer, ParquetMetadata footer) throws ExecutionSetupException {
-    this(fragmentContext, DEFAULT_BATCH_LENGTH_IN_BITS, path, rowGroupIndex, fs, codecFactoryExposer, footer);
+                             CodecFactoryExposer codecFactoryExposer, ParquetMetadata footer, FieldReference ref) throws ExecutionSetupException {
+    this(fragmentContext, DEFAULT_BATCH_LENGTH_IN_BITS, path, rowGroupIndex, fs, codecFactoryExposer, footer, ref);
   }
 
 
   public ParquetRecordReader(FragmentContext fragmentContext, long batchSize,
                              String path, int rowGroupIndex, FileSystem fs,
-                             CodecFactoryExposer codecFactoryExposer, ParquetMetadata footer) throws ExecutionSetupException {
+                             CodecFactoryExposer codecFactoryExposer, ParquetMetadata footer, FieldReference ref) throws ExecutionSetupException {
     this.allocator = fragmentContext.getAllocator();
 
     hadoopPath = new Path(path);
     fileSystem = fs;
+    this.ref = ref;
     this.codecFactoryExposer = codecFactoryExposer;
     this.rowGroupIndex = rowGroupIndex;
     this.batchSize = batchSize;
@@ -122,7 +125,7 @@ public class ParquetRecordReader implements RecordReader {
     // loop to add up the length of the fixed width columns and build the schema
     for (int i = 0; i < columns.size(); ++i) {
       column = columns.get(i);
-
+      logger.debug("Found Parquet column {} of type {}", column.getPath(), column.getType());
       // sum the lengths of all of the fixed length fields
       if (column.getType() != PrimitiveType.PrimitiveTypeName.BINARY) {
         // There is not support for the fixed binary type yet in parquet, leaving a task here as a reminder
@@ -151,7 +154,7 @@ public class ParquetRecordReader implements RecordReader {
       for (int i = 0; i < columns.size(); ++i) {
         column = columns.get(i);
         columnChunkMetaData = footer.getBlocks().get(0).getColumns().get(i);
-        field = MaterializedField.create(new SchemaPath(toFieldName(column.getPath()), ExpressionPosition.UNKNOWN),
+        field = MaterializedField.create(toFieldName(column.getPath()),
             toMajorType(column.getType(), getDataMode(column, footer.getFileMetaData().getSchema())));
         fieldFixedLength = column.getType() != PrimitiveType.PrimitiveTypeName.BINARY;
         ValueVector v = TypeHelper.getNewVector(field, allocator);
@@ -259,8 +262,13 @@ public class ParquetRecordReader implements RecordReader {
     }
   }
 
-  private static String toFieldName(String[] paths) {
-    return join(SEPERATOR, paths);
+  private SchemaPath toFieldName(String[] paths) {
+    if(this.ref == null){
+      return new SchemaPath(Joiner.on('/').join(paths), ExpressionPosition.UNKNOWN);
+    }else{
+      return ref.getChild(paths);
+    }
+    
   }
 
   private TypeProtos.DataMode getDataMode(ColumnDescriptor column, MessageType schema) {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
index 1e5d203..7cefcf4 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
@@ -17,39 +17,56 @@
  ******************************************************************************/
 package org.apache.drill.exec.store.parquet;
 
-import com.fasterxml.jackson.annotation.*;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterators;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.FieldReference;
 import org.apache.drill.common.logical.StorageEngineConfig;
 import org.apache.drill.exec.exception.SetupException;
 import org.apache.drill.exec.physical.OperatorCost;
 import org.apache.drill.exec.physical.ReadEntryFromHDFS;
-import org.apache.drill.exec.physical.base.*;
+import org.apache.drill.exec.physical.base.AbstractBase;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.base.PhysicalVisitor;
+import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.physical.base.SubScan;
 import org.apache.drill.exec.store.StorageEngineRegistry;
 
-import java.util.*;
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterators;
 
 // Class containing information for reading a single parquet row group form HDFS
 @JsonTypeName("parquet-row-group-scan")
 public class ParquetRowGroupScan extends AbstractBase implements SubScan {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetRowGroupScan.class);
 
-  public StorageEngineConfig engineConfig;
-  private ParquetStorageEngine parquetStorageEngine;
-  private List<RowGroupReadEntry> rowGroupReadEntries;
+  public final StorageEngineConfig engineConfig;
+  private final ParquetStorageEngine parquetStorageEngine;
+  private final List<RowGroupReadEntry> rowGroupReadEntries;
+  private final FieldReference ref;
 
   @JsonCreator
   public ParquetRowGroupScan(@JacksonInject StorageEngineRegistry registry, @JsonProperty("engineConfig") StorageEngineConfig engineConfig,
-                             @JsonProperty("rowGroupReadEntries") LinkedList<RowGroupReadEntry> rowGroupReadEntries) throws SetupException {
+                             @JsonProperty("rowGroupReadEntries") LinkedList<RowGroupReadEntry> rowGroupReadEntries, @JsonProperty("ref") FieldReference ref) throws ExecutionSetupException {
     parquetStorageEngine = (ParquetStorageEngine) registry.getEngine(engineConfig);
     this.rowGroupReadEntries = rowGroupReadEntries;
+    this.engineConfig = engineConfig;
+    this.ref = ref;
   }
 
   public ParquetRowGroupScan(ParquetStorageEngine engine, ParquetStorageEngineConfig config,
-                              List<RowGroupReadEntry> rowGroupReadEntries) throws SetupException {
+                              List<RowGroupReadEntry> rowGroupReadEntries, FieldReference ref) {
     parquetStorageEngine = engine;
     engineConfig = config;
     this.rowGroupReadEntries = rowGroupReadEntries;
+    this.ref = ref;
   }
 
   public List<RowGroupReadEntry> getRowGroupReadEntries() {
@@ -65,6 +82,11 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan {
     return null;
   }
 
+  
+  public FieldReference getRef() {
+    return ref;
+  }
+
   @Override
   public Size getSize() {
     return null;
@@ -88,13 +110,7 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan {
   @Override
   public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) {
     Preconditions.checkArgument(children.isEmpty());
-    try {
-      return new ParquetRowGroupScan(parquetStorageEngine, (ParquetStorageEngineConfig) engineConfig, rowGroupReadEntries);
-    } catch (SetupException e) {
-      // TODO - handle this
-      e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
-    }
-    return null;
+    return new ParquetRowGroupScan(parquetStorageEngine, (ParquetStorageEngineConfig) engineConfig, rowGroupReadEntries, ref);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
index addd288..a09f1ac 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
@@ -63,7 +63,8 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan
             new ParquetRecordReader(
                 context, e.getPath(), e.getRowGroupIndex(), rowGroupScan.getStorageEngine().getFileSystem(),
                 rowGroupScan.getStorageEngine().getCodecFactoryExposer(),
-                ParquetFileReader.readFooter( rowGroupScan.getStorageEngine().getFileSystem().getConf(), new Path(e.getPath()))
+                ParquetFileReader.readFooter( rowGroupScan.getStorageEngine().getFileSystem().getConf(), new Path(e.getPath())),
+                rowGroupScan.getRef()
             )
         );
       } catch (IOException e1) {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
new file mode 100644
index 0000000..3e4b02a
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
@@ -0,0 +1,44 @@
+package org.apache.drill.exec.store.parquet;
+
+import java.io.IOException;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.physical.ReadEntryWithPath;
+import org.apache.drill.exec.store.SchemaProvider;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import com.beust.jcommander.internal.Lists;
+
+public class ParquetSchemaProvider implements SchemaProvider{
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetSchemaProvider.class);
+
+  public static final String HADOOP_DEFAULT_NAME = "fs.default.name";
+  final ParquetStorageEngineConfig configuration;
+  final FileSystem fs;
+  final Configuration conf;
+
+  public ParquetSchemaProvider(ParquetStorageEngineConfig configuration, DrillConfig config){
+    this.configuration = configuration;
+    try {
+      this.conf = new Configuration();
+      this.conf.set(HADOOP_DEFAULT_NAME, "file:///");
+      this.fs = FileSystem.get(conf);
+    } catch (IOException ie) {
+      throw new RuntimeException("Error setting up filesystem");
+    }
+  }
+
+  @Override
+  public Object getSelectionBaseOnName(String tableName) {
+    try{
+//      if(!fs.exists(new Path(tableName))) return null;
+      ReadEntryWithPath re = new ReadEntryWithPath(tableName);
+      return Lists.newArrayList(re);
+    }catch(Exception e){
+      logger.warn(String.format("Failure while checking table name %s.", tableName), e);
+      return null;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
index cf5b5d8..1c8bbe6 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
@@ -22,10 +22,12 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 
+import com.beust.jcommander.internal.Lists;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.logical.StorageEngineConfig;
 import org.apache.drill.common.logical.data.Scan;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.physical.ReadEntry;
@@ -34,6 +36,7 @@ import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
 import org.apache.drill.exec.server.DrillbitContext;
 import org.apache.drill.exec.store.AbstractStorageEngine;
 import org.apache.drill.exec.store.RecordReader;
+import org.apache.drill.exec.store.SchemaProvider;
 import org.apache.drill.exec.store.mock.MockStorageEngine;
 
 import com.google.common.collect.ListMultimap;
@@ -53,35 +56,25 @@ public class ParquetStorageEngine extends AbstractStorageEngine{
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockStorageEngine.class);
 
   private final DrillbitContext context;
-  private final ParquetStorageEngineConfig configuration;
-  private FileSystem fs;
-  private Configuration conf;
   static final ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter();
   private CodecFactoryExposer codecFactoryExposer;
   final ParquetMetadata footer;
-  public static final String HADOOP_DEFAULT_NAME = "fs.default.name";
-
+  private final ParquetSchemaProvider schemaProvider;
 
   public ParquetStorageEngine(ParquetStorageEngineConfig configuration, DrillbitContext context){
     this.context = context;
-    this.configuration = configuration;
+    this.schemaProvider = new ParquetSchemaProvider(configuration, context.getConfig());
+    codecFactoryExposer = new CodecFactoryExposer(schemaProvider.conf);
+
     this.footer = null;
-    try {
-      this.conf = new Configuration();
-      this.conf.set(HADOOP_DEFAULT_NAME, configuration.getDfsName());
-      this.fs = FileSystem.get(conf);
-      codecFactoryExposer = new CodecFactoryExposer(conf);
-    } catch (IOException ie) {
-      throw new RuntimeException("Error setting up filesystem");
-    }
   }
 
   public Configuration getHadoopConfig() {
-    return this.conf;
+    return schemaProvider.conf;
   }
 
   public FileSystem getFileSystem() {
-    return this.fs;
+    return schemaProvider.fs;
   }
 
   public DrillbitContext getContext() {
@@ -99,7 +92,7 @@ public class ParquetStorageEngine extends AbstractStorageEngine{
     ArrayList<ReadEntryWithPath> readEntries = scan.getSelection().getListWith(new ObjectMapper(),
         new TypeReference<ArrayList<ReadEntryWithPath>>() {});
 
-    return new ParquetGroupScan(readEntries, this);
+    return new ParquetGroupScan(readEntries, this, scan.getOutputReference());
   }
 
   @Override
@@ -116,4 +109,9 @@ public class ParquetStorageEngine extends AbstractStorageEngine{
   public CodecFactoryExposer getCodecFactoryExposer() {
     return codecFactoryExposer;
   }
+
+  @Override
+  public ParquetSchemaProvider getSchemaProvider() {
+    return schemaProvider;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngineConfig.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngineConfig.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngineConfig.java
index ad55f13..494269e 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngineConfig.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngineConfig.java
@@ -17,15 +17,17 @@
  ******************************************************************************/
 package org.apache.drill.exec.store.parquet;
 
+import java.util.HashMap;
+
+import org.apache.drill.common.logical.StorageEngineConfigBase;
+import org.apache.drill.exec.store.DistributedStorageEngine;
+
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
-import org.apache.drill.common.logical.StorageEngineConfigBase;
-
-import java.util.HashMap;
 
 @JsonTypeName("parquet")
-public class ParquetStorageEngineConfig extends StorageEngineConfigBase {
+public class ParquetStorageEngineConfig extends StorageEngineConfigBase implements DistributedStorageEngine{
 
   public String getDfsName() {
     return dfsName;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
new file mode 100644
index 0000000..ed9b3b1
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java
@@ -0,0 +1,21 @@
+package org.apache.drill.exec.vector;
+
+import io.netty.buffer.UnpooledByteBufAllocator;
+
+import com.google.common.base.Charsets;
+
+
+public class ValueHolderHelper {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueHolderHelper.class);
+  
+  public static VarCharHolder getVarCharHolder(String s){
+    VarCharHolder vch = new VarCharHolder();
+    
+    byte[] b = s.getBytes(Charsets.UTF_8);
+    vch.start = 0;
+    vch.end = b.length;
+    vch.buffer = UnpooledByteBufAllocator.DEFAULT.buffer();
+    vch.buffer.setBytes(0, b);
+    return vch;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/FragmentRunnerListener.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/FragmentRunnerListener.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/FragmentRunnerListener.java
index d978470..71742ed 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/FragmentRunnerListener.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/FragmentRunnerListener.java
@@ -21,8 +21,6 @@ import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
 import org.apache.drill.exec.proto.ExecProtos.FragmentStatus.FragmentState;
 
 public interface FragmentRunnerListener {
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FragmentRunnerListener.class);
-  
   void fail(FragmentHandle handle, String message, Throwable excep);
   void stateChanged(FragmentHandle handle, FragmentState newState);
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
index fa51b09..de8d31c 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/WorkManager.java
@@ -148,7 +148,7 @@ public class WorkManager implements Closeable{
   public void run() {
     try {
     while(true){
-      logger.debug("Polling for pending work tasks.");
+//      logger.debug("Polling for pending work tasks.");
       Runnable r = pendingTasks.poll(10, TimeUnit.SECONDS);
       if(r != null){
         logger.debug("Starting pending task {}", r);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
index bd64938..26f0192 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
@@ -25,6 +25,7 @@ import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.logical.LogicalPlan;
 import org.apache.drill.exec.exception.FragmentSetupException;
+import org.apache.drill.exec.exception.OptimizerException;
 import org.apache.drill.exec.ops.QueryContext;
 import org.apache.drill.exec.opt.BasicOptimizer;
 import org.apache.drill.exec.physical.PhysicalPlan;
@@ -145,32 +146,38 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
    */
   public void run() {
     // convert a run query request into action
-
-    switch (queryRequest.getType()) {
-
-    case LOGICAL:
-      parseAndRunLogicalPlan(queryRequest.getPlan());
-      break;
-    case PHYSICAL:
-      parseAndRunPhysicalPlan(queryRequest.getPlan());
-      break;
-    case SQL:
-      runSQL(queryRequest.getPlan());
-      break;
-    default:
-      throw new UnsupportedOperationException();
+    try{
+      switch (queryRequest.getType()) {
+      
+      case LOGICAL:
+        parseAndRunLogicalPlan(queryRequest.getPlan());
+        break;
+      case PHYSICAL:
+        parseAndRunPhysicalPlan(queryRequest.getPlan());
+        break;
+      case SQL:
+        runSQL(queryRequest.getPlan());
+        break;
+      default:
+        throw new UnsupportedOperationException();
+      }
+    }catch(Exception ex){
+      fail("Failure while setting up Foreman.", ex);
     }
   }
 
   private void parseAndRunLogicalPlan(String json) {
+    
     try {
       LogicalPlan logicalPlan = context.getPlanReader().readLogicalPlan(json);
-      logger.debug("Logical {}", logicalPlan.unparse(DrillConfig.create()));
+      if(logger.isDebugEnabled()) logger.debug("Logical {}", logicalPlan.unparse(context.getConfig()));
       PhysicalPlan physicalPlan = convert(logicalPlan);
-      //logger.debug("Physical {}", new ObjectMapper().writeValueAsString(physicalPlan));
+      if(logger.isDebugEnabled()) logger.debug("Physical {}", context.getConfig().getMapper().writeValueAsString(physicalPlan));
       runPhysicalPlan(physicalPlan);
     } catch (IOException e) {
       fail("Failure while parsing logical plan.", e);
+    } catch (OptimizerException e) {
+      fail("Failure while converting logical plan to physical plan.", e);
     }
   }
 
@@ -231,7 +238,7 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
     throw new UnsupportedOperationException();
   }
 
-  private PhysicalPlan convert(LogicalPlan plan) {
+  private PhysicalPlan convert(LogicalPlan plan) throws OptimizerException {
     return new BasicOptimizer(DrillConfig.create(), context).optimize(new BasicOptimizer.BasicOptimizationContext(), plan);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/resources/drill-module.conf b/sandbox/prototype/exec/java-exec/src/main/resources/drill-module.conf
index d9e1ef5..c0735cc 100644
--- a/sandbox/prototype/exec/java-exec/src/main/resources/drill-module.conf
+++ b/sandbox/prototype/exec/java-exec/src/main/resources/drill-module.conf
@@ -11,10 +11,19 @@ drill.exec: {
   	bit.port : 31011,
   	use.ip : false
   },
+  operator: {
+    packages += "org.apache.drill.exec.physical.config"
+  },
   optimizer: {
     implementation: "org.apache.drill.exec.opt.IdentityOptimizer"
   },
-  
+  functions: ["org.apache.drill.expr.fn.impl"],
+  storage: {
+    packages += "org.apache.drill.exec.store"  
+  }
+  metrics : { 
+    context: "drillbit"
+  },
   zk: {
 	connect: "localhost:2181",
 	root: "/drill",

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
new file mode 100644
index 0000000..9a71dde
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java
@@ -0,0 +1,273 @@
+package org.apache.drill.exec.physical.impl;
+
+import java.util.List;
+
+import mockit.Injectable;
+import mockit.NonStrictExpectations;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.logical.LogicalPlan;
+import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.cache.DistributedCache;
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.coord.ClusterCoordinator;
+import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.ops.QueryContext;
+import org.apache.drill.exec.opt.BasicOptimizer;
+import org.apache.drill.exec.physical.PhysicalPlan;
+import org.apache.drill.exec.physical.base.FragmentRoot;
+import org.apache.drill.exec.planner.PhysicalPlanReader;
+import org.apache.drill.exec.proto.CoordinationProtos;
+import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
+import org.apache.drill.exec.proto.UserBitShared.QueryId;
+import org.apache.drill.exec.proto.UserProtos;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.rpc.bit.BitCom;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection;
+import org.apache.drill.exec.server.BootStrapContext;
+import org.apache.drill.exec.server.Drillbit;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.server.RemoteServiceSet;
+import org.apache.drill.exec.store.StorageEngineRegistry;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VarBinaryHolder;
+import org.apache.drill.exec.vector.VarBinaryVector;
+import org.junit.AfterClass;
+import org.junit.Test;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import com.google.common.io.Resources;
+import com.yammer.metrics.MetricRegistry;
+
+
+public class TestOptiqPlans {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestOptiqPlans.class);
+  DrillConfig c = DrillConfig.create();
+  
+  @Test
+  public void orderBy(@Injectable final BootStrapContext ctxt, @Injectable UserClientConnection connection, @Injectable ClusterCoordinator coord, @Injectable BitCom com, @Injectable DistributedCache cache) throws Throwable{
+    SimpleRootExec exec = doLogicalTest(ctxt, connection, "/logical_order.json", coord, com, cache);
+  }
+  
+  @Test
+  public void stringFilter(@Injectable final BootStrapContext ctxt, @Injectable UserClientConnection connection, @Injectable ClusterCoordinator coord, @Injectable BitCom com, @Injectable DistributedCache cache) throws Throwable{
+    SimpleRootExec exec = doLogicalTest(ctxt, connection, "/logical_string_filter.json", coord, com, cache);
+  }
+  
+  @Test
+  public void groupBy(@Injectable final BootStrapContext bitContext, @Injectable UserClientConnection connection, @Injectable ClusterCoordinator coord, @Injectable BitCom com, @Injectable DistributedCache cache) throws Throwable{
+    SimpleRootExec exec = doLogicalTest(bitContext, connection, "/logical_group.json", coord, com, cache);
+  }
+  
+  private SimpleRootExec doLogicalTest(final BootStrapContext context, UserClientConnection connection, String file, ClusterCoordinator coord, BitCom com, DistributedCache cache) throws Exception{
+    new NonStrictExpectations(){{
+      context.getMetrics(); result = new MetricRegistry("test");
+      context.getAllocator(); result = BufferAllocator.getAllocator(c);
+      context.getConfig(); result = c;
+    }};
+    RemoteServiceSet lss = RemoteServiceSet.getLocalServiceSet();
+    DrillbitContext bitContext = new DrillbitContext(DrillbitEndpoint.getDefaultInstance(), context, coord, com, cache);
+    QueryContext qc = new QueryContext(QueryId.getDefaultInstance(), bitContext);
+    PhysicalPlanReader reader = bitContext.getPlanReader();
+    LogicalPlan plan = reader.readLogicalPlan(Files.toString(FileUtils.getResourceAsFile(file), Charsets.UTF_8));
+    PhysicalPlan pp = new BasicOptimizer(DrillConfig.create(), qc).optimize(new BasicOptimizer.BasicOptimizationContext(), plan);
+    
+
+    
+    FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
+    FragmentContext fctxt = new FragmentContext(bitContext, FragmentHandle.getDefaultInstance(), connection, null, registry);
+    SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(fctxt, (FragmentRoot) pp.getSortedOperators(false).iterator().next()));
+    return exec;
+    
+  }
+  
+  @Test
+  public void testFilterPlan() throws Exception {
+    RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+    DrillConfig config = DrillConfig.create();
+
+    try(Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator());){
+      bit1.run();
+      client.connect();
+      List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, Resources.toString(Resources.getResource("physical_filter.json"),Charsets.UTF_8));
+      RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
+      for (QueryResultBatch b : results) {
+        System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
+        loader.load(b.getHeader().getDef(), b.getData());
+        for (VectorWrapper vw : loader) {
+          System.out.println(vw.getValueVector().getField().getName());
+          ValueVector vv = vw.getValueVector();
+          for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
+            Object o = vv.getAccessor().getObject(i);
+            System.out.println(vv.getAccessor().getObject(i));
+          }
+        }
+      }
+      client.close();
+    }
+  }
+
+  @Test
+  public void testJoinPlan() throws Exception {
+    RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+    DrillConfig config = DrillConfig.create();
+
+    try(Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator());){
+      bit1.run();
+      client.connect();
+      List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, Resources.toString(Resources.getResource("physical_join.json"),Charsets.UTF_8));
+      RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
+      for (QueryResultBatch b : results) {
+        System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
+        loader.load(b.getHeader().getDef(), b.getData());
+        for (VectorWrapper vw : loader) {
+          System.out.println(vw.getValueVector().getField().getName());
+          ValueVector vv = vw.getValueVector();
+          for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
+            Object o = vv.getAccessor().getObject(i);
+            System.out.println(vv.getAccessor().getObject(i));
+          }
+        }
+      }
+      client.close();
+    }
+  }
+  
+  @Test
+  public void testFilterString() throws Exception {
+    RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+    DrillConfig config = DrillConfig.create();
+
+    try(Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator());){
+      bit1.run();
+      client.connect();
+      List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.LOGICAL, Resources.toString(Resources.getResource("logical_string_filter.json"),Charsets.UTF_8));
+      RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
+      for (QueryResultBatch b : results) {
+        System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
+        loader.load(b.getHeader().getDef(), b.getData());
+        for (VectorWrapper vw : loader) {
+          System.out.println(vw.getValueVector().getField().getName());
+          ValueVector vv = vw.getValueVector();
+          for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
+            Object o = vv.getAccessor().getObject(i);
+            if(vv instanceof VarBinaryVector){
+              VarBinaryVector.Accessor x = ((VarBinaryVector) vv).getAccessor();
+              VarBinaryHolder vbh = new VarBinaryHolder();
+              x.get(i, vbh);
+              System.out.printf("%d..%d", vbh.start, vbh.end);
+              
+              System.out.println("[" + new String( (byte[]) vv.getAccessor().getObject(i)) + "]");  
+            }else{
+              System.out.println(vv.getAccessor().getObject(i));
+            }
+            
+          }
+        }
+      }
+      client.close();
+    }
+  }
+  
+  
+  @Test
+  public void testLogicalJsonScan() throws Exception {
+    RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+    DrillConfig config = DrillConfig.create();
+
+    try(Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator());){
+      bit1.run();
+      client.connect();
+      List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.LOGICAL, Resources.toString(Resources.getResource("logical_json_scan.json"),Charsets.UTF_8));
+      RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
+      for (QueryResultBatch b : results) {
+        System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
+        loader.load(b.getHeader().getDef(), b.getData());
+        for (VectorWrapper vw : loader) {
+          System.out.println(vw.getValueVector().getField().getName());
+          ValueVector vv = vw.getValueVector();
+          for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
+            Object o = vv.getAccessor().getObject(i);
+            if(vv instanceof VarBinaryVector){
+              VarBinaryVector.Accessor x = ((VarBinaryVector) vv).getAccessor();
+              VarBinaryHolder vbh = new VarBinaryHolder();
+              x.get(i, vbh);
+              System.out.printf("%d..%d", vbh.start, vbh.end);
+              
+              System.out.println("[" + new String( (byte[]) vv.getAccessor().getObject(i)) + "]");  
+            }else{
+              System.out.println(vv.getAccessor().getObject(i));
+            }
+            
+          }
+        }
+      }
+      client.close();
+    }
+  }
+  
+  @Test
+  public void testOrderVarbinary() throws Exception {
+    RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+    DrillConfig config = DrillConfig.create();
+
+    try(Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator());){
+      bit1.run();
+      client.connect();
+      List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL, Resources.toString(Resources.getResource("physical_order_varbinary.json"),Charsets.UTF_8));
+      RecordBatchLoader loader = new RecordBatchLoader(bit1.getContext().getAllocator());
+      for (QueryResultBatch b : results) {
+        System.out.println(String.format("Got %d results", b.getHeader().getRowCount()));
+        loader.load(b.getHeader().getDef(), b.getData());
+        for (VectorWrapper vw : loader) {
+          System.out.println(vw.getValueVector().getField().getName());
+          ValueVector vv = vw.getValueVector();
+          for (int i = 0; i < vv.getAccessor().getValueCount(); i++) {
+            Object o = vv.getAccessor().getObject(i);
+            if(vv instanceof VarBinaryVector){
+              VarBinaryVector.Accessor x = ((VarBinaryVector) vv).getAccessor();
+              VarBinaryHolder vbh = new VarBinaryHolder();
+              x.get(i, vbh);
+              System.out.printf("%d..%d", vbh.start, vbh.end);
+              
+              System.out.println("[" + new String( (byte[]) vv.getAccessor().getObject(i)) + "]");  
+            }else{
+              System.out.println(vv.getAccessor().getObject(i));
+            }
+            
+          }
+        }
+      }
+      client.close();
+    }
+  }
+  
+  private SimpleRootExec doPhysicalTest(final DrillbitContext bitContext, UserClientConnection connection, String file) throws Exception{
+    new NonStrictExpectations(){{
+      bitContext.getMetrics(); result = new MetricRegistry("test");
+      bitContext.getAllocator(); result = BufferAllocator.getAllocator(c);
+      bitContext.getConfig(); result = c;
+    }};
+    
+    StorageEngineRegistry reg = new StorageEngineRegistry(bitContext);
+    
+    PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance(), reg);
+    PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile(file), Charsets.UTF_8));
+    FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
+    FragmentContext context = new FragmentContext(bitContext, FragmentHandle.getDefaultInstance(), connection, null, registry);
+    SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
+    return exec;
+  }
+  
+  @AfterClass
+  public static void tearDown() throws Exception{
+    // pause to get logger to catch up.
+    Thread.sleep(1000);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
index fc5bc81..e66ec49 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java
@@ -71,8 +71,8 @@ public class ParquetRecordReaderTest {
   private boolean VERBOSE_DEBUG = false;
   private boolean checkValues = true;
 
-  static final int numberRowGroups = 20;
-  static final int recordsPerRowGroup = 300000;
+  static final int numberRowGroups = 1;
+  static final int recordsPerRowGroup = 300;
   static final String fileName = "/tmp/parquet_test_file_many_types";
 
   @BeforeClass

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java
index 1f1f01b..2ff80d6 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestFileGenerator.java
@@ -87,13 +87,13 @@ public class TestFileGenerator {
 
   public static HashMap<String, FieldInfo> getFieldMap(int recordsPerRowGroup) {
     HashMap<String, FieldInfo> fields = new HashMap<>();
-    fields.put("integer/", new FieldInfo(recordsPerRowGroup, "int32", "integer", 32, intVals, TypeProtos.MinorType.INT));
-    fields.put("bigInt/", new FieldInfo(recordsPerRowGroup, "int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT));
-    fields.put("f/", new FieldInfo(recordsPerRowGroup, "float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4));
-    fields.put("d/", new FieldInfo(recordsPerRowGroup, "double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8));
-    fields.put("b/", new FieldInfo(recordsPerRowGroup, "boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT));
-    fields.put("bin/", new FieldInfo(recordsPerRowGroup, "binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY));
-    fields.put("bin2/", new FieldInfo(recordsPerRowGroup, "binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY));
+    fields.put("integer", new FieldInfo(recordsPerRowGroup, "int32", "integer", 32, intVals, TypeProtos.MinorType.INT));
+    fields.put("bigInt", new FieldInfo(recordsPerRowGroup, "int64", "bigInt", 64, longVals, TypeProtos.MinorType.BIGINT));
+    fields.put("f", new FieldInfo(recordsPerRowGroup, "float", "f", 32, floatVals, TypeProtos.MinorType.FLOAT4));
+    fields.put("d", new FieldInfo(recordsPerRowGroup, "double", "d", 64, doubleVals, TypeProtos.MinorType.FLOAT8));
+    fields.put("b", new FieldInfo(recordsPerRowGroup, "boolean", "b", 1, boolVals, TypeProtos.MinorType.BIT));
+    fields.put("bin", new FieldInfo(recordsPerRowGroup, "binary", "bin", -1, binVals, TypeProtos.MinorType.VARBINARY));
+    fields.put("bin2", new FieldInfo(recordsPerRowGroup, "binary", "bin2", -1, bin2Vals, TypeProtos.MinorType.VARBINARY));
     return fields;
   }
 
@@ -102,8 +102,9 @@ public class TestFileGenerator {
 
     int currentBooleanByte = 0;
     WrapAroundCounter booleanBitCounter = new WrapAroundCounter(7);
+    
     Configuration configuration = new Configuration();
-    configuration.set(ParquetStorageEngine.HADOOP_DEFAULT_NAME, "file:///");
+    configuration.set(ParquetSchemaProvider.HADOOP_DEFAULT_NAME, "file:///");
     // "message m { required int32 integer; required int64 integer64; required boolean b; required float f; required double d;}"
 
     FileSystem fs = FileSystem.get(configuration);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1a2352c0/sandbox/prototype/exec/java-exec/src/test/resources/logical_group.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/logical_group.json b/sandbox/prototype/exec/java-exec/src/test/resources/logical_group.json
new file mode 100644
index 0000000..f4dd169
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/logical_group.json
@@ -0,0 +1,90 @@
+{
+  head : {
+    type : "APACHE_DRILL_LOGICAL",
+    version : 1,
+    generator : {
+      type : "optiq",
+      info : "na"
+    }
+  },
+  storage : {
+    parquet-local : {
+      type : "parquet",
+      dfsName : "file:///"
+    }
+  },
+  query : [ {
+    op : "scan",
+    @id : 1,
+    memo : "initial_scan",
+    storageengine : "parquet-local",
+    selection : [ {
+      path : "/tmp/parquet"
+    } ],
+    ref : "_MAP"
+  }, {
+    op : "project",
+    @id : 2,
+    input : 1,
+    projections : [ {
+      ref : "$f0",
+      expr : "_MAP.b"
+    }, {
+      ref : "$f1",
+      expr : "_MAP.d"
+    }, {
+      ref : "$f2",
+      expr : "1"
+    }, {
+      ref : "$f3",
+      expr : "2"
+    } ]
+  }, {
+    op : "segment",
+    @id : 3,
+    input : 2,
+    exprs : [ "$f0", "$f1" ],
+    name : "segment"
+  }, {
+    op : "collapsingaggregate",
+    @id : 4,
+    input : 3,
+    within : "segment",
+    target : null,
+    carryovers : [ "$f0", "$f1" ],
+    aggregations : [ {
+      ref : "EXPR$0",
+      expr : "count($f2) "
+    }, {
+      ref : "EXPR$1",
+      expr : "count($f3) "
+    } ]
+  }, {
+    op : "project",
+    @id : 5,
+    input : 4,
+    projections : [ {
+      ref : "EXPR$0",
+      expr : "EXPR$0"
+    }, {
+      ref : "EXPR$1",
+      expr : "EXPR$1"
+    }, {
+      ref : "C",
+      expr : "$f0"
+    }, {
+      ref : "X",
+      expr : "$f1"
+    } ]
+  }, {
+    op : "store",
+    @id : 6,
+    memo : "output sink",
+    input : 5,
+    target : {
+      number : 0
+    },
+    partition : null,
+    storageEngine : "queue"
+  } ]
+}
\ No newline at end of file


[14/18] git commit: Add code generated comparison and cast functions.

Posted by ja...@apache.org.
Add code generated comparison and cast functions.


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/4a603de7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/4a603de7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/4a603de7

Branch: refs/heads/master
Commit: 4a603de7edc1040e2295b7f43dd4f011fb6bb057
Parents: e897fe3
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sat Aug 31 18:28:30 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Sep 1 15:16:22 2013 -0700

----------------------------------------------------------------------
 sandbox/prototype/exec/java-exec/pom.xml        |    9 +-
 .../exec/java-exec/src/main/codegen/config.fmpp |    4 +-
 .../java-exec/src/main/codegen/data/Casts.tdd   |   17 +
 .../src/main/codegen/data/CompareTypes.tdd      |    6 +
 .../main/codegen/templates/CastFunctions.java   |   38 +
 .../codegen/templates/ComparisonFunctions.java  |  179 +++
 .../drill/exec/expr/fn/FunctionConverter.java   |   60 +-
 .../exec/expr/fn/impl/ComparatorFunctions.java  |  154 ---
 .../exec/expr/fn/impl/ComparisonFunctions.java  | 1094 +++++++++---------
 .../store/parquet/ParquetStorageEngine.java     |    2 +
 10 files changed, 844 insertions(+), 719 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4a603de7/sandbox/prototype/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/pom.xml b/sandbox/prototype/exec/java-exec/pom.xml
index bd29966..d7df0b8 100644
--- a/sandbox/prototype/exec/java-exec/pom.xml
+++ b/sandbox/prototype/exec/java-exec/pom.xml
@@ -228,7 +228,7 @@
         <executions>
           <execution>
             <id>copy-resources</id>
-            <phase>validate</phase>
+            <phase>generate-sources</phase>
             <goals>
               <goal>copy-resources</goal>
             </goals>
@@ -239,6 +239,11 @@
                   <directory>src/main/java</directory>
                   <filtering>true</filtering>
                 </resource>
+                <resource>
+                  <directory>target/generated-sources</directory>
+<!--                   <include>*/org</include> -->
+                  <filtering>true</filtering>
+                </resource>
               </resources>
             </configuration>
           </execution>
@@ -301,7 +306,7 @@
         <executions>
           <execution>
             <id>generate-sources</id>
-            <phase>generate-sources</phase>
+            <phase>validate</phase>
             <goals>
               <goal>generate</goal>
             </goals>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4a603de7/sandbox/prototype/exec/java-exec/src/main/codegen/config.fmpp
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/config.fmpp b/sandbox/prototype/exec/java-exec/src/main/codegen/config.fmpp
index fc37ea1..ea27863 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/config.fmpp
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/config.fmpp
@@ -1,5 +1,7 @@
 data: {
-    vv: tdd(../data/ValueVectorTypes.tdd)
+    vv: tdd(../data/ValueVectorTypes.tdd),
+    compare: tdd(../data/CompareTypes.tdd),
+    cast: tdd(../data/Casts.tdd)
 }
 freemarkerLinks: {
     includes: includes/

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4a603de7/sandbox/prototype/exec/java-exec/src/main/codegen/data/Casts.tdd
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/data/Casts.tdd b/sandbox/prototype/exec/java-exec/src/main/codegen/data/Casts.tdd
new file mode 100644
index 0000000..c6ee258
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/data/Casts.tdd
@@ -0,0 +1,17 @@
+{
+  types: [
+    {from: "Int", to: "BigInt"},
+    {from: "Float4", to: "Float8" }, 
+    {from: "Int", to: "Float4" },
+    {from: "BigInt", to: "Float4" },
+    {from: "Int", to: "Float8" },
+    {from: "BigInt", to: "Float8" },
+    {to: "Int", from: "BigInt", explicit: "int"},
+    {to: "Float4", from: "Float8" , explicit: "float"}, 
+    {to: "Int", from: "Float4" , explicit: "int"},
+    {to: "BigInt", from: "Float4" , explicit: "long"},
+    {to: "Int", from: "Float8" , explicit: "int"},
+    {to: "BigInt", from: "Float8" , explicit: "long"}
+    
+  ]
+} 
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4a603de7/sandbox/prototype/exec/java-exec/src/main/codegen/data/CompareTypes.tdd
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/data/CompareTypes.tdd b/sandbox/prototype/exec/java-exec/src/main/codegen/data/CompareTypes.tdd
new file mode 100644
index 0000000..9c26a3f
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/data/CompareTypes.tdd
@@ -0,0 +1,6 @@
+{
+  types: [
+    {comparables: ["Float4", "Float8", "Int", "BigInt", "NullableFloat4", "NullableFloat8", "NullableInt", "NullableBigInt"], mode: "fixed"}, 
+    {comparables: ["VarChar", "VarBinary", "NullableVarChar", "NullableVarBinary"], mode: "var"} 
+  ]
+} 
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4a603de7/sandbox/prototype/exec/java-exec/src/main/codegen/templates/CastFunctions.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/CastFunctions.java b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/CastFunctions.java
new file mode 100644
index 0000000..4320c2a
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/CastFunctions.java
@@ -0,0 +1,38 @@
+<@pp.dropOutputFile />
+
+
+
+<#list cast.types as type>
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gcast/Cast${type.from}${type.to}.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.expr.fn.impl.gcast;
+
+import org.apache.drill.exec.expr.DrillSimpleFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.*;
+import org.apache.drill.exec.record.RecordBatch;
+
+@SuppressWarnings("unused")
+@FunctionTemplate(name = "cast", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls=NullHandling.NULL_IF_NULL)
+public class Cast${type.from}${type.to} implements DrillSimpleFunc{
+
+  @Param ${type.from}Holder in;
+  @Output ${type.to}Holder out;
+
+  public void setup(RecordBatch b) {}
+
+  public void eval() {
+    <#if type.explicit??>
+    out.value = (${type.explicit}) in.value;
+    <#else>
+    out.value = in.value;
+    </#if>
+  }
+}
+</#list>
+

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4a603de7/sandbox/prototype/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
new file mode 100644
index 0000000..a5b560d
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
@@ -0,0 +1,179 @@
+<@pp.dropOutputFile />
+
+<#macro compareBlock mode left right output>
+
+outside:{
+  <#if left?starts_with("Nullable")>
+  <#if right?starts_with("Nullable")>
+  <#-- Both are nullable. -->
+  if(left.isSet == 0){
+    if(right.isSet == 0){
+      ${output} = 0;
+      break outside;
+    }else{
+      ${output} = 1;
+    }
+  }else if(right.isSet == 0){
+    ${output} = -1;
+    break outside;
+  }
+  <#else>
+  <#-- Left is nullable but right is not. -->
+  if(left.isSet == 0){
+    ${output} = 1;
+    break outside;
+  }
+  </#if>
+<#elseif right?starts_with("Nullable")>
+  if(right.isSet == 0){
+    ${output} = -1;
+    break outside;
+  }
+  </#if>
+  
+  
+<#if mode == "var">
+      
+      for (int l = left.start, r = right.start; l < left.end && r < right.end; l++, r++) {
+        byte leftByte = left.buffer.getByte(l);
+        byte rightByte = right.buffer.getByte(r);
+        if (leftByte != rightByte) {
+          ${output} = ((leftByte & 0xFF) - (rightByte & 0xFF)) > 0 ? 1 : -1;
+          break outside;
+        }
+      }
+      
+      int l = (left.end - left.start) - (right.end - right.start);
+      if (l > 0) {
+        ${output} = 1;
+        break outside;
+      } else if (l == 0) {
+        ${output} = 0;
+        break outside;
+      } else {
+        ${output} = -1;
+        break outside;
+      }
+<#elseif mode == "fixed">
+    ${output} = left.value < right.value ? -1 : ((left.value == right.value)? 0 : 1);
+</#if>
+}
+</#macro>
+
+<#list compare.types as type>
+<#list type.comparables as left>
+<#list type.comparables as right>
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/GCompare${left}${right}.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.expr.fn.impl;
+
+import org.apache.drill.exec.expr.DrillSimpleFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.holders.*;
+import org.apache.drill.exec.record.RecordBatch;
+
+@SuppressWarnings("unused")
+public class GCompare${left}${right}{
+
+  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+  public static class Compare${left}${right} implements DrillSimpleFunc {
+
+      @Param ${left}Holder left;
+      @Param ${right}Holder right;
+      @Output IntHolder out;
+
+      public void setup(RecordBatch b) {}
+
+      public void eval() {
+        <@compareBlock mode=type.mode left=left right=right output="out.value" />
+      }
+  }
+  
+  @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+  public static class LessThan${left}${right} implements DrillSimpleFunc {
+
+      @Param ${left}Holder left;
+      @Param ${right}Holder right;
+      @Output BitHolder out;
+
+      public void setup(RecordBatch b) {}
+
+      public void eval() {
+        int cmp;
+        <@compareBlock mode=type.mode left=left right=right output="cmp" />
+        out.value = cmp == -1 ? 1 : 0;
+      }
+  }
+  
+  @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+  public static class LessThanE${left}${right} implements DrillSimpleFunc {
+
+      @Param ${left}Holder left;
+      @Param ${right}Holder right;
+      @Output BitHolder out;
+
+      public void setup(RecordBatch b) {}
+
+      public void eval() {
+        int cmp;
+        <@compareBlock mode=type.mode left=left right=right output="cmp" />
+        out.value = cmp < 1 ? 1 : 0;
+      }
+  }
+  
+  @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+  public static class GreaterThan${left}${right} implements DrillSimpleFunc {
+
+      @Param ${left}Holder left;
+      @Param ${right}Holder right;
+      @Output BitHolder out;
+
+      public void setup(RecordBatch b) {}
+
+      public void eval() {
+        int cmp;
+        <@compareBlock mode=type.mode left=left right=right output="cmp" />
+        out.value = cmp == 1 ? 1 : 0;
+      }
+  }
+  
+  @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+  public static class GreaterThanE${left}${right} implements DrillSimpleFunc {
+
+      @Param ${left}Holder left;
+      @Param ${right}Holder right;
+      @Output BitHolder out;
+
+      public void setup(RecordBatch b) {}
+
+      public void eval() {
+        int cmp;
+        <@compareBlock mode=type.mode left=left right=right output="cmp" />
+        out.value = cmp > -1 ? 1 : 0;
+      }
+  }
+  
+  @FunctionTemplate(name = "equals", scope = FunctionTemplate.FunctionScope.SIMPLE)
+  public static class Equals${left}${right} implements DrillSimpleFunc {
+
+      @Param ${left}Holder left;
+      @Param ${right}Holder right;
+      @Output BitHolder out;
+
+      public void setup(RecordBatch b) {}
+
+      public void eval() {
+        int cmp;
+        <@compareBlock mode=type.mode left=left right=right output="cmp" />
+        out.value = cmp == 0 ? 1 : 0;
+      }
+  }
+}
+</#list>
+</#list>
+</#list>
+

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4a603de7/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
index 33f61de..0c1ab8a 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
@@ -17,14 +17,15 @@ import org.apache.drill.exec.expr.annotations.Param;
 import org.apache.drill.exec.expr.annotations.Workspace;
 import org.apache.drill.exec.expr.fn.DrillFuncHolder.ValueReference;
 import org.apache.drill.exec.expr.fn.DrillFuncHolder.WorkspaceReference;
+import org.apache.drill.exec.expr.fn.impl.GCompareBigIntNullableBigInt;
 import org.apache.drill.exec.expr.holders.ValueHolder;
 import org.codehaus.commons.compiler.CompileException;
-import org.codehaus.janino.Java;
 import org.codehaus.janino.Java.CompilationUnit;
 import org.codehaus.janino.Parser;
 import org.codehaus.janino.Scanner;
 import org.mortbay.util.IO;
 
+import com.beust.jcommander.internal.Maps;
 import com.google.common.collect.Lists;
 import com.google.common.io.InputSupplier;
 import com.google.common.io.Resources;
@@ -35,6 +36,38 @@ import com.google.common.io.Resources;
 public class FunctionConverter {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionConverter.class);
   
+  private Map<String, CompilationUnit> functionUnits = Maps.newHashMap();
+  
+  private CompilationUnit get(Class<?> c) throws IOException{
+    String path = c.getName();
+    path = path.replaceFirst("\\$.*", "");
+    path = path.replace(".", File.separator);
+    path = "/" + path + ".java";
+    CompilationUnit cu = functionUnits.get(path);
+    if(cu != null) return cu;
+
+    URL u = Resources.getResource(c, path);
+    InputSupplier<InputStream> supplier = Resources.newInputStreamSupplier(u);
+    try(InputStream is = supplier.getInput()){
+      if(is == null){
+        throw new IOException(String.format("Failure trying to located source code for Class %s, tried to read on classpath location %s", c.getName(), path));
+      }
+      String body = IO.toString(is);
+      
+      //TODO: Hack to remove annotations so Janino doesn't choke.  Need to reconsider this problem...
+      body = body.replaceAll("@\\w+(?:\\([^\\\\]*?\\))?", "");
+      try{
+        cu = new Parser(new Scanner(null, new StringReader(body))).parseCompilationUnit();
+        functionUnits.put(path, cu);
+        return cu;
+      } catch (CompileException e) {
+        logger.warn("Failure while parsing function class:\n{}", body, e);
+        return null;
+      }
+      
+    }
+    
+  }
   
   public <T extends DrillFunc> DrillFuncHolder getHolder(Class<T> clazz){
     FunctionTemplate template = clazz.getAnnotation(FunctionTemplate.class);
@@ -112,8 +145,9 @@ public class FunctionConverter {
    
     CompilationUnit cu;
     try {
-      cu = getClassBody(clazz);
-    } catch (CompileException | IOException e) {
+      cu = get(clazz);
+      if(cu == null) return null;
+    } catch (IOException e) {
       return failure("Failure while getting class body.", e, clazz);
     }
     
@@ -144,7 +178,7 @@ public class FunctionConverter {
   
   
   
-  private Java.CompilationUnit getClassBody(Class<?> c) throws CompileException, IOException{
+  private String getClassBody(Class<?> c) throws CompileException, IOException{
     String path = c.getName();
     path = path.replaceFirst("\\$.*", "");
     path = path.replace(".", File.separator);
@@ -158,12 +192,14 @@ public class FunctionConverter {
       String body = IO.toString(is);
       
       //TODO: Hack to remove annotations so Janino doesn't choke.  Need to reconsider this problem...
-      body = body.replaceAll("@(?:Output|Param|Workspace|Override|FunctionTemplate\\([^\\\\]*?\\))", "");
-      return new Parser(new Scanner(null, new StringReader(body))).parseCompilationUnit();
+      //return body.replaceAll("@(?:Output|Param|Workspace|Override|SuppressWarnings\\([^\\\\]*?\\)|FunctionTemplate\\([^\\\\]*?\\))", "");
+      return body.replaceAll("@(?:\\([^\\\\]*?\\))?", "");
     }
     
   }
   
+  
+  
   @SuppressWarnings("unchecked")
   private <T> T getStaticFieldValue(String fieldName, Class<?> valueType, Class<T> c) throws NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException{
       Field f = valueType.getDeclaredField(fieldName);
@@ -182,12 +218,12 @@ public class FunctionConverter {
   }
 
   private DrillFuncHolder failure(String message, Class<?> clazz){
-    logger.warn("Failure loading function class {}. " + message, clazz.getName());
+    logger.warn("Failure loading function class [{}]. Message: {}", clazz.getName(), message);
     return null;
   }
 
   private DrillFuncHolder failure(String message, Throwable t, Class<?> clazz){
-    logger.warn("Failure loading function class {}. " + message, t, clazz.getName());
+    logger.warn("Failure loading function class [{}]. Message: {}", clazz.getName(), message, t);
     return null;
   }
   
@@ -195,12 +231,6 @@ public class FunctionConverter {
     return failure(message, clazz, field.getName());
   }
   
-  public static void main(String[] args) throws Exception{
-    
-    URL u = Resources.getResource(FunctionConverter.class, "/org/apache/drill/exec/expr/fn/impl/MathFunctions.java");
-    InputStream is = Resources.newInputStreamSupplier(u).getInput();
-    String s = IO.toString(is);
-    System.out.println(s);
-  }
+
   
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4a603de7/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java
index ddd2752..b8a186f 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java
@@ -22,161 +22,7 @@ import org.apache.drill.exec.record.RecordBatch;
 public class ComparatorFunctions {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComparatorFunctions.class);
   
-  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-  public static class IntComparator implements DrillSimpleFunc {
 
-      @Param IntHolder left;
-      @Param IntHolder right;
-      @Output IntHolder out;
-
-      public void setup(RecordBatch b) {}
-
-      public void eval() {
-        out.value = left.value < right.value ? -1 : ((left.value == right.value)? 0 : 1);
-      }
-  }
-  
-  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-  public static class NullableIntComparator implements DrillSimpleFunc {
-
-      @Param NullableIntHolder left;
-      @Param NullableIntHolder right;
-      @Output IntHolder out;
-
-      public void setup(RecordBatch b) {}
-
-      public void eval() {
-        
-        if(left.isSet == 0){
-          if(right.isSet == 0){
-            out.value = 0; 
-          }else{
-            out.value = -1;
-          }
-        }else if(right.isSet == 0){
-          out.value = 1;
-        }else{
-          out.value = left.value < right.value ? -1 : ((left.value == right.value)? 0 : 1);  
-        }
-        
-      }
-  }
-  
-  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-  public static class NullableBigIntComparator implements DrillSimpleFunc {
-
-      @Param NullableBigIntHolder left;
-      @Param NullableBigIntHolder right;
-      @Output IntHolder out;
-
-      public void setup(RecordBatch b) {}
-
-      public void eval() {
-        
-        if(left.isSet == 0){
-          if(right.isSet == 0){
-            out.value = 0; 
-          }else{
-            out.value = -1;
-          }
-        }else if(right.isSet == 0){
-          out.value = 1;
-        }else{
-          out.value = left.value < right.value ? -1 : ((left.value == right.value)? 0 : 1);  
-        }
-        
-      }
-  }
-  
-  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-  public static class VarBinaryComparator implements DrillSimpleFunc {
-
-      @Param VarBinaryHolder left;
-      @Param VarBinaryHolder right;
-      @Output IntHolder out;
-
-      public void setup(RecordBatch b) {}
-
-      public void eval() {
-        out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right);
-      }
-     
-  }
-  
-  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-  public static class NullableVarCharComparator implements DrillSimpleFunc {
-
-      @Param NullableVarCharHolder left;
-      @Param NullableVarCharHolder right;
-      @Output IntHolder out;
-
-      public void setup(RecordBatch b) {}
-
-      public void eval() {
-        out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right);
-      }
-     
-  }
-
-  
-  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-  public static class VarCharComparator implements DrillSimpleFunc {
-
-      @Param VarCharHolder left;
-      @Param VarCharHolder right;
-      @Output IntHolder out;
-
-      public void setup(RecordBatch b) {}
-
-      public void eval() {
-        out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right);
-      }
-     
-  }  
-
-  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-  public static class VarBinaryCharComparator implements DrillSimpleFunc {
-
-      @Param VarBinaryHolder left;
-      @Param VarCharHolder right;
-      @Output IntHolder out;
-
-      public void setup(RecordBatch b) {}
-
-      public void eval() {
-        out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right);
-      }
-     
-  }    
-  
-  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-  public static class VarBinaryCharComparator2 implements DrillSimpleFunc {
-
-      @Param VarCharHolder left;
-      @Param VarBinaryHolder right;
-      @Output IntHolder out;
-
-      public void setup(RecordBatch b) {}
-
-      public void eval() {
-        out.value = -org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(right, left);
-      }
-     
-  }      
-  
-  @FunctionTemplate(name = "compare_to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-  public static class Long implements DrillSimpleFunc {
-
-      @Param BigIntHolder left;
-      @Param BigIntHolder right;
-      @Output IntHolder out;
-
-      public void setup(RecordBatch b) {}
-
-      public void eval() {
-        out.value = left.value < right.value ? -1 : ((left.value == right.value)? 0 : 1);
-      }
-  }
   public static final FunctionDefinition COMPARE_TO = FunctionDefinition.simple("compare_to", new ArgumentValidators.AllowedTypeList(2, Types.required(MinorType.INT)), new OutputTypeDeterminer.FixedType(Types.required(MinorType.INT)));
   public static class Provider implements CallProvider{
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4a603de7/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctions.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctions.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctions.java
index 6bc6313..0202b37 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctions.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctions.java
@@ -16,552 +16,552 @@ import org.apache.drill.exec.record.RecordBatch;
 public class ComparisonFunctions {
     static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MathFunctions.class);
 
-    private ComparisonFunctions() {}
-
-    @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class IntEqual implements DrillSimpleFunc {
-
-        @Param IntHolder left;
-        @Param IntHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value == right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class BigIntEqual implements DrillSimpleFunc {
-
-        @Param BigIntHolder left;
-        @Param BigIntHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value == right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class Float4Equal implements DrillSimpleFunc {
-
-        @Param Float4Holder left;
-        @Param Float4Holder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value == right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class Float8Equal implements DrillSimpleFunc {
-
-        @Param Float8Holder left;
-        @Param Float8Holder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value == right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "not equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class IntNotEqual implements DrillSimpleFunc {
-
-        @Param IntHolder left;
-        @Param IntHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value != right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "not equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class BigIntNotEqual implements DrillSimpleFunc {
-
-        @Param BigIntHolder left;
-        @Param BigIntHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value != right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "not equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class Float4NotEqual implements DrillSimpleFunc {
-
-        @Param Float4Holder left;
-        @Param Float4Holder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value != right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "not equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class Float8NotEqual implements DrillSimpleFunc {
-
-        @Param Float8Holder left;
-        @Param Float8Holder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value != right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class IntGreaterThan implements DrillSimpleFunc {
-
-        @Param IntHolder left;
-        @Param IntHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value > right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class BigIntGreaterThan implements DrillSimpleFunc {
-
-        @Param BigIntHolder left;
-        @Param BigIntHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value > right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class Float4GreaterThan implements DrillSimpleFunc {
-
-        @Param Float4Holder left;
-        @Param Float4Holder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value > right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class Float8GreaterThan implements DrillSimpleFunc {
-
-        @Param Float8Holder left;
-        @Param Float8Holder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value > right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class IntGreaterThanEqual implements DrillSimpleFunc {
-
-        @Param IntHolder left;
-        @Param IntHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value >= right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class BigIntGreaterThanEqual implements DrillSimpleFunc {
-
-        @Param BigIntHolder left;
-        @Param BigIntHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value >= right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class Float4GreaterThanEqual implements DrillSimpleFunc {
-
-        @Param Float4Holder left;
-        @Param Float4Holder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value >= right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class Float8GreaterThanEqual implements DrillSimpleFunc {
-
-        @Param Float8Holder left;
-        @Param Float8Holder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value >= right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class IntLessThan implements DrillSimpleFunc {
-
-        @Param IntHolder left;
-        @Param IntHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value < right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class BigIntLessThan implements DrillSimpleFunc {
-
-        @Param BigIntHolder left;
-        @Param BigIntHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value < right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class Float4LessThan implements DrillSimpleFunc {
-
-        @Param Float4Holder left;
-        @Param Float4Holder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value < right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class Float8LessThan implements DrillSimpleFunc {
-
-        @Param Float8Holder left;
-        @Param Float8Holder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value < right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class IntLessThanEqual implements DrillSimpleFunc {
-
-        @Param IntHolder left;
-        @Param IntHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value <= right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class BigIntLessThanEqual implements DrillSimpleFunc {
-
-        @Param BigIntHolder left;
-        @Param BigIntHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value <= right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class Float4LessThanEqual implements DrillSimpleFunc {
-
-        @Param Float4Holder left;
-        @Param Float4Holder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value <= right.value) ? 1 : 0;
-        }
-    }
-
-    @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class Float8LessThanEqual implements DrillSimpleFunc {
-
-        @Param Float8Holder left;
-        @Param Float8Holder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = (left.value <= right.value) ? 1 : 0;
-        }
-    }
-    
-    @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class VarBinaryEqual implements DrillSimpleFunc {
-
-        @Param VarBinaryHolder left;
-        @Param VarBinaryHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == 0 ? 1 : 0;
-        }
-    }
-    
-    @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class VarCharEqual implements DrillSimpleFunc {
-
-        @Param VarCharHolder left;
-        @Param VarCharHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == 0 ? 1 : 0;
-        }
-    }
-    
-    @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class VarBinaryCharEqual implements DrillSimpleFunc {
-
-        @Param VarBinaryHolder left;
-        @Param VarCharHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == 0 ? 1 : 0;
-        }
-    }
-    
-    @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class VarBinaryGTE implements DrillSimpleFunc {
-
-        @Param VarBinaryHolder left;
-        @Param VarBinaryHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) > -1 ? 1 : 0;
-        }
-    }
-    
-    @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class VarCharGTE implements DrillSimpleFunc {
-
-        @Param VarCharHolder left;
-        @Param VarCharHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) > -1 ? 1 : 0;
-        }
-    }
-    
-    @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class VarBinaryCharGTE implements DrillSimpleFunc {
-
-        @Param VarBinaryHolder left;
-        @Param VarCharHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) > -1 ? 1 : 0;
-        }
-    }
-    
-    @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class VarBinaryGT implements DrillSimpleFunc {
-
-        @Param VarBinaryHolder left;
-        @Param VarBinaryHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == 1 ? 1 : 0;
-        }
-    }
-    
-    @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class VarCharGT implements DrillSimpleFunc {
-
-        @Param VarCharHolder left;
-        @Param VarCharHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == 1 ? 1 : 0;
-        }
-    }
-    
-    @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class VarBinaryCharGT implements DrillSimpleFunc {
-
-        @Param VarBinaryHolder left;
-        @Param VarCharHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == 1? 1 : 0;
-        }
-    }
-    
-    @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class VarBinaryLTE implements DrillSimpleFunc {
-
-        @Param VarBinaryHolder left;
-        @Param VarBinaryHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) < 1 ? 1 : 0;
-        }
-    }
-    
-    @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class VarCharLTE implements DrillSimpleFunc {
-
-        @Param VarCharHolder left;
-        @Param VarCharHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) < 1 ? 1 : 0;
-        }
-    }
-    
-    @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class VarBinaryCharLTE implements DrillSimpleFunc {
-
-        @Param VarBinaryHolder left;
-        @Param VarCharHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) < 1? 1 : 0;
-        }
-    }
-    
-    @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class VarBinaryLT implements DrillSimpleFunc {
-
-        @Param VarBinaryHolder left;
-        @Param VarBinaryHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == -1 ? 1 : 0;
-        }
-    }
-    
-    @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class VarCharLT implements DrillSimpleFunc {
-
-        @Param VarCharHolder left;
-        @Param VarCharHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == -1 ? 1 : 0;
-        }
-    }
-    
-    @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
-    public static class VarBinaryCharLT implements DrillSimpleFunc {
-
-        @Param VarBinaryHolder left;
-        @Param VarCharHolder right;
-        @Output BitHolder out;
-
-        public void setup(RecordBatch b) {}
-
-        public void eval() {
-            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == -1 ? 1 : 0;
-        }
-    }
+//    private ComparisonFunctions() {}
+//
+//    @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class IntEqual implements DrillSimpleFunc {
+//
+//        @Param IntHolder left;
+//        @Param IntHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value == right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class BigIntEqual implements DrillSimpleFunc {
+//
+//        @Param BigIntHolder left;
+//        @Param BigIntHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value == right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class Float4Equal implements DrillSimpleFunc {
+//
+//        @Param Float4Holder left;
+//        @Param Float4Holder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value == right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class Float8Equal implements DrillSimpleFunc {
+//
+//        @Param Float8Holder left;
+//        @Param Float8Holder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value == right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "not equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class IntNotEqual implements DrillSimpleFunc {
+//
+//        @Param IntHolder left;
+//        @Param IntHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value != right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "not equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class BigIntNotEqual implements DrillSimpleFunc {
+//
+//        @Param BigIntHolder left;
+//        @Param BigIntHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value != right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "not equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class Float4NotEqual implements DrillSimpleFunc {
+//
+//        @Param Float4Holder left;
+//        @Param Float4Holder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value != right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "not equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class Float8NotEqual implements DrillSimpleFunc {
+//
+//        @Param Float8Holder left;
+//        @Param Float8Holder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value != right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class IntGreaterThan implements DrillSimpleFunc {
+//
+//        @Param IntHolder left;
+//        @Param IntHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value > right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class BigIntGreaterThan implements DrillSimpleFunc {
+//
+//        @Param BigIntHolder left;
+//        @Param BigIntHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value > right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class Float4GreaterThan implements DrillSimpleFunc {
+//
+//        @Param Float4Holder left;
+//        @Param Float4Holder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value > right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class Float8GreaterThan implements DrillSimpleFunc {
+//
+//        @Param Float8Holder left;
+//        @Param Float8Holder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value > right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class IntGreaterThanEqual implements DrillSimpleFunc {
+//
+//        @Param IntHolder left;
+//        @Param IntHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value >= right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class BigIntGreaterThanEqual implements DrillSimpleFunc {
+//
+//        @Param BigIntHolder left;
+//        @Param BigIntHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value >= right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class Float4GreaterThanEqual implements DrillSimpleFunc {
+//
+//        @Param Float4Holder left;
+//        @Param Float4Holder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value >= right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class Float8GreaterThanEqual implements DrillSimpleFunc {
+//
+//        @Param Float8Holder left;
+//        @Param Float8Holder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value >= right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class IntLessThan implements DrillSimpleFunc {
+//
+//        @Param IntHolder left;
+//        @Param IntHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value < right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class BigIntLessThan implements DrillSimpleFunc {
+//
+//        @Param BigIntHolder left;
+//        @Param BigIntHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value < right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class Float4LessThan implements DrillSimpleFunc {
+//
+//        @Param Float4Holder left;
+//        @Param Float4Holder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value < right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class Float8LessThan implements DrillSimpleFunc {
+//
+//        @Param Float8Holder left;
+//        @Param Float8Holder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value < right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class IntLessThanEqual implements DrillSimpleFunc {
+//
+//        @Param IntHolder left;
+//        @Param IntHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value <= right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class BigIntLessThanEqual implements DrillSimpleFunc {
+//
+//        @Param BigIntHolder left;
+//        @Param BigIntHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value <= right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class Float4LessThanEqual implements DrillSimpleFunc {
+//
+//        @Param Float4Holder left;
+//        @Param Float4Holder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value <= right.value) ? 1 : 0;
+//        }
+//    }
+//
+//    @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class Float8LessThanEqual implements DrillSimpleFunc {
+//
+//        @Param Float8Holder left;
+//        @Param Float8Holder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = (left.value <= right.value) ? 1 : 0;
+//        }
+//    }
+//    
+//    @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class VarBinaryEqual implements DrillSimpleFunc {
+//
+//        @Param VarBinaryHolder left;
+//        @Param VarBinaryHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == 0 ? 1 : 0;
+//        }
+//    }
+//    
+//    @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class VarCharEqual implements DrillSimpleFunc {
+//
+//        @Param VarCharHolder left;
+//        @Param VarCharHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == 0 ? 1 : 0;
+//        }
+//    }
+//    
+//    @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class VarBinaryCharEqual implements DrillSimpleFunc {
+//
+//        @Param VarBinaryHolder left;
+//        @Param VarCharHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == 0 ? 1 : 0;
+//        }
+//    }
+//    
+//    @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class VarBinaryGTE implements DrillSimpleFunc {
+//
+//        @Param VarBinaryHolder left;
+//        @Param VarBinaryHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) > -1 ? 1 : 0;
+//        }
+//    }
+//    
+//    @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class VarCharGTE implements DrillSimpleFunc {
+//
+//        @Param VarCharHolder left;
+//        @Param VarCharHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) > -1 ? 1 : 0;
+//        }
+//    }
+//    
+//    @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class VarBinaryCharGTE implements DrillSimpleFunc {
+//
+//        @Param VarBinaryHolder left;
+//        @Param VarCharHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) > -1 ? 1 : 0;
+//        }
+//    }
+//    
+//    @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class VarBinaryGT implements DrillSimpleFunc {
+//
+//        @Param VarBinaryHolder left;
+//        @Param VarBinaryHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == 1 ? 1 : 0;
+//        }
+//    }
+//    
+//    @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class VarCharGT implements DrillSimpleFunc {
+//
+//        @Param VarCharHolder left;
+//        @Param VarCharHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == 1 ? 1 : 0;
+//        }
+//    }
+//    
+//    @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class VarBinaryCharGT implements DrillSimpleFunc {
+//
+//        @Param VarBinaryHolder left;
+//        @Param VarCharHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == 1? 1 : 0;
+//        }
+//    }
+//    
+//    @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class VarBinaryLTE implements DrillSimpleFunc {
+//
+//        @Param VarBinaryHolder left;
+//        @Param VarBinaryHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) < 1 ? 1 : 0;
+//        }
+//    }
+//    
+//    @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class VarCharLTE implements DrillSimpleFunc {
+//
+//        @Param VarCharHolder left;
+//        @Param VarCharHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) < 1 ? 1 : 0;
+//        }
+//    }
+//    
+//    @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class VarBinaryCharLTE implements DrillSimpleFunc {
+//
+//        @Param VarBinaryHolder left;
+//        @Param VarCharHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) < 1? 1 : 0;
+//        }
+//    }
+//    
+//    @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class VarBinaryLT implements DrillSimpleFunc {
+//
+//        @Param VarBinaryHolder left;
+//        @Param VarBinaryHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == -1 ? 1 : 0;
+//        }
+//    }
+//    
+//    @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class VarCharLT implements DrillSimpleFunc {
+//
+//        @Param VarCharHolder left;
+//        @Param VarCharHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == -1 ? 1 : 0;
+//        }
+//    }
+//    
+//    @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+//    public static class VarBinaryCharLT implements DrillSimpleFunc {
+//
+//        @Param VarBinaryHolder left;
+//        @Param VarCharHolder right;
+//        @Output BitHolder out;
+//
+//        public void setup(RecordBatch b) {}
+//
+//        public void eval() {
+//            out.value = org.apache.drill.exec.expr.fn.impl.VarHelpers.compare(left, right) == -1 ? 1 : 0;
+//        }
+//    }
     
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/4a603de7/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
index 1c8bbe6..1629aba 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetStorageEngine.java
@@ -70,6 +70,8 @@ public class ParquetStorageEngine extends AbstractStorageEngine{
   }
 
   public Configuration getHadoopConfig() {
+    double y = 5;
+    int x = (int) y;
     return schemaProvider.conf;
   }
 


[18/18] git commit: Separate out JDBC drivers between ref (RefDriver) and full exec so that they don't fight with each other in the same JVM. Add ClassPathFileSystem to support reading from classpath Update merge join to support left and right by flippin

Posted by ja...@apache.org.
Separate out JDBC drivers between ref (RefDriver) and full exec so that they don't fight with each other in the same JVM.
Add ClassPathFileSystem to support reading from classpath
Update merge join to support left and right by flipping right.


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/651b3e9f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/651b3e9f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/651b3e9f

Branch: refs/heads/master
Commit: 651b3e9fd7032310eede1550d315f8c450fe3d1a
Parents: 6f300a3
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sun Sep 1 14:13:30 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Sep 1 15:16:22 2013 -0700

----------------------------------------------------------------------
 .../drill/common/expression/PathSegment.java    |   4 +-
 .../apache/drill/common/logical/data/Join.java  |   2 +-
 .../common/logical/data/JoinCondition.java      |   4 +
 .../exec/physical/config/MergeJoinPOP.java      |  15 +++
 .../physical/impl/join/MergeJoinCreator.java    |   9 +-
 .../drill/exec/store/ClassPathFileSystem.java   | 103 +++++++++++++++++++
 .../drill/exec/store/ResourceInputStream.java   |  89 ++++++++++++++++
 .../exec/store/json/JSONStorageEngine.java      |   2 +
 .../exec/store/json/JsonSchemaProvider.java     |   2 +
 .../store/parquet/ParquetSchemaProvider.java    |   2 +
 sandbox/prototype/pom.xml                       |  29 +++++-
 sandbox/prototype/sqlparser/pom.xml             |  20 +---
 .../org/apache/drill/jdbc/DrillHandler.java     |   7 +-
 .../main/java/org/apache/drill/jdbc/Driver.java |   2 +-
 .../java/org/apache/drill/jdbc/RefDriver.java   |  71 +++++++++++++
 .../org/apache/drill/optiq/DrillJoinRel.java    |   7 +-
 .../apache/drill/optiq/DrillPrepareImpl.java    |   2 +-
 .../src/main/resources/storage-engines.json     |  10 ++
 .../apache/drill/jdbc/test/FullEngineTest.java  |  32 ++----
 .../org/apache/drill/jdbc/test/JdbcAssert.java  |  10 +-
 .../org/apache/drill/jdbc/test/JdbcTest.java    |   2 +-
 21 files changed, 366 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/PathSegment.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
index 28e820b..c6de1ab 100644
--- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
+++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
@@ -76,11 +76,11 @@ public abstract class PathSegment{
   
   
   public static class NameSegment extends PathSegment{
-    private final CharSequence path;
+    private final String path;
     
     public NameSegment(CharSequence n, ValueExpressions.CollisionBehavior collision){
       super(collision);
-      this.path = n;
+      this.path = n.toString();
     }
     
     public CharSequence getPath(){

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/Join.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/Join.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/Join.java
index 865c78e..f97dab1 100644
--- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/Join.java
+++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/Join.java
@@ -36,7 +36,7 @@ public class Join extends LogicalOperatorBase {
   private final JoinCondition[] conditions;
 
   public static enum JoinType{
-    LEFT, INNER, OUTER;
+    LEFT, INNER, OUTER, RIGHT;
     
     public static JoinType resolve(String val){
       for(JoinType jt : JoinType.values()){

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/JoinCondition.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/JoinCondition.java b/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/JoinCondition.java
index 49646c1..01c332b 100644
--- a/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/JoinCondition.java
+++ b/sandbox/prototype/common/src/main/java/org/apache/drill/common/logical/data/JoinCondition.java
@@ -47,5 +47,9 @@ public class JoinCondition {
   public LogicalExpression getRight() {
     return right;
   }
+  
+  public JoinCondition flip(){
+    return new JoinCondition(relationship, right, left);
+  }
 
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java
index 0ffd218..b234f15 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MergeJoinPOP.java
@@ -4,6 +4,7 @@ import java.util.Iterator;
 import java.util.List;
 
 import org.apache.drill.common.logical.data.Join;
+import org.apache.drill.common.logical.data.Join.JoinType;
 import org.apache.drill.common.logical.data.JoinCondition;
 import org.apache.drill.exec.physical.OperatorCost;
 import org.apache.drill.exec.physical.base.AbstractBase;
@@ -11,6 +12,7 @@ import org.apache.drill.exec.physical.base.PhysicalOperator;
 import org.apache.drill.exec.physical.base.PhysicalVisitor;
 import org.apache.drill.exec.physical.base.Size;
 
+import com.beust.jcommander.internal.Lists;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
@@ -81,4 +83,17 @@ public class MergeJoinPOP extends AbstractBase{
   public List<JoinCondition> getConditions() {
     return conditions;
   }
+  
+  public MergeJoinPOP flipIfRight(){
+    if(joinType == JoinType.RIGHT){
+      List<JoinCondition> flippedConditions = Lists.newArrayList(conditions.size());
+      for(JoinCondition c : conditions){
+        flippedConditions.add(c.flip());
+      }
+      return new MergeJoinPOP(right, left, flippedConditions, JoinType.LEFT);
+    }else{
+      return this;
+    }
+
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinCreator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinCreator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinCreator.java
index 1b65227..9940219 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinCreator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinCreator.java
@@ -19,7 +19,9 @@
 package org.apache.drill.exec.physical.impl.join;
 
 import com.google.common.base.Preconditions;
+
 import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.logical.data.Join.JoinType;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.physical.config.MergeJoinPOP;
 import org.apache.drill.exec.physical.impl.BatchCreator;
@@ -33,6 +35,11 @@ public class MergeJoinCreator implements BatchCreator<MergeJoinPOP> {
   @Override
   public RecordBatch getBatch(FragmentContext context, MergeJoinPOP config, List<RecordBatch> children) throws ExecutionSetupException {
     Preconditions.checkArgument(children.size() == 2);
-    return new MergeJoinBatch(config, context, children.get(0), children.get(1));
+    if(config.getJoinType() == JoinType.RIGHT){
+      return new MergeJoinBatch(config.flipIfRight(), context, children.get(1), children.get(0));
+    }else{
+      return new MergeJoinBatch(config, context, children.get(0), children.get(1));  
+    }
+    
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java
new file mode 100644
index 0000000..85bf4f8
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/ClassPathFileSystem.java
@@ -0,0 +1,103 @@
+package org.apache.drill.exec.store;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Progressable;
+
+import com.google.common.io.Resources;
+
+
+public class ClassPathFileSystem extends FileSystem{
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ClassPathFileSystem.class);
+
+  static final String ERROR_MSG = "ClassPathFileSystem is read only.";
+  
+  private Path working;
+  
+  @Override
+  public FSDataOutputStream append(Path arg0, int arg1, Progressable arg2) throws IOException {
+    throw new IOException(ERROR_MSG);
+  }
+
+  @Override
+  public FSDataOutputStream create(Path arg0, FsPermission arg1, boolean arg2, int arg3, short arg4, long arg5,
+      Progressable arg6) throws IOException {
+    throw new IOException(ERROR_MSG);
+  }
+
+  @Override
+  public boolean delete(Path arg0) throws IOException {
+    throw new IOException(ERROR_MSG);
+  }
+
+  @Override
+  public boolean delete(Path arg0, boolean arg1) throws IOException {
+    throw new IOException(ERROR_MSG);
+  }
+
+  @Override
+  public FileStatus getFileStatus(Path arg0) throws IOException {
+    URL url = Resources.getResource(arg0.toString());
+    if(url == null){
+      throw new IOException(String.format("Unable to find path %s.", arg0.toString()));
+    }
+    return new FileStatus(-1, false, 1, 8096, System.currentTimeMillis(), arg0);
+  }
+
+  @Override
+  public URI getUri() {
+    try {
+      return new URI("classpath:///");
+    } catch (URISyntaxException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public Path getWorkingDirectory() {
+    return working;
+  }
+
+  @Override
+  public FileStatus[] listStatus(Path arg0) throws IOException {
+    throw new IOException("ClassPathFileSystem doesn't currently support listing files.");
+  }
+
+  @Override
+  public boolean mkdirs(Path arg0, FsPermission arg1) throws IOException {
+    throw new IOException(ERROR_MSG);
+  }
+
+  @Override
+  public FSDataInputStream open(Path arg0, int arg1) throws IOException {
+    URL url = Resources.getResource(arg0.toString());
+    if(url == null){
+      throw new IOException(String.format("Unable to find path %s.", arg0.getName()));
+    }
+    ResourceInputStream ris = new ResourceInputStream(Resources.toByteArray(url));
+    return new FSDataInputStream(ris);
+  }
+
+  @Override
+  public boolean rename(Path arg0, Path arg1) throws IOException {
+    throw new IOException(ERROR_MSG);
+  }
+
+  @Override
+  public void setWorkingDirectory(Path arg0) {
+    this.working = arg0;
+  }
+  
+  public static void main(String[] args) throws Exception{
+    URI uri = new URI("classpath:///");
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/ResourceInputStream.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/ResourceInputStream.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/ResourceInputStream.java
new file mode 100644
index 0000000..6d83cf9
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/ResourceInputStream.java
@@ -0,0 +1,89 @@
+package org.apache.drill.exec.store;
+
+import java.io.ByteArrayInputStream;
+import java.io.EOFException;
+import java.io.IOException;
+
+import org.apache.hadoop.fs.PositionedReadable;
+import org.apache.hadoop.fs.Seekable;
+
+public class ResourceInputStream extends ByteArrayInputStream implements Seekable, PositionedReadable {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ResourceInputStream.class);
+
+
+  public ResourceInputStream(byte[] bytes) {
+    super(bytes);
+  }
+
+  
+  
+
+  @Override
+  public void readFully(long position, byte[] buffer) throws IOException {
+    int l = read(position, buffer, 0, buffer.length);
+    if (l < buffer.length) {
+      throw new EOFException();
+    }
+  }
+
+  public int read(long position, byte b[], int off, int len) {
+    int start = (int) position;
+    if (b == null) {
+        throw new NullPointerException();
+    } else if (off < 0 || len < 0 || len > b.length - off) {
+        throw new IndexOutOfBoundsException();
+    }
+
+    if (start >= count) {
+        return -1;
+    }
+
+    int avail = count - start;
+    if (len > avail) {
+        len = avail;
+    }
+    if (len <= 0) {
+        return 0;
+    }
+    System.arraycopy(buf, start, b, off, len);
+    return len;
+}
+  
+  @Override
+  public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
+    int l = read(position, buffer, offset, length);
+    if (l < length) throw new EOFException();
+  }
+
+  @Override
+  public long getPos() throws IOException {
+    return pos;
+  }
+
+  
+  @Override
+  public int read(byte[] b) throws IOException {
+    int l = read(pos, b, 0, b.length);
+    pos += l;
+    return l;
+  }
+
+  @Override
+  public boolean seekToNewSource(long arg0) throws IOException {
+    seek(arg0);
+    return true;
+  }
+
+
+
+
+  @Override
+  public void seek(long arg0) throws IOException {
+    if(buf.length > arg0){
+      pos = (int) arg0;
+    }else{
+      throw new EOFException();
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java
index b0a2e3b..f5370ca 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JSONStorageEngine.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
 import org.apache.drill.common.logical.data.Scan;
 import org.apache.drill.exec.server.DrillbitContext;
 import org.apache.drill.exec.store.AbstractStorageEngine;
+import org.apache.drill.exec.store.ClassPathFileSystem;
 import org.apache.drill.exec.store.SchemaProvider;
 import org.apache.drill.exec.store.json.JSONGroupScan.ScanEntry;
 import org.apache.hadoop.conf.Configuration;
@@ -44,6 +45,7 @@ public class JSONStorageEngine extends AbstractStorageEngine {
     
     try {
       this.conf = new Configuration();
+      this.conf.set("fs.classpath.impl", ClassPathFileSystem.class.getName());
       this.conf.set(HADOOP_DEFAULT_NAME, config.getDfsName());
       this.fileSystem = FileSystem.get(conf);
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JsonSchemaProvider.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JsonSchemaProvider.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JsonSchemaProvider.java
index 065652a..e7f027e 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JsonSchemaProvider.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/json/JsonSchemaProvider.java
@@ -4,6 +4,7 @@ import java.io.IOException;
 
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.exec.physical.ReadEntryWithPath;
+import org.apache.drill.exec.store.ClassPathFileSystem;
 import org.apache.drill.exec.store.SchemaProvider;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -23,6 +24,7 @@ public class JsonSchemaProvider implements SchemaProvider{
     try {
       this.conf = new Configuration();
       this.conf.set(HADOOP_DEFAULT_NAME, "file:///");
+      this.conf.set("fs.classpath.impl", ClassPathFileSystem.class.getName());
       this.fs = FileSystem.get(conf);
     } catch (IOException ie) {
       throw new RuntimeException("Error setting up filesystem");

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
index 3e4b02a..9442841 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
@@ -4,6 +4,7 @@ import java.io.IOException;
 
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.exec.physical.ReadEntryWithPath;
+import org.apache.drill.exec.store.ClassPathFileSystem;
 import org.apache.drill.exec.store.SchemaProvider;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -23,6 +24,7 @@ public class ParquetSchemaProvider implements SchemaProvider{
     this.configuration = configuration;
     try {
       this.conf = new Configuration();
+      this.conf.set("fs.classpath.impl", ClassPathFileSystem.class.getName());
       this.conf.set(HADOOP_DEFAULT_NAME, "file:///");
       this.fs = FileSystem.get(conf);
     } catch (IOException ie) {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/pom.xml b/sandbox/prototype/pom.xml
index 2f2eae2..ccb8d76 100644
--- a/sandbox/prototype/pom.xml
+++ b/sandbox/prototype/pom.xml
@@ -26,16 +26,35 @@
 
   <repositories>
     <repository>
+      <id>conjars</id>
+      <name>Conjars</name>
+      <url>http://conjars.org/repo</url>
+      <layout>default</layout>
       <releases>
         <enabled>true</enabled>
-        <updatePolicy>always</updatePolicy>
+        <updatePolicy>never</updatePolicy>
         <checksumPolicy>warn</checksumPolicy>
       </releases>
-      <id>conjars</id>
-      <name>Conjars</name>
-      <url>http://conjars.org/repo</url>
+      <snapshots>
+        <enabled>false</enabled>
+      </snapshots>      
+    </repository>
+    
+    <repository>
+      <id>pentaho</id>
+      <name>Pentaho</name>
+      <url>http://repo.pentaho.org/artifactory/repo</url>
       <layout>default</layout>
+      <releases>
+        <enabled>true</enabled>
+        <updatePolicy>never</updatePolicy>
+        <checksumPolicy>warn</checksumPolicy>
+      </releases>
+      <snapshots>
+        <enabled>false</enabled>
+      </snapshots>
     </repository>
+    
     <repository>
       <id>sonatype-nexus-snapshots</id>
       <url>https://oss.sonatype.org/content/repositories/snapshots</url>
@@ -234,7 +253,7 @@
       <artifactId>guava</artifactId>
       <version>14.0.1</version>
     </dependency>
-
+    
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/sqlparser/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/pom.xml b/sandbox/prototype/sqlparser/pom.xml
index 212362c..a9d6cfb 100644
--- a/sandbox/prototype/sqlparser/pom.xml
+++ b/sandbox/prototype/sqlparser/pom.xml
@@ -11,20 +11,6 @@
   <artifactId>sqlparser</artifactId>
   <name>sqlparser</name>
 
-  <repositories>
-    <repository>
-      <releases>
-        <enabled>true</enabled>
-        <updatePolicy>always</updatePolicy>
-        <checksumPolicy>warn</checksumPolicy>
-      </releases>
-      <id>conjars</id>
-      <name>Conjars</name>
-      <url>http://conjars.org/repo</url>
-      <layout>default</layout>
-    </repository>
-  </repositories>
-
   <dependencies>
     <dependency>
       <groupId>net.hydromatic</groupId>
@@ -59,7 +45,11 @@
       <version>0.3</version>
       <scope>test</scope>
     </dependency>
-
+    <dependency>
+      <groupId>pentaho</groupId>
+      <artifactId>mondrian-data-foodmart-json</artifactId>
+      <version>0.2</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.drill.exec</groupId>
       <artifactId>ref</artifactId>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
index 380993b..6b45843 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
@@ -37,13 +37,18 @@ public class DrillHandler extends HandlerImpl {
   private Drillbit bit;
   private DrillConfig config = DrillConfig.create();
   private SchemaProviderRegistry registry;
+  private final boolean ref;
+  
+  public DrillHandler(boolean ref){
+    this.ref = ref;
+  }
 
   public void onConnectionInit(OptiqConnection connection) throws SQLException {
     super.onConnectionInit(connection);
 
     final Properties p = connection.getProperties();
     
-    if (p.getProperty("ref") != null) {
+    if (ref) {
       final String model = p.getProperty("model");
       if (model != null) {
         if (model != null) {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/Driver.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/Driver.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/Driver.java
index 7812c2c..425bcae 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/Driver.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/Driver.java
@@ -62,7 +62,7 @@ public class Driver extends UnregisteredDriver {
   
   @Override
   protected Handler createHandler() {
-    this.handler = new DrillHandler();
+    this.handler = new DrillHandler(false);
     return handler;
   }
   

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/RefDriver.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/RefDriver.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/RefDriver.java
new file mode 100644
index 0000000..e2e0f79
--- /dev/null
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/jdbc/RefDriver.java
@@ -0,0 +1,71 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.jdbc;
+
+import net.hydromatic.linq4j.function.Function0;
+import net.hydromatic.optiq.jdbc.DriverVersion;
+import net.hydromatic.optiq.jdbc.Handler;
+import net.hydromatic.optiq.jdbc.OptiqPrepare;
+import net.hydromatic.optiq.jdbc.UnregisteredDriver;
+
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.optiq.DrillPrepareImpl;
+
+/**
+ * JDBC driver for Apache Drill.
+ */
+public class RefDriver extends UnregisteredDriver {
+  public static final String CONNECT_STRING_PREFIX = "jdbc:drillref:";
+
+  private volatile DrillHandler handler;
+  
+  static {
+    new RefDriver().register();
+  }
+
+  protected String getConnectStringPrefix() {
+    return CONNECT_STRING_PREFIX;
+  }
+
+  protected DriverVersion createDriverVersion() {
+    return new DrillDriverVersion();
+  }
+
+  @Override
+  protected Function0<OptiqPrepare> createPrepareFactory() {
+    return new Function0<OptiqPrepare>() {
+      @Override
+      public OptiqPrepare apply() {
+        return new DrillPrepareImpl(null);
+      }
+    };
+  }
+
+  public DrillClient getClient(){
+    return handler.getClient();
+  }
+  
+  @Override
+  protected Handler createHandler() {
+    this.handler = new DrillHandler(true);
+    return handler;
+  }
+  
+}
+
+// End Driver.java

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRel.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRel.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRel.java
index f367e6d..2659063 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRel.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillJoinRel.java
@@ -47,10 +47,7 @@ public class DrillJoinRel extends JoinRelBase implements DrillRel {
   public DrillJoinRel(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right, RexNode condition,
       JoinRelType joinType) throws InvalidRelException {
     super(cluster, traits, left, right, condition, joinType, Collections.<String> emptySet());
-    switch (joinType) {
-    case RIGHT:
-      throw new InvalidRelException("DrillJoinRel does not support RIGHT join");
-    }
+
     RexNode remaining = RelOptUtil.splitJoinCondition(left, right, condition, leftKeys, rightKeys);
     if (!remaining.isAlwaysTrue()) {
       throw new InvalidRelException("DrillJoinRel only supports equi-join");
@@ -148,6 +145,8 @@ public class DrillJoinRel extends JoinRelBase implements DrillRel {
     switch (joinType) {
     case LEFT:
       return "left";
+    case RIGHT:
+      return "right";
     case INNER:
       return "inner";
     case FULL:

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillPrepareImpl.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillPrepareImpl.java b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillPrepareImpl.java
index b80bc8c..f4c508f 100644
--- a/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillPrepareImpl.java
+++ b/sandbox/prototype/sqlparser/src/main/java/org/apache/drill/optiq/DrillPrepareImpl.java
@@ -37,7 +37,7 @@ public class DrillPrepareImpl extends OptiqPrepareImpl {
   @Override
   protected RelOptPlanner createPlanner() {
     final RelOptPlanner planner = super.createPlanner();
-    planner.addRule(EnumerableDrillRule.getInstance(driver.getClient()));
+    planner.addRule(EnumerableDrillRule.getInstance(driver == null ? null : driver.getClient()));
 
     // Enable when https://issues.apache.org/jira/browse/DRILL-57 fixed
     if (false) {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/sqlparser/src/main/resources/storage-engines.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/main/resources/storage-engines.json b/sandbox/prototype/sqlparser/src/main/resources/storage-engines.json
index 1eead25..0dff70c 100644
--- a/sandbox/prototype/sqlparser/src/main/resources/storage-engines.json
+++ b/sandbox/prototype/sqlparser/src/main/resources/storage-engines.json
@@ -5,11 +5,21 @@
         "type":"parquet",
         "dfsName" : "file:///"
       },
+    "parquet-cp" :
+      {
+        "type":"parquet",
+        "dfsName" : "classpath:///"
+      },
     "jsonl" :
       {
         "type":"json",
         "dfsName" : "file:///"
       },
+    "json-cp" :
+      {
+        "type":"json",
+        "dfsName" : "classpath:///"
+      },
     "parquet" :
       {
         "type":"parquet",

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
index e88dc0d..a1a6cf2 100644
--- a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
+++ b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
@@ -1,8 +1,5 @@
 package org.apache.drill.jdbc.test;
 
-import java.io.IOException;
-
-import org.junit.BeforeClass;
 import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
@@ -10,42 +7,29 @@ import org.junit.rules.TestName;
 import org.junit.rules.TestRule;
 import org.junit.rules.Timeout;
 
-import com.google.common.base.Charsets;
-import com.google.common.io.Resources;
-
 
 public class FullEngineTest {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FullEngineTest.class);
 
-  private static String MODEL_FULL_ENGINE;
-
   // Determine if we are in Eclipse Debug mode.
   static final boolean IS_DEBUG = java.lang.management.ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0;
 
   // Set a timeout unless we're debugging.
   @Rule public TestRule globalTimeout = IS_DEBUG ? new TestName() : new Timeout(10000);
   
-//  @BeforeClass
-//  public static void setupFixtures() throws IOException {
-//    MODEL_FULL_ENGINE = Resources.toString(Resources.getResource("full-model.json"), Charsets.UTF_8);
-//  }
-  
-  
-  
   @Test
+  @Ignore // since this is a specifically located file.
   public void fullSelectStarEngine() throws Exception {
     JdbcAssert.withFull("parquet-local")
     // .sql("select cast(_MAP['red'] as bigint) + 1 as red_inc from donuts ")
         .sql("select _MAP['d'] as d, _MAP['b'] as b from \"/tmp/parquet_test_file_many_types\" ").displayResults(50);
   }
 
-//  @Test
-//  public void fullEngine() throws Exception {
-//    JdbcAssert
-//        .withModel(MODEL_FULL_ENGINE, "DONUTS")
-//        // .sql("select cast(_MAP['red'] as bigint) + 1 as red_inc from donuts ")
-//        .sql(
-//            "select cast(_MAP['RED'] as bigint)  as RED, cast(_MAP['GREEN'] as bigint)  as GREEN from 'monkeys' where cast(_MAP['red'] as BIGINT) > 1 ")
-//        .displayResults(50);
-//  }
+  @Test
+  public void setCPRead() throws Exception {
+    JdbcAssert.withFull("json-cp")
+    // .sql("select cast(_MAP['red'] as bigint) + 1 as red_inc from donuts ")
+        .sql("select * from \"department.json\" ").displayResults(50);
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
index 54af383..17b267e 100644
--- a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
+++ b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcAssert.java
@@ -122,8 +122,14 @@ public class JdbcAssert {
       this.info = info;
       this.connectionFactory = new ConnectionFactory() {
         public Connection createConnection() throws Exception {
-          Class.forName("org.apache.drill.jdbc.Driver");
-          String connect = ref ? "jdbc:drill:ref=true" : "jdbc:drill:";
+          String connect = ref ? "jdbc:drillref:" : "jdbc:drill:";
+          if(ref){
+            Class.forName("org.apache.drill.jdbc.RefDriver");
+          }else{
+            Class.forName("org.apache.drill.jdbc.Driver");  
+          }
+          
+          
           return DriverManager.getConnection(connect, ModelAndSchema.this.info);  
         }
       };

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/651b3e9f/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
index d9dd11a..37e81b7 100644
--- a/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
+++ b/sandbox/prototype/sqlparser/src/test/java/org/apache/drill/jdbc/test/JdbcTest.java
@@ -65,7 +65,7 @@ public class JdbcTest {
   @Test
   public void testConnect() throws Exception {
     Class.forName("org.apache.drill.jdbc.Driver");
-    final Connection connection = DriverManager.getConnection("jdbc:drill:ref=true&schema=DONUTS");
+    final Connection connection = DriverManager.getConnection("jdbc:drillref:schema=DONUTS");
     connection.close();
   }
 


[04/18] git commit: fix parquet query join errors

Posted by ja...@apache.org.
fix parquet query join errors


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/cc329aa4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/cc329aa4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/cc329aa4

Branch: refs/heads/master
Commit: cc329aa4418cf6c363f7da7363594f5174655fe1
Parents: d553749
Author: Ben Becker <be...@gmail.com>
Authored: Fri Aug 30 14:59:08 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sun Sep 1 15:16:21 2013 -0700

----------------------------------------------------------------------
 .../exec/physical/impl/join/JoinStatus.java     | 20 ++++----
 .../exec/physical/impl/join/JoinTemplate.java   |  2 +-
 .../exec/physical/impl/join/MergeJoinBatch.java | 51 ++------------------
 3 files changed, 15 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cc329aa4/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinStatus.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinStatus.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinStatus.java
index c755e5f..84a11fc 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinStatus.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinStatus.java
@@ -112,14 +112,15 @@ public final class JoinStatus {
    * Side effect: advances to next left batch if current left batch size is exceeded.
    */
   public final boolean isLeftPositionAllowed(){
-    if(!isLeftPositionInCurrentBatch()){
+    if (lastLeft == IterOutcome.NONE)
+      return false;
+    if (!isLeftPositionInCurrentBatch()) {
       leftPosition = 0;
       lastLeft = left.next();
       return lastLeft == IterOutcome.OK;
-    }else{
-      lastLeft = IterOutcome.OK;
-      return true;
     }
+    lastLeft = IterOutcome.OK;
+    return true;
   }
 
   /**
@@ -129,16 +130,15 @@ public final class JoinStatus {
   public final boolean isRightPositionAllowed(){
     if (rightSourceMode == RightSourceMode.SV4)
       return svRightPosition < sv4.getCount();
-
-    if(!isRightPositionInCurrentBatch()){
+    if (lastRight == IterOutcome.NONE)
+      return false;
+    if (!isRightPositionInCurrentBatch()) {
       rightPosition = 0;
       lastRight = right.next();
       return lastRight == IterOutcome.OK;
-    }else{
-      lastRight = IterOutcome.OK;
-      return true;
     }
-    
+    lastRight = IterOutcome.OK;
+    return true;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cc329aa4/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
index d6fc802..3d6dfce 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
@@ -119,7 +119,7 @@ public abstract class JoinTemplate implements JoinWorker {
           // If the left key has duplicates and we're about to cross a boundary in the right batch, add the
           // right table's record batch to the sv4 builder before calling next.  These records will need to be
           // copied again for each duplicate left key.
-          if (status.isLeftRepeating() && !status.isNextRightPositionInCurrentBatch()) {
+          if (status.isLeftRepeating() && !status.isRightPositionInCurrentBatch()) {
             status.outputBatch.addRightToBatchBuilder();
             crossedBatchBoundaries = true;
           }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cc329aa4/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
index 02a1961..5a62c1f 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
@@ -27,6 +27,7 @@ import org.apache.drill.exec.physical.impl.filter.ReturnValueExpression;
 import org.apache.drill.exec.physical.impl.join.JoinWorker.JoinOutcome;
 import org.apache.drill.exec.record.*;
 import org.apache.drill.exec.vector.*;
+import org.apache.drill.exec.vector.allocator.VectorAllocator;
 
 /**
  * A merge join combining to incoming in-order batches.
@@ -107,7 +108,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
     
     // we do this in the here instead of the constructor because don't necessary want to start consuming on construction.
     status.ensureInitial();
-
+    
     // loop so we can start over again if we find a new batch was created.
     while(true){
 
@@ -367,13 +368,13 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
     // add fields from both batches
     for (VectorWrapper<?> w : left) {
       ValueVector outgoingVector = TypeHelper.getNewVector(w.getField(), context.getAllocator());
-      getAllocator(w.getValueVector(), outgoingVector).alloc(left.getRecordCount() * 16);
+      VectorAllocator.getAllocator(outgoingVector, (int) Math.ceil(w.getValueVector().getBufferSize() / left.getRecordCount())).alloc(left.getRecordCount() * 16);
       container.add(outgoingVector);
     }
 
     for (VectorWrapper<?> w : right) {
       ValueVector outgoingVector = TypeHelper.getNewVector(w.getField(), context.getAllocator());
-      getAllocator(w.getValueVector(), outgoingVector).alloc(right.getRecordCount() * 16);
+      VectorAllocator.getAllocator(outgoingVector, (int) Math.ceil(w.getValueVector().getBufferSize() / right.getRecordCount())).alloc(right.getRecordCount() * 16);
       container.add(outgoingVector);
     }
 
@@ -381,48 +382,4 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
     logger.debug("Built joined schema: {}", container.getSchema());
   }
 
-  private VectorAllocator getAllocator(ValueVector in, ValueVector outgoing){
-    if(outgoing instanceof FixedWidthVector){
-      return new FixedVectorAllocator((FixedWidthVector) outgoing);
-    }else if(outgoing instanceof VariableWidthVector && in instanceof VariableWidthVector){
-      return new VariableVectorAllocator( (VariableWidthVector) in, (VariableWidthVector) outgoing);
-    }else{
-      throw new UnsupportedOperationException();
-    }
-  }
-
-  private class FixedVectorAllocator implements VectorAllocator{
-    FixedWidthVector out;
-
-    public FixedVectorAllocator(FixedWidthVector out) {
-      super();
-      this.out = out;
-    }
-
-    public void alloc(int recordCount){
-      out.allocateNew(recordCount);
-      out.getMutator().setValueCount(recordCount);
-    }
-  }
-
-  private class VariableVectorAllocator implements VectorAllocator{
-    VariableWidthVector in;
-    VariableWidthVector out;
-
-    public VariableVectorAllocator(VariableWidthVector in, VariableWidthVector out) {
-      super();
-      this.in = in;
-      this.out = out;
-    }
-
-    public void alloc(int recordCount){
-      out.allocateNew(in.getByteCapacity() * 4, recordCount);
-      out.getMutator().setValueCount(recordCount);
-    }
-  }
-
-  public interface VectorAllocator{
-    public void alloc(int recordCount);
-  }
-
 }