You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2014/04/20 03:18:54 UTC

[46/51] [abbrv] git commit: Move to Optiq 0.6 Also includes: -improve exception catching -move schema path parsing to Antlr -close zookeeper connection on if client created -enhance BaseTestQuery and have other query tests utilize it -Various test fixes

Move to Optiq 0.6
Also includes:
-improve exception catching
-move schema path parsing to Antlr
-close zookeeper connection on if client created
-enhance BaseTestQuery and have other query tests utilize it
-Various test fixes for better memory release.  still needs client allocator to be closed.
-refactor DrillSqlWorker and create multiple SqlHandlers
-Add PojoRecordReader and DirectPlan capabilities
-Update Antlr to use same quoting rules as SQL: single quote for quoted strings, back ticks for identifiers
-Move back to old Sorts until bugs are fixed
-Refector SelectionVector management within Prels
-Add support for NO_EXCHANGES option
-Extract SchemaFactories to use Optiq's new Schema handling capabilities
-Add basic handling of cancel in UserServer
-Remove output requirement from Project
-Add start of usercredentials to User communication


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ecaa838f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ecaa838f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ecaa838f

Branch: refs/heads/master
Commit: ecaa838fe7194c197884f0a1b6fb6f8a9683cede
Parents: ba0a639
Author: Jacques Nadeau <ja...@apache.org>
Authored: Mon Apr 14 07:37:02 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sat Apr 19 18:07:12 2014 -0700

----------------------------------------------------------------------
 .../drill/common/expression/parser/ExprLexer.g  |  25 +-
 .../drill/common/expression/parser/ExprParser.g |  20 +-
 .../apache/drill/common/config/DrillConfig.java |  48 +-
 .../expression/ExpressionStringBuilder.java     |  64 ++-
 .../drill/common/expression/FieldReference.java |  31 +-
 .../common/expression/LogicalExpression.java    |   2 +
 .../drill/common/expression/PathSegment.java    | 110 ++--
 .../drill/common/expression/SchemaPath.java     | 179 ++++---
 .../common/expression/ValueExpressions.java     |  56 +-
 .../drill/common/logical/data/Project.java      |  22 +-
 .../org/apache/drill/common/util/TestTools.java |  46 +-
 .../common/expression/parser/TreeTest.java      | 102 +++-
 .../exec/store/hbase/HBaseRecordReader.java     |  40 +-
 .../exec/store/hbase/HBaseSchemaFactory.java    |   6 +-
 .../exec/store/hbase/HBaseStoragePlugin.java    |   7 +-
 distribution/src/resources/drill-env.sh         |   4 +-
 .../apache/drill/exec/client/DrillClient.java   |  46 +-
 .../org/apache/drill/exec/client/DumpCat.java   |  12 +-
 .../exec/client/PrintingResultsListener.java    | 100 ++++
 .../drill/exec/client/QuerySubmitter.java       | 184 +++----
 .../apache/drill/exec/compile/MergeAdapter.java |  42 +-
 .../org/apache/drill/exec/memory/Accountor.java |  17 +-
 .../drill/exec/memory/AtomicRemainder.java      |  15 +-
 .../apache/drill/exec/ops/FragmentContext.java  |  15 +-
 .../org/apache/drill/exec/ops/Multitimer.java   |  42 ++
 .../org/apache/drill/exec/ops/QueryContext.java |  59 ++-
 .../org/apache/drill/exec/ops/QuerySetup.java   |  26 +
 .../drill/exec/physical/impl/OutputMutator.java |   4 +
 .../drill/exec/physical/impl/ScanBatch.java     |  12 +-
 .../impl/TopN/PriorityQueueTemplate.java        |  10 +-
 .../physical/impl/common/HashTableTemplate.java | 125 +++--
 .../OrderedPartitionRecordBatch.java            |  42 +-
 .../partitionsender/OutgoingRecordBatch.java    |  14 +-
 .../impl/project/ProjectRecordBatch.java        |  53 +-
 .../exec/physical/impl/sort/SortBatch.java      |  40 +-
 .../impl/sort/SortRecordBatchBuilder.java       |  34 +-
 .../IteratorValidatorBatchIterator.java         |  35 +-
 .../physical/impl/xsort/ExternalSortBatch.java  |  12 +-
 .../planner/common/DrillProjectRelBase.java     |   4 +-
 .../exec/planner/logical/DrillProjectRel.java   |   4 +-
 .../exec/planner/logical/DrillRuleSets.java     |  27 +-
 .../drill/exec/planner/logical/ExprHelper.java  |   8 +-
 .../physical/DrillDistributionTrait.java        |  35 +-
 .../drill/exec/planner/physical/FilterPrel.java |  15 +-
 .../physical/HashToRandomExchangePrel.java      |  28 +-
 .../drill/exec/planner/physical/LimitPrel.java  |  14 +-
 .../exec/planner/physical/MergeJoinPrel.java    |  57 +-
 .../planner/physical/PhysicalPlanCreator.java   |  24 +-
 .../exec/planner/physical/PlanningSettings.java |  57 ++
 .../drill/exec/planner/physical/PrelUtil.java   |  39 +-
 .../exec/planner/physical/ProjectPrel.java      |  18 +-
 .../drill/exec/planner/physical/ScanPrel.java   |  11 +-
 .../drill/exec/planner/physical/ScreenPrel.java |  18 +-
 .../physical/SingleMergeExchangePrel.java       |  30 +-
 .../drill/exec/planner/physical/SortPrel.java   |  26 +-
 .../exec/planner/physical/StreamAggPrel.java    |  19 +-
 .../drill/exec/planner/physical/TopNPrel.java   |  16 +-
 .../planner/physical/UnionExchangePrel.java     |  20 +-
 .../drill/exec/planner/sql/DirectPlan.java      |  63 +++
 .../exec/planner/sql/DrillOperatorTable.java    |  26 +-
 .../exec/planner/sql/DrillParserFactory.java    |  82 ---
 .../drill/exec/planner/sql/DrillSqlWorker.java  | 213 ++------
 .../planner/sql/handlers/DefaultSqlHandler.java | 173 ++++++
 .../planner/sql/handlers/ExplainHandler.java    | 116 ++++
 .../planner/sql/handlers/SetOptionHandler.java  |  58 ++
 .../sql/handlers/SimpleCommandResult.java       |  32 ++
 .../exec/planner/sql/handlers/SqlHandler.java   |  32 ++
 .../exec/planner/types/RelDataTypeHolder.java   |  17 +-
 .../drill/exec/record/MaterializedField.java    |  80 ++-
 .../drill/exec/record/RecordBatchLoader.java    |  15 +-
 .../drill/exec/record/VectorContainer.java      |  11 +-
 .../apache/drill/exec/rpc/user/DrillUser.java   |  36 ++
 .../apache/drill/exec/rpc/user/UserServer.java  |  29 +-
 .../apache/drill/exec/rpc/user/UserSession.java |  80 +++
 .../org/apache/drill/exec/schema/Field.java     |   5 +-
 .../drill/exec/server/DrillbitContext.java      |  30 +-
 .../apache/drill/exec/store/SchemaFactory.java  |   7 +-
 .../apache/drill/exec/store/StoragePlugin.java  |   9 +-
 .../drill/exec/store/StoragePluginRegistry.java | 162 +-----
 .../drill/exec/store/dfs/FileSystemPlugin.java  |  24 +-
 .../exec/store/dfs/FileSystemSchemaFactory.java |  16 +-
 .../exec/store/direct/DirectBatchCreator.java   |  37 ++
 .../exec/store/direct/DirectGroupScan.java      |  81 +++
 .../drill/exec/store/direct/DirectSubScan.java  |  37 ++
 .../exec/store/easy/json/JSONRecordReader.java  |   7 +-
 .../drill/exec/store/hive/HiveRecordReader.java |   6 +-
 .../exec/store/hive/HiveStoragePlugin.java      |   9 +-
 .../store/hive/schema/HiveSchemaFactory.java    |  24 +-
 .../exec/store/ischema/EmptyVectorSet.java      |  72 +--
 .../store/ischema/InfoSchemaStoragePlugin.java  |  21 +-
 .../drill/exec/store/ischema/OptiqProvider.java |  24 +-
 .../drill/exec/store/mock/MockRecordReader.java |  12 +-
 .../exec/store/mock/MockStorageEngine.java      |   9 +-
 .../exec/store/parquet/ParquetRecordReader.java |   6 +-
 .../drill/exec/store/pojo/AbstractWriter.java   |  63 +++
 .../drill/exec/store/pojo/PojoRecordReader.java | 131 +++++
 .../drill/exec/store/pojo/PojoWriter.java       |  29 +
 .../apache/drill/exec/store/pojo/Writers.java   | 127 +++++
 .../apache/drill/exec/util/BatchPrinter.java    |   6 +-
 .../org/apache/drill/exec/util/VectorUtil.java  |   8 +-
 .../apache/drill/exec/work/foreman/Foreman.java | 158 ++----
 .../apache/drill/exec/work/user/UserWorker.java |  13 +-
 .../java/org/apache/drill/BaseTestQuery.java    | 152 ++++--
 .../java/org/apache/drill/PlanningBase.java     | 114 ++++
 .../org/apache/drill/TestAltSortQueries.java    |  48 +-
 .../org/apache/drill/TestExampleQueries.java    |  52 +-
 .../org/apache/drill/TestTpchDistributed.java   | 154 ++++++
 .../java/org/apache/drill/TestTpchExplain.java  | 155 ++++++
 .../java/org/apache/drill/TestTpchPlanning.java | 150 ++++++
 .../java/org/apache/drill/TestTpchQueries.java  | 158 ------
 .../org/apache/drill/TestTpchSingleMode.java    | 157 ++++++
 .../drill/exec/cache/TestVectorCache.java       |   2 +-
 .../exec/physical/impl/SimpleRootExec.java      |  15 +-
 .../physical/impl/TestBroadcastExchange.java    |   8 +-
 .../exec/physical/impl/TestCastFunctions.java   | 151 +++---
 .../physical/impl/TestCastVarCharToBigInt.java  |   5 +
 .../impl/TestDistributedFragmentRun.java        |  10 +-
 .../physical/impl/TestExtractFunctions.java     |   5 +
 .../physical/impl/TestHashToRandomExchange.java |   1 +
 .../exec/physical/impl/TestOptiqPlans.java      |  13 +-
 .../physical/impl/TestSimpleFragmentRun.java    |  13 +-
 .../exec/physical/impl/TestUnionExchange.java   |   1 +
 .../exec/physical/impl/TopN/TestSimpleTopN.java |  10 +-
 .../drill/exec/physical/impl/agg/TestAgg.java   |  34 +-
 .../exec/physical/impl/join/TestMergeJoin.java  |  20 +-
 .../impl/join/TestMergeJoinMulCondition.java    |  10 +-
 .../impl/mergereceiver/TestMergingReceiver.java |  14 +-
 .../TestOrderedPartitionExchange.java           |   3 +
 .../impl/xsort/TestSimpleExternalSort.java      |   8 +-
 .../drill/exec/record/vector/TestDateTypes.java |  30 ++
 .../drill/exec/record/vector/TestLoad.java      |   4 +-
 .../drill/exec/store/TestOrphanSchema.java      |  22 +-
 .../drill/exec/store/TestOutputMutator.java     |  15 +
 .../drill/exec/store/ischema/OrphanSchema.java  |  33 +-
 .../exec/store/ischema/TestOrphanSchema.java    |  49 +-
 .../exec/store/ischema/TestTableProvider.java   |  57 +-
 .../exec/store/json/JSONRecordReaderTest.java   |  17 +-
 .../store/parquet/ParquetRecordReaderTest.java  | 131 ++---
 .../store/parquet/ParquetResultListener.java    |  30 +-
 .../store/parquet/TestParquetPhysicalPlan.java  |   2 +-
 .../exec/work/batch/TestSpoolingBuffer.java     |   1 +
 .../src/test/resources/agg/hashagg/q6.json      |   4 +-
 .../src/test/resources/agg/hashagg/q7_1.json    |   6 +-
 .../src/test/resources/agg/hashagg/q7_2.json    |   6 +-
 .../src/test/resources/agg/hashagg/q8.json      |   4 +-
 .../src/test/resources/agg/hashagg/q8_1.json    |   4 +-
 .../functions/cast/testCastBigInt.json          |   2 +-
 .../functions/cast/testCastFloat4.json          |   4 +-
 .../functions/cast/testCastFloat8.json          |   4 +-
 .../resources/functions/cast/testCastInt.json   |   2 +-
 .../functions/cast/testCastNested.json          |   2 +-
 .../functions/cast/testCastNumException.json    |   2 +-
 .../functions/cast/testICastConstant.json       |  36 +-
 .../functions/simple_math_functions.json        |   8 +-
 .../functions/string/testCharLength.json        |  12 +-
 .../resources/functions/string/testConcat.json  |   8 +-
 .../resources/functions/string/testLeft.json    |  14 +-
 .../resources/functions/string/testLike.json    |   8 +-
 .../resources/functions/string/testLower.json   |   6 +-
 .../resources/functions/string/testLpad.json    |  20 +-
 .../resources/functions/string/testLtrim.json   |  12 +-
 .../functions/string/testPosition.json          |  16 +-
 .../functions/string/testRegexpReplace.json     |   6 +-
 .../resources/functions/string/testReplace.json |  12 +-
 .../resources/functions/string/testRight.json   |  14 +-
 .../resources/functions/string/testRpad.json    |  20 +-
 .../resources/functions/string/testRtrim.json   |  12 +-
 .../resources/functions/string/testSimilar.json |   8 +-
 .../resources/functions/string/testUpper.json   |   6 +-
 .../test/resources/join/mj_multi_condition.json |  12 +-
 .../test/resources/logical_string_filter.json   |   2 +-
 .../src/test/resources/simple_plan.json         |  12 +-
 pom.xml                                         |   8 +-
 .../org/apache/drill/exec/proto/BitControl.java | 218 +++++++-
 .../apache/drill/exec/proto/UserBitShared.java  | 531 ++++++++++++++++++-
 .../org/apache/drill/exec/proto/UserProtos.java | 258 +++++++--
 protocol/src/main/protobuf/BitControl.proto     |   1 +
 protocol/src/main/protobuf/User.proto           |   2 +
 protocol/src/main/protobuf/UserBitShared.proto  |   6 +-
 sqlparser/pom.xml                               |   8 +
 .../drill/jdbc/DrillColumnMetaDataList.java     |  14 +-
 .../apache/drill/jdbc/test/TestJdbcQuery.java   |  28 +-
 182 files changed, 5178 insertions(+), 2401 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
----------------------------------------------------------------------
diff --git a/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g b/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
index 78c356a..700bd72 100644
--- a/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
+++ b/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
@@ -54,7 +54,7 @@ INTERVAL : 'interval' | 'INTERVAL';
 INTERVALYEAR : 'intervalyear' | 'INTERVALYEAR';
 INTERVALDAY : 'intervalday' | 'INTERVALDAY';
 
-
+Period : '.';
 Or       : '||' | 'or' | 'OR' | 'Or';
 And      : '&&' | 'and' | 'AND' ;
 Equals   : '==' | '=';
@@ -100,15 +100,21 @@ Number
 //  ;
   
 Identifier
-  :  ('a'..'z' | 'A'..'Z' | '_' | '$') ('a'..'z' | 'A'..'Z' | '_' | '$' | Digit)* ('.' ('a'..'z' | 'A'..'Z' | '_' | '$' ) ('a'..'z' | 'A'..'Z' | '_' | '$' | Digit)*)*
+  : ('a'..'z' | 'A'..'Z' | '_' | '$') ('a'..'z' | 'A'..'Z' | '_' | '$' | Digit)*
+  ;
+
+QuotedIdentifier
+@after {
+  setText(getText().substring(1, getText().length()-1).replaceAll("\\\\(.)", "$1"));
+}
+  :  '`'  (~('`' | '\\')  | '\\' ('\\' | '`'))* '`' 
   ;
 
 String
 @after {
   setText(getText().substring(1, getText().length()-1).replaceAll("\\\\(.)", "$1"));
 }
-  :  '"'  (~('"' | '\\')  | '\\' ('\\' | '"'))* '"' 
-  |  '\'' (~('\'' | '\\') | '\\' ('\\' | '\''))* '\''
+  :  '\'' (~('\'' | '\\') | '\\' ('\\' | '\''))* '\''
   ;
 
 Comment
@@ -128,3 +134,14 @@ fragment Int
 fragment Digit 
   :  '0'..'9'
   ;
+
+FallThrough
+	@after{
+	  throw new RuntimeException(java.lang.String.format(
+	      "Encountered an illegal char on line \%d, column \%d: '\%s'", 
+	      getLine(), getCharPositionInLine(), getText()
+	    )
+	  );
+	}
+  :
+  ;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
----------------------------------------------------------------------
diff --git a/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g b/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
index 2e2f3d3..e4eb149 100644
--- a/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
+++ b/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
@@ -35,6 +35,8 @@ package org.apache.drill.common.expression.parser;
 import org.antlr.runtime.BitSet;
 import java.util.*;
 import org.apache.drill.common.expression.*;
+import org.apache.drill.common.expression.PathSegment.NameSegment;
+import org.apache.drill.common.expression.PathSegment.ArraySegment;
 import org.apache.drill.common.types.*;
 import org.apache.drill.common.types.TypeProtos.*;
 import org.apache.drill.common.exceptions.ExpressionParsingException;
@@ -267,12 +269,28 @@ atom returns [LogicalExpression e]
   |  lookup {$e = $lookup.e; }
   ;
 
+pathSegment returns [NameSegment seg]
+  : s1=nameSegment {$seg = $s1.seg;}
+  ;
+
+nameSegment returns [NameSegment seg]
+  : QuotedIdentifier ( (Period s1=pathSegment) | s2=arraySegment)? {$seg = new NameSegment($QuotedIdentifier.text, ($s1.seg == null ? $s2.seg : $s1.seg) ); }
+  | Identifier ( (Period s1=pathSegment) | s2=arraySegment)? {$seg = new NameSegment($Identifier.text, ($s1.seg == null ? $s2.seg : $s1.seg) ); }
+  ;
+  
+arraySegment returns [PathSegment seg]
+  :  OBracket Number CBracket ( (Period s1=pathSegment) | s2=arraySegment)? {$seg = new ArraySegment($Number.text, ($s1.seg == null ? $s2.seg : $s1.seg) ); }
+  ;
+
 
 lookup returns [LogicalExpression e]
   :  functionCall {$e = $functionCall.e ;}
   | castCall {$e = $castCall.e; }
-  | Identifier {$e = new SchemaPath($Identifier.text, pos($Identifier) ); }
+  | pathSegment {$e = new SchemaPath($pathSegment.seg, pos($pathSegment.start) ); }
   | String {$e = new ValueExpressions.QuotedString($String.text, pos($String) ); }
   | OParen expression CParen  {$e = $expression.e; }
   | SingleQuote Identifier SingleQuote {$e = new SchemaPath($Identifier.text, pos($Identifier) ); }
   ;
+  
+  
+  

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
index 6fc048d..7fb0e39 100644
--- a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
+++ b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
@@ -27,6 +27,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
 
 import org.apache.drill.common.exceptions.DrillConfigurationException;
 import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.logical.FormatPluginConfigBase;
 import org.apache.drill.common.logical.StoragePluginConfigBase;
 import org.apache.drill.common.logical.data.LogicalOperatorBase;
@@ -48,11 +49,11 @@ public final class DrillConfig extends NestedConfig{
   private final ObjectMapper mapper;
   private final ImmutableList<String> startupArguments;
   @SuppressWarnings("restriction")  private static final long MAX_DIRECT_MEMORY = sun.misc.VM.maxDirectMemory();
-  
+
   @SuppressWarnings("unchecked")
   private volatile List<Queue<Object>> sinkQueues = new CopyOnWriteArrayList<Queue<Object>>(new Queue[1]);
 
-  
+
   @SuppressWarnings("restriction")
   @VisibleForTesting
   public DrillConfig(Config config) {
@@ -60,9 +61,10 @@ public final class DrillConfig extends NestedConfig{
 
     mapper = new ObjectMapper();
     SimpleModule deserModule = new SimpleModule("LogicalExpressionDeserializationModule")
-      .addDeserializer(LogicalExpression.class, new LogicalExpression.De(this));
-    
-    
+      .addDeserializer(LogicalExpression.class, new LogicalExpression.De(this))
+      .addDeserializer(SchemaPath.class, new SchemaPath.De(this));
+
+
     mapper.registerModule(deserModule);
     mapper.enable(SerializationFeature.INDENT_OUTPUT);
     mapper.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
@@ -71,10 +73,10 @@ public final class DrillConfig extends NestedConfig{
     mapper.registerSubtypes(LogicalOperatorBase.getSubTypes(this));
     mapper.registerSubtypes(StoragePluginConfigBase.getSubTypes(this));
     mapper.registerSubtypes(FormatPluginConfigBase.getSubTypes(this));
-    
+
     RuntimeMXBean bean = ManagementFactory.getRuntimeMXBean();
     this.startupArguments = ImmutableList.copyOf(bean.getInputArguments());
-    
+
   };
 
 
@@ -82,15 +84,15 @@ public final class DrillConfig extends NestedConfig{
   public List<String> getStartupArguments(){
     return startupArguments;
   }
-  
+
   /**
-   * Create a DrillConfig object using the default config file name 
+   * Create a DrillConfig object using the default config file name
    * @return The new DrillConfig object.
    */
   public static DrillConfig create() {
     return create(null);
   }
-  
+
   /**
    * <p>
    * DrillConfig loads up Drill configuration information. It does this utilizing a combination of classpath scanning
@@ -106,15 +108,15 @@ public final class DrillConfig extends NestedConfig{
    * <li>Check a single copy of "drill-default.conf". If multiple copies are on the classpath, behavior is
    * indeterminate.</li>
    * </ul>
-   * 
+   *
    * </p>
    *  @param overrideFileName The name of the file to use for override purposes.
    *  @return A merged Config object.
    */
   public static DrillConfig create(String overrideFileName) {
-    
+
     overrideFileName = overrideFileName == null ? CommonConstants.CONFIG_OVERRIDE : overrideFileName;
-    
+
     // first we load defaults.
     Config fallback = ConfigFactory.load(CommonConstants.CONFIG_DEFAULT);
     Collection<URL> urls = PathScanner.getConfigURLs();
@@ -126,7 +128,7 @@ public final class DrillConfig extends NestedConfig{
     Config c = ConfigFactory.load(overrideFileName).withFallback(fallback).resolve();
     return new DrillConfig(c);
   }
-  
+
   public <T> Class<T> getClassAt(String location, Class<T> clazz) throws DrillConfigurationException{
     String className = this.getString(location);
     if(className == null) throw new DrillConfigurationException(String.format("No class defined at location '%s'.  Expected a definition of the class []", location, clazz.getCanonicalName()));
@@ -142,9 +144,9 @@ public final class DrillConfig extends NestedConfig{
       if(ex instanceof DrillConfigurationException) throw (DrillConfigurationException) ex;
       throw new DrillConfigurationException(String.format("Failure while initializing class [%s] described at configuration value '%s'.", className, location), ex);
     }
-    
+
   }
-  
+
   public <T> T getInstanceOf(String location, Class<T> clazz) throws DrillConfigurationException{
     Class<T> c = getClassAt(location, clazz);
     try{
@@ -154,18 +156,18 @@ public final class DrillConfig extends NestedConfig{
       throw new DrillConfigurationException(String.format("Failure while instantiating class [%s] located at '%s.", clazz.getCanonicalName(), location), ex);
     }
   }
-  
 
-  
+
+
   public void setSinkQueues(int number, Queue<Object> queue){
     sinkQueues.set(number, queue);
   }
-  
+
   public Queue<Object> getQueue(int number){
     if(sinkQueues.size() <= number || number < 0 || sinkQueues == null) throw new IllegalArgumentException(String.format("Queue %d is not available.", number));
     return sinkQueues.get(number);
   }
-  
+
   public ObjectMapper getMapper(){
     return mapper;
   }
@@ -174,13 +176,13 @@ public final class DrillConfig extends NestedConfig{
   public String toString(){
     return this.root().render();
   }
-  
+
   public static void main(String[] args)  throws Exception{
     //"-XX:MaxDirectMemorySize"
     DrillConfig config = DrillConfig.create();
-    
+
   }
-  
+
   public static long getMaxDirectMemory(){
     return MAX_DIRECT_MEMORY;
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java b/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
index 787e4af..89fa437 100644
--- a/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
+++ b/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
@@ -44,6 +44,19 @@ import com.google.common.collect.ImmutableList;
 public class ExpressionStringBuilder extends AbstractExprVisitor<Void, StringBuilder, RuntimeException>{
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpressionStringBuilder.class);
 
+  static final ExpressionStringBuilder INSTANCE = new ExpressionStringBuilder();
+
+  public static String toString(LogicalExpression expr){
+    StringBuilder sb = new StringBuilder();
+    expr.accept(INSTANCE, sb);
+    return sb.toString();
+  }
+
+  public static void toString(LogicalExpression expr, StringBuilder sb){
+    expr.accept(INSTANCE, sb);
+  }
+
+
   @Override
   public Void visitFunctionCall(FunctionCall call, StringBuilder sb) throws RuntimeException {
     ImmutableList<LogicalExpression> args = call.args;
@@ -85,7 +98,26 @@ public class ExpressionStringBuilder extends AbstractExprVisitor<Void, StringBui
 
   @Override
   public Void visitSchemaPath(SchemaPath path, StringBuilder sb) throws RuntimeException {
-    sb.append(path.getPath());
+    PathSegment seg = path.getRootSegment();
+    if(seg.isArray()) throw new IllegalStateException("Drill doesn't currently support top level arrays");
+    sb.append('`');
+    sb.append(seg.getNameSegment().getPath());
+    sb.append('`');
+
+    while( (seg = seg.getChild()) != null){
+      if(seg.isNamed()){
+        sb.append('.');
+        sb.append('`');
+        sb.append(seg.getNameSegment().getPath());
+        sb.append('`');
+      }else{
+        sb.append('[');
+        sb.append(seg.getArraySegment().getIndex());
+        sb.append(']');
+      }
+    }
+
+
     return null;
   }
 
@@ -121,33 +153,33 @@ public class ExpressionStringBuilder extends AbstractExprVisitor<Void, StringBui
     TIMESTAMP_FORMAT = new DateTimeFormatterBuilder().append(dateFormatter).appendOptional(optionalTime).appendOptional(optionalSec).toFormatter();
 
   }
-  
+
   @Override
   public Void visitTimeStampConstant(TimeStampExpression lExpr, StringBuilder sb) throws RuntimeException {
-    sb.append("cast( \"");
+    sb.append("cast( '");
     try {
       TIMESTAMP_FORMAT.printTo(sb, lExpr.getTimeStamp());
     } catch (IOException e) {
       throw new RuntimeException(e);
     }
     sb.append(lExpr.getTimeStamp());
-    sb.append(" \" as TIMESTAMP)");
+    sb.append(" ' as TIMESTAMP)");
     return null;
   }
 
   @Override
   public Void visitIntervalYearConstant(IntervalYearExpression lExpr, StringBuilder sb) throws RuntimeException {
-    sb.append("cast( \"");
+    sb.append("cast( '");
     sb.append(Period.years(lExpr.getIntervalYear()).toString());
-    sb.append("\" as INTERVALYEAR)");
+    sb.append("' as INTERVALYEAR)");
     return null;
   }
 
   @Override
   public Void visitIntervalDayConstant(IntervalDayExpression lExpr, StringBuilder sb) throws RuntimeException {
-    sb.append("cast( \"");
+    sb.append("cast( '");
     sb.append(Period.days(lExpr.getIntervalDay()).plusMillis(lExpr.getIntervalMillis()).toString());
-    sb.append("\" as INTERVALDAY)");
+    sb.append("' as INTERVALDAY)");
     return null;
   }
 
@@ -165,21 +197,21 @@ public class ExpressionStringBuilder extends AbstractExprVisitor<Void, StringBui
 
   @Override
   public Void visitQuotedStringConstant(QuotedString e, StringBuilder sb) throws RuntimeException {
-    sb.append("\"");
+    sb.append("'");
     sb.append(e.value);
-    sb.append("\"");
+    sb.append("'");
     return null;
   }
 
   @Override
   public Void visitCastExpression(CastExpression e, StringBuilder sb) throws RuntimeException {
     MajorType mt = e.getMajorType();
-    
+
     sb.append("cast( (");
     e.getInput().accept(this, sb);
     sb.append(" ) as ");
     sb.append(mt.getMinorType().name());
-    
+
     switch(mt.getMinorType()){
     case FLOAT4:
     case FLOAT8:
@@ -228,8 +260,8 @@ public class ExpressionStringBuilder extends AbstractExprVisitor<Void, StringBui
     sb.append(intExpr.getInt());
     return null;
   }
-  
-  
-  
-  
+
+
+
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/FieldReference.java b/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
index 358a865..bcc6b52 100644
--- a/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
+++ b/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 
 import org.apache.drill.common.expression.FieldReference.De;
 import org.apache.drill.common.expression.FieldReference.Se;
+import org.apache.drill.common.expression.PathSegment.NameSegment;
 import org.apache.drill.common.types.TypeProtos.MajorType;
 
 import com.fasterxml.jackson.core.JsonGenerationException;
@@ -41,18 +42,32 @@ public class FieldReference extends SchemaPath {
 
   public FieldReference(SchemaPath sp){
     super(sp);
+    checkData();
   }
-  
+
+  private void checkData(){
+    if(getRootSegment().getChild() != null) throw new UnsupportedOperationException("Field references must be singular names.");
+
+  }
+
+
+  private void checkSimpleString(CharSequence value){
+    if(value.toString().contains(".")) throw new UnsupportedOperationException("Field references must be singular names.");
+  }
+
   public FieldReference(CharSequence value){
-    super(value, ExpressionPosition.UNKNOWN);
+    this(value, ExpressionPosition.UNKNOWN);
+    checkSimpleString(value);
   }
-  
+
   public FieldReference(CharSequence value, ExpressionPosition pos) {
-    super(value, pos);
+    super(new NameSegment(value), pos);
+    checkData();
+    checkSimpleString(value);
   }
 
   public FieldReference(String value, ExpressionPosition pos, MajorType dataType) {
-    super(value, pos);
+    this(value, pos);
     this.overrideType = dataType;
   }
 
@@ -74,7 +89,9 @@ public class FieldReference extends SchemaPath {
     @Override
     public FieldReference deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException,
         JsonProcessingException {
-      return new FieldReference(this._parseString(jp, ctxt), ExpressionPosition.UNKNOWN);
+      String ref = this._parseString(jp, ctxt);
+      ref = ref.replace("`", "");
+      return new FieldReference(ref, ExpressionPosition.UNKNOWN);
     }
 
   }
@@ -88,7 +105,7 @@ public class FieldReference extends SchemaPath {
     @Override
     public void serialize(FieldReference value, JsonGenerator jgen, SerializerProvider provider) throws IOException,
         JsonGenerationException {
-      jgen.writeString(value.getPath().toString());
+      jgen.writeString('`' + value.getRootSegment().getNameSegment().getPath() + '`');
     }
 
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java b/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java
index 4df2c8e..cceaef0 100644
--- a/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java
+++ b/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java
@@ -103,4 +103,6 @@ public interface LogicalExpression extends Iterable<LogicalExpression>{
 
   }
 
+
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/PathSegment.java b/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
index ac1aeac..448e50f 100644
--- a/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
+++ b/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
@@ -17,38 +17,42 @@
  */
 package org.apache.drill.common.expression;
 
-import org.apache.drill.common.expression.ValueExpressions.CollisionBehavior;
 
 
 
 public abstract class PathSegment{
 
-  protected final ValueExpressions.CollisionBehavior collision;
   protected PathSegment child;
 
-  protected PathSegment(CollisionBehavior collision){
-    this.collision = collision;
-  }
-  
+  public abstract PathSegment cloneWithNewChild(PathSegment segment);
   public abstract PathSegment clone();
-  
+
   public static class ArraySegment extends PathSegment{
     private final int index;
-    
-    public ArraySegment(int index, ValueExpressions.CollisionBehavior collision){
-      super(collision);
+
+    public ArraySegment(String numberAsText, PathSegment child){
+      this.child = child;
+      this.index = Integer.parseInt(numberAsText);
+    }
+
+    public ArraySegment(int index, PathSegment child){
+      this.child = child;
+      this.index = index;
+    }
+
+    public ArraySegment(int index){
       if(index < 0 ) throw new IllegalArgumentException();
       this.index = index;
     }
-    
+
     public int getIndex(){
       return index;
     }
-    
+
     public boolean isArray(){
       return true;
     }
-    
+
     public boolean isNamed(){
       return false;
     }
@@ -60,37 +64,49 @@ public abstract class PathSegment{
 
     @Override
     public String toString() {
-      return "ArraySegment [index=" + index + ", getCollisionBehavior()=" + getCollisionBehavior() + ", getChild()="
-          + getChild() + "]";
+      return "ArraySegment [index=" + index + ", getChild()=" + getChild() + "]";
     }
 
     @Override
     public PathSegment clone() {
-      PathSegment seg = new ArraySegment(index, collision);
+      PathSegment seg = new ArraySegment(index);
       if(child != null) seg.setChild(child.clone());
       return seg;
     }
-    
+
+    public ArraySegment cloneWithNewChild(PathSegment newChild){
+      ArraySegment seg = new ArraySegment(index);
+      if(child != null){
+        seg.setChild(child.cloneWithNewChild(newChild));
+      }else{
+        seg.setChild(newChild);
+      }
+      return seg;
+    }
   }
-  
-  
-  
+
+
+
   public static class NameSegment extends PathSegment{
     private final String path;
-    
-    public NameSegment(CharSequence n, ValueExpressions.CollisionBehavior collision){
-      super(collision);
+
+    public NameSegment(CharSequence n, PathSegment child){
+      this.child = child;
+      this.path = n.toString();
+    }
+
+    public NameSegment(CharSequence n){
       this.path = n.toString();
     }
-    
+
     public String getPath(){
-      return path;        
+      return path;
     }
-    
+
     public boolean isArray(){
       return false;
     }
-    
+
     public boolean isNamed(){
       return true;
     }
@@ -102,15 +118,14 @@ public abstract class PathSegment{
 
     @Override
     public String toString() {
-      return "NameSegment [path=" + path + ", getCollisionBehavior()=" + getCollisionBehavior() + ", getChild()="
-          + getChild() + "]";
+      return "NameSegment [path=" + path + ", getChild()=" + getChild() + "]";
     }
 
     @Override
     public int hashCode() {
       final int prime = 31;
       int result = 1;
-      result = prime * result + ((path == null) ? 0 : path.hashCode());
+      result = prime * result + ((path == null) ? 0 : path.toLowerCase().hashCode());
       return result;
     }
 
@@ -132,16 +147,27 @@ public abstract class PathSegment{
     }
 
     @Override
-    public PathSegment clone() {
-      PathSegment s = new NameSegment(this.path, this.collision);
+    public NameSegment clone() {
+      NameSegment s = new NameSegment(this.path);
       if(child != null) s.setChild(child.clone());
       return s;
     }
-    
-    
-    
+
+
+    @Override
+    public NameSegment cloneWithNewChild(PathSegment newChild) {
+      NameSegment s = new NameSegment(this.path);
+      if(child != null){
+        s.setChild(child.cloneWithNewChild(newChild));
+      }else{
+        s.setChild(newChild);
+      }
+      return s;
+    }
+
+
   }
-  
+
   public NameSegment getNameSegment(){
     throw new UnsupportedOperationException();
   }
@@ -150,12 +176,8 @@ public abstract class PathSegment{
   }
   public abstract boolean isArray();
   public abstract boolean isNamed();
-  
-  
-  public ValueExpressions.CollisionBehavior getCollisionBehavior(){
-    return collision;
-  }
-  
+
+
   public boolean isLastPath(){
     return child == null;
   }
@@ -168,6 +190,6 @@ public abstract class PathSegment{
     this.child = child;
   }
 
-  
-  
+
+
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java b/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
index 76838dc..a3f93a3 100644
--- a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
+++ b/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
@@ -17,98 +17,79 @@
  */
 package org.apache.drill.common.expression;
 
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 
-import org.apache.drill.common.expression.ValueExpressions.CollisionBehavior;
+import org.antlr.runtime.ANTLRStringStream;
+import org.antlr.runtime.CommonTokenStream;
+import org.antlr.runtime.RecognitionException;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.expression.PathSegment.NameSegment;
+import org.apache.drill.common.expression.parser.ExprLexer;
+import org.apache.drill.common.expression.parser.ExprParser;
+import org.apache.drill.common.expression.parser.ExprParser.parse_return;
 import org.apache.drill.common.expression.visitors.ExprVisitor;
 import org.apache.drill.common.types.TypeProtos.MajorType;
 import org.apache.drill.common.types.Types;
 
-import com.google.common.base.Joiner;
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+import com.fasterxml.jackson.databind.ser.std.StdSerializer;
 import com.google.common.collect.Iterators;
-import com.google.protobuf.DescriptorProtos.UninterpretedOption.NamePart;
 
 public class SchemaPath extends LogicalExpressionBase {
 
-  // reads well in RegexBuddy
-  private static final String ENTIRE_REGEX = "^\n" + "(?:                # first match required\n"
-      + "\\[\\d+\\]             # array index only\n" + "|\n" + "'?\n" + "[^\\.\\[\\+\\-\\!\\]\\}]+  # identifier\n"
-      + "'?\n" + "(?:\\[\\d+\\])?\n" + ")\n" + "[\\+\\-\\!\\]\\}]?\n" +
+  private final NameSegment rootSegment;
 
-      "# secondary matches (starts with dot)\n" + "(?:\n" + "\\.\n" + "(?:                # first match required\n"
-      + "\\[\\d+\\]             # array index only\n" + "|\n" + "'?\n" + "[^\\.\\[\\+\\-\\!\\]\\}]+  # identifier\n"
-      + "'?\n" + "(?:\\[\\d+\\])?\n" + ")\n" + "[\\+\\-\\!\\]\\}]?\n" +
 
-      ")*$";
+  public static SchemaPath getSimplePath(String name){
+    return getCompoundPath(name);
+  }
+
+  public static SchemaPath getCompoundPath(String... strings){
+    List<String> paths = Arrays.asList(strings);
+    Collections.reverse(paths);
+    NameSegment s = null;
+    for(String p : paths){
+      s = new NameSegment(p);
+    }
+    return new SchemaPath(s);
+  }
 
-  // reads well in RegexBuddy
-  private static final String SEGMENT_REGEX = "(?:\n" + "\\[(\\d+)\\]\n" + "|\n" + "'?\n"
-      + "([^\\.\\[\\+\\-\\!\\]\\}]+)  # identifier\n" + "'?\n" + ")\n"
-      + "([\\+\\-\\!\\]\\}]?)         # collision type";
-  private static final int GROUP_INDEX = 1;
-  private static final int GROUP_PATH_SEGMENT = 2;
-  private static final int GROUP_COLLISION = 3;
 
-  private final static Pattern SEGMENT_PATTERN = Pattern.compile(SEGMENT_REGEX, Pattern.COMMENTS);
-  private final static Pattern ENTIRE_PATTERN = Pattern.compile(ENTIRE_REGEX, Pattern.COMMENTS);
 
-  private final CharSequence originalPath;
-  private final PathSegment rootSegment;
+  /**
+   *
+   * @param simpleName
+   */
+  @Deprecated
+  public SchemaPath(String simpleName, ExpressionPosition pos){
+    super(pos);
+    this.rootSegment = new NameSegment(simpleName);
+    if(simpleName.contains(".")) throw new IllegalStateException("This is deprecated and only supports simpe paths.");
+  }
 
   public SchemaPath(SchemaPath path){
     super(path.getPosition());
-    this.originalPath = path.originalPath;
     this.rootSegment = path.rootSegment;
   }
 
-  public SchemaPath(String str){
-    this(str, ExpressionPosition.UNKNOWN);
+  public SchemaPath(NameSegment rootSegment){
+    super(ExpressionPosition.UNKNOWN);
+    this.rootSegment = rootSegment;
   }
-  
-  public SchemaPath(CharSequence str, ExpressionPosition pos) {
-    super(pos);
 
-    if (!ENTIRE_PATTERN.matcher(str).matches())
-      throw new IllegalArgumentException("Identifier doesn't match expected pattern.");
-    this.originalPath = str;
-    Matcher m = SEGMENT_PATTERN.matcher(str);
-    PathSegment r = null;
-    PathSegment previous = null;
-    PathSegment current;
-    while (m.find()) {
-      CollisionBehavior col = (m.start(GROUP_COLLISION) != -1) ? CollisionBehavior.find(m.group(GROUP_COLLISION))
-          : CollisionBehavior.DEFAULT;
-
-      if (m.start(GROUP_INDEX) != -1) {
-        String d = m.group(GROUP_INDEX);
-        current = new PathSegment.ArraySegment(Integer.parseInt(d), col);
-      } else {
-        String i = m.group(GROUP_PATH_SEGMENT);
-        current = new PathSegment.NameSegment(i, col);
-      }
-      if (previous == null) {
-        r = current;
-      } else {
-        previous.setChild(current);
-      }
-      previous = current;
-    }
-
-    rootSegment = r;
-
-  }
-  
-  private SchemaPath(SchemaPath parent, String[] childPaths){
-    super(ExpressionPosition.UNKNOWN);
-    SchemaPath p = new SchemaPath(Joiner.on('.').join(childPaths), ExpressionPosition.UNKNOWN);
-    this.originalPath = parent.originalPath + "." + p.originalPath;
-    PathSegment seg = parent.getRootSegment().clone();
-    this.rootSegment = seg;
-    while(!seg.isLastPath()) seg = seg.getChild();
-    seg.setChild(p.getRootSegment());
+  public SchemaPath(NameSegment rootSegment, ExpressionPosition pos){
+    super(pos);
+    this.rootSegment = rootSegment;
   }
 
   @Override
@@ -116,16 +97,13 @@ public class SchemaPath extends LogicalExpressionBase {
     return visitor.visitSchemaPath(this, value);
   }
 
-  public SchemaPath getChild(String... childPaths){
-    return new SchemaPath(this, childPaths);
-  }
-  
-  public PathSegment getRootSegment() {
-    return rootSegment;
+  public SchemaPath getChild(String childPath){
+    rootSegment.cloneWithNewChild(new NameSegment(childPath));
+    return new SchemaPath(rootSegment);
   }
 
-  public CharSequence getPath() {
-    return originalPath;
+  public NameSegment getRootSegment() {
+    return rootSegment;
   }
 
   @Override
@@ -147,7 +125,7 @@ public class SchemaPath extends LogicalExpressionBase {
       return true;
     if (obj == null)
       return false;
-    if (getClass() != obj.getClass())
+    if ( !(obj instanceof SchemaPath))
       return false;
     SchemaPath other = (SchemaPath) obj;
     if (rootSegment == null) {
@@ -163,9 +141,54 @@ public class SchemaPath extends LogicalExpressionBase {
     return Iterators.emptyIterator();
   }
 
+
   @Override
   public String toString() {
-    return "SchemaPath [rootSegment=" + rootSegment + "]";
+    String expr = ExpressionStringBuilder.toString(this);
+    return "SchemaPath ["+ expr + "]";
+  }
+
+  public String toExpr(){
+    return ExpressionStringBuilder.toString(this);
+  }
+
+
+  public static class De extends StdDeserializer<SchemaPath> {
+    DrillConfig config;
+
+    public De(DrillConfig config) {
+      super(LogicalExpression.class);
+      this.config = config;
+    }
+
+    @Override
+    public SchemaPath deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException,
+        JsonProcessingException {
+      String expr = jp.getText();
+
+      if (expr == null || expr.isEmpty())
+        return null;
+      try {
+        // logger.debug("Parsing expression string '{}'", expr);
+        ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
+        CommonTokenStream tokens = new CommonTokenStream(lexer);
+        ExprParser parser = new ExprParser(tokens);
+
+        //TODO: move functionregistry and error collector to injectables.
+        //ctxt.findInjectableValue(valueId, forProperty, beanInstance)
+        parse_return ret = parser.parse();
+
+        // ret.e.resolveAndValidate(expr, errorCollector);
+        if(ret.e instanceof SchemaPath){
+          return (SchemaPath) ret.e;
+        }else{
+          throw new IllegalStateException("Schema path is not a valid format.");
+        }
+      } catch (RecognitionException e) {
+        throw new RuntimeException(e);
+      }
+    }
+
   }
 
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java b/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
index 4b83e7d..1d217f2 100644
--- a/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
+++ b/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
@@ -33,22 +33,22 @@ public class ValueExpressions {
   public static LogicalExpression getBigInt(long l){
     return new LongExpression(l);
   }
-  
+
   public static LogicalExpression getInt(int i){
     return new IntExpression(i, ExpressionPosition.UNKNOWN);
   }
-  
+
   public static LogicalExpression getFloat8(double d){
     return new DoubleExpression(d, ExpressionPosition.UNKNOWN);
   }
   public static LogicalExpression getFloat4(float f){
     return new DoubleExpression(f, ExpressionPosition.UNKNOWN);
   }
-  
+
   public static LogicalExpression getBit(boolean b){
     return new BooleanExpression(Boolean.toString(b), ExpressionPosition.UNKNOWN);
   }
-  
+
   public static LogicalExpression getChar(String s){
     return new QuotedString(s, ExpressionPosition.UNKNOWN);
   }
@@ -72,7 +72,7 @@ public class ValueExpressions {
       return new IntervalDayExpression(intervalInMillis);
   }
 
-  
+
 
   public static LogicalExpression getNumericExpression(String s, ExpressionPosition ep) {
     try {
@@ -109,7 +109,7 @@ public class ValueExpressions {
     }
 
     protected abstract V parseValue(String s);
-    
+
     @Override
     public Iterator<LogicalExpression> iterator() {
       return Iterators.emptyIterator();
@@ -119,13 +119,13 @@ public class ValueExpressions {
   }
 
   public static class BooleanExpression extends ValueExpression<Boolean> {
-    
-    
+
+
     public BooleanExpression(String value, ExpressionPosition pos) {
       super(value, pos);
     }
 
-    
+
     @Override
     protected Boolean parseValue(String s) {
       return Boolean.parseBoolean(s);
@@ -250,7 +250,7 @@ public class ValueExpressions {
     public LongExpression(long l) {
       this(l, ExpressionPosition.UNKNOWN);
     }
-    
+
       public LongExpression(long l, ExpressionPosition pos) {
       super(pos);
       this.l = l;
@@ -269,7 +269,7 @@ public class ValueExpressions {
     public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
       return visitor.visitLongConstant(this, value);
     }
-    
+
     @Override
     public Iterator<LogicalExpression> iterator() {
       return Iterators.emptyIterator();
@@ -427,10 +427,10 @@ public class ValueExpressions {
 
     private static final MajorType INTERVALDAY_CONSTANT = Types.required(MinorType.INTERVALDAY);
     private static final long MILLIS_IN_DAY = 1000 * 60 * 60 * 24;
-    
+
     private int days;
     private int millis;
-    
+
     public IntervalDayExpression(long intervalInMillis) {
       this((int) (intervalInMillis / MILLIS_IN_DAY), (int) (intervalInMillis % MILLIS_IN_DAY), ExpressionPosition.UNKNOWN);
     }
@@ -490,34 +490,4 @@ public class ValueExpressions {
     }
   }
 
-  public static enum CollisionBehavior {
-    SKIP("-"), // keep the old value.
-    FAIL("!"), // give up on the record
-    REPLACE("+"), // replace the old value with the new value.
-    ARRAYIFY("]"), // replace the current position with an array. Then place the
-                   // old and new value in the array.
-    OBJECTIFY("}"), // replace the current position with a map. Give the two
-                    // values names of 'old' and 'new'.
-    MERGE_OVERRIDE("%"); // do your best to do a deep merge of the old and new
-                         // values.
-
-    private String identifier;
-
-    private CollisionBehavior(String identifier) {
-      this.identifier = identifier;
-    }
-
-    public static final CollisionBehavior DEFAULT = FAIL;
-
-    public static final CollisionBehavior find(String c) {
-      if (c == null || c.isEmpty())
-        return DEFAULT;
-
-      for (CollisionBehavior b : values()) {
-        if (b.identifier.equals(c))
-          return b;
-      }
-      return DEFAULT;
-    }
-  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/java/org/apache/drill/common/logical/data/Project.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Project.java b/common/src/main/java/org/apache/drill/common/logical/data/Project.java
index 645dc32..ab2ef35 100644
--- a/common/src/main/java/org/apache/drill/common/logical/data/Project.java
+++ b/common/src/main/java/org/apache/drill/common/logical/data/Project.java
@@ -43,17 +43,17 @@ public class Project extends SingleInputOperator {
     if (selections == null || selections.length == 0)
       throw new ExpressionParsingException(
           "Project did not provide any projection selections.  At least one projection must be provided.");
-    for (int i = 0; i < selections.length; i++) {
-      PathSegment segment = selections[i].getRef().getRootSegment();
-      CharSequence path = segment.getNameSegment().getPath();
-      if (!segment.isNamed() || !path.equals("output"))
-        throw new ExpressionParsingException(
-            String
-                .format(
-                    "Outputs for projections always have to start with named path of output. First segment was named '%s' or was named [%s]",
-                    path, segment.isNamed()));
-
-    }
+//    for (int i = 0; i < selections.length; i++) {
+//      PathSegment segment = selections[i].getRef().getRootSegment();
+//      CharSequence path = segment.getNameSegment().getPath();
+//      if (!segment.isNamed() || !path.equals("output"))
+//        throw new ExpressionParsingException(
+//            String
+//                .format(
+//                    "Outputs for projections always have to start with named path of output. First segment was named '%s' or was named [%s]",
+//                    path, segment.isNamed()));
+//
+//    }
   }
 
   @JsonProperty("projections")

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/java/org/apache/drill/common/util/TestTools.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/TestTools.java b/common/src/main/java/org/apache/drill/common/util/TestTools.java
index 0d3cc3e..6e0c241 100644
--- a/common/src/main/java/org/apache/drill/common/util/TestTools.java
+++ b/common/src/main/java/org/apache/drill/common/util/TestTools.java
@@ -21,23 +21,57 @@ import java.nio.file.Paths;
 
 import org.junit.rules.TestName;
 import org.junit.rules.TestRule;
+import org.junit.rules.TestWatcher;
 import org.junit.rules.Timeout;
+import org.junit.runner.Description;
+import org.slf4j.Logger;
 
 public class TestTools {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTools.class);
-  
-  static final boolean IS_DEBUG = java.lang.management.ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0;
+
+  static final boolean IS_DEBUG = java.lang.management.ManagementFactory.getRuntimeMXBean().getInputArguments()
+      .toString().indexOf("-agentlib:jdwp") > 0;
   static final String WORKING_PATH = Paths.get("").toAbsolutePath().toString();
-  
-  public static TestRule getTimeoutRule(){
+
+  public static TestRule getTimeoutRule() {
     return getTimeoutRule(10000);
   }
 
-  public static TestRule getTimeoutRule(int timeout){
+  public static TestRule getTimeoutRule(int timeout) {
     return IS_DEBUG ? new TestName() : new Timeout(timeout);
   }
 
-  public static String getWorkingPath(){
+  public static String getWorkingPath() {
     return WORKING_PATH;
   }
+
+  public static TestLogReporter getTestLogReporter(final Logger logger) {
+    return new TestLogReporter(logger);
+  }
+
+  public static class TestLogReporter extends TestWatcher{
+    private int failureCount = 0;
+
+    final Logger logger;
+
+    public TestLogReporter(Logger logger) {
+      super();
+      this.logger = logger;
+    }
+
+    @Override
+    protected void failed(Throwable e, Description description) {
+      logger.error("Test Failed: " + description.getDisplayName(), e);
+      failureCount++;
+    }
+
+    @Override
+    public void succeeded(Description description) {
+      logger.info("Test Succeeded: " + description.getDisplayName());
+    }
+
+    public void sleepIfFailure() throws InterruptedException{
+      if(failureCount > 0) Thread.sleep(2000);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java b/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
index f49b042..de090df 100644
--- a/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
+++ b/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
@@ -17,46 +17,98 @@
  */
 package org.apache.drill.common.expression.parser;
 
-import java.io.File;
+import java.io.IOException;
 
 import org.antlr.runtime.ANTLRStringStream;
 import org.antlr.runtime.CommonTokenStream;
-import org.antlr.runtime.tree.CommonTree;
-import org.antlr.runtime.tree.DOTTreeGenerator;
-import org.antlr.stringtemplate.StringTemplate;
+import org.antlr.runtime.RecognitionException;
+import org.apache.drill.common.expression.ExpressionStringBuilder;
 import org.apache.drill.common.expression.LogicalExpression;
 import org.apache.drill.common.expression.parser.ExprParser.parse_return;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.google.common.base.Charsets;
-import com.google.common.io.Files;
+import org.junit.Assert;
+import org.junit.Test;
 
 public class TreeTest {
-  public static void main(String[] args) throws Exception {
 
-    ExprLexer lexer = new ExprLexer(
-        new ANTLRStringStream(
-            "if ($F1) then case when (_MAP.R_NAME = 'AFRICA') then 2 else 4 end else if(4==3) then 1 else if(x==3) then 7 else (if(2==1) then 6 else 4 end) end"));
-    // ExprLexer lexer = new ExprLexer(new
-    // ANTLRStringStream("if ('blue.red') then 'orange' else if (false) then 1 else 0 end"));
-    // ExprLexer lexer = new ExprLexer(new ANTLRStringStream("2+2"));
 
+  @Test
+  public void testIfWithCase() throws Exception{
+    testExpressionParsing("if ($F1) then case when (_MAP.R_NAME = 'AFRICA') then 2 else 4 end else if(4==3) then 1 else if(x==3) then 7 else (if(2==1) then 6 else 4 end) end");
+  }
+
+  @Test
+  public void testAdd() throws Exception{
+    testExpressionParsing("2+2");
+  }
+
+  @Test
+  public void testIf() throws Exception{
+    testExpressionParsing("if ('blue.red') then 'orange' else if (false) then 1 else 0 end");
+  }
+
+  @Test
+  public void testQuotedIdentifier() throws Exception{
+    testExpressionParsing("`hello friend`.`goodbye`");
+  }
+
+  @Test
+  public void testSpecialQuoted() throws Exception{
+    testExpressionParsing("`*0` + `*` ");
+  }
+
+  @Test
+  public void testQuotedIdentifier2() throws Exception{
+    testExpressionParsing("`hello friend`.goodbye");
+  }
+
+  @Test
+  public void testComplexIdentifier() throws Exception{
+    testExpressionParsing("goodbye[4].`hello`");
+  }
+
+  private LogicalExpression parseExpression(String expr) throws RecognitionException, IOException{
+
+    ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
     CommonTokenStream tokens = new CommonTokenStream(lexer);
 
+//    tokens.fill();
+//    for(Token t : (List<Token>) tokens.getTokens()){
+//      System.out.println(t + "" + t.getType());
+//    }
+//    tokens.rewind();
+
     ExprParser parser = new ExprParser(tokens);
     parse_return ret = parser.parse();
-    LogicalExpression e = ret.e;
-    ObjectMapper mapper = new ObjectMapper();
-    mapper.enable(SerializationFeature.INDENT_OUTPUT);
-    System.out.println(mapper.writeValueAsString(e));
 
-    // print the tree
-    CommonTree tree = (CommonTree) ret.getTree();
-    DOTTreeGenerator gen = new DOTTreeGenerator();
-    StringTemplate st = gen.toDOT(tree);
+    return ret.e;
 
-    Files.write(st.toString(), new File("/Users/jnadeau/Documents/tree.dot"), Charsets.UTF_8);
+  }
+
+  private String serializeExpression(LogicalExpression expr){
 
+    ExpressionStringBuilder b = new ExpressionStringBuilder();
+    StringBuilder sb = new StringBuilder();
+    expr.accept(b, sb);
+    return sb.toString();
   }
+
+  /**
+   * Attempt to parse an expression.  Once parsed, convert it to a string and then parse it again to make sure serialization works.
+   * @param expr
+   * @throws RecognitionException
+   * @throws IOException
+   */
+  private void testExpressionParsing(String expr) throws RecognitionException, IOException{
+    System.out.println("-----" + expr + "-----");
+    LogicalExpression e = parseExpression(expr);
+
+    String newStringExpr = serializeExpression(e);
+    System.out.println(newStringExpr);
+    LogicalExpression e2 = parseExpression(newStringExpr);
+    //Assert.assertEquals(e, e2);
+
+  }
+
+
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRecordReader.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRecordReader.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRecordReader.java
index 0211de0..8e1e0ac 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRecordReader.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRecordReader.java
@@ -17,16 +17,17 @@
  */
 package org.apache.drill.exec.store.hbase;
 
-import com.google.common.base.Preconditions;
-import com.google.common.base.Stopwatch;
-import com.google.common.collect.Lists;
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hasher;
-import com.google.common.hash.Hashing;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
 import org.apache.drill.common.exceptions.DrillRuntimeException;
 import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.expression.ExpressionPosition;
 import org.apache.drill.common.expression.PathSegment;
+import org.apache.drill.common.expression.PathSegment.NameSegment;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.common.types.Types;
@@ -48,15 +49,19 @@ import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.util.Bytes;
 
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.TimeUnit;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Lists;
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
 
 public class HBaseRecordReader implements RecordReader {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HBaseRecordReader.class);
 
   private static final String ROW_KEY = "row_key";
   private static final int TARGET_RECORD_COUNT = 4000;
+  private static final SchemaPath ROW_KEY_PATH = SchemaPath.getSimplePath(ROW_KEY);
 
   private List<SchemaPath> columns;
   private OutputMutator outputMutator;
@@ -75,16 +80,16 @@ public class HBaseRecordReader implements RecordReader {
     this.context = context;
     if (columns != null && columns.size() != 0) {
       for (SchemaPath column : columns) {
-        if (column.getPath().toString().equalsIgnoreCase(ROW_KEY)) {
-          rowKeySchemaPath = new SchemaPath(ROW_KEY);
+        if (column.getRootSegment().getPath().toString().equalsIgnoreCase(ROW_KEY)) {
+          rowKeySchemaPath = ROW_KEY_PATH;
           continue;
         }
-        PathSegment root = column.getRootSegment();
+        NameSegment root = column.getRootSegment();
         assert root != null;
         PathSegment child = root.getChild();
-        byte[] family = root.getNameSegment().getPath().toString().getBytes();
+        byte[] family = root.getPath().toString().getBytes();
         if (child != null) {
-          Preconditions.checkArgument(child.getChild() == null, "Unsupported column name: " + column.getPath());
+          Preconditions.checkArgument(child.getChild() == null, "Unsupported column name: " + column.toString());
           byte[] qualifier = child.getNameSegment().getPath().toString().getBytes();
           scan.addColumn(family, qualifier);
         } else {
@@ -96,7 +101,7 @@ public class HBaseRecordReader implements RecordReader {
       if (this.columns == null) {
         this.columns = Lists.newArrayList();
       }
-      rowKeySchemaPath = new SchemaPath(ROW_KEY);
+      rowKeySchemaPath = ROW_KEY_PATH;
       this.columns.add(rowKeySchemaPath);
     }
 
@@ -127,7 +132,8 @@ public class HBaseRecordReader implements RecordReader {
           MaterializedField field = MaterializedField.create(column, Types.optional(TypeProtos.MinorType.VARBINARY));
           NullableVarBinaryVector v = new NullableVarBinaryVector(field, context.getAllocator());
           output.addField(v);
-          vvMap.put(new FamilyQualifierWrapper(column.getPath().toString()), v);
+          String fullyQualified = column.getRootSegment().getPath() + "." + column.getRootSegment().getChild().getNameSegment().getPath();
+          vvMap.put(new FamilyQualifierWrapper(fullyQualified), v);
         }
       } catch (SchemaChangeException e) {
         throw new ExecutionSetupException(e);
@@ -206,7 +212,7 @@ public class HBaseRecordReader implements RecordReader {
   }
 
   private NullableVarBinaryVector addNewVector(String column) {
-    MaterializedField field = MaterializedField.create(new SchemaPath(column, ExpressionPosition.UNKNOWN), Types.optional(TypeProtos.MinorType.VARBINARY));
+    MaterializedField field = MaterializedField.create(SchemaPath.getCompoundPath(column.split("\\.")), Types.optional(TypeProtos.MinorType.VARBINARY));
     NullableVarBinaryVector v = new NullableVarBinaryVector(field, context.getAllocator());
     VectorAllocator.getAllocator(v, 100).alloc(TARGET_RECORD_COUNT);
     vvMap.put(new FamilyQualifierWrapper(column), v);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
index 1376210..991685c 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
@@ -26,6 +26,7 @@ import net.hydromatic.optiq.SchemaPlus;
 
 import org.apache.drill.exec.planner.logical.DrillTable;
 import org.apache.drill.exec.planner.logical.DynamicDrillTable;
+import org.apache.drill.exec.rpc.user.DrillUser;
 import org.apache.drill.exec.store.AbstractSchema;
 import org.apache.drill.exec.store.SchemaFactory;
 import org.apache.hadoop.hbase.HTableDescriptor;
@@ -38,18 +39,17 @@ public class HBaseSchemaFactory implements SchemaFactory {
 
   final String schemaName;
   final HBaseStoragePlugin plugin;
-  
+
   public HBaseSchemaFactory(HBaseStoragePlugin plugin, String name) throws IOException {
     this.plugin = plugin;
     this.schemaName = name;
   }
 
   @Override
-  public Schema add(SchemaPlus parent) {
+  public void registerSchemas(DrillUser user, SchemaPlus parent) {
     HBaseSchema schema = new HBaseSchema(schemaName);
     SchemaPlus hPlus = parent.add(schemaName, schema);
     schema.setHolder(hPlus);
-    return schema;
   }
 
   class HBaseSchema extends AbstractSchema {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java
index 1fab650..a82c6c3 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java
@@ -19,11 +19,10 @@ package org.apache.drill.exec.store.hbase;
 
 import java.io.IOException;
 
-import net.hydromatic.optiq.Schema;
 import net.hydromatic.optiq.SchemaPlus;
 
 import org.apache.drill.common.JSONOptions;
-import org.apache.drill.common.logical.StoragePluginConfig;
+import org.apache.drill.exec.rpc.user.DrillUser;
 import org.apache.drill.exec.server.DrillbitContext;
 import org.apache.drill.exec.store.AbstractStoragePlugin;
 
@@ -63,8 +62,8 @@ public class HBaseStoragePlugin extends AbstractStoragePlugin {
   }
 
   @Override
-  public Schema createAndAddSchema(SchemaPlus parent) {
-    return schemaFactory.add(parent);
+  public void registerSchemas(DrillUser user, SchemaPlus parent) {
+    schemaFactory.registerSchemas(user, parent);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/distribution/src/resources/drill-env.sh
----------------------------------------------------------------------
diff --git a/distribution/src/resources/drill-env.sh b/distribution/src/resources/drill-env.sh
index 02a7022..d9fdb6f 100644
--- a/distribution/src/resources/drill-env.sh
+++ b/distribution/src/resources/drill-env.sh
@@ -13,4 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 DRILL_MAX_DIRECT_MEMORY="8G"
-export DRILL_JAVA_OPTS="-XX:MaxDirectMemorySize=$DRILL_MAX_DIRECT_MEMORY"
+DRILL_INIT_HEAP="1g"
+DRILL_MAX_HEAP="4g"
+export DRILL_JAVA_OPTS="-Xms$DRILL_INIT_HEAP -Xmx$DRILL_MAX_HEAP -XX:MaxDirectMemorySize=$DRILL_MAX_DIRECT_MEMORY"

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
index 1297cb3..73f2e29 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
@@ -59,7 +59,7 @@ import com.google.common.util.concurrent.SettableFuture;
  */
 public class DrillClient implements Closeable, ConnectionThrottle{
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillClient.class);
-  
+
   DrillConfig config;
   private UserClient client;
   private volatile ClusterCoordinator clusterCoordinator;
@@ -67,7 +67,8 @@ public class DrillClient implements Closeable, ConnectionThrottle{
   private final TopLevelAllocator allocator = new TopLevelAllocator(Long.MAX_VALUE);
   private int reconnectTimes;
   private int reconnectDelay;
-  
+  private final boolean ownsZkConnection;
+
   public DrillClient() {
     this(DrillConfig.create());
   }
@@ -79,25 +80,26 @@ public class DrillClient implements Closeable, ConnectionThrottle{
   public DrillClient(DrillConfig config) {
     this(config, null);
   }
-  
+
   public DrillClient(DrillConfig config, ClusterCoordinator coordinator){
+    this.ownsZkConnection = coordinator == null;
     this.config = config;
     this.clusterCoordinator = coordinator;
     this.reconnectTimes = config.getInt(ExecConstants.BIT_RETRY_TIMES);
     this.reconnectDelay = config.getInt(ExecConstants.BIT_RETRY_DELAY);
   }
-  
+
   public DrillConfig getConfig(){
     return config;
   }
-  
+
   @Override
   public void setAutoRead(boolean enableAutoRead) {
     client.setAutoRead(enableAutoRead);
   }
 
-  
-  
+
+
   /**
    * Connects the client to a Drillbit server
    *
@@ -106,11 +108,11 @@ public class DrillClient implements Closeable, ConnectionThrottle{
   public void connect() throws RpcException {
     connect((String) null);
   }
-    
+
   public synchronized void connect(String connect) throws RpcException {
     if (connected) return;
 
-    if (clusterCoordinator == null) {
+    if (ownsZkConnection) {
       try {
         this.clusterCoordinator = new ZKClusterCoordinator(this.config, connect);
         this.clusterCoordinator.start(10000);
@@ -171,6 +173,16 @@ public class DrillClient implements Closeable, ConnectionThrottle{
    */
   public void close(){
     this.client.close();
+    if(ownsZkConnection){
+      try {
+        this.clusterCoordinator.close();
+      } catch (IOException e) {
+        logger.warn("Error while closing Cluster Coordinator.", e);
+      }
+    }
+
+    // TODO: fix tests that fail when this is called.
+    //allocator.close();
     connected = false;
   }
 
@@ -187,12 +199,12 @@ public class DrillClient implements Closeable, ConnectionThrottle{
     client.submitQuery(listener,query);
     return listener.getResults();
   }
-  
+
   public void cancelQuery(QueryId id){
     client.send(RpcType.CANCEL_QUERY, id, Ack.class);
   }
-  
-  
+
+
   /**
    * Submits a Logical plan for direct execution (bypasses parsing)
    *
@@ -203,7 +215,7 @@ public class DrillClient implements Closeable, ConnectionThrottle{
   public void runQuery(QueryType type, String plan, UserResultsListener resultsListener){
     client.submitQuery(resultsListener, newBuilder().setResultsMode(STREAM_FULL).setType(type).setPlan(plan).build());
   }
-  
+
   private class ListHoldingResultsListener implements UserResultsListener {
     private Vector<QueryResultBatch> results = new Vector<QueryResultBatch>();
     private SettableFuture<List<QueryResultBatch>> future = SettableFuture.create();
@@ -245,7 +257,7 @@ public class DrillClient implements Closeable, ConnectionThrottle{
         future.set(results);
       }
     }
-  
+
     public List<QueryResultBatch> getResults() throws RpcException{
       try{
         return future.get();
@@ -258,7 +270,7 @@ public class DrillClient implements Closeable, ConnectionThrottle{
     public void queryIdArrived(QueryId queryId) {
     }
   }
-  
+
   private class FutureHandler extends AbstractCheckedFuture<Void, RpcException> implements RpcConnectionHandler<ServerConnection>, DrillRpcFuture<Void>{
 
     protected FutureHandler() {
@@ -278,7 +290,7 @@ public class DrillClient implements Closeable, ConnectionThrottle{
     private SettableFuture<Void> getInner(){
       return (SettableFuture<Void>) delegate();
     }
-    
+
     @Override
     protected RpcException mapException(Exception e) {
       return RpcException.mapException(e);
@@ -288,7 +300,7 @@ public class DrillClient implements Closeable, ConnectionThrottle{
     public ByteBuf getBuffer() {
       return null;
     }
-    
+
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/exec/java-exec/src/main/java/org/apache/drill/exec/client/DumpCat.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DumpCat.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DumpCat.java
index cfffed9..c69a43e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DumpCat.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DumpCat.java
@@ -68,7 +68,7 @@ public class DumpCat {
       System.out.println(String.format("Trace file %s not created", o.location));
       System.exit(-1);
     }
- 
+
     FileInputStream input = new FileInputStream(file.getAbsoluteFile());
 
     if (o.batch < 0) {
@@ -142,11 +142,11 @@ public class DumpCat {
       String avgRecSizeStr = null;
       if (this.rows>0)
         avgRecSizeStr = String.format("Average Record Size : %d ", this.dataSize/this.rows);
-      else 
+      else
         avgRecSizeStr = "Average Record Size : 0";
-      
+
       return String.format("Records : %d / %d \n", this.selectedRows, this.rows) +
-             avgRecSizeStr + 
+             avgRecSizeStr +
              String.format("\n Total Data Size : %d", this.dataSize);
     }
   }
@@ -188,7 +188,7 @@ public class DumpCat {
 
        prevSchema = vectorContainer.getSchema();
        batchNum ++;
- 
+
        vectorContainer.zeroVectors();
     }
 
@@ -253,7 +253,7 @@ public class DumpCat {
       for (VectorWrapper w : vectorContainer) {
         MaterializedField field = w.getValueVector().getField();
         System.out.println (String.format("name : %s, minor_type : %s, data_mode : %s",
-                                          field.getName(),
+                                          field.toExpr(),
                                           field.getType().getMinorType().toString(),
                                           field.isNullable() ? "nullable":"non-nullable"
                           ));

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java
new file mode 100644
index 0000000..f161782
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.client;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.drill.exec.client.QuerySubmitter.Format;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.TopLevelAllocator;
+import org.apache.drill.exec.proto.UserBitShared.QueryId;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.rpc.user.ConnectionThrottle;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.apache.drill.exec.rpc.user.UserResultsListener;
+import org.apache.drill.exec.util.VectorUtil;
+
+public class PrintingResultsListener implements UserResultsListener {
+  AtomicInteger count = new AtomicInteger();
+  private CountDownLatch latch = new CountDownLatch(1);
+  RecordBatchLoader loader;
+  Format format;
+  int    columnWidth;
+  BufferAllocator allocator = new TopLevelAllocator();
+  volatile Exception exception;
+
+  public PrintingResultsListener(Format format, int columnWidth) {
+    loader = new RecordBatchLoader(allocator);
+    this.format = format;
+    this.columnWidth = columnWidth;
+  }
+
+  @Override
+  public void submissionFailed(RpcException ex) {
+    exception = ex;
+    latch.countDown();
+  }
+
+  @Override
+  public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) {
+    int rows = result.getHeader().getRowCount();
+    if (result.getData() != null) {
+      count.addAndGet(rows);
+      try {
+        loader.load(result.getHeader().getDef(), result.getData());
+      } catch (SchemaChangeException e) {
+        submissionFailed(new RpcException(e));
+      }
+
+      switch(format) {
+        case TABLE:
+          VectorUtil.showVectorAccessibleContent(loader, columnWidth);
+          break;
+        case TSV:
+          VectorUtil.showVectorAccessibleContent(loader, "\t");
+          break;
+        case CSV:
+          VectorUtil.showVectorAccessibleContent(loader, ",");
+          break;
+      }
+
+    }
+
+    boolean isLastChunk = result.getHeader().getIsLastChunk();
+    result.release();
+
+    if (isLastChunk) {
+      allocator.close();
+      latch.countDown();
+    }
+
+  }
+
+  public int await() throws Exception {
+    latch.await();
+    if(exception != null) throw exception;
+    return count.get();
+  }
+
+  @Override
+  public void queryIdArrived(QueryId queryId) {
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
index 4551e53..0234b7e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
@@ -20,24 +20,11 @@ package org.apache.drill.exec.client;
 import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.nio.file.Paths;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Sets;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.exec.coord.ZKClusterCoordinator;
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.proto.UserBitShared.QueryId;
 import org.apache.drill.exec.proto.UserProtos;
-import org.apache.drill.exec.record.RecordBatchLoader;
-import org.apache.drill.exec.rpc.RpcException;
-import org.apache.drill.exec.rpc.user.ConnectionThrottle;
-import org.apache.drill.exec.rpc.user.QueryResultBatch;
-import org.apache.drill.exec.rpc.user.UserResultsListener;
-import org.apache.drill.exec.server.BootStrapContext;
 import org.apache.drill.exec.server.Drillbit;
 import org.apache.drill.exec.server.RemoteServiceSet;
 import org.apache.drill.exec.util.VectorUtil;
@@ -46,6 +33,7 @@ import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.ParameterException;
 import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
 import com.google.common.base.Stopwatch;
 
 public class QuerySubmitter {
@@ -115,10 +103,13 @@ public class QuerySubmitter {
     Preconditions.checkArgument(!(planLocation == null && queryString == null), "Must provide either query file or query string");
     Preconditions.checkArgument(!(planLocation != null && queryString != null), "Must provide either query file or query string, not both");
 
+    RemoteServiceSet serviceSet = null;
+    Drillbit[] drillbits = null;
+
     try{
       if (local) {
-        RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
-        Drillbit[] drillbits = new Drillbit[bits];
+        serviceSet = RemoteServiceSet.getLocalServiceSet();
+        drillbits = new Drillbit[bits];
         for (int i = 0; i < bits; i++) {
           drillbits[i] = new Drillbit(config, serviceSet);
           drillbits[i].run();
@@ -130,126 +121,81 @@ public class QuerySubmitter {
         client = new DrillClient(config, clusterCoordinator);
       }
       client.connect();
-      QueryResultsListener listener;
+
       String plan;
       if (queryString == null) {
         plan = Charsets.UTF_8.decode(ByteBuffer.wrap(Files.readAllBytes(Paths.get(planLocation)))).toString();
       } else {
         plan = queryString;
       }
-      String[] queries;
-      UserProtos.QueryType queryType;
-      type = type.toLowerCase();
-      switch(type) {
-        case "sql":
-          queryType = UserProtos.QueryType.SQL;
-          queries = plan.trim().split(";");
-          break;
-        case "logical":
-          queryType = UserProtos.QueryType.LOGICAL;
-          queries = new String[]{ plan };
-          break;
-        case "physical":
-          queryType = UserProtos.QueryType.PHYSICAL;
-          queries = new String[]{ plan };
-          break;
-        default:
-          System.out.println("Invalid query type: " + type);
-          return -1;
-      }
-      Format outputFormat;
-      format = format.toLowerCase();
-      switch(format) {
-        case "csv":
-          outputFormat = Format.CSV;
-          break;
-        case "tsv":
-          outputFormat = Format.TSV;
-          break;
-        case "table":
-          outputFormat = Format.TABLE;
-          break;
-        default:
-          System.out.println("Invalid format type: " + format);
-          return -1;
-      }
-      Stopwatch watch = new Stopwatch();
-      for (String query : queries) {
-        listener = new QueryResultsListener(outputFormat, width);
-        watch.start();
-        client.runQuery(queryType, query, listener);
-        int rows = listener.await();
-        System.out.println(String.format("%d record%s selected (%f seconds)", rows, rows > 1 ? "s" : "", (float) watch.elapsed(TimeUnit.MILLISECONDS) / (float) 1000));
-        if (query != queries[queries.length - 1]) {
-          System.out.println();
-        }
-        watch.stop();
-        watch.reset();
-      }
-      return 0;
+      return submitQuery(client, plan, type, format, width);
+
     }finally{
       if(client != null) client.close();
+      if(local){
+        for(Drillbit b : drillbits){
+          b.close();
+        }
+        serviceSet.close();
+      }
     }
   }
 
-  private class QueryResultsListener implements UserResultsListener {
-    AtomicInteger count = new AtomicInteger();
-    private CountDownLatch latch = new CountDownLatch(1);
-    RecordBatchLoader loader = new RecordBatchLoader(new BootStrapContext(DrillConfig.create()).getAllocator());
-    Format format;
-    int    columnWidth;
-    volatile Exception exception;
-
-    public QueryResultsListener(Format format, int columnWidth) {
-      this.format = format;
-      this.columnWidth = columnWidth;
+  public int submitQuery(DrillClient client, String plan, String type, String format, int width) throws Exception {
+
+    PrintingResultsListener listener;
+
+    String[] queries;
+    UserProtos.QueryType queryType;
+    type = type.toLowerCase();
+    switch(type) {
+      case "sql":
+        queryType = UserProtos.QueryType.SQL;
+        queries = plan.trim().split(";");
+        break;
+      case "logical":
+        queryType = UserProtos.QueryType.LOGICAL;
+        queries = new String[]{ plan };
+        break;
+      case "physical":
+        queryType = UserProtos.QueryType.PHYSICAL;
+        queries = new String[]{ plan };
+        break;
+      default:
+        System.out.println("Invalid query type: " + type);
+        return -1;
     }
 
-    @Override
-    public void submissionFailed(RpcException ex) {
-      exception = ex;
-      latch.countDown();
+    Format outputFormat;
+    format = format.toLowerCase();
+    switch(format) {
+      case "csv":
+        outputFormat = Format.CSV;
+        break;
+      case "tsv":
+        outputFormat = Format.TSV;
+        break;
+      case "table":
+        outputFormat = Format.TABLE;
+        break;
+      default:
+        System.out.println("Invalid format type: " + format);
+        return -1;
     }
-
-    @Override
-    public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) {
-      int rows = result.getHeader().getRowCount();
-      if (result.getData() != null) {
-        count.addAndGet(rows);
-        try {
-          loader.load(result.getHeader().getDef(), result.getData());
-        } catch (SchemaChangeException e) {
-          submissionFailed(new RpcException(e));
-        }
-
-        switch(format) {
-          case TABLE:
-            VectorUtil.showVectorAccessibleContent(loader, columnWidth);
-            break;
-          case TSV:
-            VectorUtil.showVectorAccessibleContent(loader, "\t");
-            break;
-          case CSV:
-            VectorUtil.showVectorAccessibleContent(loader, ",");
-            break;
-        }
-        
+    Stopwatch watch = new Stopwatch();
+    for (String query : queries) {
+      listener = new PrintingResultsListener(outputFormat, width);
+      watch.start();
+      client.runQuery(queryType, query, listener);
+      int rows = listener.await();
+      System.out.println(String.format("%d record%s selected (%f seconds)", rows, rows > 1 ? "s" : "", (float) watch.elapsed(TimeUnit.MILLISECONDS) / (float) 1000));
+      if (query != queries[queries.length - 1]) {
+        System.out.println();
       }
-      
-      if (result.getHeader().getIsLastChunk()) {
-        latch.countDown();
-      }
-      result.release();
-    }
-
-    public int await() throws Exception {
-      latch.await();
-      if(exception != null) throw exception;
-      return count.get();
+      watch.stop();
+      watch.reset();
     }
+    return 0;
 
-    @Override
-    public void queryIdArrived(QueryId queryId) {
-    }
   }
 }