You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2014/04/20 03:18:54 UTC
[46/51] [abbrv] git commit: Move to Optiq 0.6 Also includes: -improve
exception catching -move schema path parsing to Antlr -close zookeeper
connection on if client created -enhance BaseTestQuery and have other query
tests utilize it -Various test fixes
Move to Optiq 0.6
Also includes:
-improve exception catching
-move schema path parsing to Antlr
-close zookeeper connection on if client created
-enhance BaseTestQuery and have other query tests utilize it
-Various test fixes for better memory release. still needs client allocator to be closed.
-refactor DrillSqlWorker and create multiple SqlHandlers
-Add PojoRecordReader and DirectPlan capabilities
-Update Antlr to use same quoting rules as SQL: single quote for quoted strings, back ticks for identifiers
-Move back to old Sorts until bugs are fixed
-Refector SelectionVector management within Prels
-Add support for NO_EXCHANGES option
-Extract SchemaFactories to use Optiq's new Schema handling capabilities
-Add basic handling of cancel in UserServer
-Remove output requirement from Project
-Add start of usercredentials to User communication
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ecaa838f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ecaa838f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ecaa838f
Branch: refs/heads/master
Commit: ecaa838fe7194c197884f0a1b6fb6f8a9683cede
Parents: ba0a639
Author: Jacques Nadeau <ja...@apache.org>
Authored: Mon Apr 14 07:37:02 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sat Apr 19 18:07:12 2014 -0700
----------------------------------------------------------------------
.../drill/common/expression/parser/ExprLexer.g | 25 +-
.../drill/common/expression/parser/ExprParser.g | 20 +-
.../apache/drill/common/config/DrillConfig.java | 48 +-
.../expression/ExpressionStringBuilder.java | 64 ++-
.../drill/common/expression/FieldReference.java | 31 +-
.../common/expression/LogicalExpression.java | 2 +
.../drill/common/expression/PathSegment.java | 110 ++--
.../drill/common/expression/SchemaPath.java | 179 ++++---
.../common/expression/ValueExpressions.java | 56 +-
.../drill/common/logical/data/Project.java | 22 +-
.../org/apache/drill/common/util/TestTools.java | 46 +-
.../common/expression/parser/TreeTest.java | 102 +++-
.../exec/store/hbase/HBaseRecordReader.java | 40 +-
.../exec/store/hbase/HBaseSchemaFactory.java | 6 +-
.../exec/store/hbase/HBaseStoragePlugin.java | 7 +-
distribution/src/resources/drill-env.sh | 4 +-
.../apache/drill/exec/client/DrillClient.java | 46 +-
.../org/apache/drill/exec/client/DumpCat.java | 12 +-
.../exec/client/PrintingResultsListener.java | 100 ++++
.../drill/exec/client/QuerySubmitter.java | 184 +++----
.../apache/drill/exec/compile/MergeAdapter.java | 42 +-
.../org/apache/drill/exec/memory/Accountor.java | 17 +-
.../drill/exec/memory/AtomicRemainder.java | 15 +-
.../apache/drill/exec/ops/FragmentContext.java | 15 +-
.../org/apache/drill/exec/ops/Multitimer.java | 42 ++
.../org/apache/drill/exec/ops/QueryContext.java | 59 ++-
.../org/apache/drill/exec/ops/QuerySetup.java | 26 +
.../drill/exec/physical/impl/OutputMutator.java | 4 +
.../drill/exec/physical/impl/ScanBatch.java | 12 +-
.../impl/TopN/PriorityQueueTemplate.java | 10 +-
.../physical/impl/common/HashTableTemplate.java | 125 +++--
.../OrderedPartitionRecordBatch.java | 42 +-
.../partitionsender/OutgoingRecordBatch.java | 14 +-
.../impl/project/ProjectRecordBatch.java | 53 +-
.../exec/physical/impl/sort/SortBatch.java | 40 +-
.../impl/sort/SortRecordBatchBuilder.java | 34 +-
.../IteratorValidatorBatchIterator.java | 35 +-
.../physical/impl/xsort/ExternalSortBatch.java | 12 +-
.../planner/common/DrillProjectRelBase.java | 4 +-
.../exec/planner/logical/DrillProjectRel.java | 4 +-
.../exec/planner/logical/DrillRuleSets.java | 27 +-
.../drill/exec/planner/logical/ExprHelper.java | 8 +-
.../physical/DrillDistributionTrait.java | 35 +-
.../drill/exec/planner/physical/FilterPrel.java | 15 +-
.../physical/HashToRandomExchangePrel.java | 28 +-
.../drill/exec/planner/physical/LimitPrel.java | 14 +-
.../exec/planner/physical/MergeJoinPrel.java | 57 +-
.../planner/physical/PhysicalPlanCreator.java | 24 +-
.../exec/planner/physical/PlanningSettings.java | 57 ++
.../drill/exec/planner/physical/PrelUtil.java | 39 +-
.../exec/planner/physical/ProjectPrel.java | 18 +-
.../drill/exec/planner/physical/ScanPrel.java | 11 +-
.../drill/exec/planner/physical/ScreenPrel.java | 18 +-
.../physical/SingleMergeExchangePrel.java | 30 +-
.../drill/exec/planner/physical/SortPrel.java | 26 +-
.../exec/planner/physical/StreamAggPrel.java | 19 +-
.../drill/exec/planner/physical/TopNPrel.java | 16 +-
.../planner/physical/UnionExchangePrel.java | 20 +-
.../drill/exec/planner/sql/DirectPlan.java | 63 +++
.../exec/planner/sql/DrillOperatorTable.java | 26 +-
.../exec/planner/sql/DrillParserFactory.java | 82 ---
.../drill/exec/planner/sql/DrillSqlWorker.java | 213 ++------
.../planner/sql/handlers/DefaultSqlHandler.java | 173 ++++++
.../planner/sql/handlers/ExplainHandler.java | 116 ++++
.../planner/sql/handlers/SetOptionHandler.java | 58 ++
.../sql/handlers/SimpleCommandResult.java | 32 ++
.../exec/planner/sql/handlers/SqlHandler.java | 32 ++
.../exec/planner/types/RelDataTypeHolder.java | 17 +-
.../drill/exec/record/MaterializedField.java | 80 ++-
.../drill/exec/record/RecordBatchLoader.java | 15 +-
.../drill/exec/record/VectorContainer.java | 11 +-
.../apache/drill/exec/rpc/user/DrillUser.java | 36 ++
.../apache/drill/exec/rpc/user/UserServer.java | 29 +-
.../apache/drill/exec/rpc/user/UserSession.java | 80 +++
.../org/apache/drill/exec/schema/Field.java | 5 +-
.../drill/exec/server/DrillbitContext.java | 30 +-
.../apache/drill/exec/store/SchemaFactory.java | 7 +-
.../apache/drill/exec/store/StoragePlugin.java | 9 +-
.../drill/exec/store/StoragePluginRegistry.java | 162 +-----
.../drill/exec/store/dfs/FileSystemPlugin.java | 24 +-
.../exec/store/dfs/FileSystemSchemaFactory.java | 16 +-
.../exec/store/direct/DirectBatchCreator.java | 37 ++
.../exec/store/direct/DirectGroupScan.java | 81 +++
.../drill/exec/store/direct/DirectSubScan.java | 37 ++
.../exec/store/easy/json/JSONRecordReader.java | 7 +-
.../drill/exec/store/hive/HiveRecordReader.java | 6 +-
.../exec/store/hive/HiveStoragePlugin.java | 9 +-
.../store/hive/schema/HiveSchemaFactory.java | 24 +-
.../exec/store/ischema/EmptyVectorSet.java | 72 +--
.../store/ischema/InfoSchemaStoragePlugin.java | 21 +-
.../drill/exec/store/ischema/OptiqProvider.java | 24 +-
.../drill/exec/store/mock/MockRecordReader.java | 12 +-
.../exec/store/mock/MockStorageEngine.java | 9 +-
.../exec/store/parquet/ParquetRecordReader.java | 6 +-
.../drill/exec/store/pojo/AbstractWriter.java | 63 +++
.../drill/exec/store/pojo/PojoRecordReader.java | 131 +++++
.../drill/exec/store/pojo/PojoWriter.java | 29 +
.../apache/drill/exec/store/pojo/Writers.java | 127 +++++
.../apache/drill/exec/util/BatchPrinter.java | 6 +-
.../org/apache/drill/exec/util/VectorUtil.java | 8 +-
.../apache/drill/exec/work/foreman/Foreman.java | 158 ++----
.../apache/drill/exec/work/user/UserWorker.java | 13 +-
.../java/org/apache/drill/BaseTestQuery.java | 152 ++++--
.../java/org/apache/drill/PlanningBase.java | 114 ++++
.../org/apache/drill/TestAltSortQueries.java | 48 +-
.../org/apache/drill/TestExampleQueries.java | 52 +-
.../org/apache/drill/TestTpchDistributed.java | 154 ++++++
.../java/org/apache/drill/TestTpchExplain.java | 155 ++++++
.../java/org/apache/drill/TestTpchPlanning.java | 150 ++++++
.../java/org/apache/drill/TestTpchQueries.java | 158 ------
.../org/apache/drill/TestTpchSingleMode.java | 157 ++++++
.../drill/exec/cache/TestVectorCache.java | 2 +-
.../exec/physical/impl/SimpleRootExec.java | 15 +-
.../physical/impl/TestBroadcastExchange.java | 8 +-
.../exec/physical/impl/TestCastFunctions.java | 151 +++---
.../physical/impl/TestCastVarCharToBigInt.java | 5 +
.../impl/TestDistributedFragmentRun.java | 10 +-
.../physical/impl/TestExtractFunctions.java | 5 +
.../physical/impl/TestHashToRandomExchange.java | 1 +
.../exec/physical/impl/TestOptiqPlans.java | 13 +-
.../physical/impl/TestSimpleFragmentRun.java | 13 +-
.../exec/physical/impl/TestUnionExchange.java | 1 +
.../exec/physical/impl/TopN/TestSimpleTopN.java | 10 +-
.../drill/exec/physical/impl/agg/TestAgg.java | 34 +-
.../exec/physical/impl/join/TestMergeJoin.java | 20 +-
.../impl/join/TestMergeJoinMulCondition.java | 10 +-
.../impl/mergereceiver/TestMergingReceiver.java | 14 +-
.../TestOrderedPartitionExchange.java | 3 +
.../impl/xsort/TestSimpleExternalSort.java | 8 +-
.../drill/exec/record/vector/TestDateTypes.java | 30 ++
.../drill/exec/record/vector/TestLoad.java | 4 +-
.../drill/exec/store/TestOrphanSchema.java | 22 +-
.../drill/exec/store/TestOutputMutator.java | 15 +
.../drill/exec/store/ischema/OrphanSchema.java | 33 +-
.../exec/store/ischema/TestOrphanSchema.java | 49 +-
.../exec/store/ischema/TestTableProvider.java | 57 +-
.../exec/store/json/JSONRecordReaderTest.java | 17 +-
.../store/parquet/ParquetRecordReaderTest.java | 131 ++---
.../store/parquet/ParquetResultListener.java | 30 +-
.../store/parquet/TestParquetPhysicalPlan.java | 2 +-
.../exec/work/batch/TestSpoolingBuffer.java | 1 +
.../src/test/resources/agg/hashagg/q6.json | 4 +-
.../src/test/resources/agg/hashagg/q7_1.json | 6 +-
.../src/test/resources/agg/hashagg/q7_2.json | 6 +-
.../src/test/resources/agg/hashagg/q8.json | 4 +-
.../src/test/resources/agg/hashagg/q8_1.json | 4 +-
.../functions/cast/testCastBigInt.json | 2 +-
.../functions/cast/testCastFloat4.json | 4 +-
.../functions/cast/testCastFloat8.json | 4 +-
.../resources/functions/cast/testCastInt.json | 2 +-
.../functions/cast/testCastNested.json | 2 +-
.../functions/cast/testCastNumException.json | 2 +-
.../functions/cast/testICastConstant.json | 36 +-
.../functions/simple_math_functions.json | 8 +-
.../functions/string/testCharLength.json | 12 +-
.../resources/functions/string/testConcat.json | 8 +-
.../resources/functions/string/testLeft.json | 14 +-
.../resources/functions/string/testLike.json | 8 +-
.../resources/functions/string/testLower.json | 6 +-
.../resources/functions/string/testLpad.json | 20 +-
.../resources/functions/string/testLtrim.json | 12 +-
.../functions/string/testPosition.json | 16 +-
.../functions/string/testRegexpReplace.json | 6 +-
.../resources/functions/string/testReplace.json | 12 +-
.../resources/functions/string/testRight.json | 14 +-
.../resources/functions/string/testRpad.json | 20 +-
.../resources/functions/string/testRtrim.json | 12 +-
.../resources/functions/string/testSimilar.json | 8 +-
.../resources/functions/string/testUpper.json | 6 +-
.../test/resources/join/mj_multi_condition.json | 12 +-
.../test/resources/logical_string_filter.json | 2 +-
.../src/test/resources/simple_plan.json | 12 +-
pom.xml | 8 +-
.../org/apache/drill/exec/proto/BitControl.java | 218 +++++++-
.../apache/drill/exec/proto/UserBitShared.java | 531 ++++++++++++++++++-
.../org/apache/drill/exec/proto/UserProtos.java | 258 +++++++--
protocol/src/main/protobuf/BitControl.proto | 1 +
protocol/src/main/protobuf/User.proto | 2 +
protocol/src/main/protobuf/UserBitShared.proto | 6 +-
sqlparser/pom.xml | 8 +
.../drill/jdbc/DrillColumnMetaDataList.java | 14 +-
.../apache/drill/jdbc/test/TestJdbcQuery.java | 28 +-
182 files changed, 5178 insertions(+), 2401 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
----------------------------------------------------------------------
diff --git a/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g b/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
index 78c356a..700bd72 100644
--- a/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
+++ b/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprLexer.g
@@ -54,7 +54,7 @@ INTERVAL : 'interval' | 'INTERVAL';
INTERVALYEAR : 'intervalyear' | 'INTERVALYEAR';
INTERVALDAY : 'intervalday' | 'INTERVALDAY';
-
+Period : '.';
Or : '||' | 'or' | 'OR' | 'Or';
And : '&&' | 'and' | 'AND' ;
Equals : '==' | '=';
@@ -100,15 +100,21 @@ Number
// ;
Identifier
- : ('a'..'z' | 'A'..'Z' | '_' | '$') ('a'..'z' | 'A'..'Z' | '_' | '$' | Digit)* ('.' ('a'..'z' | 'A'..'Z' | '_' | '$' ) ('a'..'z' | 'A'..'Z' | '_' | '$' | Digit)*)*
+ : ('a'..'z' | 'A'..'Z' | '_' | '$') ('a'..'z' | 'A'..'Z' | '_' | '$' | Digit)*
+ ;
+
+QuotedIdentifier
+@after {
+ setText(getText().substring(1, getText().length()-1).replaceAll("\\\\(.)", "$1"));
+}
+ : '`' (~('`' | '\\') | '\\' ('\\' | '`'))* '`'
;
String
@after {
setText(getText().substring(1, getText().length()-1).replaceAll("\\\\(.)", "$1"));
}
- : '"' (~('"' | '\\') | '\\' ('\\' | '"'))* '"'
- | '\'' (~('\'' | '\\') | '\\' ('\\' | '\''))* '\''
+ : '\'' (~('\'' | '\\') | '\\' ('\\' | '\''))* '\''
;
Comment
@@ -128,3 +134,14 @@ fragment Int
fragment Digit
: '0'..'9'
;
+
+FallThrough
+ @after{
+ throw new RuntimeException(java.lang.String.format(
+ "Encountered an illegal char on line \%d, column \%d: '\%s'",
+ getLine(), getCharPositionInLine(), getText()
+ )
+ );
+ }
+ :
+ ;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
----------------------------------------------------------------------
diff --git a/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g b/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
index 2e2f3d3..e4eb149 100644
--- a/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
+++ b/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
@@ -35,6 +35,8 @@ package org.apache.drill.common.expression.parser;
import org.antlr.runtime.BitSet;
import java.util.*;
import org.apache.drill.common.expression.*;
+import org.apache.drill.common.expression.PathSegment.NameSegment;
+import org.apache.drill.common.expression.PathSegment.ArraySegment;
import org.apache.drill.common.types.*;
import org.apache.drill.common.types.TypeProtos.*;
import org.apache.drill.common.exceptions.ExpressionParsingException;
@@ -267,12 +269,28 @@ atom returns [LogicalExpression e]
| lookup {$e = $lookup.e; }
;
+pathSegment returns [NameSegment seg]
+ : s1=nameSegment {$seg = $s1.seg;}
+ ;
+
+nameSegment returns [NameSegment seg]
+ : QuotedIdentifier ( (Period s1=pathSegment) | s2=arraySegment)? {$seg = new NameSegment($QuotedIdentifier.text, ($s1.seg == null ? $s2.seg : $s1.seg) ); }
+ | Identifier ( (Period s1=pathSegment) | s2=arraySegment)? {$seg = new NameSegment($Identifier.text, ($s1.seg == null ? $s2.seg : $s1.seg) ); }
+ ;
+
+arraySegment returns [PathSegment seg]
+ : OBracket Number CBracket ( (Period s1=pathSegment) | s2=arraySegment)? {$seg = new ArraySegment($Number.text, ($s1.seg == null ? $s2.seg : $s1.seg) ); }
+ ;
+
lookup returns [LogicalExpression e]
: functionCall {$e = $functionCall.e ;}
| castCall {$e = $castCall.e; }
- | Identifier {$e = new SchemaPath($Identifier.text, pos($Identifier) ); }
+ | pathSegment {$e = new SchemaPath($pathSegment.seg, pos($pathSegment.start) ); }
| String {$e = new ValueExpressions.QuotedString($String.text, pos($String) ); }
| OParen expression CParen {$e = $expression.e; }
| SingleQuote Identifier SingleQuote {$e = new SchemaPath($Identifier.text, pos($Identifier) ); }
;
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
index 6fc048d..7fb0e39 100644
--- a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
+++ b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
@@ -27,6 +27,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
import org.apache.drill.common.exceptions.DrillConfigurationException;
import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.FormatPluginConfigBase;
import org.apache.drill.common.logical.StoragePluginConfigBase;
import org.apache.drill.common.logical.data.LogicalOperatorBase;
@@ -48,11 +49,11 @@ public final class DrillConfig extends NestedConfig{
private final ObjectMapper mapper;
private final ImmutableList<String> startupArguments;
@SuppressWarnings("restriction") private static final long MAX_DIRECT_MEMORY = sun.misc.VM.maxDirectMemory();
-
+
@SuppressWarnings("unchecked")
private volatile List<Queue<Object>> sinkQueues = new CopyOnWriteArrayList<Queue<Object>>(new Queue[1]);
-
+
@SuppressWarnings("restriction")
@VisibleForTesting
public DrillConfig(Config config) {
@@ -60,9 +61,10 @@ public final class DrillConfig extends NestedConfig{
mapper = new ObjectMapper();
SimpleModule deserModule = new SimpleModule("LogicalExpressionDeserializationModule")
- .addDeserializer(LogicalExpression.class, new LogicalExpression.De(this));
-
-
+ .addDeserializer(LogicalExpression.class, new LogicalExpression.De(this))
+ .addDeserializer(SchemaPath.class, new SchemaPath.De(this));
+
+
mapper.registerModule(deserModule);
mapper.enable(SerializationFeature.INDENT_OUTPUT);
mapper.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
@@ -71,10 +73,10 @@ public final class DrillConfig extends NestedConfig{
mapper.registerSubtypes(LogicalOperatorBase.getSubTypes(this));
mapper.registerSubtypes(StoragePluginConfigBase.getSubTypes(this));
mapper.registerSubtypes(FormatPluginConfigBase.getSubTypes(this));
-
+
RuntimeMXBean bean = ManagementFactory.getRuntimeMXBean();
this.startupArguments = ImmutableList.copyOf(bean.getInputArguments());
-
+
};
@@ -82,15 +84,15 @@ public final class DrillConfig extends NestedConfig{
public List<String> getStartupArguments(){
return startupArguments;
}
-
+
/**
- * Create a DrillConfig object using the default config file name
+ * Create a DrillConfig object using the default config file name
* @return The new DrillConfig object.
*/
public static DrillConfig create() {
return create(null);
}
-
+
/**
* <p>
* DrillConfig loads up Drill configuration information. It does this utilizing a combination of classpath scanning
@@ -106,15 +108,15 @@ public final class DrillConfig extends NestedConfig{
* <li>Check a single copy of "drill-default.conf". If multiple copies are on the classpath, behavior is
* indeterminate.</li>
* </ul>
- *
+ *
* </p>
* @param overrideFileName The name of the file to use for override purposes.
* @return A merged Config object.
*/
public static DrillConfig create(String overrideFileName) {
-
+
overrideFileName = overrideFileName == null ? CommonConstants.CONFIG_OVERRIDE : overrideFileName;
-
+
// first we load defaults.
Config fallback = ConfigFactory.load(CommonConstants.CONFIG_DEFAULT);
Collection<URL> urls = PathScanner.getConfigURLs();
@@ -126,7 +128,7 @@ public final class DrillConfig extends NestedConfig{
Config c = ConfigFactory.load(overrideFileName).withFallback(fallback).resolve();
return new DrillConfig(c);
}
-
+
public <T> Class<T> getClassAt(String location, Class<T> clazz) throws DrillConfigurationException{
String className = this.getString(location);
if(className == null) throw new DrillConfigurationException(String.format("No class defined at location '%s'. Expected a definition of the class []", location, clazz.getCanonicalName()));
@@ -142,9 +144,9 @@ public final class DrillConfig extends NestedConfig{
if(ex instanceof DrillConfigurationException) throw (DrillConfigurationException) ex;
throw new DrillConfigurationException(String.format("Failure while initializing class [%s] described at configuration value '%s'.", className, location), ex);
}
-
+
}
-
+
public <T> T getInstanceOf(String location, Class<T> clazz) throws DrillConfigurationException{
Class<T> c = getClassAt(location, clazz);
try{
@@ -154,18 +156,18 @@ public final class DrillConfig extends NestedConfig{
throw new DrillConfigurationException(String.format("Failure while instantiating class [%s] located at '%s.", clazz.getCanonicalName(), location), ex);
}
}
-
-
+
+
public void setSinkQueues(int number, Queue<Object> queue){
sinkQueues.set(number, queue);
}
-
+
public Queue<Object> getQueue(int number){
if(sinkQueues.size() <= number || number < 0 || sinkQueues == null) throw new IllegalArgumentException(String.format("Queue %d is not available.", number));
return sinkQueues.get(number);
}
-
+
public ObjectMapper getMapper(){
return mapper;
}
@@ -174,13 +176,13 @@ public final class DrillConfig extends NestedConfig{
public String toString(){
return this.root().render();
}
-
+
public static void main(String[] args) throws Exception{
//"-XX:MaxDirectMemorySize"
DrillConfig config = DrillConfig.create();
-
+
}
-
+
public static long getMaxDirectMemory(){
return MAX_DIRECT_MEMORY;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java b/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
index 787e4af..89fa437 100644
--- a/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
+++ b/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
@@ -44,6 +44,19 @@ import com.google.common.collect.ImmutableList;
public class ExpressionStringBuilder extends AbstractExprVisitor<Void, StringBuilder, RuntimeException>{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpressionStringBuilder.class);
+ static final ExpressionStringBuilder INSTANCE = new ExpressionStringBuilder();
+
+ public static String toString(LogicalExpression expr){
+ StringBuilder sb = new StringBuilder();
+ expr.accept(INSTANCE, sb);
+ return sb.toString();
+ }
+
+ public static void toString(LogicalExpression expr, StringBuilder sb){
+ expr.accept(INSTANCE, sb);
+ }
+
+
@Override
public Void visitFunctionCall(FunctionCall call, StringBuilder sb) throws RuntimeException {
ImmutableList<LogicalExpression> args = call.args;
@@ -85,7 +98,26 @@ public class ExpressionStringBuilder extends AbstractExprVisitor<Void, StringBui
@Override
public Void visitSchemaPath(SchemaPath path, StringBuilder sb) throws RuntimeException {
- sb.append(path.getPath());
+ PathSegment seg = path.getRootSegment();
+ if(seg.isArray()) throw new IllegalStateException("Drill doesn't currently support top level arrays");
+ sb.append('`');
+ sb.append(seg.getNameSegment().getPath());
+ sb.append('`');
+
+ while( (seg = seg.getChild()) != null){
+ if(seg.isNamed()){
+ sb.append('.');
+ sb.append('`');
+ sb.append(seg.getNameSegment().getPath());
+ sb.append('`');
+ }else{
+ sb.append('[');
+ sb.append(seg.getArraySegment().getIndex());
+ sb.append(']');
+ }
+ }
+
+
return null;
}
@@ -121,33 +153,33 @@ public class ExpressionStringBuilder extends AbstractExprVisitor<Void, StringBui
TIMESTAMP_FORMAT = new DateTimeFormatterBuilder().append(dateFormatter).appendOptional(optionalTime).appendOptional(optionalSec).toFormatter();
}
-
+
@Override
public Void visitTimeStampConstant(TimeStampExpression lExpr, StringBuilder sb) throws RuntimeException {
- sb.append("cast( \"");
+ sb.append("cast( '");
try {
TIMESTAMP_FORMAT.printTo(sb, lExpr.getTimeStamp());
} catch (IOException e) {
throw new RuntimeException(e);
}
sb.append(lExpr.getTimeStamp());
- sb.append(" \" as TIMESTAMP)");
+ sb.append(" ' as TIMESTAMP)");
return null;
}
@Override
public Void visitIntervalYearConstant(IntervalYearExpression lExpr, StringBuilder sb) throws RuntimeException {
- sb.append("cast( \"");
+ sb.append("cast( '");
sb.append(Period.years(lExpr.getIntervalYear()).toString());
- sb.append("\" as INTERVALYEAR)");
+ sb.append("' as INTERVALYEAR)");
return null;
}
@Override
public Void visitIntervalDayConstant(IntervalDayExpression lExpr, StringBuilder sb) throws RuntimeException {
- sb.append("cast( \"");
+ sb.append("cast( '");
sb.append(Period.days(lExpr.getIntervalDay()).plusMillis(lExpr.getIntervalMillis()).toString());
- sb.append("\" as INTERVALDAY)");
+ sb.append("' as INTERVALDAY)");
return null;
}
@@ -165,21 +197,21 @@ public class ExpressionStringBuilder extends AbstractExprVisitor<Void, StringBui
@Override
public Void visitQuotedStringConstant(QuotedString e, StringBuilder sb) throws RuntimeException {
- sb.append("\"");
+ sb.append("'");
sb.append(e.value);
- sb.append("\"");
+ sb.append("'");
return null;
}
@Override
public Void visitCastExpression(CastExpression e, StringBuilder sb) throws RuntimeException {
MajorType mt = e.getMajorType();
-
+
sb.append("cast( (");
e.getInput().accept(this, sb);
sb.append(" ) as ");
sb.append(mt.getMinorType().name());
-
+
switch(mt.getMinorType()){
case FLOAT4:
case FLOAT8:
@@ -228,8 +260,8 @@ public class ExpressionStringBuilder extends AbstractExprVisitor<Void, StringBui
sb.append(intExpr.getInt());
return null;
}
-
-
-
-
+
+
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/FieldReference.java b/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
index 358a865..bcc6b52 100644
--- a/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
+++ b/common/src/main/java/org/apache/drill/common/expression/FieldReference.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.drill.common.expression.FieldReference.De;
import org.apache.drill.common.expression.FieldReference.Se;
+import org.apache.drill.common.expression.PathSegment.NameSegment;
import org.apache.drill.common.types.TypeProtos.MajorType;
import com.fasterxml.jackson.core.JsonGenerationException;
@@ -41,18 +42,32 @@ public class FieldReference extends SchemaPath {
public FieldReference(SchemaPath sp){
super(sp);
+ checkData();
}
-
+
+ private void checkData(){
+ if(getRootSegment().getChild() != null) throw new UnsupportedOperationException("Field references must be singular names.");
+
+ }
+
+
+ private void checkSimpleString(CharSequence value){
+ if(value.toString().contains(".")) throw new UnsupportedOperationException("Field references must be singular names.");
+ }
+
public FieldReference(CharSequence value){
- super(value, ExpressionPosition.UNKNOWN);
+ this(value, ExpressionPosition.UNKNOWN);
+ checkSimpleString(value);
}
-
+
public FieldReference(CharSequence value, ExpressionPosition pos) {
- super(value, pos);
+ super(new NameSegment(value), pos);
+ checkData();
+ checkSimpleString(value);
}
public FieldReference(String value, ExpressionPosition pos, MajorType dataType) {
- super(value, pos);
+ this(value, pos);
this.overrideType = dataType;
}
@@ -74,7 +89,9 @@ public class FieldReference extends SchemaPath {
@Override
public FieldReference deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException,
JsonProcessingException {
- return new FieldReference(this._parseString(jp, ctxt), ExpressionPosition.UNKNOWN);
+ String ref = this._parseString(jp, ctxt);
+ ref = ref.replace("`", "");
+ return new FieldReference(ref, ExpressionPosition.UNKNOWN);
}
}
@@ -88,7 +105,7 @@ public class FieldReference extends SchemaPath {
@Override
public void serialize(FieldReference value, JsonGenerator jgen, SerializerProvider provider) throws IOException,
JsonGenerationException {
- jgen.writeString(value.getPath().toString());
+ jgen.writeString('`' + value.getRootSegment().getNameSegment().getPath() + '`');
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java b/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java
index 4df2c8e..cceaef0 100644
--- a/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java
+++ b/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java
@@ -103,4 +103,6 @@ public interface LogicalExpression extends Iterable<LogicalExpression>{
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/PathSegment.java b/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
index ac1aeac..448e50f 100644
--- a/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
+++ b/common/src/main/java/org/apache/drill/common/expression/PathSegment.java
@@ -17,38 +17,42 @@
*/
package org.apache.drill.common.expression;
-import org.apache.drill.common.expression.ValueExpressions.CollisionBehavior;
public abstract class PathSegment{
- protected final ValueExpressions.CollisionBehavior collision;
protected PathSegment child;
- protected PathSegment(CollisionBehavior collision){
- this.collision = collision;
- }
-
+ public abstract PathSegment cloneWithNewChild(PathSegment segment);
public abstract PathSegment clone();
-
+
public static class ArraySegment extends PathSegment{
private final int index;
-
- public ArraySegment(int index, ValueExpressions.CollisionBehavior collision){
- super(collision);
+
+ public ArraySegment(String numberAsText, PathSegment child){
+ this.child = child;
+ this.index = Integer.parseInt(numberAsText);
+ }
+
+ public ArraySegment(int index, PathSegment child){
+ this.child = child;
+ this.index = index;
+ }
+
+ public ArraySegment(int index){
if(index < 0 ) throw new IllegalArgumentException();
this.index = index;
}
-
+
public int getIndex(){
return index;
}
-
+
public boolean isArray(){
return true;
}
-
+
public boolean isNamed(){
return false;
}
@@ -60,37 +64,49 @@ public abstract class PathSegment{
@Override
public String toString() {
- return "ArraySegment [index=" + index + ", getCollisionBehavior()=" + getCollisionBehavior() + ", getChild()="
- + getChild() + "]";
+ return "ArraySegment [index=" + index + ", getChild()=" + getChild() + "]";
}
@Override
public PathSegment clone() {
- PathSegment seg = new ArraySegment(index, collision);
+ PathSegment seg = new ArraySegment(index);
if(child != null) seg.setChild(child.clone());
return seg;
}
-
+
+ public ArraySegment cloneWithNewChild(PathSegment newChild){
+ ArraySegment seg = new ArraySegment(index);
+ if(child != null){
+ seg.setChild(child.cloneWithNewChild(newChild));
+ }else{
+ seg.setChild(newChild);
+ }
+ return seg;
+ }
}
-
-
-
+
+
+
public static class NameSegment extends PathSegment{
private final String path;
-
- public NameSegment(CharSequence n, ValueExpressions.CollisionBehavior collision){
- super(collision);
+
+ public NameSegment(CharSequence n, PathSegment child){
+ this.child = child;
+ this.path = n.toString();
+ }
+
+ public NameSegment(CharSequence n){
this.path = n.toString();
}
-
+
public String getPath(){
- return path;
+ return path;
}
-
+
public boolean isArray(){
return false;
}
-
+
public boolean isNamed(){
return true;
}
@@ -102,15 +118,14 @@ public abstract class PathSegment{
@Override
public String toString() {
- return "NameSegment [path=" + path + ", getCollisionBehavior()=" + getCollisionBehavior() + ", getChild()="
- + getChild() + "]";
+ return "NameSegment [path=" + path + ", getChild()=" + getChild() + "]";
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
- result = prime * result + ((path == null) ? 0 : path.hashCode());
+ result = prime * result + ((path == null) ? 0 : path.toLowerCase().hashCode());
return result;
}
@@ -132,16 +147,27 @@ public abstract class PathSegment{
}
@Override
- public PathSegment clone() {
- PathSegment s = new NameSegment(this.path, this.collision);
+ public NameSegment clone() {
+ NameSegment s = new NameSegment(this.path);
if(child != null) s.setChild(child.clone());
return s;
}
-
-
-
+
+
+ @Override
+ public NameSegment cloneWithNewChild(PathSegment newChild) {
+ NameSegment s = new NameSegment(this.path);
+ if(child != null){
+ s.setChild(child.cloneWithNewChild(newChild));
+ }else{
+ s.setChild(newChild);
+ }
+ return s;
+ }
+
+
}
-
+
public NameSegment getNameSegment(){
throw new UnsupportedOperationException();
}
@@ -150,12 +176,8 @@ public abstract class PathSegment{
}
public abstract boolean isArray();
public abstract boolean isNamed();
-
-
- public ValueExpressions.CollisionBehavior getCollisionBehavior(){
- return collision;
- }
-
+
+
public boolean isLastPath(){
return child == null;
}
@@ -168,6 +190,6 @@ public abstract class PathSegment{
this.child = child;
}
-
-
+
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java b/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
index 76838dc..a3f93a3 100644
--- a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
+++ b/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
@@ -17,98 +17,79 @@
*/
package org.apache.drill.common.expression;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.Iterator;
import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import org.apache.drill.common.expression.ValueExpressions.CollisionBehavior;
+import org.antlr.runtime.ANTLRStringStream;
+import org.antlr.runtime.CommonTokenStream;
+import org.antlr.runtime.RecognitionException;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.expression.PathSegment.NameSegment;
+import org.apache.drill.common.expression.parser.ExprLexer;
+import org.apache.drill.common.expression.parser.ExprParser;
+import org.apache.drill.common.expression.parser.ExprParser.parse_return;
import org.apache.drill.common.expression.visitors.ExprVisitor;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.Types;
-import com.google.common.base.Joiner;
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
+import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import com.google.common.collect.Iterators;
-import com.google.protobuf.DescriptorProtos.UninterpretedOption.NamePart;
public class SchemaPath extends LogicalExpressionBase {
- // reads well in RegexBuddy
- private static final String ENTIRE_REGEX = "^\n" + "(?: # first match required\n"
- + "\\[\\d+\\] # array index only\n" + "|\n" + "'?\n" + "[^\\.\\[\\+\\-\\!\\]\\}]+ # identifier\n"
- + "'?\n" + "(?:\\[\\d+\\])?\n" + ")\n" + "[\\+\\-\\!\\]\\}]?\n" +
+ private final NameSegment rootSegment;
- "# secondary matches (starts with dot)\n" + "(?:\n" + "\\.\n" + "(?: # first match required\n"
- + "\\[\\d+\\] # array index only\n" + "|\n" + "'?\n" + "[^\\.\\[\\+\\-\\!\\]\\}]+ # identifier\n"
- + "'?\n" + "(?:\\[\\d+\\])?\n" + ")\n" + "[\\+\\-\\!\\]\\}]?\n" +
- ")*$";
+ public static SchemaPath getSimplePath(String name){
+ return getCompoundPath(name);
+ }
+
+ public static SchemaPath getCompoundPath(String... strings){
+ List<String> paths = Arrays.asList(strings);
+ Collections.reverse(paths);
+ NameSegment s = null;
+ for(String p : paths){
+ s = new NameSegment(p);
+ }
+ return new SchemaPath(s);
+ }
- // reads well in RegexBuddy
- private static final String SEGMENT_REGEX = "(?:\n" + "\\[(\\d+)\\]\n" + "|\n" + "'?\n"
- + "([^\\.\\[\\+\\-\\!\\]\\}]+) # identifier\n" + "'?\n" + ")\n"
- + "([\\+\\-\\!\\]\\}]?) # collision type";
- private static final int GROUP_INDEX = 1;
- private static final int GROUP_PATH_SEGMENT = 2;
- private static final int GROUP_COLLISION = 3;
- private final static Pattern SEGMENT_PATTERN = Pattern.compile(SEGMENT_REGEX, Pattern.COMMENTS);
- private final static Pattern ENTIRE_PATTERN = Pattern.compile(ENTIRE_REGEX, Pattern.COMMENTS);
- private final CharSequence originalPath;
- private final PathSegment rootSegment;
+ /**
+ *
+ * @param simpleName
+ */
+ @Deprecated
+ public SchemaPath(String simpleName, ExpressionPosition pos){
+ super(pos);
+ this.rootSegment = new NameSegment(simpleName);
+ if(simpleName.contains(".")) throw new IllegalStateException("This is deprecated and only supports simpe paths.");
+ }
public SchemaPath(SchemaPath path){
super(path.getPosition());
- this.originalPath = path.originalPath;
this.rootSegment = path.rootSegment;
}
- public SchemaPath(String str){
- this(str, ExpressionPosition.UNKNOWN);
+ public SchemaPath(NameSegment rootSegment){
+ super(ExpressionPosition.UNKNOWN);
+ this.rootSegment = rootSegment;
}
-
- public SchemaPath(CharSequence str, ExpressionPosition pos) {
- super(pos);
- if (!ENTIRE_PATTERN.matcher(str).matches())
- throw new IllegalArgumentException("Identifier doesn't match expected pattern.");
- this.originalPath = str;
- Matcher m = SEGMENT_PATTERN.matcher(str);
- PathSegment r = null;
- PathSegment previous = null;
- PathSegment current;
- while (m.find()) {
- CollisionBehavior col = (m.start(GROUP_COLLISION) != -1) ? CollisionBehavior.find(m.group(GROUP_COLLISION))
- : CollisionBehavior.DEFAULT;
-
- if (m.start(GROUP_INDEX) != -1) {
- String d = m.group(GROUP_INDEX);
- current = new PathSegment.ArraySegment(Integer.parseInt(d), col);
- } else {
- String i = m.group(GROUP_PATH_SEGMENT);
- current = new PathSegment.NameSegment(i, col);
- }
- if (previous == null) {
- r = current;
- } else {
- previous.setChild(current);
- }
- previous = current;
- }
-
- rootSegment = r;
-
- }
-
- private SchemaPath(SchemaPath parent, String[] childPaths){
- super(ExpressionPosition.UNKNOWN);
- SchemaPath p = new SchemaPath(Joiner.on('.').join(childPaths), ExpressionPosition.UNKNOWN);
- this.originalPath = parent.originalPath + "." + p.originalPath;
- PathSegment seg = parent.getRootSegment().clone();
- this.rootSegment = seg;
- while(!seg.isLastPath()) seg = seg.getChild();
- seg.setChild(p.getRootSegment());
+ public SchemaPath(NameSegment rootSegment, ExpressionPosition pos){
+ super(pos);
+ this.rootSegment = rootSegment;
}
@Override
@@ -116,16 +97,13 @@ public class SchemaPath extends LogicalExpressionBase {
return visitor.visitSchemaPath(this, value);
}
- public SchemaPath getChild(String... childPaths){
- return new SchemaPath(this, childPaths);
- }
-
- public PathSegment getRootSegment() {
- return rootSegment;
+ public SchemaPath getChild(String childPath){
+ rootSegment.cloneWithNewChild(new NameSegment(childPath));
+ return new SchemaPath(rootSegment);
}
- public CharSequence getPath() {
- return originalPath;
+ public NameSegment getRootSegment() {
+ return rootSegment;
}
@Override
@@ -147,7 +125,7 @@ public class SchemaPath extends LogicalExpressionBase {
return true;
if (obj == null)
return false;
- if (getClass() != obj.getClass())
+ if ( !(obj instanceof SchemaPath))
return false;
SchemaPath other = (SchemaPath) obj;
if (rootSegment == null) {
@@ -163,9 +141,54 @@ public class SchemaPath extends LogicalExpressionBase {
return Iterators.emptyIterator();
}
+
@Override
public String toString() {
- return "SchemaPath [rootSegment=" + rootSegment + "]";
+ String expr = ExpressionStringBuilder.toString(this);
+ return "SchemaPath ["+ expr + "]";
+ }
+
+ public String toExpr(){
+ return ExpressionStringBuilder.toString(this);
+ }
+
+
+ public static class De extends StdDeserializer<SchemaPath> {
+ DrillConfig config;
+
+ public De(DrillConfig config) {
+ super(LogicalExpression.class);
+ this.config = config;
+ }
+
+ @Override
+ public SchemaPath deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException,
+ JsonProcessingException {
+ String expr = jp.getText();
+
+ if (expr == null || expr.isEmpty())
+ return null;
+ try {
+ // logger.debug("Parsing expression string '{}'", expr);
+ ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
+ CommonTokenStream tokens = new CommonTokenStream(lexer);
+ ExprParser parser = new ExprParser(tokens);
+
+ //TODO: move functionregistry and error collector to injectables.
+ //ctxt.findInjectableValue(valueId, forProperty, beanInstance)
+ parse_return ret = parser.parse();
+
+ // ret.e.resolveAndValidate(expr, errorCollector);
+ if(ret.e instanceof SchemaPath){
+ return (SchemaPath) ret.e;
+ }else{
+ throw new IllegalStateException("Schema path is not a valid format.");
+ }
+ } catch (RecognitionException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java b/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
index 4b83e7d..1d217f2 100644
--- a/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
+++ b/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
@@ -33,22 +33,22 @@ public class ValueExpressions {
public static LogicalExpression getBigInt(long l){
return new LongExpression(l);
}
-
+
public static LogicalExpression getInt(int i){
return new IntExpression(i, ExpressionPosition.UNKNOWN);
}
-
+
public static LogicalExpression getFloat8(double d){
return new DoubleExpression(d, ExpressionPosition.UNKNOWN);
}
public static LogicalExpression getFloat4(float f){
return new DoubleExpression(f, ExpressionPosition.UNKNOWN);
}
-
+
public static LogicalExpression getBit(boolean b){
return new BooleanExpression(Boolean.toString(b), ExpressionPosition.UNKNOWN);
}
-
+
public static LogicalExpression getChar(String s){
return new QuotedString(s, ExpressionPosition.UNKNOWN);
}
@@ -72,7 +72,7 @@ public class ValueExpressions {
return new IntervalDayExpression(intervalInMillis);
}
-
+
public static LogicalExpression getNumericExpression(String s, ExpressionPosition ep) {
try {
@@ -109,7 +109,7 @@ public class ValueExpressions {
}
protected abstract V parseValue(String s);
-
+
@Override
public Iterator<LogicalExpression> iterator() {
return Iterators.emptyIterator();
@@ -119,13 +119,13 @@ public class ValueExpressions {
}
public static class BooleanExpression extends ValueExpression<Boolean> {
-
-
+
+
public BooleanExpression(String value, ExpressionPosition pos) {
super(value, pos);
}
-
+
@Override
protected Boolean parseValue(String s) {
return Boolean.parseBoolean(s);
@@ -250,7 +250,7 @@ public class ValueExpressions {
public LongExpression(long l) {
this(l, ExpressionPosition.UNKNOWN);
}
-
+
public LongExpression(long l, ExpressionPosition pos) {
super(pos);
this.l = l;
@@ -269,7 +269,7 @@ public class ValueExpressions {
public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
return visitor.visitLongConstant(this, value);
}
-
+
@Override
public Iterator<LogicalExpression> iterator() {
return Iterators.emptyIterator();
@@ -427,10 +427,10 @@ public class ValueExpressions {
private static final MajorType INTERVALDAY_CONSTANT = Types.required(MinorType.INTERVALDAY);
private static final long MILLIS_IN_DAY = 1000 * 60 * 60 * 24;
-
+
private int days;
private int millis;
-
+
public IntervalDayExpression(long intervalInMillis) {
this((int) (intervalInMillis / MILLIS_IN_DAY), (int) (intervalInMillis % MILLIS_IN_DAY), ExpressionPosition.UNKNOWN);
}
@@ -490,34 +490,4 @@ public class ValueExpressions {
}
}
- public static enum CollisionBehavior {
- SKIP("-"), // keep the old value.
- FAIL("!"), // give up on the record
- REPLACE("+"), // replace the old value with the new value.
- ARRAYIFY("]"), // replace the current position with an array. Then place the
- // old and new value in the array.
- OBJECTIFY("}"), // replace the current position with a map. Give the two
- // values names of 'old' and 'new'.
- MERGE_OVERRIDE("%"); // do your best to do a deep merge of the old and new
- // values.
-
- private String identifier;
-
- private CollisionBehavior(String identifier) {
- this.identifier = identifier;
- }
-
- public static final CollisionBehavior DEFAULT = FAIL;
-
- public static final CollisionBehavior find(String c) {
- if (c == null || c.isEmpty())
- return DEFAULT;
-
- for (CollisionBehavior b : values()) {
- if (b.identifier.equals(c))
- return b;
- }
- return DEFAULT;
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/java/org/apache/drill/common/logical/data/Project.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/data/Project.java b/common/src/main/java/org/apache/drill/common/logical/data/Project.java
index 645dc32..ab2ef35 100644
--- a/common/src/main/java/org/apache/drill/common/logical/data/Project.java
+++ b/common/src/main/java/org/apache/drill/common/logical/data/Project.java
@@ -43,17 +43,17 @@ public class Project extends SingleInputOperator {
if (selections == null || selections.length == 0)
throw new ExpressionParsingException(
"Project did not provide any projection selections. At least one projection must be provided.");
- for (int i = 0; i < selections.length; i++) {
- PathSegment segment = selections[i].getRef().getRootSegment();
- CharSequence path = segment.getNameSegment().getPath();
- if (!segment.isNamed() || !path.equals("output"))
- throw new ExpressionParsingException(
- String
- .format(
- "Outputs for projections always have to start with named path of output. First segment was named '%s' or was named [%s]",
- path, segment.isNamed()));
-
- }
+// for (int i = 0; i < selections.length; i++) {
+// PathSegment segment = selections[i].getRef().getRootSegment();
+// CharSequence path = segment.getNameSegment().getPath();
+// if (!segment.isNamed() || !path.equals("output"))
+// throw new ExpressionParsingException(
+// String
+// .format(
+// "Outputs for projections always have to start with named path of output. First segment was named '%s' or was named [%s]",
+// path, segment.isNamed()));
+//
+// }
}
@JsonProperty("projections")
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/main/java/org/apache/drill/common/util/TestTools.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/TestTools.java b/common/src/main/java/org/apache/drill/common/util/TestTools.java
index 0d3cc3e..6e0c241 100644
--- a/common/src/main/java/org/apache/drill/common/util/TestTools.java
+++ b/common/src/main/java/org/apache/drill/common/util/TestTools.java
@@ -21,23 +21,57 @@ import java.nio.file.Paths;
import org.junit.rules.TestName;
import org.junit.rules.TestRule;
+import org.junit.rules.TestWatcher;
import org.junit.rules.Timeout;
+import org.junit.runner.Description;
+import org.slf4j.Logger;
public class TestTools {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestTools.class);
-
- static final boolean IS_DEBUG = java.lang.management.ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0;
+
+ static final boolean IS_DEBUG = java.lang.management.ManagementFactory.getRuntimeMXBean().getInputArguments()
+ .toString().indexOf("-agentlib:jdwp") > 0;
static final String WORKING_PATH = Paths.get("").toAbsolutePath().toString();
-
- public static TestRule getTimeoutRule(){
+
+ public static TestRule getTimeoutRule() {
return getTimeoutRule(10000);
}
- public static TestRule getTimeoutRule(int timeout){
+ public static TestRule getTimeoutRule(int timeout) {
return IS_DEBUG ? new TestName() : new Timeout(timeout);
}
- public static String getWorkingPath(){
+ public static String getWorkingPath() {
return WORKING_PATH;
}
+
+ public static TestLogReporter getTestLogReporter(final Logger logger) {
+ return new TestLogReporter(logger);
+ }
+
+ public static class TestLogReporter extends TestWatcher{
+ private int failureCount = 0;
+
+ final Logger logger;
+
+ public TestLogReporter(Logger logger) {
+ super();
+ this.logger = logger;
+ }
+
+ @Override
+ protected void failed(Throwable e, Description description) {
+ logger.error("Test Failed: " + description.getDisplayName(), e);
+ failureCount++;
+ }
+
+ @Override
+ public void succeeded(Description description) {
+ logger.info("Test Succeeded: " + description.getDisplayName());
+ }
+
+ public void sleepIfFailure() throws InterruptedException{
+ if(failureCount > 0) Thread.sleep(2000);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java b/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
index f49b042..de090df 100644
--- a/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
+++ b/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
@@ -17,46 +17,98 @@
*/
package org.apache.drill.common.expression.parser;
-import java.io.File;
+import java.io.IOException;
import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.CommonTokenStream;
-import org.antlr.runtime.tree.CommonTree;
-import org.antlr.runtime.tree.DOTTreeGenerator;
-import org.antlr.stringtemplate.StringTemplate;
+import org.antlr.runtime.RecognitionException;
+import org.apache.drill.common.expression.ExpressionStringBuilder;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.parser.ExprParser.parse_return;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.google.common.base.Charsets;
-import com.google.common.io.Files;
+import org.junit.Assert;
+import org.junit.Test;
public class TreeTest {
- public static void main(String[] args) throws Exception {
- ExprLexer lexer = new ExprLexer(
- new ANTLRStringStream(
- "if ($F1) then case when (_MAP.R_NAME = 'AFRICA') then 2 else 4 end else if(4==3) then 1 else if(x==3) then 7 else (if(2==1) then 6 else 4 end) end"));
- // ExprLexer lexer = new ExprLexer(new
- // ANTLRStringStream("if ('blue.red') then 'orange' else if (false) then 1 else 0 end"));
- // ExprLexer lexer = new ExprLexer(new ANTLRStringStream("2+2"));
+ @Test
+ public void testIfWithCase() throws Exception{
+ testExpressionParsing("if ($F1) then case when (_MAP.R_NAME = 'AFRICA') then 2 else 4 end else if(4==3) then 1 else if(x==3) then 7 else (if(2==1) then 6 else 4 end) end");
+ }
+
+ @Test
+ public void testAdd() throws Exception{
+ testExpressionParsing("2+2");
+ }
+
+ @Test
+ public void testIf() throws Exception{
+ testExpressionParsing("if ('blue.red') then 'orange' else if (false) then 1 else 0 end");
+ }
+
+ @Test
+ public void testQuotedIdentifier() throws Exception{
+ testExpressionParsing("`hello friend`.`goodbye`");
+ }
+
+ @Test
+ public void testSpecialQuoted() throws Exception{
+ testExpressionParsing("`*0` + `*` ");
+ }
+
+ @Test
+ public void testQuotedIdentifier2() throws Exception{
+ testExpressionParsing("`hello friend`.goodbye");
+ }
+
+ @Test
+ public void testComplexIdentifier() throws Exception{
+ testExpressionParsing("goodbye[4].`hello`");
+ }
+
+ private LogicalExpression parseExpression(String expr) throws RecognitionException, IOException{
+
+ ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
CommonTokenStream tokens = new CommonTokenStream(lexer);
+// tokens.fill();
+// for(Token t : (List<Token>) tokens.getTokens()){
+// System.out.println(t + "" + t.getType());
+// }
+// tokens.rewind();
+
ExprParser parser = new ExprParser(tokens);
parse_return ret = parser.parse();
- LogicalExpression e = ret.e;
- ObjectMapper mapper = new ObjectMapper();
- mapper.enable(SerializationFeature.INDENT_OUTPUT);
- System.out.println(mapper.writeValueAsString(e));
- // print the tree
- CommonTree tree = (CommonTree) ret.getTree();
- DOTTreeGenerator gen = new DOTTreeGenerator();
- StringTemplate st = gen.toDOT(tree);
+ return ret.e;
- Files.write(st.toString(), new File("/Users/jnadeau/Documents/tree.dot"), Charsets.UTF_8);
+ }
+
+ private String serializeExpression(LogicalExpression expr){
+ ExpressionStringBuilder b = new ExpressionStringBuilder();
+ StringBuilder sb = new StringBuilder();
+ expr.accept(b, sb);
+ return sb.toString();
}
+
+ /**
+ * Attempt to parse an expression. Once parsed, convert it to a string and then parse it again to make sure serialization works.
+ * @param expr
+ * @throws RecognitionException
+ * @throws IOException
+ */
+ private void testExpressionParsing(String expr) throws RecognitionException, IOException{
+ System.out.println("-----" + expr + "-----");
+ LogicalExpression e = parseExpression(expr);
+
+ String newStringExpr = serializeExpression(e);
+ System.out.println(newStringExpr);
+ LogicalExpression e2 = parseExpression(newStringExpr);
+ //Assert.assertEquals(e, e2);
+
+ }
+
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRecordReader.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRecordReader.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRecordReader.java
index 0211de0..8e1e0ac 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRecordReader.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRecordReader.java
@@ -17,16 +17,17 @@
*/
package org.apache.drill.exec.store.hbase;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Stopwatch;
-import com.google.common.collect.Lists;
-import com.google.common.hash.HashFunction;
-import com.google.common.hash.Hasher;
-import com.google.common.hash.Hashing;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.PathSegment;
+import org.apache.drill.common.expression.PathSegment.NameSegment;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
@@ -48,15 +49,19 @@ import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
-import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.TimeUnit;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Lists;
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hasher;
+import com.google.common.hash.Hashing;
public class HBaseRecordReader implements RecordReader {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HBaseRecordReader.class);
private static final String ROW_KEY = "row_key";
private static final int TARGET_RECORD_COUNT = 4000;
+ private static final SchemaPath ROW_KEY_PATH = SchemaPath.getSimplePath(ROW_KEY);
private List<SchemaPath> columns;
private OutputMutator outputMutator;
@@ -75,16 +80,16 @@ public class HBaseRecordReader implements RecordReader {
this.context = context;
if (columns != null && columns.size() != 0) {
for (SchemaPath column : columns) {
- if (column.getPath().toString().equalsIgnoreCase(ROW_KEY)) {
- rowKeySchemaPath = new SchemaPath(ROW_KEY);
+ if (column.getRootSegment().getPath().toString().equalsIgnoreCase(ROW_KEY)) {
+ rowKeySchemaPath = ROW_KEY_PATH;
continue;
}
- PathSegment root = column.getRootSegment();
+ NameSegment root = column.getRootSegment();
assert root != null;
PathSegment child = root.getChild();
- byte[] family = root.getNameSegment().getPath().toString().getBytes();
+ byte[] family = root.getPath().toString().getBytes();
if (child != null) {
- Preconditions.checkArgument(child.getChild() == null, "Unsupported column name: " + column.getPath());
+ Preconditions.checkArgument(child.getChild() == null, "Unsupported column name: " + column.toString());
byte[] qualifier = child.getNameSegment().getPath().toString().getBytes();
scan.addColumn(family, qualifier);
} else {
@@ -96,7 +101,7 @@ public class HBaseRecordReader implements RecordReader {
if (this.columns == null) {
this.columns = Lists.newArrayList();
}
- rowKeySchemaPath = new SchemaPath(ROW_KEY);
+ rowKeySchemaPath = ROW_KEY_PATH;
this.columns.add(rowKeySchemaPath);
}
@@ -127,7 +132,8 @@ public class HBaseRecordReader implements RecordReader {
MaterializedField field = MaterializedField.create(column, Types.optional(TypeProtos.MinorType.VARBINARY));
NullableVarBinaryVector v = new NullableVarBinaryVector(field, context.getAllocator());
output.addField(v);
- vvMap.put(new FamilyQualifierWrapper(column.getPath().toString()), v);
+ String fullyQualified = column.getRootSegment().getPath() + "." + column.getRootSegment().getChild().getNameSegment().getPath();
+ vvMap.put(new FamilyQualifierWrapper(fullyQualified), v);
}
} catch (SchemaChangeException e) {
throw new ExecutionSetupException(e);
@@ -206,7 +212,7 @@ public class HBaseRecordReader implements RecordReader {
}
private NullableVarBinaryVector addNewVector(String column) {
- MaterializedField field = MaterializedField.create(new SchemaPath(column, ExpressionPosition.UNKNOWN), Types.optional(TypeProtos.MinorType.VARBINARY));
+ MaterializedField field = MaterializedField.create(SchemaPath.getCompoundPath(column.split("\\.")), Types.optional(TypeProtos.MinorType.VARBINARY));
NullableVarBinaryVector v = new NullableVarBinaryVector(field, context.getAllocator());
VectorAllocator.getAllocator(v, 100).alloc(TARGET_RECORD_COUNT);
vvMap.put(new FamilyQualifierWrapper(column), v);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
index 1376210..991685c 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
@@ -26,6 +26,7 @@ import net.hydromatic.optiq.SchemaPlus;
import org.apache.drill.exec.planner.logical.DrillTable;
import org.apache.drill.exec.planner.logical.DynamicDrillTable;
+import org.apache.drill.exec.rpc.user.DrillUser;
import org.apache.drill.exec.store.AbstractSchema;
import org.apache.drill.exec.store.SchemaFactory;
import org.apache.hadoop.hbase.HTableDescriptor;
@@ -38,18 +39,17 @@ public class HBaseSchemaFactory implements SchemaFactory {
final String schemaName;
final HBaseStoragePlugin plugin;
-
+
public HBaseSchemaFactory(HBaseStoragePlugin plugin, String name) throws IOException {
this.plugin = plugin;
this.schemaName = name;
}
@Override
- public Schema add(SchemaPlus parent) {
+ public void registerSchemas(DrillUser user, SchemaPlus parent) {
HBaseSchema schema = new HBaseSchema(schemaName);
SchemaPlus hPlus = parent.add(schemaName, schema);
schema.setHolder(hPlus);
- return schema;
}
class HBaseSchema extends AbstractSchema {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java
index 1fab650..a82c6c3 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseStoragePlugin.java
@@ -19,11 +19,10 @@ package org.apache.drill.exec.store.hbase;
import java.io.IOException;
-import net.hydromatic.optiq.Schema;
import net.hydromatic.optiq.SchemaPlus;
import org.apache.drill.common.JSONOptions;
-import org.apache.drill.common.logical.StoragePluginConfig;
+import org.apache.drill.exec.rpc.user.DrillUser;
import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.store.AbstractStoragePlugin;
@@ -63,8 +62,8 @@ public class HBaseStoragePlugin extends AbstractStoragePlugin {
}
@Override
- public Schema createAndAddSchema(SchemaPlus parent) {
- return schemaFactory.add(parent);
+ public void registerSchemas(DrillUser user, SchemaPlus parent) {
+ schemaFactory.registerSchemas(user, parent);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/distribution/src/resources/drill-env.sh
----------------------------------------------------------------------
diff --git a/distribution/src/resources/drill-env.sh b/distribution/src/resources/drill-env.sh
index 02a7022..d9fdb6f 100644
--- a/distribution/src/resources/drill-env.sh
+++ b/distribution/src/resources/drill-env.sh
@@ -13,4 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
DRILL_MAX_DIRECT_MEMORY="8G"
-export DRILL_JAVA_OPTS="-XX:MaxDirectMemorySize=$DRILL_MAX_DIRECT_MEMORY"
+DRILL_INIT_HEAP="1g"
+DRILL_MAX_HEAP="4g"
+export DRILL_JAVA_OPTS="-Xms$DRILL_INIT_HEAP -Xmx$DRILL_MAX_HEAP -XX:MaxDirectMemorySize=$DRILL_MAX_DIRECT_MEMORY"
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
index 1297cb3..73f2e29 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DrillClient.java
@@ -59,7 +59,7 @@ import com.google.common.util.concurrent.SettableFuture;
*/
public class DrillClient implements Closeable, ConnectionThrottle{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillClient.class);
-
+
DrillConfig config;
private UserClient client;
private volatile ClusterCoordinator clusterCoordinator;
@@ -67,7 +67,8 @@ public class DrillClient implements Closeable, ConnectionThrottle{
private final TopLevelAllocator allocator = new TopLevelAllocator(Long.MAX_VALUE);
private int reconnectTimes;
private int reconnectDelay;
-
+ private final boolean ownsZkConnection;
+
public DrillClient() {
this(DrillConfig.create());
}
@@ -79,25 +80,26 @@ public class DrillClient implements Closeable, ConnectionThrottle{
public DrillClient(DrillConfig config) {
this(config, null);
}
-
+
public DrillClient(DrillConfig config, ClusterCoordinator coordinator){
+ this.ownsZkConnection = coordinator == null;
this.config = config;
this.clusterCoordinator = coordinator;
this.reconnectTimes = config.getInt(ExecConstants.BIT_RETRY_TIMES);
this.reconnectDelay = config.getInt(ExecConstants.BIT_RETRY_DELAY);
}
-
+
public DrillConfig getConfig(){
return config;
}
-
+
@Override
public void setAutoRead(boolean enableAutoRead) {
client.setAutoRead(enableAutoRead);
}
-
-
+
+
/**
* Connects the client to a Drillbit server
*
@@ -106,11 +108,11 @@ public class DrillClient implements Closeable, ConnectionThrottle{
public void connect() throws RpcException {
connect((String) null);
}
-
+
public synchronized void connect(String connect) throws RpcException {
if (connected) return;
- if (clusterCoordinator == null) {
+ if (ownsZkConnection) {
try {
this.clusterCoordinator = new ZKClusterCoordinator(this.config, connect);
this.clusterCoordinator.start(10000);
@@ -171,6 +173,16 @@ public class DrillClient implements Closeable, ConnectionThrottle{
*/
public void close(){
this.client.close();
+ if(ownsZkConnection){
+ try {
+ this.clusterCoordinator.close();
+ } catch (IOException e) {
+ logger.warn("Error while closing Cluster Coordinator.", e);
+ }
+ }
+
+ // TODO: fix tests that fail when this is called.
+ //allocator.close();
connected = false;
}
@@ -187,12 +199,12 @@ public class DrillClient implements Closeable, ConnectionThrottle{
client.submitQuery(listener,query);
return listener.getResults();
}
-
+
public void cancelQuery(QueryId id){
client.send(RpcType.CANCEL_QUERY, id, Ack.class);
}
-
-
+
+
/**
* Submits a Logical plan for direct execution (bypasses parsing)
*
@@ -203,7 +215,7 @@ public class DrillClient implements Closeable, ConnectionThrottle{
public void runQuery(QueryType type, String plan, UserResultsListener resultsListener){
client.submitQuery(resultsListener, newBuilder().setResultsMode(STREAM_FULL).setType(type).setPlan(plan).build());
}
-
+
private class ListHoldingResultsListener implements UserResultsListener {
private Vector<QueryResultBatch> results = new Vector<QueryResultBatch>();
private SettableFuture<List<QueryResultBatch>> future = SettableFuture.create();
@@ -245,7 +257,7 @@ public class DrillClient implements Closeable, ConnectionThrottle{
future.set(results);
}
}
-
+
public List<QueryResultBatch> getResults() throws RpcException{
try{
return future.get();
@@ -258,7 +270,7 @@ public class DrillClient implements Closeable, ConnectionThrottle{
public void queryIdArrived(QueryId queryId) {
}
}
-
+
private class FutureHandler extends AbstractCheckedFuture<Void, RpcException> implements RpcConnectionHandler<ServerConnection>, DrillRpcFuture<Void>{
protected FutureHandler() {
@@ -278,7 +290,7 @@ public class DrillClient implements Closeable, ConnectionThrottle{
private SettableFuture<Void> getInner(){
return (SettableFuture<Void>) delegate();
}
-
+
@Override
protected RpcException mapException(Exception e) {
return RpcException.mapException(e);
@@ -288,7 +300,7 @@ public class DrillClient implements Closeable, ConnectionThrottle{
public ByteBuf getBuffer() {
return null;
}
-
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/exec/java-exec/src/main/java/org/apache/drill/exec/client/DumpCat.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DumpCat.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DumpCat.java
index cfffed9..c69a43e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/DumpCat.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/DumpCat.java
@@ -68,7 +68,7 @@ public class DumpCat {
System.out.println(String.format("Trace file %s not created", o.location));
System.exit(-1);
}
-
+
FileInputStream input = new FileInputStream(file.getAbsoluteFile());
if (o.batch < 0) {
@@ -142,11 +142,11 @@ public class DumpCat {
String avgRecSizeStr = null;
if (this.rows>0)
avgRecSizeStr = String.format("Average Record Size : %d ", this.dataSize/this.rows);
- else
+ else
avgRecSizeStr = "Average Record Size : 0";
-
+
return String.format("Records : %d / %d \n", this.selectedRows, this.rows) +
- avgRecSizeStr +
+ avgRecSizeStr +
String.format("\n Total Data Size : %d", this.dataSize);
}
}
@@ -188,7 +188,7 @@ public class DumpCat {
prevSchema = vectorContainer.getSchema();
batchNum ++;
-
+
vectorContainer.zeroVectors();
}
@@ -253,7 +253,7 @@ public class DumpCat {
for (VectorWrapper w : vectorContainer) {
MaterializedField field = w.getValueVector().getField();
System.out.println (String.format("name : %s, minor_type : %s, data_mode : %s",
- field.getName(),
+ field.toExpr(),
field.getType().getMinorType().toString(),
field.isNullable() ? "nullable":"non-nullable"
));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java
new file mode 100644
index 0000000..f161782
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/PrintingResultsListener.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.client;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.drill.exec.client.QuerySubmitter.Format;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.TopLevelAllocator;
+import org.apache.drill.exec.proto.UserBitShared.QueryId;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.rpc.user.ConnectionThrottle;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.apache.drill.exec.rpc.user.UserResultsListener;
+import org.apache.drill.exec.util.VectorUtil;
+
+public class PrintingResultsListener implements UserResultsListener {
+ AtomicInteger count = new AtomicInteger();
+ private CountDownLatch latch = new CountDownLatch(1);
+ RecordBatchLoader loader;
+ Format format;
+ int columnWidth;
+ BufferAllocator allocator = new TopLevelAllocator();
+ volatile Exception exception;
+
+ public PrintingResultsListener(Format format, int columnWidth) {
+ loader = new RecordBatchLoader(allocator);
+ this.format = format;
+ this.columnWidth = columnWidth;
+ }
+
+ @Override
+ public void submissionFailed(RpcException ex) {
+ exception = ex;
+ latch.countDown();
+ }
+
+ @Override
+ public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) {
+ int rows = result.getHeader().getRowCount();
+ if (result.getData() != null) {
+ count.addAndGet(rows);
+ try {
+ loader.load(result.getHeader().getDef(), result.getData());
+ } catch (SchemaChangeException e) {
+ submissionFailed(new RpcException(e));
+ }
+
+ switch(format) {
+ case TABLE:
+ VectorUtil.showVectorAccessibleContent(loader, columnWidth);
+ break;
+ case TSV:
+ VectorUtil.showVectorAccessibleContent(loader, "\t");
+ break;
+ case CSV:
+ VectorUtil.showVectorAccessibleContent(loader, ",");
+ break;
+ }
+
+ }
+
+ boolean isLastChunk = result.getHeader().getIsLastChunk();
+ result.release();
+
+ if (isLastChunk) {
+ allocator.close();
+ latch.countDown();
+ }
+
+ }
+
+ public int await() throws Exception {
+ latch.await();
+ if(exception != null) throw exception;
+ return count.get();
+ }
+
+ @Override
+ public void queryIdArrived(QueryId queryId) {
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ecaa838f/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
index 4551e53..0234b7e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/client/QuerySubmitter.java
@@ -20,24 +20,11 @@ package org.apache.drill.exec.client;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Paths;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Sets;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.exec.coord.ZKClusterCoordinator;
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.proto.UserBitShared.QueryId;
import org.apache.drill.exec.proto.UserProtos;
-import org.apache.drill.exec.record.RecordBatchLoader;
-import org.apache.drill.exec.rpc.RpcException;
-import org.apache.drill.exec.rpc.user.ConnectionThrottle;
-import org.apache.drill.exec.rpc.user.QueryResultBatch;
-import org.apache.drill.exec.rpc.user.UserResultsListener;
-import org.apache.drill.exec.server.BootStrapContext;
import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.server.RemoteServiceSet;
import org.apache.drill.exec.util.VectorUtil;
@@ -46,6 +33,7 @@ import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
import com.beust.jcommander.ParameterException;
import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
import com.google.common.base.Stopwatch;
public class QuerySubmitter {
@@ -115,10 +103,13 @@ public class QuerySubmitter {
Preconditions.checkArgument(!(planLocation == null && queryString == null), "Must provide either query file or query string");
Preconditions.checkArgument(!(planLocation != null && queryString != null), "Must provide either query file or query string, not both");
+ RemoteServiceSet serviceSet = null;
+ Drillbit[] drillbits = null;
+
try{
if (local) {
- RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
- Drillbit[] drillbits = new Drillbit[bits];
+ serviceSet = RemoteServiceSet.getLocalServiceSet();
+ drillbits = new Drillbit[bits];
for (int i = 0; i < bits; i++) {
drillbits[i] = new Drillbit(config, serviceSet);
drillbits[i].run();
@@ -130,126 +121,81 @@ public class QuerySubmitter {
client = new DrillClient(config, clusterCoordinator);
}
client.connect();
- QueryResultsListener listener;
+
String plan;
if (queryString == null) {
plan = Charsets.UTF_8.decode(ByteBuffer.wrap(Files.readAllBytes(Paths.get(planLocation)))).toString();
} else {
plan = queryString;
}
- String[] queries;
- UserProtos.QueryType queryType;
- type = type.toLowerCase();
- switch(type) {
- case "sql":
- queryType = UserProtos.QueryType.SQL;
- queries = plan.trim().split(";");
- break;
- case "logical":
- queryType = UserProtos.QueryType.LOGICAL;
- queries = new String[]{ plan };
- break;
- case "physical":
- queryType = UserProtos.QueryType.PHYSICAL;
- queries = new String[]{ plan };
- break;
- default:
- System.out.println("Invalid query type: " + type);
- return -1;
- }
- Format outputFormat;
- format = format.toLowerCase();
- switch(format) {
- case "csv":
- outputFormat = Format.CSV;
- break;
- case "tsv":
- outputFormat = Format.TSV;
- break;
- case "table":
- outputFormat = Format.TABLE;
- break;
- default:
- System.out.println("Invalid format type: " + format);
- return -1;
- }
- Stopwatch watch = new Stopwatch();
- for (String query : queries) {
- listener = new QueryResultsListener(outputFormat, width);
- watch.start();
- client.runQuery(queryType, query, listener);
- int rows = listener.await();
- System.out.println(String.format("%d record%s selected (%f seconds)", rows, rows > 1 ? "s" : "", (float) watch.elapsed(TimeUnit.MILLISECONDS) / (float) 1000));
- if (query != queries[queries.length - 1]) {
- System.out.println();
- }
- watch.stop();
- watch.reset();
- }
- return 0;
+ return submitQuery(client, plan, type, format, width);
+
}finally{
if(client != null) client.close();
+ if(local){
+ for(Drillbit b : drillbits){
+ b.close();
+ }
+ serviceSet.close();
+ }
}
}
- private class QueryResultsListener implements UserResultsListener {
- AtomicInteger count = new AtomicInteger();
- private CountDownLatch latch = new CountDownLatch(1);
- RecordBatchLoader loader = new RecordBatchLoader(new BootStrapContext(DrillConfig.create()).getAllocator());
- Format format;
- int columnWidth;
- volatile Exception exception;
-
- public QueryResultsListener(Format format, int columnWidth) {
- this.format = format;
- this.columnWidth = columnWidth;
+ public int submitQuery(DrillClient client, String plan, String type, String format, int width) throws Exception {
+
+ PrintingResultsListener listener;
+
+ String[] queries;
+ UserProtos.QueryType queryType;
+ type = type.toLowerCase();
+ switch(type) {
+ case "sql":
+ queryType = UserProtos.QueryType.SQL;
+ queries = plan.trim().split(";");
+ break;
+ case "logical":
+ queryType = UserProtos.QueryType.LOGICAL;
+ queries = new String[]{ plan };
+ break;
+ case "physical":
+ queryType = UserProtos.QueryType.PHYSICAL;
+ queries = new String[]{ plan };
+ break;
+ default:
+ System.out.println("Invalid query type: " + type);
+ return -1;
}
- @Override
- public void submissionFailed(RpcException ex) {
- exception = ex;
- latch.countDown();
+ Format outputFormat;
+ format = format.toLowerCase();
+ switch(format) {
+ case "csv":
+ outputFormat = Format.CSV;
+ break;
+ case "tsv":
+ outputFormat = Format.TSV;
+ break;
+ case "table":
+ outputFormat = Format.TABLE;
+ break;
+ default:
+ System.out.println("Invalid format type: " + format);
+ return -1;
}
-
- @Override
- public void resultArrived(QueryResultBatch result, ConnectionThrottle throttle) {
- int rows = result.getHeader().getRowCount();
- if (result.getData() != null) {
- count.addAndGet(rows);
- try {
- loader.load(result.getHeader().getDef(), result.getData());
- } catch (SchemaChangeException e) {
- submissionFailed(new RpcException(e));
- }
-
- switch(format) {
- case TABLE:
- VectorUtil.showVectorAccessibleContent(loader, columnWidth);
- break;
- case TSV:
- VectorUtil.showVectorAccessibleContent(loader, "\t");
- break;
- case CSV:
- VectorUtil.showVectorAccessibleContent(loader, ",");
- break;
- }
-
+ Stopwatch watch = new Stopwatch();
+ for (String query : queries) {
+ listener = new PrintingResultsListener(outputFormat, width);
+ watch.start();
+ client.runQuery(queryType, query, listener);
+ int rows = listener.await();
+ System.out.println(String.format("%d record%s selected (%f seconds)", rows, rows > 1 ? "s" : "", (float) watch.elapsed(TimeUnit.MILLISECONDS) / (float) 1000));
+ if (query != queries[queries.length - 1]) {
+ System.out.println();
}
-
- if (result.getHeader().getIsLastChunk()) {
- latch.countDown();
- }
- result.release();
- }
-
- public int await() throws Exception {
- latch.await();
- if(exception != null) throw exception;
- return count.get();
+ watch.stop();
+ watch.reset();
}
+ return 0;
- @Override
- public void queryIdArrived(QueryId queryId) {
- }
}
}