You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by vo...@apache.org on 2020/03/12 14:50:27 UTC
[drill] branch master updated (a8c9a0f -> cc51f24)
This is an automated email from the ASF dual-hosted git repository.
volodymyr pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git.
from a8c9a0f DRILL-7603: Allow default schema to be set for HTTP queries
new 2b7ab27 DRILL-7634: Rollup of code cleanup changes
new cc51f24 DRILL-7631: Updates to the Json Structure Parser
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../exceptions/DrillConfigurationException.java | 8 +-
.../apache/drill/common/exceptions/DrillError.java | 8 +-
.../drill/common/exceptions/DrillIOException.java | 8 +-
.../exceptions/RetryAfterSpillException.java | 5 +-
.../common/exceptions/UserRemoteException.java | 6 +-
.../java/org/apache/drill/common/types/Types.java | 5 -
.../store/mapr/db/binary/MapRDBFilterBuilder.java | 33 +--
.../mapr/db/json/CompareFunctionsProcessor.java | 21 +-
.../store/mapr/db/json/JsonConditionBuilder.java | 82 +++---
.../exec/store/mapr/db/json/JsonScanSpec.java | 6 +-
.../store/mapr/db/json/OjaiFunctionsProcessor.java | 14 +-
.../store/hbase/CompareFunctionsProcessor.java | 67 ++---
.../drill/exec/store/hbase/HBaseFilterBuilder.java | 36 +--
.../drill/exec/store/hbase/HBaseRegexParser.java | 6 +-
.../drill/exec/expr/fn/HiveFunctionRegistry.java | 22 +-
.../drill/exec/store/kafka/KafkaNodeProcessor.java | 27 +-
.../exec/store/kafka/KafkaPartitionScanSpec.java | 11 +-
.../store/kafka/KafkaPartitionScanSpecBuilder.java | 28 +-
.../store/mongo/MongoCompareFunctionProcessor.java | 24 +-
.../drill/exec/store/mongo/MongoFilterBuilder.java | 40 ++-
.../codegen/templates/ComparisonFunctions.java | 310 ++++++++++-----------
.../apache/drill/exec/expr/BooleanPredicate.java | 9 +-
.../org/apache/drill/exec/expr/CloneVisitor.java | 2 +-
.../drill/exec/expr/ComparisonPredicate.java | 14 +-
.../apache/drill/exec/expr/EvaluationVisitor.java | 14 +-
.../exec/expr/ExpressionTreeMaterializer.java | 33 ++-
.../org/apache/drill/exec/expr/FilterBuilder.java | 34 +--
.../apache/drill/exec/expr/StatisticsProvider.java | 3 +-
.../drill/exec/expr/fn/DrillAggFuncHolder.java | 75 +++--
.../exec/expr/fn/FunctionGenerationHelper.java | 22 +-
.../expr/fn/FunctionImplementationRegistry.java | 4 +-
.../drill/exec/expr/fn/impl/BitFunctions.java | 14 +-
.../drill/exec/expr/fn/impl/StringFunctions.java | 6 +-
.../expr/fn/interpreter/InterpreterEvaluator.java | 44 ++-
.../drill/exec/ops/ExecutorFragmentContext.java | 2 +-
.../exec/physical/base/AbstractDbGroupScan.java | 1 -
.../exec/physical/base/AbstractDbSubScan.java | 2 +-
.../exec/physical/base/AbstractGroupScan.java | 6 +-
.../base/AbstractGroupScanWithMetadata.java | 4 +-
.../physical/base/AbstractPhysicalVisitor.java | 25 +-
.../drill/exec/physical/base/FragmentLeaf.java | 4 +-
.../org/apache/drill/exec/physical/base/Leaf.java | 6 +-
.../apache/drill/exec/physical/base/ScanStats.java | 7 +-
.../exec/physical/config/SingleMergeExchange.java | 5 +-
.../exec/physical/impl/aggregate/HashAggBatch.java | 6 +-
.../impl/scan/project/ResolvedMapColumn.java | 1 -
.../exec/physical/impl/window/WindowFunction.java | 39 ++-
.../physical/resultSet/impl/BuildFromSchema.java | 3 +-
.../physical/resultSet/impl/ColumnBuilder.java | 2 +-
.../exec/physical/resultSet/impl/ColumnState.java | 2 +-
.../exec/planner/AbstractPartitionDescriptor.java | 18 +-
.../exec/planner/DFSFilePartitionLocation.java | 4 +-
.../planner/FileSystemPartitionDescriptor.java | 24 +-
.../drill/exec/planner/PartitionDescriptor.java | 8 +-
.../apache/drill/exec/planner/PlannerPhase.java | 27 +-
.../drill/exec/planner/common/DrillRelOptUtil.java | 17 +-
.../exec/planner/common/DrillScanRelBase.java | 2 +-
.../planner/fragment/DistributionAffinity.java | 22 +-
.../fragment/HardAffinityFragmentParallelizer.java | 10 +-
.../drill/exec/planner/fragment/Materializer.java | 18 +-
.../exec/planner/fragment/MemoryCalculator.java | 4 +-
.../drill/exec/planner/fragment/PlanningSet.java | 13 +-
.../exec/planner/fragment/QueryParallelizer.java | 6 +-
.../exec/planner/fragment/SimpleParallelizer.java | 56 ++--
.../fragment/SoftAffinityFragmentParallelizer.java | 10 +-
.../apache/drill/exec/planner/fragment/Stats.java | 2 +-
.../drill/exec/planner/fragment/Wrapper.java | 61 ++--
.../fragment/contrib/SplittingParallelizer.java | 51 ++--
.../drill/exec/planner/index/PathInExpr.java | 30 +-
.../generators/AbstractIndexPlanGenerator.java | 19 +-
.../drill/exec/planner/logical/DrillOptiq.java | 48 ++--
.../planner/logical/DrillPushLimitToScanRule.java | 6 +-
.../drill/exec/planner/logical/ExprHelper.java | 8 +-
.../planner/physical/DrillDistributionTrait.java | 3 +-
.../physical/DrillDistributionTraitDef.java | 3 +-
.../drill/exec/planner/physical/ExchangePrel.java | 5 +-
.../drill/exec/planner/physical/HashPrelUtil.java | 5 +-
.../drill/exec/planner/physical/JoinPrel.java | 19 +-
.../planner/physical/MetadataControllerPrel.java | 3 +-
.../exec/planner/physical/NestedLoopJoinPrule.java | 2 -
.../exec/planner/physical/PhysicalPlanCreator.java | 8 +-
.../apache/drill/exec/planner/physical/Prel.java | 5 +-
.../drill/exec/planner/physical/ScanPrel.java | 16 +-
.../drill/exec/planner/physical/ScanPrule.java | 2 +-
.../planner/physical/SingleMergeExchangePrel.java | 5 +-
.../drill/exec/planner/physical/SortPrel.java | 2 +-
.../drill/exec/planner/physical/SortPrule.java | 14 +-
.../exec/planner/physical/SubsetTransformer.java | 22 +-
.../drill/exec/planner/physical/TopNPrel.java | 4 +-
.../exec/planner/physical/UnionExchangePrel.java | 2 +-
.../visitor/ExcessiveExchangeIdentifier.java | 10 +-
.../physical/visitor/MemoryEstimationVisitor.java | 9 +-
.../drill/exec/planner/sql/TypeInferenceUtils.java | 7 +-
.../planner/sql/handlers/DefaultSqlHandler.java | 20 +-
.../exec/planner/sql/handlers/ExplainHandler.java | 31 ++-
.../exec/resolver/DefaultFunctionResolver.java | 2 +-
.../drill/exec/resolver/ExactFunctionResolver.java | 2 +-
.../drill/exec/store/dfs/FileSystemPlugin.java | 15 +-
.../apache/drill/exec/store/dfs/FormatPlugin.java | 16 +-
.../exec/store/dfs/easy/EasyFormatPlugin.java | 14 +-
.../drill/exec/store/dfs/easy/EasyGroupScan.java | 9 +-
.../easy/json/parser/AbstractElementParser.java | 3 +
.../exec/store/easy/json/parser/ArrayListener.java | 89 ++----
.../exec/store/easy/json/parser/ArrayParser.java | 63 ++++-
.../store/easy/json/parser/DummyValueParser.java | 2 -
.../easy/json/parser/JsonStructureParser.java | 12 +
.../exec/store/easy/json/parser/JsonType.java | 25 --
.../store/easy/json/parser/JsonValueParser.java | 116 ++++++++
.../store/easy/json/parser/ObjectListener.java | 110 ++++----
.../exec/store/easy/json/parser/ObjectParser.java | 60 +++-
.../exec/store/easy/json/parser/TokenIterator.java | 16 ++
.../exec/store/easy/json/parser/ValueDef.java | 85 ++++++
.../store/easy/json/parser/ValueDefFactory.java | 99 +++++++
.../exec/store/easy/json/parser/ValueFactory.java | 222 ---------------
.../exec/store/easy/json/parser/ValueListener.java | 37 +--
.../exec/store/easy/json/parser/ValueParser.java | 55 ++--
.../drill/exec/store/ischema/InfoSchemaFilter.java | 20 +-
.../store/ischema/InfoSchemaFilterBuilder.java | 37 +--
.../exec/store/parquet/FilterEvaluatorUtils.java | 10 +-
.../exec/store/schedule/AssignmentCreator.java | 35 ++-
.../drill/exec/work/metadata/MetadataProvider.java | 13 +-
.../impl/TestResultSetLoaderMapArray.java | 1 -
.../impl/TestResultSetLoaderRepeatedList.java | 31 +--
.../{ => easy}/json/parser/BaseTestJsonParser.java | 128 ++++-----
.../json/parser/TestJsonParserArrays.java | 86 +++---
.../json/parser/TestJsonParserBasics.java | 92 ++++--
.../json/parser/TestJsonParserErrors.java | 5 +-
.../json/parser/TestJsonParserObjects.java | 9 +-
.../easy/json/parser/TestJsonParserUnknowns.java | 294 +++++++++++++++++++
.../easy/text/compliant/TestCsvWithHeaders.java | 4 +-
.../store/parquet/TestParquetFilterPushDown.java | 6 +-
.../drill/exec/record/MaterializedField.java | 32 +--
.../record/metadata/AbstractColumnMetadata.java | 5 +-
.../exec/record/metadata/PropertyAccessor.java | 2 +-
.../accessor/reader/AbstractScalarReader.java | 2 +-
.../accessor/writer/AbstractTupleWriter.java | 12 +-
.../vector/accessor/writer/ObjectArrayWriter.java | 2 -
.../org/apache/drill/common/FunctionNames.java | 54 ++++
.../drill/common/expression/BooleanOperator.java | 8 +-
.../common/expression/ExpressionStringBuilder.java | 4 +-
.../drill/common/expression/FunctionCall.java | 21 +-
.../common/expression/FunctionCallFactory.java | 56 ++--
.../apache/drill/common/expression/SchemaPath.java | 3 +-
.../drill/common/expression/TypedFieldExpr.java | 7 +-
.../drill/common/expression/ValueExpressions.java | 96 +++----
.../expression/visitors/AggregateChecker.java | 3 +-
.../visitors/ConditionalExprOptimizer.java | 2 +-
.../expression/visitors/ConstantChecker.java | 2 +-
.../expression/visitors/ExpressionValidator.java | 3 +-
.../common/parser/LogicalExpressionParser.java | 6 +-
150 files changed, 2251 insertions(+), 1709 deletions(-)
delete mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/JsonType.java
create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/JsonValueParser.java
create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ValueDef.java
create mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ValueDefFactory.java
delete mode 100644 exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ValueFactory.java
rename exec/java-exec/src/test/java/org/apache/drill/exec/store/{ => easy}/json/parser/BaseTestJsonParser.java (71%)
rename exec/java-exec/src/test/java/org/apache/drill/exec/store/{ => easy}/json/parser/TestJsonParserArrays.java (82%)
rename exec/java-exec/src/test/java/org/apache/drill/exec/store/{ => easy}/json/parser/TestJsonParserBasics.java (74%)
rename exec/java-exec/src/test/java/org/apache/drill/exec/store/{ => easy}/json/parser/TestJsonParserErrors.java (94%)
rename exec/java-exec/src/test/java/org/apache/drill/exec/store/{ => easy}/json/parser/TestJsonParserObjects.java (94%)
create mode 100644 exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/parser/TestJsonParserUnknowns.java
create mode 100644 logical/src/main/java/org/apache/drill/common/FunctionNames.java
[drill] 01/02: DRILL-7634: Rollup of code cleanup changes
Posted by vo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
volodymyr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit 2b7ab27933166257449161d7f8e0387ee3e142a4
Author: Paul Rogers <pa...@yahoo.com>
AuthorDate: Sun Feb 9 18:24:47 2020 -0800
DRILL-7634: Rollup of code cleanup changes
Collection of code cleanup changes. The most significant
is to create constants for function names.
closes #2020
---
.../exceptions/DrillConfigurationException.java | 8 +-
.../apache/drill/common/exceptions/DrillError.java | 8 +-
.../drill/common/exceptions/DrillIOException.java | 8 +-
.../exceptions/RetryAfterSpillException.java | 5 +-
.../common/exceptions/UserRemoteException.java | 6 +-
.../java/org/apache/drill/common/types/Types.java | 5 -
.../store/mapr/db/binary/MapRDBFilterBuilder.java | 33 +--
.../mapr/db/json/CompareFunctionsProcessor.java | 21 +-
.../store/mapr/db/json/JsonConditionBuilder.java | 82 +++---
.../exec/store/mapr/db/json/JsonScanSpec.java | 6 +-
.../store/mapr/db/json/OjaiFunctionsProcessor.java | 14 +-
.../store/hbase/CompareFunctionsProcessor.java | 67 ++---
.../drill/exec/store/hbase/HBaseFilterBuilder.java | 36 +--
.../drill/exec/store/hbase/HBaseRegexParser.java | 6 +-
.../drill/exec/expr/fn/HiveFunctionRegistry.java | 22 +-
.../drill/exec/store/kafka/KafkaNodeProcessor.java | 27 +-
.../exec/store/kafka/KafkaPartitionScanSpec.java | 11 +-
.../store/kafka/KafkaPartitionScanSpecBuilder.java | 28 +-
.../store/mongo/MongoCompareFunctionProcessor.java | 24 +-
.../drill/exec/store/mongo/MongoFilterBuilder.java | 40 ++-
.../codegen/templates/ComparisonFunctions.java | 310 ++++++++++-----------
.../apache/drill/exec/expr/BooleanPredicate.java | 9 +-
.../org/apache/drill/exec/expr/CloneVisitor.java | 2 +-
.../drill/exec/expr/ComparisonPredicate.java | 14 +-
.../apache/drill/exec/expr/EvaluationVisitor.java | 14 +-
.../exec/expr/ExpressionTreeMaterializer.java | 33 ++-
.../org/apache/drill/exec/expr/FilterBuilder.java | 34 +--
.../apache/drill/exec/expr/StatisticsProvider.java | 3 +-
.../drill/exec/expr/fn/DrillAggFuncHolder.java | 75 +++--
.../exec/expr/fn/FunctionGenerationHelper.java | 22 +-
.../expr/fn/FunctionImplementationRegistry.java | 4 +-
.../drill/exec/expr/fn/impl/BitFunctions.java | 14 +-
.../drill/exec/expr/fn/impl/StringFunctions.java | 6 +-
.../expr/fn/interpreter/InterpreterEvaluator.java | 44 ++-
.../drill/exec/ops/ExecutorFragmentContext.java | 2 +-
.../exec/physical/base/AbstractDbGroupScan.java | 1 -
.../exec/physical/base/AbstractDbSubScan.java | 2 +-
.../exec/physical/base/AbstractGroupScan.java | 6 +-
.../base/AbstractGroupScanWithMetadata.java | 4 +-
.../physical/base/AbstractPhysicalVisitor.java | 25 +-
.../drill/exec/physical/base/FragmentLeaf.java | 4 +-
.../org/apache/drill/exec/physical/base/Leaf.java | 6 +-
.../apache/drill/exec/physical/base/ScanStats.java | 7 +-
.../exec/physical/config/SingleMergeExchange.java | 5 +-
.../exec/physical/impl/aggregate/HashAggBatch.java | 6 +-
.../impl/scan/project/ResolvedMapColumn.java | 1 -
.../exec/physical/impl/window/WindowFunction.java | 39 ++-
.../physical/resultSet/impl/BuildFromSchema.java | 3 +-
.../physical/resultSet/impl/ColumnBuilder.java | 2 +-
.../exec/physical/resultSet/impl/ColumnState.java | 2 +-
.../exec/planner/AbstractPartitionDescriptor.java | 18 +-
.../exec/planner/DFSFilePartitionLocation.java | 4 +-
.../planner/FileSystemPartitionDescriptor.java | 24 +-
.../drill/exec/planner/PartitionDescriptor.java | 8 +-
.../apache/drill/exec/planner/PlannerPhase.java | 27 +-
.../drill/exec/planner/common/DrillRelOptUtil.java | 17 +-
.../exec/planner/common/DrillScanRelBase.java | 2 +-
.../planner/fragment/DistributionAffinity.java | 22 +-
.../fragment/HardAffinityFragmentParallelizer.java | 10 +-
.../drill/exec/planner/fragment/Materializer.java | 18 +-
.../exec/planner/fragment/MemoryCalculator.java | 4 +-
.../drill/exec/planner/fragment/PlanningSet.java | 13 +-
.../exec/planner/fragment/QueryParallelizer.java | 6 +-
.../exec/planner/fragment/SimpleParallelizer.java | 56 ++--
.../fragment/SoftAffinityFragmentParallelizer.java | 10 +-
.../apache/drill/exec/planner/fragment/Stats.java | 2 +-
.../drill/exec/planner/fragment/Wrapper.java | 61 ++--
.../fragment/contrib/SplittingParallelizer.java | 51 ++--
.../drill/exec/planner/index/PathInExpr.java | 30 +-
.../generators/AbstractIndexPlanGenerator.java | 19 +-
.../drill/exec/planner/logical/DrillOptiq.java | 48 ++--
.../planner/logical/DrillPushLimitToScanRule.java | 6 +-
.../drill/exec/planner/logical/ExprHelper.java | 8 +-
.../planner/physical/DrillDistributionTrait.java | 3 +-
.../physical/DrillDistributionTraitDef.java | 3 +-
.../drill/exec/planner/physical/ExchangePrel.java | 5 +-
.../drill/exec/planner/physical/HashPrelUtil.java | 5 +-
.../drill/exec/planner/physical/JoinPrel.java | 19 +-
.../planner/physical/MetadataControllerPrel.java | 3 +-
.../exec/planner/physical/NestedLoopJoinPrule.java | 2 -
.../exec/planner/physical/PhysicalPlanCreator.java | 8 +-
.../apache/drill/exec/planner/physical/Prel.java | 5 +-
.../drill/exec/planner/physical/ScanPrel.java | 16 +-
.../drill/exec/planner/physical/ScanPrule.java | 2 +-
.../planner/physical/SingleMergeExchangePrel.java | 5 +-
.../drill/exec/planner/physical/SortPrel.java | 2 +-
.../drill/exec/planner/physical/SortPrule.java | 14 +-
.../exec/planner/physical/SubsetTransformer.java | 22 +-
.../drill/exec/planner/physical/TopNPrel.java | 4 +-
.../exec/planner/physical/UnionExchangePrel.java | 2 +-
.../visitor/ExcessiveExchangeIdentifier.java | 10 +-
.../physical/visitor/MemoryEstimationVisitor.java | 9 +-
.../drill/exec/planner/sql/TypeInferenceUtils.java | 7 +-
.../planner/sql/handlers/DefaultSqlHandler.java | 20 +-
.../exec/planner/sql/handlers/ExplainHandler.java | 31 ++-
.../exec/resolver/DefaultFunctionResolver.java | 2 +-
.../drill/exec/resolver/ExactFunctionResolver.java | 2 +-
.../drill/exec/store/dfs/FileSystemPlugin.java | 15 +-
.../apache/drill/exec/store/dfs/FormatPlugin.java | 16 +-
.../exec/store/dfs/easy/EasyFormatPlugin.java | 14 +-
.../drill/exec/store/dfs/easy/EasyGroupScan.java | 9 +-
.../drill/exec/store/ischema/InfoSchemaFilter.java | 20 +-
.../store/ischema/InfoSchemaFilterBuilder.java | 37 +--
.../exec/store/parquet/FilterEvaluatorUtils.java | 10 +-
.../exec/store/schedule/AssignmentCreator.java | 35 ++-
.../drill/exec/work/metadata/MetadataProvider.java | 13 +-
.../impl/TestResultSetLoaderMapArray.java | 1 -
.../impl/TestResultSetLoaderRepeatedList.java | 31 +--
.../easy/text/compliant/TestCsvWithHeaders.java | 4 +-
.../store/parquet/TestParquetFilterPushDown.java | 6 +-
.../drill/exec/record/MaterializedField.java | 32 +--
.../record/metadata/AbstractColumnMetadata.java | 5 +-
.../exec/record/metadata/PropertyAccessor.java | 2 +-
.../accessor/reader/AbstractScalarReader.java | 2 +-
.../accessor/writer/AbstractTupleWriter.java | 12 +-
.../vector/accessor/writer/ObjectArrayWriter.java | 2 -
.../org/apache/drill/common/FunctionNames.java | 54 ++++
.../drill/common/expression/BooleanOperator.java | 8 +-
.../common/expression/ExpressionStringBuilder.java | 4 +-
.../drill/common/expression/FunctionCall.java | 21 +-
.../common/expression/FunctionCallFactory.java | 56 ++--
.../apache/drill/common/expression/SchemaPath.java | 3 +-
.../drill/common/expression/TypedFieldExpr.java | 7 +-
.../drill/common/expression/ValueExpressions.java | 96 +++----
.../expression/visitors/AggregateChecker.java | 3 +-
.../visitors/ConditionalExprOptimizer.java | 2 +-
.../expression/visitors/ConstantChecker.java | 2 +-
.../expression/visitors/ExpressionValidator.java | 3 +-
.../common/parser/LogicalExpressionParser.java | 6 +-
129 files changed, 1200 insertions(+), 1152 deletions(-)
diff --git a/common/src/main/java/org/apache/drill/common/exceptions/DrillConfigurationException.java b/common/src/main/java/org/apache/drill/common/exceptions/DrillConfigurationException.java
index 3f88e28..dbe67c5 100644
--- a/common/src/main/java/org/apache/drill/common/exceptions/DrillConfigurationException.java
+++ b/common/src/main/java/org/apache/drill/common/exceptions/DrillConfigurationException.java
@@ -17,10 +17,10 @@
*/
package org.apache.drill.common.exceptions;
+@SuppressWarnings("serial")
public class DrillConfigurationException extends DrillException {
- public DrillConfigurationException() {
- super();
- }
+
+ public DrillConfigurationException() { }
public DrillConfigurationException(String message, Throwable cause, boolean enableSuppression,
boolean writableStackTrace) {
@@ -38,6 +38,4 @@ public class DrillConfigurationException extends DrillException {
public DrillConfigurationException(Throwable cause) {
super(cause);
}
-
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillConfigurationException.class);
}
diff --git a/common/src/main/java/org/apache/drill/common/exceptions/DrillError.java b/common/src/main/java/org/apache/drill/common/exceptions/DrillError.java
index 202d2e2..decaf23 100644
--- a/common/src/main/java/org/apache/drill/common/exceptions/DrillError.java
+++ b/common/src/main/java/org/apache/drill/common/exceptions/DrillError.java
@@ -17,12 +17,10 @@
*/
package org.apache.drill.common.exceptions;
+@SuppressWarnings("serial")
public class DrillError extends Error {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillError.class);
- public DrillError() {
- super();
- }
+ public DrillError() { }
public DrillError(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
super(message, cause, enableSuppression, writableStackTrace);
@@ -39,6 +37,4 @@ public class DrillError extends Error {
public DrillError(Throwable cause) {
super(cause);
}
-
-
}
diff --git a/common/src/main/java/org/apache/drill/common/exceptions/DrillIOException.java b/common/src/main/java/org/apache/drill/common/exceptions/DrillIOException.java
index ef422ae..7859d9a 100644
--- a/common/src/main/java/org/apache/drill/common/exceptions/DrillIOException.java
+++ b/common/src/main/java/org/apache/drill/common/exceptions/DrillIOException.java
@@ -19,12 +19,10 @@ package org.apache.drill.common.exceptions;
import java.io.IOException;
+@SuppressWarnings("serial")
public class DrillIOException extends IOException{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillIOException.class);
- public DrillIOException() {
- super();
- }
+ public DrillIOException() { }
public DrillIOException(String message, Throwable cause) {
super(message, cause);
@@ -37,6 +35,4 @@ public class DrillIOException extends IOException{
public DrillIOException(Throwable cause) {
super(cause);
}
-
-
}
diff --git a/common/src/main/java/org/apache/drill/common/exceptions/RetryAfterSpillException.java b/common/src/main/java/org/apache/drill/common/exceptions/RetryAfterSpillException.java
index 3bda97f..36a2524 100644
--- a/common/src/main/java/org/apache/drill/common/exceptions/RetryAfterSpillException.java
+++ b/common/src/main/java/org/apache/drill/common/exceptions/RetryAfterSpillException.java
@@ -18,9 +18,10 @@
package org.apache.drill.common.exceptions;
/**
- * A special exception to be caught by caller, who is supposed to free memory by spilling and try again
- *
+ * Special exception to be caught by caller, who is supposed to free memory by
+ * spilling and try again
*/
+@SuppressWarnings("serial")
public class RetryAfterSpillException extends Exception {
}
diff --git a/common/src/main/java/org/apache/drill/common/exceptions/UserRemoteException.java b/common/src/main/java/org/apache/drill/common/exceptions/UserRemoteException.java
index 4835b29..ea7f61f 100644
--- a/common/src/main/java/org/apache/drill/common/exceptions/UserRemoteException.java
+++ b/common/src/main/java/org/apache/drill/common/exceptions/UserRemoteException.java
@@ -22,9 +22,11 @@ import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
import static org.apache.drill.common.util.DrillExceptionUtil.getThrowable;
/**
- * Wraps a DrillPBError object so we don't need to rebuilt it multiple times when sending it to the client. It also
- * gives access to the original exception className and message.
+ * Wraps a DrillPBError object so we don't need to rebuilt it multiple times
+ * when sending it to the client. It also gives access to the original exception
+ * className and message.
*/
+@SuppressWarnings("serial")
public class UserRemoteException extends UserException {
private final DrillPBError error;
diff --git a/common/src/main/java/org/apache/drill/common/types/Types.java b/common/src/main/java/org/apache/drill/common/types/Types.java
index 596199a..b23fd6a 100644
--- a/common/src/main/java/org/apache/drill/common/types/Types.java
+++ b/common/src/main/java/org/apache/drill/common/types/Types.java
@@ -844,7 +844,6 @@ public class Types {
* @return true if the two types have the same minor type, mode,
* precision and scale
*/
-
public static boolean isSameType(MajorType type1, MajorType type2) {
return isSameTypeAndMode(type1, type2) &&
type1.getScale() == type2.getScale() &&
@@ -870,7 +869,6 @@ public class Types {
* But, unset fields are equivalent to 0. Can't use the protobuf-provided
* isEquals() which treats set and unset fields as different.
*/
-
public static boolean isEquivalent(MajorType type1, MajorType type2) {
if (!isSameType(type1, type2)) {
@@ -878,7 +876,6 @@ public class Types {
}
// Subtypes are only for unions and are seldom used.
-
if (type1.getMinorType() != MinorType.UNION) {
return true;
}
@@ -896,7 +893,6 @@ public class Types {
}
// Now it gets slow because subtype lists are not ordered.
-
final List<MinorType> copy1 = new ArrayList<>(subtypes1);
final List<MinorType> copy2 = new ArrayList<>(subtypes2);
Collections.sort(copy1);
@@ -913,7 +909,6 @@ public class Types {
* @return string key to use for this type in a union vector type
* map
*/
-
public static String typeKey(MinorType type) {
return type.name().toLowerCase();
}
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MapRDBFilterBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MapRDBFilterBuilder.java
index cdef510..f94e0a8 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MapRDBFilterBuilder.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/binary/MapRDBFilterBuilder.java
@@ -18,7 +18,9 @@
package org.apache.drill.exec.store.mapr.db.binary;
import java.util.Arrays;
+import java.util.List;
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.common.expression.BooleanOperator;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.LogicalExpression;
@@ -39,7 +41,6 @@ import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.drill.shaded.guava.com.google.common.base.Charsets;
-import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList;
public class MapRDBFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void, RuntimeException> implements DrillHBaseConstants {
@@ -59,7 +60,7 @@ public class MapRDBFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void
public HBaseScanSpec parseTree() {
HBaseScanSpec parsedSpec = le.accept(this, null);
if (parsedSpec != null) {
- parsedSpec = mergeScanSpecs("booleanAnd", this.groupScan.getHBaseScanSpec(), parsedSpec);
+ parsedSpec = mergeScanSpecs(FunctionNames.AND, this.groupScan.getHBaseScanSpec(), parsedSpec);
/*
* If RowFilter is THE filter attached to the scan specification,
* remove it since its effect is also achieved through startRow and stopRow.
@@ -93,7 +94,7 @@ public class MapRDBFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void
public HBaseScanSpec visitFunctionCall(FunctionCall call, Void value) throws RuntimeException {
HBaseScanSpec nodeScanSpec = null;
String functionName = call.getName();
- ImmutableList<LogicalExpression> args = call.args;
+ List<LogicalExpression> args = call.args();
if (MaprDBCompareFunctionsProcessor.isCompareFunction(functionName)) {
/*
@@ -111,8 +112,8 @@ public class MapRDBFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void
}
} else {
switch (functionName) {
- case "booleanAnd":
- case "booleanOr":
+ case FunctionNames.AND:
+ case FunctionNames.OR:
HBaseScanSpec firstScanSpec = args.get(0).accept(this, null);
for (int i = 1; i < args.size(); ++i) {
HBaseScanSpec nextScanSpec = args.get(i).accept(this, null);
@@ -120,7 +121,7 @@ public class MapRDBFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void
nodeScanSpec = mergeScanSpecs(functionName, firstScanSpec, nextScanSpec);
} else {
allExpressionsConverted = false;
- if ("booleanAnd".equals(functionName)) {
+ if (FunctionNames.AND.equals(functionName)) {
nodeScanSpec = firstScanSpec == null ? nextScanSpec : firstScanSpec;
}
}
@@ -143,12 +144,12 @@ public class MapRDBFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void
byte[] stopRow = HConstants.EMPTY_END_ROW;
switch (functionName) {
- case "booleanAnd":
+ case FunctionNames.AND:
newFilter = HBaseUtils.andFilterAtIndex(leftScanSpec.getFilter(), -1, rightScanSpec.getFilter()); //HBaseUtils.LAST_FILTER
startRow = HBaseUtils.maxOfStartRows(leftScanSpec.getStartRow(), rightScanSpec.getStartRow());
stopRow = HBaseUtils.minOfStopRows(leftScanSpec.getStopRow(), rightScanSpec.getStopRow());
break;
- case "booleanOr":
+ case FunctionNames.OR:
newFilter = HBaseUtils.orFilterAtIndex(leftScanSpec.getFilter(), -1, rightScanSpec.getFilter()); //HBaseUtils.LAST_FILTER
startRow = HBaseUtils.minOfStartRows(leftScanSpec.getStartRow(), rightScanSpec.getStartRow());
stopRow = HBaseUtils.maxOfStopRows(leftScanSpec.getStopRow(), rightScanSpec.getStopRow());
@@ -184,7 +185,7 @@ public class MapRDBFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void
byte[] startRow = HConstants.EMPTY_START_ROW;
byte[] stopRow = HConstants.EMPTY_END_ROW;
switch (functionName) {
- case "equal":
+ case FunctionNames.EQ:
compareOp = CompareOp.EQUAL;
if (isRowKey) {
startRow = fieldValue;
@@ -193,10 +194,10 @@ public class MapRDBFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void
compareOp = CompareOp.EQUAL;
}
break;
- case "not_equal":
+ case FunctionNames.NE:
compareOp = CompareOp.NOT_EQUAL;
break;
- case "greater_than_or_equal_to":
+ case FunctionNames.GE:
if (sortOrderAscending) {
compareOp = CompareOp.GREATER_OR_EQUAL;
if (isRowKey) {
@@ -210,7 +211,7 @@ public class MapRDBFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void
}
}
break;
- case "greater_than":
+ case FunctionNames.GT:
if (sortOrderAscending) {
compareOp = CompareOp.GREATER;
if (isRowKey) {
@@ -224,7 +225,7 @@ public class MapRDBFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void
}
}
break;
- case "less_than_or_equal_to":
+ case FunctionNames.LE:
if (sortOrderAscending) {
compareOp = CompareOp.LESS_OR_EQUAL;
if (isRowKey) {
@@ -238,7 +239,7 @@ public class MapRDBFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void
}
}
break;
- case "less_than":
+ case FunctionNames.LT:
if (sortOrderAscending) {
compareOp = CompareOp.LESS;
if (isRowKey) {
@@ -252,7 +253,7 @@ public class MapRDBFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void
}
}
break;
- case "isnull":
+ case FunctionNames.IS_NULL:
case "isNull":
case "is null":
if (isRowKey) {
@@ -262,7 +263,7 @@ public class MapRDBFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void
compareOp = CompareOp.EQUAL;
comparator = new NullComparator();
break;
- case "isnotnull":
+ case FunctionNames.IS_NOT_NULL:
case "isNotNull":
case "is not null":
if (isRowKey) {
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/CompareFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/CompareFunctionsProcessor.java
index 0285384..204ddae 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/CompareFunctionsProcessor.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/CompareFunctionsProcessor.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.store.mapr.db.json;
import static com.mapr.db.rowcol.DBValueBuilderImpl.KeyValueBuilder;
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
@@ -111,8 +112,8 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr
private static CompareFunctionsProcessor processWithEvaluator(FunctionCall call, CompareFunctionsProcessor evaluator) {
String functionName = call.getName();
- LogicalExpression nameArg = call.args.get(0);
- LogicalExpression valueArg = call.args.size() >= 2 ? call.args.get(1) : null;
+ LogicalExpression nameArg = call.arg(0);
+ LogicalExpression valueArg = call.argCount() >= 2 ? call.arg(1) : null;
if (VALUE_EXPRESSION_CLASSES.contains(nameArg.getClass())) {
LogicalExpression swapArg = valueArg;
@@ -258,20 +259,20 @@ class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, LogicalExpr
ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
COMPARE_FUNCTIONS_TRANSPOSE_MAP = builder
// unary functions
- .put("isnotnull", "isnotnull")
+ .put(FunctionNames.IS_NOT_NULL, FunctionNames.IS_NOT_NULL)
.put("isNotNull", "isNotNull")
.put("is not null", "is not null")
- .put("isnull", "isnull")
+ .put(FunctionNames.IS_NULL, FunctionNames.IS_NULL)
.put("isNull", "isNull")
.put("is null", "is null")
// binary functions
.put("like", "like")
- .put("equal", "equal")
- .put("not_equal", "not_equal")
- .put("greater_than_or_equal_to", "less_than_or_equal_to")
- .put("greater_than", "less_than")
- .put("less_than_or_equal_to", "greater_than_or_equal_to")
- .put("less_than", "greater_than")
+ .put(FunctionNames.EQ, FunctionNames.EQ)
+ .put(FunctionNames.NE, FunctionNames.NE)
+ .put(FunctionNames.GE, FunctionNames.LE)
+ .put(FunctionNames.GT, FunctionNames.LT)
+ .put(FunctionNames.LE, FunctionNames.GE)
+ .put(FunctionNames.LT, FunctionNames.GT)
.build();
}
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonConditionBuilder.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonConditionBuilder.java
index dc531ac..1e93cf7 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonConditionBuilder.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonConditionBuilder.java
@@ -17,6 +17,9 @@
*/
package org.apache.drill.exec.store.mapr.db.json;
+import java.util.List;
+
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.common.expression.BooleanOperator;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.LogicalExpression;
@@ -27,7 +30,6 @@ import org.ojai.Value;
import org.ojai.store.QueryCondition;
import org.ojai.store.QueryCondition.Op;
-import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList;
import com.mapr.db.impl.MapRDBImpl;
public class JsonConditionBuilder extends AbstractExprVisitor<JsonScanSpec, Void, RuntimeException> implements DrillHBaseConstants {
@@ -47,7 +49,7 @@ public class JsonConditionBuilder extends AbstractExprVisitor<JsonScanSpec, Void
public JsonScanSpec parseTree() {
JsonScanSpec parsedSpec = le.accept(this, null);
if (parsedSpec != null) {
- parsedSpec.mergeScanSpec("booleanAnd", this.groupScan.getScanSpec());
+ parsedSpec.mergeScanSpec(FunctionNames.AND, this.groupScan.getScanSpec());
}
return parsedSpec;
}
@@ -81,7 +83,7 @@ public class JsonConditionBuilder extends AbstractExprVisitor<JsonScanSpec, Void
public JsonScanSpec visitFunctionCall(FunctionCall call, Void value) throws RuntimeException {
JsonScanSpec nodeScanSpec = null;
String functionName = call.getName();
- ImmutableList<LogicalExpression> args = call.args;
+ List<LogicalExpression> args = call.args();
if (CompareFunctionsProcessor.isCompareFunction(functionName)) {
CompareFunctionsProcessor processor;
@@ -95,34 +97,34 @@ public class JsonConditionBuilder extends AbstractExprVisitor<JsonScanSpec, Void
}
} else {
switch(functionName) {
- case "booleanAnd":
- case "booleanOr":
- nodeScanSpec = args.get(0).accept(this, null);
- for (int i = 1; i < args.size(); ++i) {
- JsonScanSpec nextScanSpec = args.get(i).accept(this, null);
- if (nodeScanSpec != null && nextScanSpec != null) {
- nodeScanSpec.mergeScanSpec(functionName, nextScanSpec);
- } else {
- allExpressionsConverted = false;
- if ("booleanAnd".equals(functionName)) {
- nodeScanSpec = nodeScanSpec == null ? nextScanSpec : nodeScanSpec;
+ case FunctionNames.AND:
+ case FunctionNames.OR:
+ nodeScanSpec = args.get(0).accept(this, null);
+ for (int i = 1; i < args.size(); ++i) {
+ JsonScanSpec nextScanSpec = args.get(i).accept(this, null);
+ if (nodeScanSpec != null && nextScanSpec != null) {
+ nodeScanSpec.mergeScanSpec(functionName, nextScanSpec);
+ } else {
+ allExpressionsConverted = false;
+ if (FunctionNames.AND.equals(functionName)) {
+ nodeScanSpec = nodeScanSpec == null ? nextScanSpec : nodeScanSpec;
+ }
}
}
+ break;
+
+ case "ojai_sizeof":
+ case "ojai_typeof":
+ case "ojai_nottypeof":
+ case "ojai_matches":
+ case "ojai_notmatches":
+ case "ojai_condition": {
+ final OjaiFunctionsProcessor processor = OjaiFunctionsProcessor.process(call);
+ if (processor != null) {
+ return new JsonScanSpec(groupScan.getTableName(), groupScan.getIndexDesc(),
+ processor.getCondition());
+ }
}
- break;
-
- case "ojai_sizeof":
- case "ojai_typeof":
- case "ojai_nottypeof":
- case "ojai_matches":
- case "ojai_notmatches":
- case "ojai_condition": {
- final OjaiFunctionsProcessor processor = OjaiFunctionsProcessor.process(call);
- if (processor != null) {
- return new JsonScanSpec(groupScan.getTableName(), groupScan.getIndexDesc(),
- processor.getCondition());
- }
- }
}
}
@@ -191,61 +193,61 @@ public class JsonConditionBuilder extends AbstractExprVisitor<JsonScanSpec, Void
QueryCondition cond = null;
switch (functionName) {
- case "equal":
+ case FunctionNames.EQ:
cond = MapRDBImpl.newCondition();
setIsCondition(cond, fieldPath, Op.EQUAL, fieldValue);
break;
- case "not_equal":
+ case FunctionNames.NE:
cond = MapRDBImpl.newCondition();
setIsCondition(cond, fieldPath, Op.NOT_EQUAL, fieldValue);
break;
- case "less_than":
+ case FunctionNames.LT:
cond = MapRDBImpl.newCondition();
setIsCondition(cond, fieldPath, Op.LESS, fieldValue);
break;
- case "less_than_or_equal_to":
+ case FunctionNames.LE:
cond = MapRDBImpl.newCondition();
setIsCondition(cond, fieldPath, Op.LESS_OR_EQUAL, fieldValue);
break;
- case "greater_than":
+ case FunctionNames.GT:
cond = MapRDBImpl.newCondition();
setIsCondition(cond, fieldPath, Op.GREATER, fieldValue);
break;
- case "greater_than_or_equal_to":
+ case FunctionNames.GE:
cond = MapRDBImpl.newCondition();
setIsCondition(cond, fieldPath, Op.GREATER_OR_EQUAL, fieldValue);
break;
- case "isnull":
+ case FunctionNames.IS_NULL:
// 'field is null' should be transformed to 'field not exists OR typeof(field) = NULL'
QueryCondition orCond = MapRDBImpl.newCondition().or();
cond = orCond.notExists(fieldPath).typeOf(fieldPath, Value.Type.NULL).close();
break;
- case "isnotnull":
+ case FunctionNames.IS_NOT_NULL:
// 'field is not null should be transformed to 'field exists AND typeof(field) != NULL'
QueryCondition andCond = MapRDBImpl.newCondition().and();
cond = andCond.exists(fieldPath).notTypeOf(fieldPath, Value.Type.NULL).close();
break;
- case "istrue":
+ case FunctionNames.IS_TRUE:
cond = MapRDBImpl.newCondition().is(fieldPath, Op.EQUAL, true);
break;
- case "isnotfalse":
+ case FunctionNames.IS_NOT_FALSE:
cond = MapRDBImpl.newCondition().is(fieldPath, Op.NOT_EQUAL, false);
break;
- case "isfalse":
+ case FunctionNames.IS_FALSE:
cond = MapRDBImpl.newCondition().is(fieldPath, Op.EQUAL, false);
break;
- case "isnottrue":
+ case FunctionNames.IS_NOT_TRUE:
cond = MapRDBImpl.newCondition().is(fieldPath, Op.NOT_EQUAL, true);
break;
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonScanSpec.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonScanSpec.java
index 8d06f17..1a9878f 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonScanSpec.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/JsonScanSpec.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.store.mapr.db.json;
import java.nio.ByteBuffer;
import java.util.List;
+import org.apache.drill.common.FunctionNames;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HConstants;
import org.ojai.store.QueryCondition;
@@ -119,10 +120,10 @@ public class JsonScanSpec {
if (this.condition != null && scanSpec.getCondition() != null) {
QueryCondition newCond = MapRDBImpl.newCondition();
switch (functionName) {
- case "booleanAnd":
+ case FunctionNames.AND:
newCond.and();
break;
- case "booleanOr":
+ case FunctionNames.OR:
newCond.or();
break;
default:
@@ -148,5 +149,4 @@ public class JsonScanSpec {
+ fidInfo
+ "]";
}
-
}
diff --git a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/OjaiFunctionsProcessor.java b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/OjaiFunctionsProcessor.java
index d07c97b..151e21d 100644
--- a/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/OjaiFunctionsProcessor.java
+++ b/contrib/format-maprdb/src/main/java/org/apache/drill/exec/store/mapr/db/json/OjaiFunctionsProcessor.java
@@ -139,12 +139,12 @@ class OjaiFunctionsProcessor extends AbstractExprVisitor<Void, Void, RuntimeExce
@Override
public Void visitFunctionCall(FunctionCall call, Void v) throws RuntimeException {
final String functionName = call.getName();
- final String fieldName = FieldPathHelper.schemaPath2FieldPath(getSchemaPathArg(call.args.get(0))).asPathString();
+ final String fieldName = FieldPathHelper.schemaPath2FieldPath(getSchemaPathArg(call.arg(0))).asPathString();
switch(functionName) {
case "ojai_sizeof": {
// ojai_sizeof(field, "<rel-op>", <int-value>)
- final String relOp = getStringArg(call.args.get(1));
- final long size = getLongArg(call.args.get(2));
+ final String relOp = getStringArg(call.arg(1));
+ final long size = getLongArg(call.arg(2));
queryCond = MapRDBImpl.newCondition()
.sizeOf(fieldName, STRING_TO_RELOP.get(relOp), size)
.build();
@@ -154,7 +154,7 @@ class OjaiFunctionsProcessor extends AbstractExprVisitor<Void, Void, RuntimeExce
case "ojai_typeof":
case "ojai_nottypeof": {
// ojai_[not]typeof(field, <type-code>);
- final int typeCode = getIntArg(call.args.get(1));
+ final int typeCode = getIntArg(call.arg(1));
final Value.Type typeValue = Value.Type.valueOf(typeCode);
queryCond = MapRDBImpl.newCondition();
if (functionName.equals("ojai_typeof")) {
@@ -169,8 +169,7 @@ class OjaiFunctionsProcessor extends AbstractExprVisitor<Void, Void, RuntimeExce
case "ojai_matches":
case "ojai_notmatches": {
// ojai_[not]matches(field, <regex>);
- final SchemaPath schemaPath = getSchemaPathArg(call.args.get(0));
- final String regex = getStringArg(call.args.get(1));
+ final String regex = getStringArg(call.arg(1));
if (functionName.equals("ojai_matches")) {
queryCond = MapRDBImpl.newCondition()
.matches(fieldName, regex);
@@ -184,8 +183,7 @@ class OjaiFunctionsProcessor extends AbstractExprVisitor<Void, Void, RuntimeExce
case "ojai_condition": {
// ojai_condition(field, <serialized-condition>);
- final SchemaPath schemaPath = getSchemaPathArg(call.args.get(0));
- final String condString = getStringArg(call.args.get(1));
+ final String condString = getStringArg(call.arg(1));
final byte[] condBytes = Base64.decodeBase64(condString);
final ByteBuffer condBuffer = ByteBuffer.wrap(condBytes);
queryCond = ConditionImpl.parseFrom(condBuffer);
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/CompareFunctionsProcessor.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/CompareFunctionsProcessor.java
index 25c65f0..6adbb44 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/CompareFunctionsProcessor.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/CompareFunctionsProcessor.java
@@ -25,6 +25,7 @@ import java.nio.ByteOrder;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.common.expression.CastExpression;
import org.apache.drill.common.expression.ConvertExpression;
import org.apache.drill.common.expression.FunctionCall;
@@ -59,7 +60,7 @@ public class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, Logi
private byte[] value;
private boolean success;
- private boolean isEqualityFn;
+ private final boolean isEqualityFn;
private SchemaPath path;
private String functionName;
private boolean sortOrderAscending;
@@ -86,8 +87,8 @@ public class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, Logi
protected static <T extends CompareFunctionsProcessor> T createFunctionsProcessorInstanceInternal(FunctionCall call,
boolean nullComparatorSupported,
T evaluator) {
- LogicalExpression nameArg = call.args.get(0);
- LogicalExpression valueArg = call.args.size() >= 2 ? call.args.get(1) : null;
+ LogicalExpression nameArg = call.arg(0);
+ LogicalExpression valueArg = call.argCount() >= 2 ? call.arg(1) : null;
if (valueArg != null) { // binary function
if (VALUE_EXPRESSION_CLASSES.contains(nameArg.getClass())) {
LogicalExpression swapArg = valueArg;
@@ -96,7 +97,7 @@ public class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, Logi
evaluator.setFunctionName(COMPARE_FUNCTIONS_TRANSPOSE_MAP.get(evaluator.getFunctionName()));
}
evaluator.setSuccess(nameArg.accept(evaluator, valueArg));
- } else if (nullComparatorSupported && call.args.get(0) instanceof SchemaPath) {
+ } else if (nullComparatorSupported && call.arg(0) instanceof SchemaPath) {
evaluator.setSuccess(true);
evaluator.setPath((SchemaPath) nameArg);
}
@@ -198,9 +199,9 @@ public class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, Logi
return false;
}
- LogicalExpression nameArg = call.args.get(0);
- LogicalExpression valueArg1 = call.args.size() >= 2 ? call.args.get(1) : null;
- LogicalExpression valueArg2 = call.args.size() >= 3 ? call.args.get(2) : null;
+ LogicalExpression nameArg = call.arg(0);
+ LogicalExpression valueArg1 = call.argCount() >= 2 ? call.arg(1) : null;
+ LogicalExpression valueArg2 = call.argCount() >= 3 ? call.arg(2) : null;
if (!(nameArg instanceof SchemaPath)
|| (valueArg1 == null) || !(valueArg1 instanceof IntExpression)
@@ -381,21 +382,21 @@ public class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, Logi
// For TIME_EPOCH_BE/BIGINT_BE encoding, the operators that we push-down are =, <>, <, <=, >, >=
switch (functionName) {
- case "equal":
+ case FunctionNames.EQ:
rowKeyPrefixFilter = new PrefixFilter(ByteBuffer.allocate(4).putInt(val).array());
rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val).array();
rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val + 1).array();
return true;
- case "greater_than_or_equal_to":
+ case FunctionNames.GE:
rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val).array();
return true;
- case "greater_than":
+ case FunctionNames.GT:
rowKeyPrefixStartRow = ByteBuffer.allocate(4).putInt(val + 1).array();
return true;
- case "less_than_or_equal_to":
+ case FunctionNames.LE:
rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val + 1).array();
return true;
- case "less_than":
+ case FunctionNames.LT:
rowKeyPrefixStopRow = ByteBuffer.allocate(4).putInt(val).array();
return true;
}
@@ -437,21 +438,21 @@ public class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, Logi
// For TIME_EPOCH_BE/BIGINT_BE encoding, the operators that we push-down are =, <>, <, <=, >, >=
switch (functionName) {
- case "equal":
+ case FunctionNames.EQ:
rowKeyPrefixFilter = new PrefixFilter(ByteBuffer.allocate(8).putLong(val).array());
rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val).array();
rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val + 1).array();
return true;
- case "greater_than_or_equal_to":
+ case FunctionNames.GE:
rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val).array();
return true;
- case "greater_than":
+ case FunctionNames.GT:
rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(val + 1).array();
return true;
- case "less_than_or_equal_to":
+ case FunctionNames.LE:
rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val + 1).array();
return true;
- case "less_than":
+ case FunctionNames.LT:
rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(val).array();
return true;
}
@@ -472,25 +473,25 @@ public class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, Logi
long dateToSet;
// For DATE encoding, the operators that we push-down are =, <>, <, <=, >, >=
switch (functionName) {
- case "equal":
+ case FunctionNames.EQ:
long startDate = ((DateExpression) valueArg).getDate();
rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(startDate).array();
long stopDate = ((DateExpression) valueArg).getDate() + MILLISECONDS_IN_A_DAY;
rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(stopDate).array();
return true;
- case "greater_than_or_equal_to":
+ case FunctionNames.GE:
dateToSet = ((DateExpression) valueArg).getDate();
rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(dateToSet).array();
return true;
- case "greater_than":
+ case FunctionNames.GT:
dateToSet = ((DateExpression) valueArg).getDate() + MILLISECONDS_IN_A_DAY;
rowKeyPrefixStartRow = ByteBuffer.allocate(8).putLong(dateToSet).array();
return true;
- case "less_than_or_equal_to":
+ case FunctionNames.LE:
dateToSet = ((DateExpression) valueArg).getDate() + MILLISECONDS_IN_A_DAY;
rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(dateToSet).array();
return true;
- case "less_than":
+ case FunctionNames.LT:
dateToSet = ((DateExpression) valueArg).getDate();
rowKeyPrefixStopRow = ByteBuffer.allocate(8).putLong(dateToSet).array();
return true;
@@ -522,7 +523,9 @@ public class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, Logi
Matcher matcher = convertFromPattern.matcher(call.getName());
if (matcher.find()) {
// convert function call to ConvertExpression
- ConvertExpression convert = new ConvertExpression(ConvertExpression.CONVERT_FROM, matcher.group(1), call.args.get(0), call.getPosition());
+ ConvertExpression convert = new ConvertExpression(
+ ConvertExpression.CONVERT_FROM, matcher.group(1),
+ call.arg(0), call.getPosition());
return visitConvertExpression(convert, valueArg);
}
return false;
@@ -554,20 +557,20 @@ public class CompareFunctionsProcessor extends AbstractExprVisitor<Boolean, Logi
ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
COMPARE_FUNCTIONS_TRANSPOSE_MAP = builder
// unary functions
- .put("isnotnull", "isnotnull")
+ .put(FunctionNames.IS_NOT_NULL, FunctionNames.IS_NOT_NULL)
.put("isNotNull", "isNotNull")
.put("is not null", "is not null")
- .put("isnull", "isnull")
+ .put(FunctionNames.IS_NULL, FunctionNames.IS_NULL)
.put("isNull", "isNull")
.put("is null", "is null")
// binary functions
- .put("like", "like")
- .put("equal", "equal")
- .put("not_equal", "not_equal")
- .put("greater_than_or_equal_to", "less_than_or_equal_to")
- .put("greater_than", "less_than")
- .put("less_than_or_equal_to", "greater_than_or_equal_to")
- .put("less_than", "greater_than")
+ .put(FunctionNames.LIKE, FunctionNames.LIKE)
+ .put(FunctionNames.EQ, FunctionNames.EQ)
+ .put(FunctionNames.NE, FunctionNames.NE)
+ .put(FunctionNames.GE, FunctionNames.LE)
+ .put(FunctionNames.GT, FunctionNames.LT)
+ .put(FunctionNames.LE, FunctionNames.GE)
+ .put(FunctionNames.LT, FunctionNames.GT)
.build();
}
}
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java
index 21ac8c6..19e8758 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseFilterBuilder.java
@@ -18,7 +18,9 @@
package org.apache.drill.exec.store.hbase;
import java.util.Arrays;
+import java.util.List;
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.common.expression.BooleanOperator;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.LogicalExpression;
@@ -35,7 +37,6 @@ import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.drill.shaded.guava.com.google.common.base.Charsets;
-import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList;
public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void, RuntimeException> implements DrillHBaseConstants {
@@ -45,7 +46,7 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
private boolean allExpressionsConverted = true;
- private static Boolean nullComparatorSupported = null;
+ private static Boolean nullComparatorSupported;
HBaseFilterBuilder(HBaseGroupScan groupScan, LogicalExpression le) {
this.groupScan = groupScan;
@@ -55,7 +56,7 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
public HBaseScanSpec parseTree() {
HBaseScanSpec parsedSpec = le.accept(this, null);
if (parsedSpec != null) {
- parsedSpec = mergeScanSpecs("booleanAnd", this.groupScan.getHBaseScanSpec(), parsedSpec);
+ parsedSpec = mergeScanSpecs(FunctionNames.AND, this.groupScan.getHBaseScanSpec(), parsedSpec);
/*
* If RowFilter is THE filter attached to the scan specification,
* remove it since its effect is also achieved through startRow and stopRow.
@@ -88,7 +89,7 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
public HBaseScanSpec visitFunctionCall(FunctionCall call, Void value) throws RuntimeException {
HBaseScanSpec nodeScanSpec = null;
String functionName = call.getName();
- ImmutableList<LogicalExpression> args = call.args;
+ List<LogicalExpression> args = call.args();
if (CompareFunctionsProcessor.isCompareFunction(functionName)) {
/*
@@ -106,8 +107,8 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
}
} else {
switch (functionName) {
- case "booleanAnd":
- case "booleanOr":
+ case FunctionNames.AND:
+ case FunctionNames.OR:
HBaseScanSpec firstScanSpec = args.get(0).accept(this, null);
for (int i = 1; i < args.size(); ++i) {
HBaseScanSpec nextScanSpec = args.get(i).accept(this, null);
@@ -115,7 +116,7 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
nodeScanSpec = mergeScanSpecs(functionName, firstScanSpec, nextScanSpec);
} else {
allExpressionsConverted = false;
- if ("booleanAnd".equals(functionName)) {
+ if (FunctionNames.AND.equals(functionName)) {
nodeScanSpec = firstScanSpec == null ? nextScanSpec : firstScanSpec;
}
}
@@ -138,12 +139,12 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
byte[] stopRow = HConstants.EMPTY_END_ROW;
switch (functionName) {
- case "booleanAnd":
+ case FunctionNames.AND:
newFilter = HBaseUtils.andFilterAtIndex(leftScanSpec.filter, HBaseUtils.LAST_FILTER, rightScanSpec.filter);
startRow = HBaseUtils.maxOfStartRows(leftScanSpec.startRow, rightScanSpec.startRow);
stopRow = HBaseUtils.minOfStopRows(leftScanSpec.stopRow, rightScanSpec.stopRow);
break;
- case "booleanOr":
+ case FunctionNames.OR:
newFilter = HBaseUtils.orFilterAtIndex(leftScanSpec.filter, HBaseUtils.LAST_FILTER, rightScanSpec.filter);
startRow = HBaseUtils.minOfStartRows(leftScanSpec.startRow, rightScanSpec.startRow);
stopRow = HBaseUtils.maxOfStopRows(leftScanSpec.stopRow, rightScanSpec.stopRow);
@@ -188,10 +189,10 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
compareOp = CompareOp.EQUAL;
}
break;
- case "not_equal":
+ case FunctionNames.NE:
compareOp = CompareOp.NOT_EQUAL;
break;
- case "greater_than_or_equal_to":
+ case FunctionNames.GE:
if (sortOrderAscending) {
compareOp = CompareOp.GREATER_OR_EQUAL;
if (isRowKey) {
@@ -205,7 +206,7 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
}
}
break;
- case "greater_than":
+ case FunctionNames.GT:
if (sortOrderAscending) {
compareOp = CompareOp.GREATER;
if (isRowKey) {
@@ -219,7 +220,7 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
}
}
break;
- case "less_than_or_equal_to":
+ case FunctionNames.LE:
if (sortOrderAscending) {
compareOp = CompareOp.LESS_OR_EQUAL;
if (isRowKey) {
@@ -233,7 +234,7 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
}
}
break;
- case "less_than":
+ case FunctionNames.LT:
if (sortOrderAscending) {
compareOp = CompareOp.LESS;
if (isRowKey) {
@@ -247,7 +248,7 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
}
}
break;
- case "isnull":
+ case FunctionNames.IS_NULL:
case "isNull":
case "is null":
if (isRowKey) {
@@ -257,7 +258,7 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
compareOp = CompareOp.EQUAL;
comparator = new NullComparator();
break;
- case "isnotnull":
+ case FunctionNames.IS_NOT_NULL:
case "isNotNull":
case "is not null":
if (isRowKey) {
@@ -266,7 +267,7 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
compareOp = CompareOp.NOT_EQUAL;
comparator = new NullComparator();
break;
- case "like":
+ case FunctionNames.LIKE:
/*
* Convert the LIKE operand to Regular Expression pattern so that we can
* apply RegexStringComparator()
@@ -346,5 +347,4 @@ public class HBaseFilterBuilder extends AbstractExprVisitor<HBaseScanSpec, Void,
// else
return null;
}
-
}
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRegexParser.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRegexParser.java
index 99e7d58..3d8f9c1 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRegexParser.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseRegexParser.java
@@ -152,12 +152,12 @@ public class HBaseRegexParser {
}
private static String likeString(FunctionCall call) {
- return ((QuotedString) call.args.get(1)).value;
+ return ((QuotedString) call.arg(1)).value;
}
private static Character escapeString(FunctionCall call) {
- if (call.args.size() > 2) {
- return ((QuotedString) call.args.get(2)).value.charAt(0);
+ if (call.argCount() > 2) {
+ return ((QuotedString) call.arg(2)).value.charAt(0);
}
return null;
}
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java
index 23cbd58..99c999f 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java
@@ -42,18 +42,20 @@ import org.apache.drill.exec.planner.sql.HiveUDFOperator;
import org.apache.drill.exec.planner.sql.HiveUDFOperatorWithoutInference;
import org.apache.drill.exec.planner.sql.TypeInferenceUtils;
import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableMap;
+import org.apache.drill.shaded.guava.com.google.common.collect.Multimap;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.hive.ql.udf.UDFType;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.drill.shaded.guava.com.google.common.collect.ArrayListMultimap;
import org.apache.drill.shaded.guava.com.google.common.collect.Sets;
public class HiveFunctionRegistry implements PluggableFunctionRegistry {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveFunctionRegistry.class);
+ private static final Logger logger = LoggerFactory.getLogger(HiveFunctionRegistry.class);
/**
* Map for renaming UDFs. Keys of the map represent UDF names which should be replaced
@@ -65,9 +67,9 @@ public class HiveFunctionRegistry implements PluggableFunctionRegistry {
OracleSqlOperatorTable.TRANSLATE3.getName().toLowerCase())
.build();
- private ArrayListMultimap<String, Class<? extends GenericUDF>> methodsGenericUDF = ArrayListMultimap.create();
- private ArrayListMultimap<String, Class<? extends UDF>> methodsUDF = ArrayListMultimap.create();
- private HashSet<Class<?>> nonDeterministicUDFs = new HashSet<>();
+ private final Multimap<String, Class<? extends GenericUDF>> methodsGenericUDF = ArrayListMultimap.create();
+ private final Multimap<String, Class<? extends UDF>> methodsUDF = ArrayListMultimap.create();
+ private final Set<Class<?>> nonDeterministicUDFs = new HashSet<>();
/**
* Scan the classpath for implementation of GenericUDF/UDF interfaces,
@@ -116,7 +118,7 @@ public class HiveFunctionRegistry implements PluggableFunctionRegistry {
}
}
- private <I> void register(Class<? extends I> clazz, ArrayListMultimap<String, Class<? extends I>> methods) {
+ private <I> void register(Class<? extends I> clazz, Multimap<String, Class<? extends I>> methods) {
Description desc = clazz.getAnnotation(Description.class);
Stream<String> namesStream;
if (desc != null) {
@@ -168,11 +170,11 @@ public class HiveFunctionRegistry implements PluggableFunctionRegistry {
*/
private HiveFuncHolder resolveFunction(FunctionCall call, boolean varCharToStringReplacement) {
HiveFuncHolder holder;
- MajorType[] argTypes = new MajorType[call.args.size()];
- ObjectInspector[] argOIs = new ObjectInspector[call.args.size()];
- for (int i=0; i<call.args.size(); i++) {
+ MajorType[] argTypes = new MajorType[call.argCount()];
+ ObjectInspector[] argOIs = new ObjectInspector[call.argCount()];
+ for (int i=0; i<call.argCount(); i++) {
try {
- argTypes[i] = call.args.get(i).getMajorType();
+ argTypes[i] = call.arg(i).getMajorType();
argOIs[i] = ObjectInspectorHelper.getDrillObjectInspector(argTypes[i].getMode(), argTypes[i].getMinorType(),
varCharToStringReplacement);
} catch(Exception e) {
diff --git a/contrib/storage-kafka/src/main/java/org/apache/drill/exec/store/kafka/KafkaNodeProcessor.java b/contrib/storage-kafka/src/main/java/org/apache/drill/exec/store/kafka/KafkaNodeProcessor.java
index c0ce1c2..6d267c1 100644
--- a/contrib/storage-kafka/src/main/java/org/apache/drill/exec/store/kafka/KafkaNodeProcessor.java
+++ b/contrib/storage-kafka/src/main/java/org/apache/drill/exec/store/kafka/KafkaNodeProcessor.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.store.kafka;
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
@@ -57,8 +58,8 @@ class KafkaNodeProcessor extends AbstractExprVisitor<Boolean, LogicalExpression,
public static KafkaNodeProcessor process(FunctionCall call) {
String functionName = call.getName();
- LogicalExpression nameArg = call.args.get(0);
- LogicalExpression valueArg = call.args.size() >= 2? call.args.get(1) : null;
+ LogicalExpression nameArg = call.arg(0);
+ LogicalExpression valueArg = call.argCount() >= 2? call.arg(1) : null;
KafkaNodeProcessor evaluator = new KafkaNodeProcessor(functionName);
if (VALUE_EXPRESSION_CLASSES.contains(nameArg.getClass())) {
@@ -99,9 +100,9 @@ class KafkaNodeProcessor extends AbstractExprVisitor<Boolean, LogicalExpression,
switch (this.path) {
case "kafkaMsgOffset":
/*
- * Do not pushdown "not_equal" on kafkaMsgOffset.
+ * Do not pushdown FunctionNames.NE on kafkaMsgOffset.
*/
- if(functionName.equals("not_equal")) {
+ if(functionName.equals(FunctionNames.NE)) {
return false;
}
case "kafkaPartitionId":
@@ -117,10 +118,10 @@ class KafkaNodeProcessor extends AbstractExprVisitor<Boolean, LogicalExpression,
break;
case "kafkaMsgTimestamp":
/*
- Only pushdown "equal", "greater_than", "greater_than_or_equal" on kafkaMsgTimestamp
+ Only pushdown FunctionNames.EQ, FunctionNames.GT, "greater_than_or_equal" on kafkaMsgTimestamp
*/
- if(!functionName.equals("equal") && !functionName.equals("greater_than")
- && !functionName.equals("greater_than_or_equal_to")) {
+ if(!functionName.equals(FunctionNames.EQ) && !functionName.equals(FunctionNames.GT)
+ && !functionName.equals(FunctionNames.GE)) {
return false;
}
@@ -172,12 +173,12 @@ class KafkaNodeProcessor extends AbstractExprVisitor<Boolean, LogicalExpression,
static {
ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
COMPARE_FUNCTIONS_TRANSPOSE_MAP = builder
- .put("equal", "equal")
- .put("not_equal", "not_equal")
- .put("greater_than_or_equal_to", "less_than_or_equal_to")
- .put("greater_than", "less_than")
- .put("less_than_or_equal_to", "greater_than_or_equal_to")
- .put("less_than", "greater_than")
+ .put(FunctionNames.EQ, FunctionNames.EQ)
+ .put(FunctionNames.NE, FunctionNames.NE)
+ .put(FunctionNames.GE, FunctionNames.LE)
+ .put(FunctionNames.GT, FunctionNames.LT)
+ .put(FunctionNames.LE, FunctionNames.GE)
+ .put(FunctionNames.LT, FunctionNames.GT)
.build();
}
}
diff --git a/contrib/storage-kafka/src/main/java/org/apache/drill/exec/store/kafka/KafkaPartitionScanSpec.java b/contrib/storage-kafka/src/main/java/org/apache/drill/exec/store/kafka/KafkaPartitionScanSpec.java
index eaef410..509f77a 100644
--- a/contrib/storage-kafka/src/main/java/org/apache/drill/exec/store/kafka/KafkaPartitionScanSpec.java
+++ b/contrib/storage-kafka/src/main/java/org/apache/drill/exec/store/kafka/KafkaPartitionScanSpec.java
@@ -17,12 +17,14 @@
*/
package org.apache.drill.exec.store.kafka;
+import org.apache.drill.common.FunctionNames;
+
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
public class KafkaPartitionScanSpec {
- private String topicName;
- private int partitionId;
+ private final String topicName;
+ private final int partitionId;
private long startOffset;
private long endOffset;
@@ -55,7 +57,7 @@ public class KafkaPartitionScanSpec {
public void mergeScanSpec(String functionName, KafkaPartitionScanSpec scanSpec) {
switch (functionName) {
- case "booleanAnd":
+ case FunctionNames.AND:
//Reduce the scan range
if (startOffset < scanSpec.startOffset) {
startOffset = scanSpec.startOffset;
@@ -65,7 +67,7 @@ public class KafkaPartitionScanSpec {
endOffset = scanSpec.endOffset;
}
break;
- case "booleanOr":
+ case FunctionNames.OR:
//Increase the scan range
if (scanSpec.startOffset < startOffset) {
startOffset = scanSpec.startOffset;
@@ -94,6 +96,7 @@ public class KafkaPartitionScanSpec {
+ startOffset + ", endOffset=" + endOffset + "]";
}
+ @Override
public KafkaPartitionScanSpec clone() {
return new KafkaPartitionScanSpec(topicName, partitionId, startOffset, endOffset);
}
diff --git a/contrib/storage-kafka/src/main/java/org/apache/drill/exec/store/kafka/KafkaPartitionScanSpecBuilder.java b/contrib/storage-kafka/src/main/java/org/apache/drill/exec/store/kafka/KafkaPartitionScanSpecBuilder.java
index 2734861..5c34dc6 100644
--- a/contrib/storage-kafka/src/main/java/org/apache/drill/exec/store/kafka/KafkaPartitionScanSpecBuilder.java
+++ b/contrib/storage-kafka/src/main/java/org/apache/drill/exec/store/kafka/KafkaPartitionScanSpecBuilder.java
@@ -17,11 +17,11 @@
*/
package org.apache.drill.exec.store.kafka;
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.common.expression.BooleanOperator;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
-import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList;
import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableMap;
import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableSet;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
@@ -83,7 +83,7 @@ public class KafkaPartitionScanSpecBuilder
@Override
public List<KafkaPartitionScanSpec> visitBooleanOperator(BooleanOperator op, Void value) {
Map<TopicPartition, KafkaPartitionScanSpec> specMap = Maps.newHashMap();
- ImmutableList<LogicalExpression> args = op.args;
+ List<LogicalExpression> args = op.args();
if (op.getName().equals("booleanOr")) {
for (LogicalExpression expr : args) {
@@ -169,7 +169,7 @@ public class KafkaPartitionScanSpecBuilder
ImmutableSet<TopicPartition> topicPartitions = fullScanSpec.keySet();
for (TopicPartition partitions : topicPartitions) {
- timesValMap.put(partitions, functionName.equals("greater_than") ? fieldValue+1 : fieldValue);
+ timesValMap.put(partitions, functionName.equals(FunctionNames.GT) ? fieldValue+1 : fieldValue);
}
Map<TopicPartition, OffsetAndTimestamp> offsetAndTimestamp = kafkaConsumer.offsetsForTimes(timesValMap);
@@ -202,7 +202,7 @@ public class KafkaPartitionScanSpecBuilder
*/
switch (functionName) {
- case "equal":
+ case FunctionNames.EQ:
for (TopicPartition tp : topicPartitions) {
if (fieldValue < fullScanSpec.get(tp).getStartOffset()) {
//Offset does not exist
@@ -216,7 +216,7 @@ public class KafkaPartitionScanSpecBuilder
}
}
break;
- case "greater_than_or_equal_to":
+ case FunctionNames.GE:
for (TopicPartition tp : topicPartitions) {
//Ensure scan range is between startOffset and endOffset,
long val = bindOffsetToRange(tp, fieldValue);
@@ -225,7 +225,7 @@ public class KafkaPartitionScanSpecBuilder
fullScanSpec.get(tp).getEndOffset()));
}
break;
- case "greater_than":
+ case FunctionNames.GT:
for (TopicPartition tp : topicPartitions) {
//Ensure scan range is between startOffset and endOffset,
long val = bindOffsetToRange(tp, fieldValue + 1);
@@ -234,7 +234,7 @@ public class KafkaPartitionScanSpecBuilder
val, fullScanSpec.get(tp).getEndOffset()));
}
break;
- case "less_than_or_equal_to":
+ case FunctionNames.LE:
for (TopicPartition tp : topicPartitions) {
//Ensure scan range is between startOffset and endOffset,
long val = bindOffsetToRange(tp, fieldValue+1);
@@ -244,7 +244,7 @@ public class KafkaPartitionScanSpecBuilder
fullScanSpec.get(tp).getStartOffset(), val));
}
break;
- case "less_than":
+ case FunctionNames.LT:
for (TopicPartition tp : topicPartitions) {
//Ensure scan range is between startOffset and endOffset,
long val = bindOffsetToRange(tp, fieldValue);
@@ -263,7 +263,7 @@ public class KafkaPartitionScanSpecBuilder
ImmutableSet<TopicPartition> topicPartitions = fullScanSpec.keySet();
switch (functionName) {
- case "equal":
+ case FunctionNames.EQ:
for (TopicPartition tp : topicPartitions) {
if (tp.partition() == fieldValue) {
scanSpecList.add(
@@ -273,7 +273,7 @@ public class KafkaPartitionScanSpecBuilder
}
}
break;
- case "not_equal":
+ case FunctionNames.NE:
for (TopicPartition tp : topicPartitions) {
if (tp.partition() != fieldValue) {
scanSpecList.add(
@@ -283,7 +283,7 @@ public class KafkaPartitionScanSpecBuilder
}
}
break;
- case "greater_than_or_equal_to":
+ case FunctionNames.GE:
for (TopicPartition tp : topicPartitions) {
if (tp.partition() >= fieldValue) {
scanSpecList.add(
@@ -293,7 +293,7 @@ public class KafkaPartitionScanSpecBuilder
}
}
break;
- case "greater_than":
+ case FunctionNames.GT:
for (TopicPartition tp : topicPartitions) {
if (tp.partition() > fieldValue) {
scanSpecList.add(
@@ -303,7 +303,7 @@ public class KafkaPartitionScanSpecBuilder
}
}
break;
- case "less_than_or_equal_to":
+ case FunctionNames.LE:
for (TopicPartition tp : topicPartitions) {
if (tp.partition() <= fieldValue) {
scanSpecList.add(
@@ -313,7 +313,7 @@ public class KafkaPartitionScanSpecBuilder
}
}
break;
- case "less_than":
+ case FunctionNames.LT:
for (TopicPartition tp : topicPartitions) {
if (tp.partition() < fieldValue) {
scanSpecList.add(
diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoCompareFunctionProcessor.java b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoCompareFunctionProcessor.java
index 1e07ee8..33e5d00 100644
--- a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoCompareFunctionProcessor.java
+++ b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoCompareFunctionProcessor.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.store.mongo;
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.common.expression.CastExpression;
import org.apache.drill.common.expression.ConvertExpression;
import org.apache.drill.common.expression.FunctionCall;
@@ -40,7 +41,7 @@ public class MongoCompareFunctionProcessor extends
AbstractExprVisitor<Boolean, LogicalExpression, RuntimeException> {
private Object value;
private boolean success;
- private boolean isEqualityFn;
+ private final boolean isEqualityFn;
private SchemaPath path;
private String functionName;
@@ -50,8 +51,8 @@ public class MongoCompareFunctionProcessor extends
public static MongoCompareFunctionProcessor process(FunctionCall call) {
String functionName = call.getName();
- LogicalExpression nameArg = call.args.get(0);
- LogicalExpression valueArg = call.args.size() == 2 ? call.args.get(1)
+ LogicalExpression nameArg = call.arg(0);
+ LogicalExpression valueArg = call.argCount() == 2 ? call.arg(1)
: null;
MongoCompareFunctionProcessor evaluator = new MongoCompareFunctionProcessor(
functionName);
@@ -65,7 +66,7 @@ public class MongoCompareFunctionProcessor extends
.get(functionName);
}
evaluator.success = nameArg.accept(evaluator, valueArg);
- } else if (call.args.get(0) instanceof SchemaPath) {
+ } else if (call.arg(0) instanceof SchemaPath) {
evaluator.success = true;
evaluator.path = (SchemaPath) nameArg;
}
@@ -247,18 +248,19 @@ public class MongoCompareFunctionProcessor extends
ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
COMPARE_FUNCTIONS_TRANSPOSE_MAP = builder
// unary functions
- .put("isnotnull", "isnotnull")
+ .put(FunctionNames.IS_NOT_NULL, FunctionNames.IS_NOT_NULL)
.put("isNotNull", "isNotNull")
.put("is not null", "is not null")
- .put("isnull", "isnull")
+ .put(FunctionNames.IS_NULL, FunctionNames.IS_NULL)
.put("isNull", "isNull")
.put("is null", "is null")
// binary functions
- .put("equal", "equal").put("not_equal", "not_equal")
- .put("greater_than_or_equal_to", "less_than_or_equal_to")
- .put("greater_than", "less_than")
- .put("less_than_or_equal_to", "greater_than_or_equal_to")
- .put("less_than", "greater_than").build();
+ .put(FunctionNames.EQ, FunctionNames.EQ)
+ .put(FunctionNames.NE, FunctionNames.NE)
+ .put(FunctionNames.GE, FunctionNames.LE)
+ .put(FunctionNames.GT, FunctionNames.LT)
+ .put(FunctionNames.LE, FunctionNames.GE)
+ .put(FunctionNames.LT, FunctionNames.GT).build();
}
}
diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoFilterBuilder.java b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoFilterBuilder.java
index 5b2eb33..8bdf2e0 100644
--- a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoFilterBuilder.java
+++ b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoFilterBuilder.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.store.mongo;
import java.io.IOException;
import java.util.List;
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.common.expression.BooleanOperator;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.LogicalExpression;
@@ -30,8 +31,6 @@ import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList;
-
public class MongoFilterBuilder extends
AbstractExprVisitor<MongoScanSpec, Void, RuntimeException> implements
DrillMongoConstants {
@@ -50,7 +49,7 @@ public class MongoFilterBuilder extends
public MongoScanSpec parseTree() {
MongoScanSpec parsedSpec = le.accept(this, null);
if (parsedSpec != null) {
- parsedSpec = mergeScanSpecs("booleanAnd", this.groupScan.getScanSpec(),
+ parsedSpec = mergeScanSpecs(FunctionNames.AND, this.groupScan.getScanSpec(),
parsedSpec);
}
return parsedSpec;
@@ -61,7 +60,7 @@ public class MongoFilterBuilder extends
Document newFilter = new Document();
switch (functionName) {
- case "booleanAnd":
+ case FunctionNames.AND:
if (leftScanSpec.getFilters() != null
&& rightScanSpec.getFilters() != null) {
newFilter = MongoUtils.andFilterAtIndex(leftScanSpec.getFilters(),
@@ -72,7 +71,7 @@ public class MongoFilterBuilder extends
newFilter = rightScanSpec.getFilters();
}
break;
- case "booleanOr":
+ case FunctionNames.OR:
newFilter = MongoUtils.orFilterAtIndex(leftScanSpec.getFilters(),
rightScanSpec.getFilters());
}
@@ -93,13 +92,13 @@ public class MongoFilterBuilder extends
@Override
public MongoScanSpec visitBooleanOperator(BooleanOperator op, Void value) {
- List<LogicalExpression> args = op.args;
+ List<LogicalExpression> args = op.args();
MongoScanSpec nodeScanSpec = null;
String functionName = op.getName();
for (int i = 0; i < args.size(); ++i) {
switch (functionName) {
- case "booleanAnd":
- case "booleanOr":
+ case FunctionNames.AND:
+ case FunctionNames.OR:
if (nodeScanSpec == null) {
nodeScanSpec = args.get(i).accept(this, null);
} else {
@@ -121,7 +120,7 @@ public class MongoFilterBuilder extends
throws RuntimeException {
MongoScanSpec nodeScanSpec = null;
String functionName = call.getName();
- ImmutableList<LogicalExpression> args = call.args;
+ List<LogicalExpression> args = call.args();
if (MongoCompareFunctionProcessor.isCompareFunction(functionName)) {
MongoCompareFunctionProcessor processor = MongoCompareFunctionProcessor
@@ -137,8 +136,8 @@ public class MongoFilterBuilder extends
}
} else {
switch (functionName) {
- case "booleanAnd":
- case "booleanOr":
+ case FunctionNames.AND:
+ case FunctionNames.OR:
MongoScanSpec leftScanSpec = args.get(0).accept(this, null);
MongoScanSpec rightScanSpec = args.get(1).accept(this, null);
if (leftScanSpec != null && rightScanSpec != null) {
@@ -146,7 +145,7 @@ public class MongoFilterBuilder extends
rightScanSpec);
} else {
allExpressionsConverted = false;
- if ("booleanAnd".equals(functionName)) {
+ if (FunctionNames.AND.equals(functionName)) {
nodeScanSpec = leftScanSpec == null ? rightScanSpec : leftScanSpec;
}
}
@@ -168,30 +167,30 @@ public class MongoFilterBuilder extends
String fieldName = field.getRootSegmentPath();
MongoCompareOp compareOp = null;
switch (functionName) {
- case "equal":
+ case FunctionNames.EQ:
compareOp = MongoCompareOp.EQUAL;
break;
- case "not_equal":
+ case FunctionNames.NE:
compareOp = MongoCompareOp.NOT_EQUAL;
break;
- case "greater_than_or_equal_to":
+ case FunctionNames.GE:
compareOp = MongoCompareOp.GREATER_OR_EQUAL;
break;
- case "greater_than":
+ case FunctionNames.GT:
compareOp = MongoCompareOp.GREATER;
break;
- case "less_than_or_equal_to":
+ case FunctionNames.LE:
compareOp = MongoCompareOp.LESS_OR_EQUAL;
break;
- case "less_than":
+ case FunctionNames.LT:
compareOp = MongoCompareOp.LESS;
break;
- case "isnull":
+ case FunctionNames.IS_NULL:
case "isNull":
case "is null":
compareOp = MongoCompareOp.IFNULL;
break;
- case "isnotnull":
+ case FunctionNames.IS_NOT_NULL:
case "isNotNull":
case "is not null":
compareOp = MongoCompareOp.IFNOTNULL;
@@ -215,5 +214,4 @@ public class MongoFilterBuilder extends
}
return null;
}
-
}
diff --git a/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java b/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
index 2113c25..2610c1e 100644
--- a/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
@@ -164,6 +164,7 @@
package org.apache.drill.exec.expr.fn.impl;
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
@@ -233,236 +234,225 @@ public class GCompare${leftTypeBase}Vs${rightTypeBase} {
<#-- Comparison function for comparison expression operator (=, <, etc.),
not for sorting and grouping relational operators. -->
- @FunctionTemplate(names = {"less_than", "<"},
+ @FunctionTemplate(names = {FunctionNames.LT, "<"},
scope = FunctionTemplate.FunctionScope.SIMPLE,
nulls = NullHandling.NULL_IF_NULL)
public static class LessThan${leftTypeBase}Vs${rightTypeBase} implements DrillSimpleFunc {
- @Param ${leftTypeBase}Holder left;
- @Param ${rightTypeBase}Holder right;
- @Output BitHolder out;
+ @Param ${leftTypeBase}Holder left;
+ @Param ${rightTypeBase}Holder right;
+ @Output BitHolder out;
- public void setup() {}
-
- public void eval() {
+ public void setup() {}
- <#if typeGroup.mode == "primitive">
- // NaN is the biggest possible value, and NaN == NaN
- if (Double.isNaN(left.value) || ( Double.isNaN(left.value) && Double.isNaN(right.value))) {
- out.value=0;
- } else if (Double.isNaN(right.value) && !Double.isNaN(left.value)) {
- out.value = 1;
- } else {
- out.value = left.value < right.value ? 1 : 0;
- }
- <#elseif typeGroup.mode == "varString"
- || typeGroup.mode == "intervalNameThis" || typeGroup.mode == "intervalDay" >
- int cmp;
- <@compareBlock mode=typeGroup.mode leftType=leftTypeBase rightType=rightTypeBase
- output="cmp" nullCompare=false nullComparesHigh=false />
- out.value = cmp == -1 ? 1 : 0;
- <#-- TODO: Refactor other comparison code to here. -->
- <#else>
- ${mode_HAS_BAD_VALUE}
- </#if>
+ public void eval() {
+ <#if typeGroup.mode == "primitive">
+ // NaN is the biggest possible value, and NaN == NaN
+ if (Double.isNaN(left.value) || ( Double.isNaN(left.value) && Double.isNaN(right.value))) {
+ out.value=0;
+ } else if (Double.isNaN(right.value) && !Double.isNaN(left.value)) {
+ out.value = 1;
+ } else {
+ out.value = left.value < right.value ? 1 : 0;
}
+ <#elseif typeGroup.mode == "varString"
+ || typeGroup.mode == "intervalNameThis" || typeGroup.mode == "intervalDay" >
+ int cmp;
+ <@compareBlock mode=typeGroup.mode leftType=leftTypeBase rightType=rightTypeBase
+ output="cmp" nullCompare=false nullComparesHigh=false />
+ out.value = cmp == -1 ? 1 : 0;
+ <#-- TODO: Refactor other comparison code to here. -->
+ <#else>
+ ${mode_HAS_BAD_VALUE}
+ </#if>
+ }
}
<#-- Comparison function for comparison expression operator (=, <, etc.),
not for sorting and grouping relational operators. -->
- @FunctionTemplate(names = {"less_than_or_equal_to", "<="},
+ @FunctionTemplate(names = {FunctionNames.LE, "<="},
scope = FunctionTemplate.FunctionScope.SIMPLE,
nulls = NullHandling.NULL_IF_NULL)
public static class LessThanEq${leftTypeBase}Vs${rightTypeBase} implements DrillSimpleFunc {
- @Param ${leftTypeBase}Holder left;
- @Param ${rightTypeBase}Holder right;
- @Output BitHolder out;
-
- public void setup() {}
+ @Param ${leftTypeBase}Holder left;
+ @Param ${rightTypeBase}Holder right;
+ @Output BitHolder out;
- public void eval() {
+ public void setup() {}
- <#if typeGroup.mode == "primitive">
- // NaN is the biggest possible value, and NaN == NaN
- if (Double.isNaN(right.value)){
- out.value = 1;
- } else if (!Double.isNaN(right.value) && Double.isNaN(left.value)) {
- out.value = 0;
- } else {
- out.value = left.value <= right.value ? 1 : 0;
- }
- <#elseif typeGroup.mode == "varString"
- || typeGroup.mode == "intervalNameThis" || typeGroup.mode == "intervalDay" >
- int cmp;
- <@compareBlock mode=typeGroup.mode leftType=leftTypeBase rightType=rightTypeBase
- output="cmp" nullCompare=false nullComparesHigh=false />
- out.value = cmp < 1 ? 1 : 0;
- <#-- TODO: Refactor other comparison code to here. -->
- <#else>
- ${mode_HAS_BAD_VALUE}
- </#if>
+ public void eval() {
+ <#if typeGroup.mode == "primitive">
+ // NaN is the biggest possible value, and NaN == NaN
+ if (Double.isNaN(right.value)){
+ out.value = 1;
+ } else if (!Double.isNaN(right.value) && Double.isNaN(left.value)) {
+ out.value = 0;
+ } else {
+ out.value = left.value <= right.value ? 1 : 0;
+ }
+ <#elseif typeGroup.mode == "varString"
+ || typeGroup.mode == "intervalNameThis" || typeGroup.mode == "intervalDay" >
+ int cmp;
+ <@compareBlock mode=typeGroup.mode leftType=leftTypeBase rightType=rightTypeBase
+ output="cmp" nullCompare=false nullComparesHigh=false />
+ out.value = cmp < 1 ? 1 : 0;
+ <#-- TODO: Refactor other comparison code to here. -->
+ <#else>
+ ${mode_HAS_BAD_VALUE}
+ </#if>
}
}
<#-- Comparison function for comparison expression operator (=, <, etc.),
not for sorting and grouping relational operators. -->
- @FunctionTemplate(names = {"greater_than", ">"},
+ @FunctionTemplate(names = {FunctionNames.GT, ">"},
scope = FunctionTemplate.FunctionScope.SIMPLE,
nulls = NullHandling.NULL_IF_NULL)
public static class GreaterThan${leftTypeBase}Vs${rightTypeBase} implements DrillSimpleFunc {
- @Param ${leftTypeBase}Holder left;
- @Param ${rightTypeBase}Holder right;
- @Output BitHolder out;
+ @Param ${leftTypeBase}Holder left;
+ @Param ${rightTypeBase}Holder right;
+ @Output BitHolder out;
- public void setup() {}
-
- public void eval() {
+ public void setup() {}
- <#if typeGroup.mode == "primitive">
- // NaN is the biggest possible value, and NaN == NaN
- if (Double.isNaN(right.value) || ( Double.isNaN(left.value) && Double.isNaN(right.value))) {
- out.value = 0;
- } else if (Double.isNaN(left.value) && !Double.isNaN(right.value)) {
- out.value = 1;
- } else {
- out.value = left.value > right.value ? 1 : 0;
- }
- <#elseif typeGroup.mode == "varString"
- || typeGroup.mode == "intervalNameThis" || typeGroup.mode == "intervalDay" >
- int cmp;
- <@compareBlock mode=typeGroup.mode leftType=leftTypeBase rightType=rightTypeBase
- output="cmp" nullCompare=false nullComparesHigh=false />
- out.value = cmp == 1 ? 1 : 0;
- <#-- TODO: Refactor other comparison code to here. -->
- <#else>
- ${mode_HAS_BAD_VALUE}
- </#if>
+ public void eval() {
+ <#if typeGroup.mode == "primitive">
+ // NaN is the biggest possible value, and NaN == NaN
+ if (Double.isNaN(right.value) || ( Double.isNaN(left.value) && Double.isNaN(right.value))) {
+ out.value = 0;
+ } else if (Double.isNaN(left.value) && !Double.isNaN(right.value)) {
+ out.value = 1;
+ } else {
+ out.value = left.value > right.value ? 1 : 0;
+ }
+ <#elseif typeGroup.mode == "varString"
+ || typeGroup.mode == "intervalNameThis" || typeGroup.mode == "intervalDay" >
+ int cmp;
+ <@compareBlock mode=typeGroup.mode leftType=leftTypeBase rightType=rightTypeBase
+ output="cmp" nullCompare=false nullComparesHigh=false />
+ out.value = cmp == 1 ? 1 : 0;
+ <#-- TODO: Refactor other comparison code to here. -->
+ <#else>
+ ${mode_HAS_BAD_VALUE}
+ </#if>
}
}
<#-- Comparison function for comparison expression operator (=, <, etc.),
not for sorting and grouping relational operators. -->
- @FunctionTemplate(names = {"greater_than_or_equal_to", ">="},
+ @FunctionTemplate(names = {FunctionNames.GE, ">="},
scope = FunctionTemplate.FunctionScope.SIMPLE,
nulls = NullHandling.NULL_IF_NULL)
public static class GreaterThanEq${leftTypeBase}Vs${rightTypeBase} implements DrillSimpleFunc {
- @Param ${leftTypeBase}Holder left;
- @Param ${rightTypeBase}Holder right;
- @Output BitHolder out;
+ @Param ${leftTypeBase}Holder left;
+ @Param ${rightTypeBase}Holder right;
+ @Output BitHolder out;
- public void setup() {}
-
- public void eval() {
-
- <#if typeGroup.mode == "primitive">
- // NaN is the biggest possible value, and NaN == NaN
- if (Double.isNaN(left.value)){
- out.value=1;
- } else if (!Double.isNaN(left.value) && Double.isNaN(right.value)) {
- out.value = 0;
- } else {
- out.value = left.value >= right.value ? 1 : 0;
- }
+ public void setup() {}
- <#elseif typeGroup.mode == "varString"
- || typeGroup.mode == "intervalNameThis" || typeGroup.mode == "intervalDay" >
- int cmp;
- <@compareBlock mode=typeGroup.mode leftType=leftTypeBase rightType=rightTypeBase
- output="cmp" nullCompare=false nullComparesHigh=false />
- out.value = cmp > -1 ? 1 : 0;
- <#-- TODO: Refactor other comparison code to here. -->
- <#else>
- ${mode_HAS_BAD_VALUE}
- </#if>
+ public void eval() {
+ <#if typeGroup.mode == "primitive">
+ // NaN is the biggest possible value, and NaN == NaN
+ if (Double.isNaN(left.value)){
+ out.value=1;
+ } else if (!Double.isNaN(left.value) && Double.isNaN(right.value)) {
+ out.value = 0;
+ } else {
+ out.value = left.value >= right.value ? 1 : 0;
}
+
+ <#elseif typeGroup.mode == "varString"
+ || typeGroup.mode == "intervalNameThis" || typeGroup.mode == "intervalDay" >
+ int cmp;
+ <@compareBlock mode=typeGroup.mode leftType=leftTypeBase rightType=rightTypeBase
+ output="cmp" nullCompare=false nullComparesHigh=false />
+ out.value = cmp > -1 ? 1 : 0;
+ <#-- TODO: Refactor other comparison code to here. -->
+ <#else>
+ ${mode_HAS_BAD_VALUE}
+ </#if>
+ }
}
<#-- Comparison function for comparison expression operator (=, <, etc.),
not for sorting and grouping relational operators. -->
- @FunctionTemplate(names = {"equal", "==", "="},
+ @FunctionTemplate(names = {FunctionNames.EQ, "==", "="},
scope = FunctionTemplate.FunctionScope.SIMPLE,
nulls = NullHandling.NULL_IF_NULL)
public static class Equals${leftTypeBase}Vs${rightTypeBase} implements DrillSimpleFunc {
- @Param ${leftTypeBase}Holder left;
- @Param ${rightTypeBase}Holder right;
- @Output BitHolder out;
+ @Param ${leftTypeBase}Holder left;
+ @Param ${rightTypeBase}Holder right;
+ @Output BitHolder out;
- public void setup() {}
-
- public void eval() {
+ public void setup() {}
- <#if typeGroup.mode == "primitive">
- // NaN is the biggest possible value, and NaN == NaN
- if (Double.isNaN(left.value) && Double.isNaN(right.value)) {
- out.value = 1;
- } else {
- out.value = left.value == right.value ? 1 : 0;
- }
- <#elseif typeGroup.mode == "varString" >
- out.value = org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.equal(
- left.buffer, left.start, left.end, right.buffer, right.start, right.end);
- <#elseif typeGroup.mode == "intervalNameThis" || typeGroup.mode == "intervalDay" >
- int cmp;
- <@compareBlock mode=typeGroup.mode leftType=leftTypeBase rightType=rightTypeBase
- output="cmp" nullCompare=false nullComparesHigh=false />
- out.value = cmp == 0 ? 1 : 0;
- <#-- TODO: Refactor other comparison code to here. -->
- <#else>
- ${mode_HAS_BAD_VALUE}
- </#if>
+ public void eval() {
+ <#if typeGroup.mode == "primitive">
+ // NaN is the biggest possible value, and NaN == NaN
+ if (Double.isNaN(left.value) && Double.isNaN(right.value)) {
+ out.value = 1;
+ } else {
+ out.value = left.value == right.value ? 1 : 0;
}
+ <#elseif typeGroup.mode == "varString" >
+ out.value = org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers.equal(
+ left.buffer, left.start, left.end, right.buffer, right.start, right.end);
+ <#elseif typeGroup.mode == "intervalNameThis" || typeGroup.mode == "intervalDay" >
+ int cmp;
+ <@compareBlock mode=typeGroup.mode leftType=leftTypeBase rightType=rightTypeBase
+ output="cmp" nullCompare=false nullComparesHigh=false />
+ out.value = cmp == 0 ? 1 : 0;
+ <#-- TODO: Refactor other comparison code to here. -->
+ <#else>
+ ${mode_HAS_BAD_VALUE}
+ </#if>
+ }
}
<#-- Comparison function for comparison expression operator (=, <, etc.),
not for sorting and grouping relational operators. -->
- @FunctionTemplate(names = {"not_equal", "<>", "!="},
+ @FunctionTemplate(names = {FunctionNames.NE, "<>", "!="},
scope = FunctionTemplate.FunctionScope.SIMPLE,
nulls = NullHandling.NULL_IF_NULL)
public static class NotEquals${leftTypeBase}Vs${rightTypeBase} implements DrillSimpleFunc {
- @Param ${leftTypeBase}Holder left;
- @Param ${rightTypeBase}Holder right;
- @Output BitHolder out;
-
- public void setup() {}
+ @Param ${leftTypeBase}Holder left;
+ @Param ${rightTypeBase}Holder right;
+ @Output BitHolder out;
- public void eval() {
+ public void setup() {}
- <#if typeGroup.mode == "primitive">
- // NaN is the biggest possible value, and NaN == NaN
- if (Double.isNaN(left.value) && Double.isNaN(right.value)) {
- out.value = 0;
- } else {
- out.value = left.value != right.value ? 1 : 0;
- }
- <#elseif typeGroup.mode == "varString"
- || typeGroup.mode == "intervalNameThis" || typeGroup.mode == "intervalDay" >
- int cmp;
- <@compareBlock mode=typeGroup.mode leftType=leftTypeBase rightType=rightTypeBase
- output="cmp" nullCompare=false nullComparesHigh=false />
- out.value = cmp == 0 ? 0 : 1;
- <#-- TODO: Refactor other comparison code to here. -->
- <#else>
- ${mode_HAS_BAD_VALUE}
- </#if>
+ public void eval() {
+ <#if typeGroup.mode == "primitive">
+ // NaN is the biggest possible value, and NaN == NaN
+ if (Double.isNaN(left.value) && Double.isNaN(right.value)) {
+ out.value = 0;
+ } else {
+ out.value = left.value != right.value ? 1 : 0;
}
+ <#elseif typeGroup.mode == "varString"
+ || typeGroup.mode == "intervalNameThis" || typeGroup.mode == "intervalDay" >
+ int cmp;
+ <@compareBlock mode=typeGroup.mode leftType=leftTypeBase rightType=rightTypeBase
+ output="cmp" nullCompare=false nullComparesHigh=false />
+ out.value = cmp == 0 ? 0 : 1;
+ <#-- TODO: Refactor other comparison code to here. -->
+ <#else>
+ ${mode_HAS_BAD_VALUE}
+ </#if>
+ }
}
-
}
-
-
</#list> <#-- 2. Pair of types-->
</#list>
-
</#list> <#-- 1. Group -->
-
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/BooleanPredicate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/BooleanPredicate.java
index 09cbec9..4af9709 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/BooleanPredicate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/BooleanPredicate.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.expr;
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.common.expression.BooleanOperator;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.LogicalExpression;
@@ -56,7 +57,7 @@ public abstract class BooleanPredicate<C extends Comparable<C>> extends BooleanO
RowsMatch resultMatch = RowsMatch.ALL;
for (LogicalExpression child : this) {
if (child instanceof FilterPredicate) {
- switch (((FilterPredicate) child).matches(evaluator)) {
+ switch (((FilterPredicate<C>) child).matches(evaluator)) {
case NONE:
return RowsMatch.NONE; // No row comply to 1 filter part => can drop RG
case SOME:
@@ -87,7 +88,7 @@ public abstract class BooleanPredicate<C extends Comparable<C>> extends BooleanO
RowsMatch resultMatch = RowsMatch.NONE;
for (LogicalExpression child : this) {
if (child instanceof FilterPredicate) {
- switch (((FilterPredicate) child).matches(evaluator)) {
+ switch (((FilterPredicate<C>) child).matches(evaluator)) {
case ALL:
return RowsMatch.ALL; // One at least is ALL => can drop filter but not RG
case SOME:
@@ -104,9 +105,9 @@ public abstract class BooleanPredicate<C extends Comparable<C>> extends BooleanO
public static <C extends Comparable<C>> LogicalExpression createBooleanPredicate(
String function, String name, List<LogicalExpression> args, ExpressionPosition pos) {
switch (function) {
- case "booleanOr":
+ case FunctionNames.OR:
return BooleanPredicate.<C>createOrPredicate(name, args, pos);
- case "booleanAnd":
+ case FunctionNames.AND:
return BooleanPredicate.<C>createAndPredicate(name, args, pos);
default:
logger.warn("Unknown Boolean '{}' predicate.", function);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CloneVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CloneVisitor.java
index a68326b..d661a77 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CloneVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/CloneVisitor.java
@@ -57,7 +57,7 @@ public class CloneVisitor extends AbstractExprVisitor<LogicalExpression,Void,Run
@Override
public LogicalExpression visitFunctionCall(FunctionCall call, Void value) throws RuntimeException {
List<LogicalExpression> args = Lists.newArrayList();
- for (LogicalExpression arg : call.args) {
+ for (LogicalExpression arg : call.args()) {
args.add(arg.accept(this, null));
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ComparisonPredicate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ComparisonPredicate.java
index 3594a6d..2b786e9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ComparisonPredicate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ComparisonPredicate.java
@@ -17,11 +17,11 @@
*/
package org.apache.drill.exec.expr;
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.LogicalExpressionBase;
import org.apache.drill.common.expression.visitors.ExprVisitor;
import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
import org.apache.drill.exec.expr.stat.RowsMatch;
import org.apache.drill.metastore.statistics.ColumnStatistics;
import org.apache.drill.metastore.statistics.ColumnStatisticsKind;
@@ -258,17 +258,17 @@ public class ComparisonPredicate<C extends Comparable<C>> extends LogicalExpress
public static <C extends Comparable<C>> LogicalExpression createComparisonPredicate(
String function, LogicalExpression left, LogicalExpression right) {
switch (function) {
- case FunctionGenerationHelper.EQ:
+ case FunctionNames.EQ:
return ComparisonPredicate.<C>createEqualPredicate(left, right);
- case FunctionGenerationHelper.GT:
+ case FunctionNames.GT:
return ComparisonPredicate.<C>createGTPredicate(left, right);
- case FunctionGenerationHelper.GE:
+ case FunctionNames.GE:
return ComparisonPredicate.<C>createGEPredicate(left, right);
- case FunctionGenerationHelper.LT:
+ case FunctionNames.LT:
return ComparisonPredicate.<C>createLTPredicate(left, right);
- case FunctionGenerationHelper.LE:
+ case FunctionNames.LE:
return ComparisonPredicate.<C>createLEPredicate(left, right);
- case FunctionGenerationHelper.NE:
+ case FunctionNames.NE:
return ComparisonPredicate.<C>createNEPredicate(left, right);
default:
return null;
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
index 1c5dc53..773b4be 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
@@ -31,6 +31,7 @@ import java.util.Stack;
import io.netty.buffer.DrillBuf;
import org.apache.calcite.util.Pair;
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.common.expression.AnyValueExpression;
import org.apache.drill.common.expression.BooleanOperator;
import org.apache.drill.common.expression.CastExpression;
@@ -250,9 +251,9 @@ public class EvaluationVisitor {
@Override
public HoldingContainer visitBooleanOperator(BooleanOperator op,
ClassGenerator<?> generator) throws RuntimeException {
- if (op.getName().equals("booleanAnd")) {
+ if (op.getName().equals(FunctionNames.AND)) {
return visitBooleanAnd(op, generator);
- } else if(op.getName().equals("booleanOr")) {
+ } else if(op.getName().equals(FunctionNames.OR)) {
return visitBooleanOr(op, generator);
} else {
throw new UnsupportedOperationException(
@@ -1033,8 +1034,8 @@ public class EvaluationVisitor {
// null true null
// null false false
// null null null
- for (int i = 0; i < op.args.size(); i++) {
- arg = op.args.get(i).accept(this, generator);
+ for (LogicalExpression expr : op.args()) {
+ arg = expr.accept(this, generator);
JBlock earlyExit = null;
if (arg.isOptional()) {
@@ -1095,9 +1096,8 @@ public class EvaluationVisitor {
// null true true
// null false null
// null null null
-
- for (int i = 0; i < op.args.size(); i++) {
- arg = op.args.get(i).accept(this, generator);
+ for (LogicalExpression expr : op.args()) {
+ arg = expr.accept(this, generator);
JBlock earlyExit = null;
if (arg.isOptional()) {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
index 2d098ad..603d766 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
@@ -256,7 +256,7 @@ public class ExpressionTreeMaterializer {
sb.append(call.getName());
sb.append("(");
boolean first = true;
- for(LogicalExpression e : call.args) {
+ for (LogicalExpression e : call.args()) {
TypeProtos.MajorType mt = e.getMajorType();
if (first) {
first = false;
@@ -384,9 +384,9 @@ public class ExpressionTreeMaterializer {
@Override
public LogicalExpression visitBooleanOperator(BooleanOperator op, FunctionLookupContext functionLookupContext) {
List<LogicalExpression> args = new ArrayList<>();
- for (int i = 0; i < op.args.size(); ++i) {
- LogicalExpression newExpr = op.args.get(i).accept(this, functionLookupContext);
- assert newExpr != null : String.format("Materialization of %s return a null expression.", op.args.get(i));
+ for (LogicalExpression expr : op.args()) {
+ LogicalExpression newExpr = expr.accept(this, functionLookupContext);
+ assert newExpr != null : String.format("Materialization of %s return a null expression.", expr);
args.add(newExpr);
}
@@ -404,9 +404,9 @@ public class ExpressionTreeMaterializer {
// Possibly convert input expressions with a rewritten expression.
List<LogicalExpression> args = new ArrayList<>();
- for (int i = 0; i < call.args.size(); ++i) {
- LogicalExpression newExpr = call.args.get(i).accept(this, functionLookupContext);
- assert newExpr != null : String.format("Materialization of %s returned a null expression.", call.args.get(i));
+ for (LogicalExpression expr : call.args()) {
+ LogicalExpression newExpr = expr.accept(this, functionLookupContext);
+ assert newExpr != null : String.format("Materialization of %s returned a null expression.", expr);
args.add(newExpr);
}
@@ -462,9 +462,8 @@ public class ExpressionTreeMaterializer {
List<LogicalExpression> argsWithCast = new ArrayList<>();
// Compare param type against arg type. Insert cast on top of arg, whenever necessary.
- for (int i = 0; i < call.args.size(); ++i) {
-
- LogicalExpression currentArg = call.args.get(i);
+ for (int i = 0; i < call.argCount(); ++i) {
+ LogicalExpression currentArg = call.arg(i);
TypeProtos.MajorType parmType = matchedFuncHolder.getParamMajorType(i);
@@ -518,8 +517,8 @@ public class ExpressionTreeMaterializer {
// Insert implicit cast function holder expressions if required
List<LogicalExpression> extArgsWithCast = new ArrayList<>();
- for (int i = 0; i < call.args.size(); ++i) {
- LogicalExpression currentArg = call.args.get(i);
+ for (int i = 0; i < call.argCount(); ++i) {
+ final LogicalExpression currentArg = call.arg(i);
TypeProtos.MajorType paramType = matchedNonDrillFuncHolder.getParamMajorType(i);
if (Types.softEquals(paramType, currentArg.getMajorType(), true)) {
@@ -531,7 +530,7 @@ public class ExpressionTreeMaterializer {
paramType = MajorType.newBuilder().setMinorType(paramType.getMinorType()).setMode(paramType.getMode()).
setScale(currentArg.getMajorType().getScale()).setPrecision(computePrecision(currentArg)).build();
}
- extArgsWithCast.add(addCastExpression(call.args.get(i), paramType, functionLookupContext, errorCollector));
+ extArgsWithCast.add(addCastExpression(currentArg, paramType, functionLookupContext, errorCollector));
}
}
@@ -539,7 +538,7 @@ public class ExpressionTreeMaterializer {
}
private boolean hasUnionInput(FunctionCall call) {
- for (LogicalExpression arg : call.args) {
+ for (LogicalExpression arg : call.args()) {
if (arg.getMajorType().getMinorType() == MinorType.UNION) {
return true;
}
@@ -555,11 +554,11 @@ public class ExpressionTreeMaterializer {
* branch of the case statement
*/
private LogicalExpression rewriteUnionFunction(FunctionCall call, FunctionLookupContext functionLookupContext) {
- LogicalExpression[] args = new LogicalExpression[call.args.size()];
- call.args.toArray(args);
+ LogicalExpression[] args = new LogicalExpression[call.argCount()];
+ call.args().toArray(args);
for (int i = 0; i < args.length; i++) {
- LogicalExpression arg = call.args.get(i);
+ LogicalExpression arg = call.arg(i);
MajorType majorType = arg.getMajorType();
if (majorType.getMinorType() != MinorType.UNION) {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/FilterBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/FilterBuilder.java
index 08205be..f4c19cb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/FilterBuilder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/FilterBuilder.java
@@ -21,6 +21,7 @@ import org.apache.drill.common.expression.fn.FunctionReplacementUtils;
import org.apache.drill.exec.expr.fn.impl.StringFunctionHelpers;
import org.apache.drill.exec.expr.holders.VarCharHolder;
import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableSet;
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.common.expression.BooleanOperator;
import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.LogicalExpression;
@@ -52,8 +53,9 @@ import java.util.List;
import java.util.Set;
/**
- * A visitor which visits a materialized logical expression, and build FilterPredicate
- * If a visitXXX method returns null, that means the corresponding filter branch is not qualified for push down.
+ * A visitor which visits a materialized logical expression, and build
+ * FilterPredicate If a visitXXX method returns null, that means the
+ * corresponding filter branch is not qualified for push down.
*/
public class FilterBuilder extends AbstractExprVisitor<LogicalExpression, Set<LogicalExpression>, RuntimeException> {
private static final Logger logger = LoggerFactory.getLogger(FilterBuilder.class);
@@ -73,16 +75,17 @@ public class FilterBuilder extends AbstractExprVisitor<LogicalExpression, Set<Lo
*
* @return metadata filter predicate
*/
- public static FilterPredicate buildFilterPredicate(LogicalExpression expr,
+ public static FilterPredicate<?> buildFilterPredicate(LogicalExpression expr,
Set<LogicalExpression> constantBoundaries,
UdfUtilities udfUtilities,
boolean omitUnsupportedExprs) {
- LogicalExpression logicalExpression = expr.accept(new FilterBuilder(udfUtilities, omitUnsupportedExprs), constantBoundaries);
+ LogicalExpression logicalExpression = expr.accept(
+ new FilterBuilder(udfUtilities, omitUnsupportedExprs), constantBoundaries);
if (logicalExpression instanceof FilterPredicate) {
- return (FilterPredicate) logicalExpression;
+ return (FilterPredicate<?>) logicalExpression;
} else if (logicalExpression instanceof TypedFieldExpr) {
// Calcite simplifies `= true` expression to field name, wrap it with is true predicate
- return (FilterPredicate) IsPredicate.createIsPredicate(FunctionGenerationHelper.IS_TRUE, logicalExpression);
+ return (FilterPredicate<?>) IsPredicate.createIsPredicate(FunctionNames.IS_TRUE, logicalExpression);
}
logger.debug("Logical expression {} was not qualified for filter push down", logicalExpression);
return null;
@@ -159,10 +162,10 @@ public class FilterBuilder extends AbstractExprVisitor<LogicalExpression, Set<Lo
List<LogicalExpression> childPredicates = new ArrayList<>();
String functionName = op.getName();
- for (LogicalExpression arg : op.args) {
+ for (LogicalExpression arg : op.args()) {
LogicalExpression childPredicate = arg.accept(this, value);
if (childPredicate == null) {
- if (functionName.equals("booleanOr") || !omitUnsupportedExprs) {
+ if (functionName.equals(FunctionNames.OR) || !omitUnsupportedExprs) {
// we can't include any leg of the OR if any of the predicates cannot be converted
// or prohibited omitting of unconverted operands
return null;
@@ -220,7 +223,8 @@ public class FilterBuilder extends AbstractExprVisitor<LogicalExpression, Set<Lo
}
@Override
- public LogicalExpression visitFunctionHolderExpression(FunctionHolderExpression funcHolderExpr, Set<LogicalExpression> value) {
+ public LogicalExpression visitFunctionHolderExpression(
+ FunctionHolderExpression funcHolderExpr, Set<LogicalExpression> value) {
FuncHolder holder = funcHolderExpr.getHolder();
if (!(holder instanceof DrillSimpleFuncHolder)) {
@@ -244,19 +248,15 @@ public class FilterBuilder extends AbstractExprVisitor<LogicalExpression, Set<Lo
if (isCompareFunction(funcName)) {
return handleCompareFunction(funcHolderExpr, value);
- }
-
- if (isIsFunction(funcName) || isNot(funcHolderExpr, funcName)) {
+ } else if (isIsFunction(funcName) || isNot(funcHolderExpr, funcName)) {
return handleIsFunction(funcHolderExpr, value);
- }
-
- if (FunctionReplacementUtils.isCastFunction(funcName)) {
+ } else if (FunctionReplacementUtils.isCastFunction(funcName)) {
List<LogicalExpression> newArgs = generateNewExpressions(funcHolderExpr.args, value);
if (newArgs == null) {
return null;
+ } else {
+ return funcHolderExpr.copy(newArgs);
}
-
- return funcHolderExpr.copy(newArgs);
} else {
return null;
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/StatisticsProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/StatisticsProvider.java
index 1ab7579..5379760 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/StatisticsProvider.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/StatisticsProvider.java
@@ -76,8 +76,9 @@ public class StatisticsProvider<T extends Comparable<T>> extends AbstractExprVis
} else if (typedFieldExpr.getMajorType().equals(Types.OPTIONAL_INT)) {
// field does not exist.
return StatisticsProvider.getColumnStatistics(null, null, rowCount, typedFieldExpr.getMajorType().getMinorType());
+ } else {
+ return null;
}
- return null;
}
@Override
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
index 992e032..5f8c6c8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
@@ -19,7 +19,6 @@ package org.apache.drill.exec.expr.fn;
import static org.apache.drill.shaded.guava.com.google.common.base.Preconditions.checkArgument;
-import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.types.TypeProtos;
@@ -44,20 +43,23 @@ import com.sun.codemodel.JMod;
import com.sun.codemodel.JVar;
class DrillAggFuncHolder extends DrillFuncHolder {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillAggFuncHolder.class);
protected String setup() {
return meth("setup");
}
+
protected String reset() {
return meth("reset", false);
}
+
protected String add() {
return meth("add");
}
+
protected String output() {
return meth("output");
}
+
protected String cleanup() {
return meth("cleanup", false);
}
@@ -81,51 +83,47 @@ class DrillAggFuncHolder extends DrillFuncHolder {
@Override
public JVar[] renderStart(ClassGenerator<?> g, HoldingContainer[] inputVariables, FieldReference fieldReference) {
if (!g.getMappingSet().isHashAggMapping()) { //Declare workspace vars for non-hash-aggregation.
- JVar[] workspaceJVars = declareWorkspaceVariables(g);
- generateBody(g, BlockType.SETUP, setup(), null, workspaceJVars, true);
- return workspaceJVars;
- } else { //Declare workspace vars and workspace vectors for hash aggregation.
-
- JVar[] workspaceJVars = declareWorkspaceVectors(g);
-
- JBlock setupBlock = g.getSetupBlock();
-
- //Loop through all workspace vectors, to get the minimum of size of all workspace vectors.
- JVar sizeVar = setupBlock.decl(g.getModel().INT, "vectorSize", JExpr.lit(Integer.MAX_VALUE));
- JClass mathClass = g.getModel().ref(Math.class);
- for (int id = 0; id < getWorkspaceVars().length; id++) {
- if (!getWorkspaceVars()[id].isInject()) {
- setupBlock.assign(sizeVar,mathClass.staticInvoke("min").arg(sizeVar).arg(g.getWorkspaceVectors().get(getWorkspaceVars()[id]).invoke("getValueCapacity")));
- }
- }
+ JVar[] workspaceJVars = declareWorkspaceVariables(g);
+ generateBody(g, BlockType.SETUP, setup(), null, workspaceJVars, true);
+ return workspaceJVars;
+ } else { //Declare workspace vars and workspace vectors for hash aggregation.
- for (int i = 0; i < getWorkspaceVars().length; i++) {
- if (!getWorkspaceVars()[i].isInject()) {
- setupBlock.assign(workspaceJVars[i], JExpr._new(g.getHolderType(getWorkspaceVars()[i].getMajorType())));
- }
- }
+ JVar[] workspaceJVars = declareWorkspaceVectors(g);
- //Use for loop to initialize entries in the workspace vectors.
- JForLoop forLoop = setupBlock._for();
- JVar ivar = forLoop.init(g.getModel().INT, "drill_internal_i", JExpr.lit(0));
- forLoop.test(ivar.lt(sizeVar));
- forLoop.update(ivar.assignPlus(JExpr.lit(1)));
+ JBlock setupBlock = g.getSetupBlock();
+
+ //Loop through all workspace vectors, to get the minimum of size of all workspace vectors.
+ JVar sizeVar = setupBlock.decl(g.getModel().INT, "vectorSize", JExpr.lit(Integer.MAX_VALUE));
+ JClass mathClass = g.getModel().ref(Math.class);
+ for (int id = 0; id < getWorkspaceVars().length; id++) {
+ if (!getWorkspaceVars()[id].isInject()) {
+ setupBlock.assign(sizeVar,mathClass.staticInvoke("min").arg(sizeVar).arg(g.getWorkspaceVectors().get(getWorkspaceVars()[id]).invoke("getValueCapacity")));
+ }
+ }
- JBlock subBlock = generateInitWorkspaceBlockHA(g, BlockType.SETUP, setup(), workspaceJVars, ivar);
- forLoop.body().add(subBlock);
- return workspaceJVars;
+ for (int i = 0; i < getWorkspaceVars().length; i++) {
+ if (!getWorkspaceVars()[i].isInject()) {
+ setupBlock.assign(workspaceJVars[i], JExpr._new(g.getHolderType(getWorkspaceVars()[i].getMajorType())));
+ }
}
+ //Use for loop to initialize entries in the workspace vectors.
+ JForLoop forLoop = setupBlock._for();
+ JVar ivar = forLoop.init(g.getModel().INT, "drill_internal_i", JExpr.lit(0));
+ forLoop.test(ivar.lt(sizeVar));
+ forLoop.update(ivar.assignPlus(JExpr.lit(1)));
+ JBlock subBlock = generateInitWorkspaceBlockHA(g, BlockType.SETUP, setup(), workspaceJVars, ivar);
+ forLoop.body().add(subBlock);
+ return workspaceJVars;
+ }
}
-
@Override
public void renderMiddle(ClassGenerator<?> g, HoldingContainer[] inputVariables, JVar[] workspaceJVars) {
addProtectedBlock(g, g.getBlock(BlockType.EVAL), add(), inputVariables, workspaceJVars, false);
}
-
@Override
public HoldingContainer renderEnd(ClassGenerator<?> classGenerator, HoldingContainer[] inputVariables,
JVar[] workspaceJVars, FunctionHolderExpression holderExpr) {
@@ -152,7 +150,6 @@ class DrillAggFuncHolder extends DrillFuncHolder {
return out;
}
-
private JVar[] declareWorkspaceVectors(ClassGenerator<?> g) {
JVar[] workspaceJVars = new JVar[getWorkspaceVars().length];
@@ -204,8 +201,9 @@ class DrillAggFuncHolder extends DrillFuncHolder {
}
}
- /*
- * This is customized version of "addProtectedBlock" for hash aggregation. It take one additional parameter "wsIndexVariable".
+ /**
+ * This is customized version of "addProtectedBlock" for hash aggregation. It
+ * take one additional parameter "wsIndexVariable".
*/
private void addProtectedBlockHA(ClassGenerator<?> g, JBlock sub, String body, HoldingContainer[] inputVariables, JVar[] workspaceJVars, JExpression wsIndexVariable) {
if (inputVariables != null) {
@@ -259,10 +257,7 @@ class DrillAggFuncHolder extends DrillFuncHolder {
}
sub.add(setMeth);
-
- JClass drillRunTimeException = g.getModel().ref(DrillRuntimeException.class);
}
-
}
@Override
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java
index 52b89db..f7c32aa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java
@@ -21,6 +21,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.ExpressionPosition;
@@ -36,24 +37,7 @@ import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.HoldingContainerExpression;
-public class FunctionGenerationHelper {
- public static final String COMPARE_TO_NULLS_HIGH = "compare_to_nulls_high";
- public static final String COMPARE_TO_NULLS_LOW = "compare_to_nulls_low";
-
- public static final String EQ = "equal";
- public static final String NE = "not_equal";
- public static final String GT = "greater_than";
- public static final String GE = "greater_than_or_equal_to";
- public static final String LT = "less_than";
- public static final String LE = "less_than_or_equal_to";
-
- public static final String IS_NULL = "isnull";
- public static final String IS_NOT_NULL = "isnotnull";
- public static final String IS_TRUE = "istrue";
- public static final String IS_NOT_TRUE = "isnottrue";
- public static final String IS_FALSE = "isfalse";
- public static final String IS_NOT_FALSE = "isnotfalse";
- public static final String NOT = "not";
+public class FunctionGenerationHelper implements FunctionNames {
/**
* Finds ordering comparator ("compare_to...") FunctionHolderExpression with
@@ -141,7 +125,7 @@ public class FunctionGenerationHelper {
List<LogicalExpression> newArgs = Lists.newArrayList();
newArgs.add(call);
newArgs.add(new IntExpression(0, ExpressionPosition.UNKNOWN));
- FunctionCall notEqual = new FunctionCall("not_equal", newArgs, ExpressionPosition.UNKNOWN);
+ FunctionCall notEqual = new FunctionCall(FunctionNames.NE, newArgs, ExpressionPosition.UNKNOWN);
IfExpression.IfCondition ifCondition = new IfCondition(notEqual, call);
IfExpression ifExpression = IfExpression.newBuilder().setIfCondition(ifCondition).setElse(comparisonFunction).build();
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
index 44d062c..f2c7281 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
@@ -204,7 +204,7 @@ public class FunctionImplementationRegistry implements FunctionLookupContext, Au
*/
private String functionReplacement(FunctionCall functionCall) {
String funcName = functionCall.getName();
- if (functionCall.args.size() == 0) {
+ if (functionCall.argCount() == 0) {
return funcName;
}
boolean castEmptyStringToNull = optionManager != null &&
@@ -212,7 +212,7 @@ public class FunctionImplementationRegistry implements FunctionLookupContext, Au
if (!castEmptyStringToNull) {
return funcName;
}
- MajorType majorType = functionCall.args.get(0).getMajorType();
+ MajorType majorType = functionCall.arg(0).getMajorType();
DataMode dataMode = majorType.getMode();
MinorType minorType = majorType.getMinorType();
if (FunctionReplacementUtils.isReplacementNeeded(funcName, minorType)) {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/BitFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/BitFunctions.java
index ade76eb..e0bcb91 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/BitFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/BitFunctions.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.expr.fn.impl;
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
@@ -34,7 +35,7 @@ import org.apache.drill.exec.expr.holders.IntHolder;
*/
public class BitFunctions {
- @FunctionTemplate(names = {"booleanOr", "or", "||", "orNoShortCircuit"},
+ @FunctionTemplate(names = {FunctionNames.OR, "or", "||", "orNoShortCircuit"},
scope = FunctionScope.SIMPLE,
nulls = NullHandling.NULL_IF_NULL)
public static class BitOr implements DrillSimpleFunc {
@@ -43,14 +44,16 @@ public class BitFunctions {
@Param BitHolder right;
@Output BitHolder out;
+ @Override
public void setup() {}
+ @Override
public void eval() {
out.value = left.value | right.value;
}
}
- @FunctionTemplate(names = {"booleanAnd", "and", "&&"},
+ @FunctionTemplate(names = {FunctionNames.AND, "and", "&&"},
scope = FunctionScope.SIMPLE,
nulls = NullHandling.NULL_IF_NULL)
public static class BitAnd implements DrillSimpleFunc {
@@ -59,15 +62,17 @@ public class BitFunctions {
@Param BitHolder right;
@Output BitHolder out;
+ @Override
public void setup() {}
+ @Override
public void eval() {
out.value = left.value & right.value;
}
}
- @FunctionTemplate(names = {"xor", "^"},
+ @FunctionTemplate(names = {FunctionNames.XOR, "^"},
scope = FunctionScope.SIMPLE,
nulls = NullHandling.NULL_IF_NULL)
public static class IntXor implements DrillSimpleFunc {
@@ -76,11 +81,12 @@ public class BitFunctions {
@Param IntHolder right;
@Output IntHolder out;
+ @Override
public void setup() {}
+ @Override
public void eval() {
out.value = left.value ^ right.value;
}
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
index aff9f27..b9c53be 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
@@ -19,6 +19,8 @@ package org.apache.drill.exec.expr.fn.impl;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.DrillBuf;
+
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
@@ -47,7 +49,7 @@ public class StringFunctions{
* String Function Implementation.
*/
- @FunctionTemplate(name = "like", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
+ @FunctionTemplate(name = FunctionNames.LIKE, scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
public static class Like implements DrillSimpleFunc {
@Param VarCharHolder input;
@@ -69,7 +71,7 @@ public class StringFunctions{
}
}
- @FunctionTemplate(name = "like", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
+ @FunctionTemplate(name = FunctionNames.LIKE, scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
public static class LikeWithEscape implements DrillSimpleFunc {
@Param VarCharHolder input;
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterEvaluator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterEvaluator.java
index 6dcd632..19cc0f2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterEvaluator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/interpreter/InterpreterEvaluator.java
@@ -27,6 +27,7 @@ import javax.inject.Inject;
import org.apache.drill.exec.expr.BasicTypeHelper;
import org.apache.drill.shaded.guava.com.google.common.base.Function;
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.expression.BooleanOperator;
import org.apache.drill.common.expression.ConvertExpression;
@@ -88,7 +89,6 @@ public class InterpreterEvaluator {
}
outVV.getMutator().setValueCount(recordCount);
-
}
/**
@@ -102,7 +102,8 @@ public class InterpreterEvaluator {
* @throws Exception if {@code args} types does not match function input arguments types
*/
public static ValueHolder evaluateFunction(DrillSimpleFunc interpreter, Object[] args, String funcName) throws Exception {
- Preconditions.checkArgument(interpreter != null, "interpreter could not be null when use interpreted model to evaluate function " + funcName);
+ Preconditions.checkArgument(interpreter != null,
+ "interpreter could not be null when use interpreted model to evaluate function " + funcName);
// the current input index to assign into the next available parameter, found using the @Param notation
// the order parameters are declared in the java class for the DrillFunc is meaningful
@@ -152,7 +153,7 @@ public class InterpreterEvaluator {
private static class InitVisitor extends AbstractExprVisitor<LogicalExpression, VectorAccessible, RuntimeException> {
- private UdfUtilities udfUtilities;
+ private final UdfUtilities udfUtilities;
protected InitVisitor(UdfUtilities udfUtilities) {
super();
@@ -161,7 +162,7 @@ public class InterpreterEvaluator {
@Override
public LogicalExpression visitFunctionHolderExpression(FunctionHolderExpression holderExpr, VectorAccessible incoming) {
- if (! (holderExpr.getHolder() instanceof DrillSimpleFuncHolder)) {
+ if (!(holderExpr.getHolder() instanceof DrillSimpleFuncHolder)) {
throw new UnsupportedOperationException("Only Drill simple UDF can be used in interpreter mode!");
}
@@ -175,7 +176,7 @@ public class InterpreterEvaluator {
DrillSimpleFunc interpreter = holder.createInterpreter();
Field[] fields = interpreter.getClass().getDeclaredFields();
for (Field f : fields) {
- if ( f.getAnnotation(Inject.class) != null ) {
+ if (f.getAnnotation(Inject.class) != null) {
f.setAccessible(true);
Class<?> fieldType = f.getType();
if (UdfUtilities.INJECTABLE_GETTER_METHODS.get(fieldType) != null) {
@@ -208,10 +209,9 @@ public class InterpreterEvaluator {
}
}
-
public static class EvalVisitor extends AbstractExprVisitor<ValueHolder, Integer, RuntimeException> {
- private VectorAccessible incoming;
- private UdfUtilities udfUtilities;
+ private final VectorAccessible incoming;
+ private final UdfUtilities udfUtilities;
protected EvalVisitor(VectorAccessible incoming, UdfUtilities udfUtilities) {
super();
@@ -314,9 +314,9 @@ public class InterpreterEvaluator {
public ValueHolder visitNullExpression(NullExpression e,Integer value) throws RuntimeException {
return visitUnknown(e, value);
}
+
// TODO - review what to do with these (2 functions above)
//********************************************
-
@Override
public ValueHolder visitFunctionHolderExpression(FunctionHolderExpression holderExpr, Integer inIndex) {
if (! (holderExpr.getHolder() instanceof DrillSimpleFuncHolder)) {
@@ -376,19 +376,19 @@ public class InterpreterEvaluator {
} catch (Exception ex) {
throw new RuntimeException("Error in evaluating function of " + holderExpr.getName(), ex);
}
-
}
-
@Override
public ValueHolder visitBooleanOperator(BooleanOperator op, Integer inIndex) {
// Apply short circuit evaluation to boolean operator.
- if (op.getName().equals("booleanAnd")) {
+ if (op.getName().equals(FunctionNames.AND)) {
return visitBooleanAnd(op, inIndex);
- }else if(op.getName().equals("booleanOr")) {
+ } else if(op.getName().equals(FunctionNames.OR)) {
return visitBooleanOr(op, inIndex);
} else {
- throw new UnsupportedOperationException("BooleanOperator can only be booleanAnd, booleanOr. You are using " + op.getName());
+ throw new UnsupportedOperationException(
+ String.format("BooleanOperator can only be %s, %s. You are using %s",
+ FunctionNames.AND, FunctionNames.OR, op.getName()));
}
}
@@ -443,7 +443,6 @@ public class InterpreterEvaluator {
});
}
-
@Override
public ValueHolder visitUnknown(LogicalExpression e, Integer inIndex) throws RuntimeException {
if (e instanceof ValueVectorReadExpression) {
@@ -451,7 +450,6 @@ public class InterpreterEvaluator {
} else {
return super.visitUnknown(e, inIndex);
}
-
}
protected ValueHolder visitValueVectorReadExpression(ValueVectorReadExpression e, Integer inIndex)
@@ -487,10 +485,10 @@ public class InterpreterEvaluator {
// 2) if none argument is false, but at least one is null, return null.
// 3) finally, return true (finalValue).
private ValueHolder visitBooleanAnd(BooleanOperator op, Integer inIndex) {
- ValueHolder [] args = new ValueHolder [op.args.size()];
+ ValueHolder [] args = new ValueHolder [op.argCount()];
boolean hasNull = false;
- for (int i = 0; i < op.args.size(); i++) {
- args[i] = op.args.get(i).accept(this, inIndex);
+ for (int i = 0; i < op.argCount(); i++) {
+ args[i] = op.arg(i).accept(this, inIndex);
Trivalent flag = isBitOn(args[i]);
@@ -518,10 +516,10 @@ public class InterpreterEvaluator {
// null false null
// null null null
private ValueHolder visitBooleanOr(BooleanOperator op, Integer inIndex) {
- ValueHolder [] args = new ValueHolder [op.args.size()];
+ ValueHolder [] args = new ValueHolder [op.argCount()];
boolean hasNull = false;
- for (int i = 0; i < op.args.size(); i++) {
- args[i] = op.args.get(i).accept(this, inIndex);
+ for (int i = 0; i < op.argCount(); i++) {
+ args[i] = op.arg(i).accept(this, inIndex);
Trivalent flag = isBitOn(args[i]);
@@ -565,8 +563,6 @@ public class InterpreterEvaluator {
private ValueHolder getConstantValueHolder(String value, MinorType type, Function<DrillBuf, ValueHolder> holderInitializer) {
return udfUtilities.getConstantValueHolder(value, type, holderInitializer);
}
-
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ExecutorFragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ExecutorFragmentContext.java
index 030e2c4..ac423cf 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ExecutorFragmentContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/ExecutorFragmentContext.java
@@ -38,7 +38,7 @@ import java.util.Set;
public interface ExecutorFragmentContext extends RootFragmentContext {
/**
- * @return The root allocator for the Drillbit.
+ * Returns the root allocator for the Drillbit.
*/
BufferAllocator getRootAllocator();
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractDbGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractDbGroupScan.java
index 403492c..d3c7795 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractDbGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractDbGroupScan.java
@@ -29,7 +29,6 @@ import org.apache.drill.exec.planner.physical.PartitionFunction;
import org.apache.drill.exec.store.AbstractStoragePlugin;
public abstract class AbstractDbGroupScan extends AbstractGroupScan implements DbGroupScan {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractDbGroupScan.class);
private static final String ROW_KEY = "_id";
private static final SchemaPath ROW_KEY_PATH = SchemaPath.getSimplePath(ROW_KEY);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractDbSubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractDbSubScan.java
index caa5831..33bd045 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractDbSubScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractDbSubScan.java
@@ -25,6 +25,7 @@ public abstract class AbstractDbSubScan extends AbstractSubScan implements DbSub
super(userName);
}
+ @Override
public boolean isRestrictedSubScan() {
return false;
}
@@ -33,5 +34,4 @@ public abstract class AbstractDbSubScan extends AbstractSubScan implements DbSub
public void addJoinForRestrictedSubScan(RowKeyJoin batch) {
throw new UnsupportedOperationException();
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java
index 600a0bd..9e82ffc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java
@@ -132,7 +132,8 @@ public abstract class AbstractGroupScan extends AbstractBase implements GroupSca
*/
@Override
public long getColumnValueCount(SchemaPath column) {
- throw new UnsupportedOperationException(String.format("%s does not have exact column value count!", this.getClass().getCanonicalName()));
+ throw new UnsupportedOperationException(String.format(
+ "%s does not have exact column value count!", getClass().getCanonicalName()));
}
@Override
@@ -195,7 +196,8 @@ public abstract class AbstractGroupScan extends AbstractBase implements GroupSca
}
@Override
- public GroupScan applyFilter(LogicalExpression filterExpr, UdfUtilities udfUtilities, FunctionImplementationRegistry functionImplementationRegistry, OptionManager optionManager) {
+ public GroupScan applyFilter(LogicalExpression filterExpr, UdfUtilities udfUtilities,
+ FunctionImplementationRegistry functionImplementationRegistry, OptionManager optionManager) {
return null;
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScanWithMetadata.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScanWithMetadata.java
index 9293a8d..b864ac9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScanWithMetadata.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScanWithMetadata.java
@@ -108,7 +108,7 @@ public abstract class AbstractGroupScanWithMetadata<P extends TableMetadataProvi
protected Set<Path> fileSet;
// whether all files, partitions or row groups of this group scan fully match the filter
- protected boolean matchAllMetadata = false;
+ protected boolean matchAllMetadata;
protected boolean usedMetastore; // false by default
@@ -640,7 +640,7 @@ public abstract class AbstractGroupScanWithMetadata<P extends TableMetadataProvi
protected abstract static class GroupScanWithMetadataFilterer<B extends GroupScanWithMetadataFilterer<B>> {
protected final AbstractGroupScanWithMetadata<? extends TableMetadataProvider> source;
- protected boolean matchAllMetadata = false;
+ protected boolean matchAllMetadata;
protected TableMetadata tableMetadata;
protected List<PartitionMetadata> partitions = Collections.emptyList();
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
index 212581d..be6eae8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
@@ -47,10 +47,9 @@ import org.apache.drill.exec.physical.config.Values;
import org.apache.drill.exec.physical.config.WindowPOP;
public abstract class AbstractPhysicalVisitor<T, X, E extends Throwable> implements PhysicalVisitor<T, X, E> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractPhysicalVisitor.class);
@Override
- public T visitExchange(Exchange exchange, X value) throws E{
+ public T visitExchange(Exchange exchange, X value) throws E {
return visitOp(exchange, value);
}
@@ -65,7 +64,7 @@ public abstract class AbstractPhysicalVisitor<T, X, E extends Throwable> impleme
}
@Override
- public T visitFilter(Filter filter, X value) throws E{
+ public T visitFilter(Filter filter, X value) throws E {
return visitOp(filter, value);
}
@@ -75,17 +74,17 @@ public abstract class AbstractPhysicalVisitor<T, X, E extends Throwable> impleme
}
@Override
- public T visitProject(Project project, X value) throws E{
+ public T visitProject(Project project, X value) throws E {
return visitOp(project, value);
}
@Override
- public T visitTrace(Trace trace, X value) throws E{
+ public T visitTrace(Trace trace, X value) throws E {
return visitOp(trace, value);
}
@Override
- public T visitSort(Sort sort, X value) throws E{
+ public T visitSort(Sort sort, X value) throws E {
return visitOp(sort, value);
}
@@ -130,22 +129,21 @@ public abstract class AbstractPhysicalVisitor<T, X, E extends Throwable> impleme
}
@Override
- public T visitGroupScan(GroupScan groupScan, X value) throws E{
+ public T visitGroupScan(GroupScan groupScan, X value) throws E {
return visitOp(groupScan, value);
}
@Override
- public T visitSubScan(SubScan subScan, X value) throws E{
+ public T visitSubScan(SubScan subScan, X value) throws E {
return visitOp(subScan, value);
}
@Override
- public T visitStore(Store store, X value) throws E{
+ public T visitStore(Store store, X value) throws E {
return visitOp(store, value);
}
-
- public T visitChildren(PhysicalOperator op, X value) throws E{
+ public T visitChildren(PhysicalOperator op, X value) throws E {
for (PhysicalOperator child : op) {
child.accept(this, value);
}
@@ -233,10 +231,9 @@ public abstract class AbstractPhysicalVisitor<T, X, E extends Throwable> impleme
}
@Override
- public T visitOp(PhysicalOperator op, X value) throws E{
+ public T visitOp(PhysicalOperator op, X value) throws E {
throw new UnsupportedOperationException(String.format(
- "The PhysicalVisitor of type %s does not currently support visiting the PhysicalOperator type %s.", this
+ "PhysicalVisitor of type %s does not support PhysicalOperator type %s.", this
.getClass().getCanonicalName(), op.getClass().getCanonicalName()));
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/FragmentLeaf.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/FragmentLeaf.java
index ef356c6..168b32c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/FragmentLeaf.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/FragmentLeaf.java
@@ -18,8 +18,8 @@
package org.apache.drill.exec.physical.base;
/**
- * A Physical Operator that can be the leaf node of one particular execution fragment. Typically includes Receivers and
- * Scans
+ * A Physical Operator that can be the leaf node of one particular execution
+ * fragment. Typically includes Receivers and Scans.
*/
public interface FragmentLeaf extends PhysicalOperator {
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Leaf.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Leaf.java
index 53fec38..a84a799 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Leaf.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Leaf.java
@@ -18,8 +18,10 @@
package org.apache.drill.exec.physical.base;
/**
- * An operator which specifically is a lowest level leaf node of a query plan across all possible fragments. Currently, the only operator that is a Leaf
- * node are GroupScan nodes. Ultimately this could include use of Cache scans and other types of atypical data production systems.
+ * Operator which specifically is a lowest level leaf node of a query plan
+ * across all possible fragments. Currently, the only operator that is a Leaf
+ * node are GroupScan nodes. Ultimately this could include use of Cache scans
+ * and other types of atypical data production systems.
*/
public interface Leaf extends FragmentLeaf {
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ScanStats.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ScanStats.java
index 596dc5b..7c5656f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ScanStats.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/ScanStats.java
@@ -47,9 +47,10 @@ public class ScanStats {
}
/**
- * Return if GroupScan knows the exact row count in the result of getSize() call.
- * By default, group scan does not know the exact row count, before it scans every rows.
- * Currently, parquet group scan will return the exact row count.
+ * Return whether GroupScan knows the exact row count in the result of getSize()
+ * call. By default, group scan does not know the exact row count, before it
+ * scans every rows. Currently, Parquet group scan will return the exact row
+ * count.
*
* @return group scan property
*/
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/SingleMergeExchange.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/SingleMergeExchange.java
index a5986f3..4fb162e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/SingleMergeExchange.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/SingleMergeExchange.java
@@ -36,7 +36,6 @@ import com.fasterxml.jackson.annotation.JsonTypeName;
@JsonTypeName("single-merge-exchange")
public class SingleMergeExchange extends AbstractExchange {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SingleMergeExchange.class);
private final List<Ordering> orderExpr;
@@ -71,7 +70,8 @@ public class SingleMergeExchange extends AbstractExchange {
@Override
public Receiver getReceiver(int minorFragmentId) {
- return new MergingReceiverPOP(senderMajorFragmentId, PhysicalOperatorUtil.getIndexOrderedEndpoints(senderLocations), orderExpr, false);
+ return new MergingReceiverPOP(senderMajorFragmentId,
+ PhysicalOperatorUtil.getIndexOrderedEndpoints(senderLocations), orderExpr, false);
}
@Override
@@ -83,5 +83,4 @@ public class SingleMergeExchange extends AbstractExchange {
public List<Ordering> getOrderings() {
return this.orderExpr;
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
index 7171e24..2f0c9e2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
@@ -445,14 +445,14 @@ public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
if (funcName.equals("sum") || funcName.equals("max") || funcName.equals("min")) {
extraNonNullColumns++;
}
- List<LogicalExpression> args = ((FunctionCall) ne.getExpr()).args;
+ List<LogicalExpression> args = ((FunctionCall) ne.getExpr()).args();
if (!args.isEmpty()) {
if (args.get(0) instanceof SchemaPath) {
columnMapping.put(outputField.getName(), ((SchemaPath) args.get(0)).getAsNamePart().getName());
} else if (args.get(0) instanceof FunctionCall) {
FunctionCall functionCall = (FunctionCall) args.get(0);
- if (functionCall.args.get(0) instanceof SchemaPath) {
- columnMapping.put(outputField.getName(), ((SchemaPath) functionCall.args.get(0)).getAsNamePart().getName());
+ if (functionCall.arg(0) instanceof SchemaPath) {
+ columnMapping.put(outputField.getName(), ((SchemaPath) functionCall.arg(0)).getAsNamePart().getName());
}
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ResolvedMapColumn.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ResolvedMapColumn.java
index 6d5614e..f590a71 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ResolvedMapColumn.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/scan/project/ResolvedMapColumn.java
@@ -55,7 +55,6 @@ public class ResolvedMapColumn extends ResolvedColumn {
// We may have to create a matching new output
// map. Determine whether it is a single map or
// an array.
-
assert schema.getType().getMinorType() == MinorType.MAP;
if (schema.getType().getMode() == DataMode.REPEATED) {
members = new ResolvedMapArray(this);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFunction.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFunction.java
index ea61a53..05d4e04 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFunction.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/window/WindowFunction.java
@@ -40,9 +40,11 @@ import org.apache.drill.exec.physical.config.WindowPOP;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TypedFieldId;
import org.apache.drill.exec.record.VectorContainer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public abstract class WindowFunction {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(WindowFunction.class);
+ private static final Logger logger = LoggerFactory.getLogger(WindowFunction.class);
public enum Type {
ROW_NUMBER,
@@ -96,9 +98,10 @@ public abstract class WindowFunction {
abstract boolean supportsCustomFrames();
/**
- * @param pop window group definition
- * @return true if this window function requires all batches of current partition to be available before processing
- * the first batch
+ * @param pop
+ * window group definition
+ * @return true if this window function requires all batches of current
+ * partition to be available before processing the first batch
*/
public boolean requiresFullPartition(final WindowPOP pop) {
return true;
@@ -264,7 +267,7 @@ public abstract class WindowFunction {
boolean materialize(final NamedExpression ne, final VectorContainer batch, final FunctionLookupContext registry)
throws SchemaChangeException {
final FunctionCall call = (FunctionCall) ne.getExpr();
- final LogicalExpression argument = call.args.get(0);
+ final LogicalExpression argument = call.arg(0);
final MaterializedField outputField = MaterializedField.create(ne.getRef().getAsNamePart().getName(), argument.getMajorType());
batch.addOrGet(outputField).allocateNew();
fieldId = batch.getValueVectorId(ne.getRef());
@@ -307,7 +310,7 @@ public abstract class WindowFunction {
boolean materialize(final NamedExpression ne, final VectorContainer batch, final FunctionLookupContext registry)
throws SchemaChangeException {
final FunctionCall call = (FunctionCall) ne.getExpr();
- final LogicalExpression input = ExpressionTreeMaterializer.materializeAndCheckErrors(call.args.get(0), batch, registry);
+ final LogicalExpression input = ExpressionTreeMaterializer.materializeAndCheckErrors(call.arg(0), batch, registry);
if (input == null) {
return false;
}
@@ -355,7 +358,7 @@ public abstract class WindowFunction {
boolean materialize(final NamedExpression ne, final VectorContainer batch, final FunctionLookupContext registry)
throws SchemaChangeException {
final FunctionCall call = (FunctionCall) ne.getExpr();
- final LogicalExpression input = ExpressionTreeMaterializer.materializeAndCheckErrors(call.args.get(0), batch, registry);
+ final LogicalExpression input = ExpressionTreeMaterializer.materializeAndCheckErrors(call.arg(0), batch, registry);
if (input == null) {
return false;
}
@@ -426,7 +429,7 @@ public abstract class WindowFunction {
boolean materialize(final NamedExpression ne, final VectorContainer batch, final FunctionLookupContext registry)
throws SchemaChangeException {
final FunctionCall call = (FunctionCall) ne.getExpr();
- final LogicalExpression input = ExpressionTreeMaterializer.materializeAndCheckErrors(call.args.get(0), batch, registry);
+ final LogicalExpression input = ExpressionTreeMaterializer.materializeAndCheckErrors(call.arg(0), batch, registry);
if (input == null) {
return false;
}
@@ -442,6 +445,7 @@ public abstract class WindowFunction {
@Override
void generateCode(ClassGenerator<WindowFramer> cg) {
+
// in DefaultFrameTemplate we call setupReadLastValue:
// setupReadLastValue(current, container)
// and readLastValue:
@@ -449,8 +453,8 @@ public abstract class WindowFunction {
//
// this will generate the the following, pseudo, code:
// write current.source_last_value[frameLastRow] to container.last_value[row]
-
- final GeneratorMapping mapping = GeneratorMapping.create("setupReadLastValue", "writeLastValue", "resetValues", "cleanup");
+ final GeneratorMapping mapping = GeneratorMapping.create(
+ "setupReadLastValue", "writeLastValue", "resetValues", "cleanup");
final MappingSet mappingSet = new MappingSet("index", "outIndex", mapping, mapping);
cg.setMappingSet(mappingSet);
@@ -487,17 +491,20 @@ public abstract class WindowFunction {
boolean materialize(final NamedExpression ne, final VectorContainer batch, final FunctionLookupContext registry)
throws SchemaChangeException {
final FunctionCall call = (FunctionCall) ne.getExpr();
- final LogicalExpression input = ExpressionTreeMaterializer.materializeAndCheckErrors(call.args.get(0), batch, registry);
+ final LogicalExpression input = ExpressionTreeMaterializer.materializeAndCheckErrors(
+ call.arg(0), batch, registry);
if (input == null) {
return false;
}
- final MaterializedField output = MaterializedField.create(ne.getRef().getAsNamePart().getName(), input.getMajorType());
+ final MaterializedField output = MaterializedField.create(
+ ne.getRef().getAsNamePart().getName(), input.getMajorType());
batch.addOrGet(output).allocateNew();
final TypedFieldId outputId = batch.getValueVectorId(ne.getRef());
// write incoming.first_value[inIndex] to outgoing.first_value[outIndex]
- writeFirstValueToFirstValue = new ValueVectorWriteExpression(outputId, new ValueVectorReadExpression(outputId), true);
+ writeFirstValueToFirstValue = new ValueVectorWriteExpression(outputId,
+ new ValueVectorReadExpression(outputId), true);
// write incoming.source[inIndex] to outgoing.first_value[outIndex]
writeInputToFirstValue = new ValueVectorWriteExpression(outputId, input, true);
return true;
@@ -517,7 +524,8 @@ public abstract class WindowFunction {
// so it basically copies the first value of current partition into the first row of internal.first_value
// this is especially useful when handling multiple batches for the same partition where we need to keep
// the first value of the partition somewhere after we release the first batch
- final GeneratorMapping mapping = GeneratorMapping.create("setupSaveFirstValue", "saveFirstValue", null, null);
+ final GeneratorMapping mapping = GeneratorMapping.create(
+ "setupSaveFirstValue", "saveFirstValue", null, null);
final MappingSet mappingSet = new MappingSet("index", "0", mapping, mapping);
cg.setMappingSet(mappingSet);
@@ -534,7 +542,8 @@ public abstract class WindowFunction {
// write internal.first_value[0] to container.first_value[outIndex]
//
// so it basically copies the value stored in internal.first_value's first row into all rows of container.first_value
- final GeneratorMapping mapping = GeneratorMapping.create("setupWriteFirstValue", "outputRow", "resetValues", "cleanup");
+ final GeneratorMapping mapping = GeneratorMapping.create(
+ "setupWriteFirstValue", "outputRow", "resetValues", "cleanup");
final MappingSet mappingSet = new MappingSet("0", "outIndex", mapping, mapping);
cg.setMappingSet(mappingSet);
cg.addExpr(writeFirstValueToFirstValue);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/impl/BuildFromSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/impl/BuildFromSchema.java
index 54d3c0f..386195a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/impl/BuildFromSchema.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/impl/BuildFromSchema.java
@@ -269,8 +269,7 @@ public class BuildFromSchema {
*/
private ObjectWriter buildRepeatedList(ParentShim parent, ColumnMetadata colSchema) {
- final ColumnMetadata seed = colSchema.cloneEmpty();
- final ObjectWriter objWriter = parent.add(seed);
+ final ObjectWriter objWriter = parent.add(colSchema.cloneEmpty());
final RepeatedListWriter listWriter = (RepeatedListWriter) objWriter.array();
final ColumnMetadata elements = colSchema.childSchema();
if (elements != null) {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/impl/ColumnBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/impl/ColumnBuilder.java
index 3c5df73..2661898 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/impl/ColumnBuilder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/impl/ColumnBuilder.java
@@ -487,7 +487,7 @@ public class ColumnBuilder {
columnSchema, vector, dummyElement);
// Create the list vector state that tracks the list vector lifecycle.
- // For a repeated list, we only care about
+
final RepeatedListVectorState vectorState = new RepeatedListVectorState(
arrayWriter, vector);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/impl/ColumnState.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/impl/ColumnState.java
index 3815744..62e17a3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/impl/ColumnState.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/resultSet/impl/ColumnState.java
@@ -289,7 +289,7 @@ public abstract class ColumnState {
// Remember that we have look-ahead values stashed away in the
// backup vector.
- state= State.LOOK_AHEAD;
+ state = State.LOOK_AHEAD;
break;
default:
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/AbstractPartitionDescriptor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/AbstractPartitionDescriptor.java
index d414827..4acef81 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/AbstractPartitionDescriptor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/AbstractPartitionDescriptor.java
@@ -25,21 +25,23 @@ import org.apache.drill.exec.store.dfs.MetadataContext;
import org.apache.hadoop.fs.Path;
/**
- * Abstract base class for file system based partition descriptors and Hive partition descriptors.
- *
+ * Abstract base class for file system based partition descriptors and Hive
+ * partition descriptors.
*/
public abstract class AbstractPartitionDescriptor implements PartitionDescriptor, Iterable<List<PartitionLocation>> {
/**
- * A sequence of sublists of partition locations combined into a single super list.
- * The size of each sublist is at most {@link PartitionDescriptor#PARTITION_BATCH_SIZE}
- * For example if the size is 3, the complete list could be: {(a, b, c), {d, e, f), (g, h)}
+ * A sequence of sublists of partition locations combined into a single super
+ * list. The size of each sublist is at most
+ * {@link PartitionDescriptor#PARTITION_BATCH_SIZE} For example if the size is
+ * 3, the complete list could be: {(a, b, c), {d, e, f), (g, h)}
*/
protected List<List<PartitionLocation>> locationSuperList;
+
/**
- * Flag to indicate if the sublists of the partition locations has been created
+ * Indicates if the sublists of the partition locations has been created
*/
- protected boolean sublistsCreated = false;
+ protected boolean sublistsCreated;
/**
* Create sublists of the partition locations, each sublist of size
@@ -64,11 +66,9 @@ public abstract class AbstractPartitionDescriptor implements PartitionDescriptor
return false;
}
-
@Override
public TableScan createTableScan(List<PartitionLocation> newPartitions, Path cacheFileRoot,
boolean isAllPruned, MetadataContext metaContext) throws Exception {
throw new UnsupportedOperationException();
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/DFSFilePartitionLocation.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/DFSFilePartitionLocation.java
index 0dfe316..55c0074 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/DFSFilePartitionLocation.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/DFSFilePartitionLocation.java
@@ -21,7 +21,7 @@ import org.apache.drill.exec.store.ColumnExplorer;
import org.apache.hadoop.fs.Path;
/**
- * Class defines a single partition in a DFS table.
+ * Defines a single partition in a DFS table.
*/
public class DFSFilePartitionLocation extends SimplePartitionLocation {
private final String[] dirs;
@@ -58,6 +58,4 @@ public class DFSFilePartitionLocation extends SimplePartitionLocation {
public String[] getDirs() {
return dirs;
}
-
}
-
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/FileSystemPartitionDescriptor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/FileSystemPartitionDescriptor.java
index 6d6f6ba..865ea04 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/FileSystemPartitionDescriptor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/FileSystemPartitionDescriptor.java
@@ -33,7 +33,6 @@ import org.apache.drill.shaded.guava.com.google.common.base.Charsets;
import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
-import org.apache.drill.shaded.guava.com.google.common.collect.Maps;
import org.apache.calcite.adapter.enumerable.EnumerableTableScan;
import org.apache.calcite.prepare.RelOptTableImpl;
@@ -58,14 +57,16 @@ import org.apache.drill.exec.vector.ValueVector;
import org.apache.hadoop.fs.Path;
-// partition descriptor for file system based tables
+/**
+ * Partition descriptor for file system based tables.
+ */
public class FileSystemPartitionDescriptor extends AbstractPartitionDescriptor {
- static final int MAX_NESTED_SUBDIRS = 10; // allow up to 10 nested sub-directories
+ static final int MAX_NESTED_SUBDIRS = 10; // allow up to 10 nested sub-directories
private final String partitionLabel;
private final int partitionLabelLength;
- private final Map<String, Integer> partitions = Maps.newHashMap();
+ private final Map<String, Integer> partitions = new HashMap<>();
private final TableScan scanRel;
private final DrillTable table;
@@ -160,14 +161,15 @@ public class FileSystemPartitionDescriptor extends AbstractPartitionDescriptor {
final Path selectionRoot = getBaseTableLocation();
- // map used to map the partition keys (dir0, dir1, ..), to the list of partitions that share the same partition keys.
+ // map used to map the partition keys (dir0, dir1, ..), to the list of partitions
+ // that share the same partition keys.
// For example,
// 1990/Q1/1.parquet, 2.parquet
// would have <1990, Q1> as key, and value as list of partition location for 1.parquet and 2.parquet.
HashMap<List<String>, List<PartitionLocation>> dirToFileMap = new HashMap<>();
- // Figure out the list of leaf subdirectories. For each leaf subdirectory, find the list of files (DFSFilePartitionLocation)
- // it contains.
+ // Figure out the list of leaf subdirectories. For each leaf subdirectory, find
+ // the list of files (DFSFilePartitionLocation) it contains.
for (Path file: fileLocationsAndStatus.getLeft()) {
DFSFilePartitionLocation dfsFilePartitionLocation = new DFSFilePartitionLocation(MAX_NESTED_SUBDIRS,
selectionRoot, file, hasDirsOnly);
@@ -267,11 +269,7 @@ public class FileSystemPartitionDescriptor extends AbstractPartitionDescriptor {
@Override
public boolean supportsMetadataCachePruning() {
final Object selection = this.table.getSelection();
- if (selection instanceof FormatSelection
- && ((FormatSelection)selection).getSelection().getCacheFileRoot() != null) {
- return true;
- }
- return false;
+ return selection instanceof FormatSelection
+ && ((FormatSelection)selection).getSelection().getCacheFileRoot() != null;
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PartitionDescriptor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PartitionDescriptor.java
index 8759078..20e9ae6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PartitionDescriptor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PartitionDescriptor.java
@@ -30,7 +30,8 @@ import java.util.List;
import java.util.Map;
/**
- * Interface used to describe partitions. Currently used by file system based partitions and hive partitions
+ * Interface used to describe partitions. Currently used by file system based
+ * partitions and hive partitions
*/
public interface PartitionDescriptor extends Iterable<List<PartitionLocation>> {
@@ -71,7 +72,7 @@ public interface PartitionDescriptor extends Iterable<List<PartitionLocation>> {
int getMaxHierarchyLevel();
/**
- * Method creates an in memory representation of all the partitions. For each level of partitioning we
+ * Creates an in memory representation of all the partitions. For each level of partitioning we
* will create a value vector which this method will populate for all the partitions with the values of the
* partitioning key
* @param vectors - Array of vectors in the container that need to be populated
@@ -90,7 +91,7 @@ public interface PartitionDescriptor extends Iterable<List<PartitionLocation>> {
TypeProtos.MajorType getVectorType(SchemaPath column, PlannerSettings plannerSettings);
/**
- * Methods create a new TableScan rel node, given the lists of new partitions or new files to SCAN.
+ * Create a new TableScan rel node, given the lists of new partitions or new files to SCAN.
* @param newPartitions
* @param wasAllPartitionsPruned
* @throws Exception
@@ -113,5 +114,4 @@ public interface PartitionDescriptor extends Iterable<List<PartitionLocation>> {
boolean supportsMetadataCachePruning();
Path getBaseTableLocation();
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java
index 49b6d38..f24c77b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java
@@ -105,6 +105,7 @@ import java.util.Set;
public enum PlannerPhase {
LOGICAL_PRUNE_AND_JOIN("Logical Planning (with join and partition pruning)") {
+ @Override
public RuleSet getRules(OptimizerRulesContext context, Collection<StoragePlugin> plugins) {
return PlannerPhase.mergedRuleSets(
getDrillBasicRules(context),
@@ -116,6 +117,7 @@ public enum PlannerPhase {
},
WINDOW_REWRITE("Window Function rewrites") {
+ @Override
public RuleSet getRules(OptimizerRulesContext context, Collection<StoragePlugin> plugins) {
return RuleSets.ofList(
RuleInstance.CALC_INSTANCE,
@@ -125,6 +127,7 @@ public enum PlannerPhase {
},
SUBQUERY_REWRITE("Sub-queries rewrites") {
+ @Override
public RuleSet getRules(OptimizerRulesContext context, Collection<StoragePlugin> plugins) {
return RuleSets.ofList(
RuleInstance.SUB_QUERY_FILTER_REMOVE_RULE,
@@ -135,6 +138,7 @@ public enum PlannerPhase {
},
LOGICAL_PRUNE("Logical Planning (with partition pruning)") {
+ @Override
public RuleSet getRules(OptimizerRulesContext context, Collection<StoragePlugin> plugins) {
return PlannerPhase.mergedRuleSets(
getDrillBasicRules(context),
@@ -145,6 +149,7 @@ public enum PlannerPhase {
},
JOIN_PLANNING("LOPT Join Planning") {
+ @Override
public RuleSet getRules(OptimizerRulesContext context, Collection<StoragePlugin> plugins) {
List<RelOptRule> rules = Lists.newArrayList();
if (context.getPlannerSettings().isJoinOptimizationEnabled()) {
@@ -160,6 +165,7 @@ public enum PlannerPhase {
},
ROWKEYJOIN_CONVERSION("Convert Join to RowKeyJoin") {
+ @Override
public RuleSet getRules(OptimizerRulesContext context, Collection<StoragePlugin> plugins) {
List<RelOptRule> rules = Lists.newArrayList();
if (context.getPlannerSettings().isRowKeyJoinConversionEnabled()) {
@@ -173,6 +179,7 @@ public enum PlannerPhase {
},
SUM_CONVERSION("Convert SUM to $SUM0") {
+ @Override
public RuleSet getRules(OptimizerRulesContext context, Collection<StoragePlugin> plugins) {
return PlannerPhase.mergedRuleSets(
RuleSets.ofList(
@@ -184,24 +191,28 @@ public enum PlannerPhase {
},
PARTITION_PRUNING("Partition Prune Planning") {
+ @Override
public RuleSet getRules(OptimizerRulesContext context, Collection<StoragePlugin> plugins) {
return PlannerPhase.mergedRuleSets(getPruneScanRules(context), getStorageRules(context, plugins, this));
}
},
PHYSICAL_PARTITION_PRUNING("Physical Partition Prune Planning") {
+ @Override
public RuleSet getRules(OptimizerRulesContext context, Collection<StoragePlugin> plugins) {
return PlannerPhase.mergedRuleSets(getPhysicalPruneScanRules(context), getStorageRules(context, plugins, this));
}
},
DIRECTORY_PRUNING("Directory Prune Planning") {
+ @Override
public RuleSet getRules(OptimizerRulesContext context, Collection<StoragePlugin> plugins) {
return PlannerPhase.mergedRuleSets(getDirPruneScanRules(context), getStorageRules(context, plugins, this));
}
},
LOGICAL("Logical Planning (no pruning or join).") {
+ @Override
public RuleSet getRules(OptimizerRulesContext context, Collection<StoragePlugin> plugins) {
return PlannerPhase.mergedRuleSets(
PlannerPhase.getDrillBasicRules(context),
@@ -211,6 +222,7 @@ public enum PlannerPhase {
},
PHYSICAL("Physical Planning") {
+ @Override
public RuleSet getRules(OptimizerRulesContext context, Collection<StoragePlugin> plugins) {
return PlannerPhase.mergedRuleSets(
PlannerPhase.getPhysicalRules(context),
@@ -220,12 +232,14 @@ public enum PlannerPhase {
},
PRE_LOGICAL_PLANNING("Planning with Hep planner only for rules, which are failed for Volcano planner") {
+ @Override
public RuleSet getRules (OptimizerRulesContext context, Collection<StoragePlugin> plugins) {
return PlannerPhase.getSetOpTransposeRules();
}
},
TRANSITIVE_CLOSURE("Transitive closure") {
+ @Override
public RuleSet getRules(OptimizerRulesContext context, Collection<StoragePlugin> plugins) {
return getJoinTransitiveClosureRules();
}
@@ -239,20 +253,20 @@ public enum PlannerPhase {
public abstract RuleSet getRules(OptimizerRulesContext context, Collection<StoragePlugin> plugins);
+ @SuppressWarnings("deprecation")
private static RuleSet getStorageRules(OptimizerRulesContext context, Collection<StoragePlugin> plugins,
PlannerPhase phase) {
final Builder<RelOptRule> rules = ImmutableSet.builder();
- for(StoragePlugin plugin : plugins){
- if(plugin instanceof AbstractStoragePlugin){
+ for (StoragePlugin plugin : plugins) {
+ if (plugin instanceof AbstractStoragePlugin) {
rules.addAll(((AbstractStoragePlugin) plugin).getOptimizerRules(context, phase));
- }else{
+ } else {
rules.addAll(plugin.getOptimizerRules(context));
}
}
return RuleSets.ofList(rules.build());
}
-
static final RelOptRule DRILL_JOIN_TO_MULTIJOIN_RULE =
new JoinToMultiJoinRule(DrillJoinRel.class, DrillRelFactories.LOGICAL_BUILDER);
static final RelOptRule DRILL_LOPT_OPTIMIZE_JOIN_RULE =
@@ -412,9 +426,7 @@ public enum PlannerPhase {
return RuleSets.ofList(pruneRules);
}
- /**
- *
- */
+
static RuleSet getIndexRules(OptimizerRulesContext optimizerRulesContext) {
final PlannerSettings ps = optimizerRulesContext.getPlannerSettings();
if (!ps.isIndexPlanningEnabled()) {
@@ -480,7 +492,6 @@ public enum PlannerPhase {
.build();
return RuleSets.ofList(pruneRules);
-
}
/**
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
index 22248fb..3cb2077 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
@@ -73,8 +73,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Utility class that is a subset of the RelOptUtil class and is a placeholder for Drill specific
- * static methods that are needed during either logical or physical planning.
+ * Utility class that is a subset of the RelOptUtil class and is a placeholder
+ * for Drill specific static methods that are needed during either logical or
+ * physical planning.
*/
public abstract class DrillRelOptUtil {
@@ -142,10 +143,12 @@ public abstract class DrillRelOptUtil {
assert fieldNames.size() == fields.size();
final List<RexNode> refs =
new AbstractList<RexNode>() {
+ @Override
public int size() {
return fields.size();
}
+ @Override
public RexNode get(int index) {
return RexInputRef.of(index, fields);
}
@@ -221,6 +224,7 @@ public abstract class DrillRelOptUtil {
try {
RexVisitor<Void> visitor =
new RexVisitorImpl<Void>(true) {
+ @Override
public Void visitCall(RexCall call) {
if (operators.contains(call.getOperator().getName().toLowerCase())) {
throw new Util.FoundOne(call); /* throw exception to interrupt tree walk (this is similar to
@@ -229,6 +233,7 @@ public abstract class DrillRelOptUtil {
return super.visitCall(call);
}
+ @Override
public Void visitInputRef(RexInputRef inputRef) {
if (projExprs.size() == 0 ) {
return super.visitInputRef(inputRef);
@@ -264,6 +269,7 @@ public abstract class DrillRelOptUtil {
if (((long) l.getValue2()) == 0) {
return true;
}
+ default:
}
}
return false;
@@ -296,6 +302,7 @@ public abstract class DrillRelOptUtil {
try {
RexVisitor<Void> visitor =
new RexVisitorImpl<Void>(true) {
+ @Override
public Void visitCall(RexCall call) {
if ("convert_fromjson".equals(call.getOperator().getName().toLowerCase())) {
throw new Util.FoundOne(call); /* throw exception to interrupt tree walk (this is similar to
@@ -327,11 +334,13 @@ public abstract class DrillRelOptUtil {
inputRefList = new ArrayList<>();
}
+ @Override
public Void visitInputRef(RexInputRef ref) {
inputRefList.add(ref);
return null;
}
+ @Override
public Void visitCall(RexCall call) {
for (RexNode operand : call.operands) {
operand.accept(this);
@@ -421,12 +430,12 @@ public abstract class DrillRelOptUtil {
}
}
+ @SuppressWarnings("deprecation")
public static boolean isProjectFlatten(RelNode project) {
assert project instanceof Project : "Rel is NOT an instance of project!";
for (RexNode rex : project.getChildExps()) {
- RexNode newExpr = rex;
if (rex instanceof RexCall) {
RexCall function = (RexCall) rex;
String functionName = function.getOperator().getName();
@@ -684,6 +693,7 @@ public abstract class DrillRelOptUtil {
try {
RexVisitor<Void> visitor =
new RexVisitorImpl<Void>(true) {
+ @Override
public Void visitCall(RexCall call) {
if (call.getKind() == SqlKind.AND || call.getKind() == SqlKind.OR) {
super.visitCall(call);
@@ -738,6 +748,7 @@ public abstract class DrillRelOptUtil {
List<RexInputRef> rexRefs = new ArrayList<>();
RexVisitor<Void> visitor =
new RexVisitorImpl<Void>(true) {
+ @Override
public Void visitInputRef(RexInputRef inputRef) {
rexRefs.add(inputRef);
return super.visitInputRef(inputRef);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillScanRelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillScanRelBase.java
index 88b8d30..96e5bfc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillScanRelBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillScanRelBase.java
@@ -37,7 +37,7 @@ import org.apache.calcite.rel.metadata.RelMetadataQuery;
* Base class for logical/physical scan rel implemented in Drill.
*/
public abstract class DrillScanRelBase extends TableScan implements DrillRelNode {
- protected GroupScan groupScan;
+ protected final GroupScan groupScan;
protected final DrillTable drillTable;
public DrillScanRelBase(RelOptCluster cluster,
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/DistributionAffinity.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/DistributionAffinity.java
index b1ab9aa..1876df5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/DistributionAffinity.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/DistributionAffinity.java
@@ -18,8 +18,10 @@
package org.apache.drill.exec.planner.fragment;
/**
- * Describes an operator's endpoint assignment requirements. Ordering is from no assignment requirement to mandatory
- * assignment requirements. Changes/new addition should keep the order of increasing restrictive assignment requirement.
+ * Describes an operator's endpoint assignment requirements. Ordering is from no
+ * assignment requirement to mandatory assignment requirements. Changes/new
+ * addition should keep the order of increasing restrictive assignment
+ * requirement.
*/
public enum DistributionAffinity {
/**
@@ -28,14 +30,15 @@ public enum DistributionAffinity {
NONE(SoftAffinityFragmentParallelizer.INSTANCE),
/**
- * Operator has soft distribution affinity to one or more endpoints. Operator performs better when fragments are
- * assigned to the endpoints with affinity, but not a mandatory requirement.
+ * Operator has soft distribution affinity to one or more endpoints. Operator
+ * performs better when fragments are assigned to the endpoints with affinity,
+ * but not a mandatory requirement.
*/
SOFT(SoftAffinityFragmentParallelizer.INSTANCE),
/**
- * Hard distribution affinity to one or more endpoints. Fragments having the operator must be scheduled on the nodes
- * with affinity.
+ * Hard distribution affinity to one or more endpoints. Fragments having the
+ * operator must be scheduled on the nodes with affinity.
*/
HARD(HardAffinityFragmentParallelizer.INSTANCE);
@@ -53,9 +56,12 @@ public enum DistributionAffinity {
}
/**
- * Is the current DistributionAffinity less restrictive than the given DistributionAffinity?
+ * Is the current DistributionAffinity less restrictive than the given
+ * DistributionAffinity?
+ *
* @param distributionAffinity
- * @return True if the current DistributionAffinity less restrictive than the given DistributionAffinity. False otherwise.
+ * @return True if the current DistributionAffinity less restrictive than the
+ * given DistributionAffinity. False otherwise.
*/
public boolean isLessRestrictiveThan(final DistributionAffinity distributionAffinity) {
return ordinal() < distributionAffinity.ordinal();
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/HardAffinityFragmentParallelizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/HardAffinityFragmentParallelizer.java
index be45d56..752b081 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/HardAffinityFragmentParallelizer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/HardAffinityFragmentParallelizer.java
@@ -23,6 +23,7 @@ import org.apache.drill.exec.physical.EndpointAffinity;
import org.apache.drill.exec.physical.PhysicalOperatorSetupException;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.Collection;
import java.util.List;
@@ -30,11 +31,12 @@ import java.util.Map;
import java.util.Map.Entry;
/**
- * Implementation of {@link FragmentParallelizer} where fragment requires running on a given set of endpoints. Width
- * per node is depended on the affinity to the endpoint and total width (calculated using costs)
+ * Implementation of {@link FragmentParallelizer} where fragment requires
+ * running on a given set of endpoints. Width per node is depended on the
+ * affinity to the endpoint and total width (calculated using costs).
*/
public class HardAffinityFragmentParallelizer implements FragmentParallelizer {
- private static final Logger logger = org.slf4j.LoggerFactory.getLogger(HardAffinityFragmentParallelizer.class);
+ private static final Logger logger = LoggerFactory.getLogger(HardAffinityFragmentParallelizer.class);
public static final HardAffinityFragmentParallelizer INSTANCE = new HardAffinityFragmentParallelizer();
@@ -53,7 +55,7 @@ public class HardAffinityFragmentParallelizer implements FragmentParallelizer {
// Go through the affinity map and extract the endpoints that have mandatory assignment requirement
final Map<DrillbitEndpoint, EndpointAffinity> endpointPool = Maps.newHashMap();
- for(Entry<DrillbitEndpoint, EndpointAffinity> entry : pInfo.getEndpointAffinityMap().entrySet()) {
+ for (Entry<DrillbitEndpoint, EndpointAffinity> entry : pInfo.getEndpointAffinityMap().entrySet()) {
if (entry.getValue().isAssignmentRequired()) {
endpointPool.put(entry.getKey(), entry.getValue());
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Materializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Materializer.java
index fdfa95c..fd040e9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Materializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Materializer.java
@@ -38,31 +38,27 @@ import org.apache.drill.exec.physical.config.UnnestPOP;
import org.apache.drill.exec.physical.config.RowKeyJoinPOP;
public class Materializer extends AbstractPhysicalVisitor<PhysicalOperator, Materializer.IndexedFragmentNode, ExecutionSetupException>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Materializer.class);
public static final Materializer INSTANCE = new Materializer();
- private Materializer() {
- }
+ private Materializer() { }
@Override
public PhysicalOperator visitExchange(Exchange exchange, IndexedFragmentNode iNode) throws ExecutionSetupException {
iNode.addAllocation(exchange);
- if(exchange == iNode.getNode().getSendingExchange()){
+ if (exchange == iNode.getNode().getSendingExchange()) {
// this is a sending exchange.
PhysicalOperator child = exchange.getChild().accept(this, iNode);
PhysicalOperator materializedSender = exchange.getSender(iNode.getMinorFragmentId(), child);
materializedSender.setOperatorId(0);
materializedSender.setCost(exchange.getCost());
-// logger.debug("Visit sending exchange, materialized {} with child {}.", materializedSender, child);
return materializedSender;
- }else{
+ } else {
// receiving exchange.
PhysicalOperator materializedReceiver = exchange.getReceiver(iNode.getMinorFragmentId());
materializedReceiver.setOperatorId(Short.MAX_VALUE & exchange.getOperatorId());
-// logger.debug("Visit receiving exchange, materialized receiver: {}.", materializedReceiver);
materializedReceiver.setCost(exchange.getCost());
return materializedReceiver;
}
@@ -95,7 +91,6 @@ public class Materializer extends AbstractPhysicalVisitor<PhysicalOperator, Mate
try {
PhysicalOperator o = store.getSpecificStore(child, iNode.getMinorFragmentId());
o.setOperatorId(Short.MAX_VALUE & store.getOperatorId());
-// logger.debug("New materialized store node {} with child {}", o, child);
return o;
} catch (PhysicalOperatorSetupException e) {
throw new FragmentSetupException("Failure while generating a specific Store materialization.");
@@ -105,9 +100,8 @@ public class Materializer extends AbstractPhysicalVisitor<PhysicalOperator, Mate
@Override
public PhysicalOperator visitOp(PhysicalOperator op, IndexedFragmentNode iNode) throws ExecutionSetupException {
iNode.addAllocation(op);
-// logger.debug("Visiting catch all: {}", op);
List<PhysicalOperator> children = Lists.newArrayList();
- for(PhysicalOperator child : op){
+ for (PhysicalOperator child : op) {
children.add(child.accept(this, iNode));
}
PhysicalOperator newOp = op.getNewWithChildren(children);
@@ -116,7 +110,6 @@ public class Materializer extends AbstractPhysicalVisitor<PhysicalOperator, Mate
return newOp;
}
-
@Override
public PhysicalOperator visitLateralJoin(LateralJoinPOP op, IndexedFragmentNode iNode) throws ExecutionSetupException {
iNode.addAllocation(op);
@@ -168,7 +161,7 @@ public class Materializer extends AbstractPhysicalVisitor<PhysicalOperator, Mate
private final BiFunction<Wrapper, Integer, DrillbitEndpoint> endpoint;
private final int minorFragmentId;
private final BiFunction<DrillbitEndpoint, PhysicalOperator, Long> memoryPerOperPerDrillbit;
- SubScan subScan = null;
+ SubScan subScan;
private final Deque<UnnestPOP> unnest = new ArrayDeque<>();
@@ -217,5 +210,4 @@ public class Materializer extends AbstractPhysicalVisitor<PhysicalOperator, Mate
return this.subScan;
}
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/MemoryCalculator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/MemoryCalculator.java
index 443ab79..e8dd4af 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/MemoryCalculator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/MemoryCalculator.java
@@ -121,13 +121,13 @@ public class MemoryCalculator extends AbstractOpWrapperVisitor<Void, RuntimeExce
@Override
public Void visitOp(PhysicalOperator op, Wrapper fragment) {
- long memoryCost = (int)Math.ceil(op.getCost().getMemoryCost());
+ long memoryCost = (int) Math.ceil(op.getCost().getMemoryCost());
if (op.isBufferedOperator(queryContext)) {
// If the operator is a buffered operator then get the memory estimates of the optimizer.
// The memory estimates of the optimizer are for the whole operator spread across all the
// minor fragments. Divide this memory estimation by fragment width to get the memory
// requirement per minor fragment.
- long memoryCostPerMinorFrag = (int)Math.ceil(memoryCost/fragment.getAssignedEndpoints().size());
+ long memoryCostPerMinorFrag = (int) Math.ceil(memoryCost/fragment.getAssignedEndpoints().size());
Map<DrillbitEndpoint, Integer> drillbitEndpointMinorFragMap = getMinorFragCountPerDrillbit(fragment);
Map<DrillbitEndpoint,
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/PlanningSet.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/PlanningSet.java
index 15ff13b..be30971 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/PlanningSet.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/PlanningSet.java
@@ -21,14 +21,15 @@ import java.util.Iterator;
import java.util.Map;
import org.apache.drill.shaded.guava.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class PlanningSet implements Iterable<Wrapper> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PlanningSet.class);
+ static final Logger logger = LoggerFactory.getLogger(PlanningSet.class);
private final Map<Fragment, Wrapper> fragmentMap = Maps.newHashMap();
- private int majorFragmentIdIndex = 0;
-
- private Wrapper rootWrapper = null;
+ private int majorFragmentIdIndex;
+ private Wrapper rootWrapper;
public Wrapper get(Fragment node) {
Wrapper wrapper = fragmentMap.get(node);
@@ -42,7 +43,8 @@ public class PlanningSet implements Iterable<Wrapper> {
// assign the upper 16 bits as the major fragment id.
majorFragmentId = node.getSendingExchange().getChild().getOperatorId() >> 16;
- // if they are not assigned, that means we mostly likely have an externally generated plan. in this case, come up with a major fragmentid.
+ // if they are not assigned, that means we mostly likely have an externally generated plan.
+ // In this case, come up with a major fragmentid.
if (majorFragmentId == 0) {
majorFragmentId = majorFragmentIdIndex;
}
@@ -84,5 +86,4 @@ public class PlanningSet implements Iterable<Wrapper> {
public Wrapper getRootWrapper(){
return rootWrapper;
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/QueryParallelizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/QueryParallelizer.java
index 097f4b2..d7e7c91 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/QueryParallelizer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/QueryParallelizer.java
@@ -28,15 +28,15 @@ import org.apache.drill.exec.work.QueryWorkUnit;
import java.util.Collection;
/**
- * This class parallelizes the query plan. Once the optimizer finishes its job by producing a
+ * Parallelizes the query plan. Once the optimizer finishes its job by producing a
* optimized plan, it is the job of this parallelizer to generate a parallel plan out of the
* optimized physical plan. It does so by using the optimizers estimates for row count etc.
* There are two kinds of parallelizers as explained below. Currently the difference in
* both of these parallelizers is only in the memory assignment for the physical operators.
- *
+ * <p>
* a) Default Parallelizer: It optimistically assumes that the whole cluster is running only the
* current query and based on heuristics assigns the optimal memory to the buffered operators.
- *
+ * <p>
* b) Queue Parallelizer: This parallelizer computes the memory that can be allocated at best based
* on the current cluster state(as to how much memory is available) and also the configuration
* of the queue that it can run on.
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
index 0d3529b..e56af09 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
@@ -52,6 +52,8 @@ import org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTes
import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
import org.apache.drill.shaded.guava.com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.drill.exec.server.options.OptionManager;
import org.apache.drill.exec.work.QueryWorkUnit;
import org.apache.drill.exec.work.QueryWorkUnit.MinorFragmentDefn;
@@ -59,13 +61,16 @@ import org.apache.drill.exec.work.foreman.ForemanSetupException;
/**
- * The simple parallelizer determines the level of parallelization of a plan based on the cost of the underlying
- * operations. It doesn't take into account system load or other factors. Based on the cost of the query, the
- * parallelization for each major fragment will be determined. Once the amount of parallelization is done, assignment
- * is done based on round robin assignment ordered by operator affinity (locality) to available execution Drillbits.
+ * The simple parallelizer determines the level of parallelization of a plan
+ * based on the cost of the underlying operations. It doesn't take into account
+ * system load or other factors. Based on the cost of the query, the
+ * parallelization for each major fragment will be determined. Once the amount
+ * of parallelization is done, assignment is done based on round robin
+ * assignment ordered by operator affinity (locality) to available execution
+ * Drillbits.
*/
public abstract class SimpleParallelizer implements QueryParallelizer {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SimpleParallelizer.class);
+ static final Logger logger = LoggerFactory.getLogger(SimpleParallelizer.class);
private final long parallelizationThreshold;
private final int maxWidthPerNode;
@@ -80,7 +85,7 @@ public abstract class SimpleParallelizer implements QueryParallelizer {
double cpu_load_average = optionManager.getOption(ExecConstants.CPU_LOAD_AVERAGE);
final long maxWidth = optionManager.getOption(ExecConstants.MAX_WIDTH_PER_NODE);
// compute the maxwidth
- this.maxWidthPerNode = ExecConstants.MAX_WIDTH_PER_NODE.computeMaxWidth(cpu_load_average,maxWidth);
+ this.maxWidthPerNode = ExecConstants.MAX_WIDTH_PER_NODE.computeMaxWidth(cpu_load_average, maxWidth);
this.maxGlobalWidth = optionManager.getOption(ExecConstants.MAX_WIDTH_GLOBAL_KEY).num_val.intValue();
this.affinityFactor = optionManager.getOption(ExecConstants.AFFINITY_FACTOR_KEY).float_val.intValue();
this.enableDynamicFC = optionManager.getBoolean(ExecConstants.ENABLE_DYNAMIC_CREDIT_BASED_FC);
@@ -114,19 +119,20 @@ public abstract class SimpleParallelizer implements QueryParallelizer {
}
public Set<Wrapper> getRootFragments(PlanningSet planningSet) {
- //The following code gets the root fragment by removing all the dependent fragments on which root fragments depend upon.
- //This is fine because the later parallelizer code traverses from these root fragments to their respective dependent
- //fragments.
+ // Gets the root fragment by removing all the dependent fragments on which
+ // root fragments depend upon. This is fine because the later parallelizer
+ // code traverses from these root fragments to their respective dependent
+ // fragments.
final Set<Wrapper> roots = Sets.newHashSet();
- for(Wrapper w : planningSet) {
+ for (Wrapper w : planningSet) {
roots.add(w);
}
- //roots will be left over with the fragments which are not depended upon by any other fragments.
- for(Wrapper wrapper : planningSet) {
+ // Roots will be left over with the fragments which are not depended upon by any other fragments.
+ for (Wrapper wrapper : planningSet) {
final List<Wrapper> fragmentDependencies = wrapper.getFragmentDependencies();
if (fragmentDependencies != null && fragmentDependencies.size() > 0) {
- for(Wrapper dependency : fragmentDependencies) {
+ for (Wrapper dependency : fragmentDependencies) {
if (roots.contains(dependency)) {
roots.remove(dependency);
}
@@ -193,7 +199,7 @@ public abstract class SimpleParallelizer implements QueryParallelizer {
* @param options List of options set by the user.
* @param foremanNode foreman node for this query plan.
* @param queryId Query ID.
- * @param activeEndpoints currently active endpoins on which this plan will run.
+ * @param activeEndpoints currently active endpoints on which this plan will run.
* @param rootFragment Root major fragment.
* @param session session context.
* @param queryContextInfo query context.
@@ -235,8 +241,6 @@ public abstract class SimpleParallelizer implements QueryParallelizer {
throw new UnsupportedOperationException("Use children classes");
}
-
-
// For every fragment, create a Wrapper in PlanningSet.
@VisibleForTesting
public void initFragmentWrappers(Fragment rootFragment, PlanningSet planningSet) {
@@ -256,14 +260,15 @@ public abstract class SimpleParallelizer implements QueryParallelizer {
private void constructFragmentDependencyGraph(Fragment rootFragment, PlanningSet planningSet) {
// Set up dependency of fragments based on the affinity of exchange that separates the fragments.
- for(Wrapper currentFragment : planningSet) {
+ for (Wrapper currentFragment : planningSet) {
ExchangeFragmentPair sendingXchgForCurrFrag = currentFragment.getNode().getSendingExchangePair();
if (sendingXchgForCurrFrag != null) {
ParallelizationDependency dependency = sendingXchgForCurrFrag.getExchange().getParallelizationDependency();
Wrapper receivingFragmentWrapper = planningSet.get(sendingXchgForCurrFrag.getNode());
- //Mostly Receivers of the current fragment depend on the sender of the child fragments. However there is a special case
- //for DeMux Exchanges where the Sender of the current fragment depends on the receiver of the parent fragment.
+ // Mostly Receivers of the current fragment depend on the sender of the child fragments.
+ // However there is a special case for DeMux Exchanges where the Sender of the current
+ // fragment depends on the receiver of the parent fragment.
if (dependency == ParallelizationDependency.RECEIVER_DEPENDS_ON_SENDER) {
receivingFragmentWrapper.addFragmentDependency(currentFragment);
} else if (dependency == ParallelizationDependency.SENDER_DEPENDS_ON_RECEIVER) {
@@ -274,16 +279,15 @@ public abstract class SimpleParallelizer implements QueryParallelizer {
planningSet.findRootWrapper(rootFragment);
}
-
/**
- * Helper method to call operation on each fragment. Traversal calls operation on child fragments before
- * calling it on the parent fragment.
+ * Call operation on each fragment. Traversal calls operation
+ * on child fragments before calling it on the parent fragment.
*/
protected void traverse(Wrapper fragmentWrapper, Consumer<Wrapper> operation) throws PhysicalOperatorSetupException {
final List<Wrapper> fragmentDependencies = fragmentWrapper.getFragmentDependencies();
if (fragmentDependencies != null && fragmentDependencies.size() > 0) {
- for(Wrapper dependency : fragmentDependencies) {
+ for (Wrapper dependency : fragmentDependencies) {
traverse(dependency, operation);
}
}
@@ -300,8 +304,8 @@ public abstract class SimpleParallelizer implements QueryParallelizer {
MinorFragmentDefn rootFragmentDefn = null;
FragmentRoot rootOperator = null;
- // now we generate all the individual plan fragments and associated assignments. Note, we need all endpoints
- // assigned before we can materialize, so we start a new loop here rather than utilizing the previous one.
+ // Generate all the individual plan fragments and associated assignments. Note, we need all endpoints
+ // assigned before we can materialize.
for (Wrapper wrapper : planningSet) {
Fragment node = wrapper.getNode();
final PhysicalOperator physicalOperatorRoot = node.getRoot();
@@ -312,7 +316,7 @@ public abstract class SimpleParallelizer implements QueryParallelizer {
"The root fragment must always have parallelization one. In the current case, the width was set to %d.",
wrapper.getWidth()));
}
- // a fragment is self driven if it doesn't rely on any other exchanges.
+ // A fragment is self-driven if it doesn't rely on any other exchanges.
boolean isLeafFragment = node.getReceivingExchangePairs().size() == 0;
// Create a minorFragment for each major fragment.
for (int minorFragmentId = 0; minorFragmentId < wrapper.getWidth(); minorFragmentId++) {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SoftAffinityFragmentParallelizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SoftAffinityFragmentParallelizer.java
index 77292e8..45e507e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SoftAffinityFragmentParallelizer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SoftAffinityFragmentParallelizer.java
@@ -34,9 +34,11 @@ import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
/**
- * Implementation of {@link FragmentParallelizer} where fragment has zero or more endpoints with affinities. Width
- * per node is depended on the affinity to the endpoint and total width (calculated using costs). Based on various
- * factors endpoints which have no affinity can be assigned to run the fragments.
+ * Implementation of {@link FragmentParallelizer} where fragment has zero or
+ * more endpoints with affinities. Width per node is depended on the affinity to
+ * the endpoint and total width (calculated using costs). Based on various
+ * factors endpoints which have no affinity can be assigned to run the
+ * fragments.
*/
public class SoftAffinityFragmentParallelizer implements FragmentParallelizer {
public static final SoftAffinityFragmentParallelizer INSTANCE = new SoftAffinityFragmentParallelizer();
@@ -116,7 +118,7 @@ public class SoftAffinityFragmentParallelizer implements FragmentParallelizer {
// Find the maximum number of slots which should go to endpoints with affinity (See DRILL-825 for details)
int affinedSlots =
- Math.max(1, (int) (Math.ceil((double)parameters.getAffinityFactor() * width / activeEndpoints.size()) * sortedAffinityList.size()));
+ Math.max(1, (int) (Math.ceil(parameters.getAffinityFactor() * width / activeEndpoints.size()) * sortedAffinityList.size()));
// Make sure affined slots is at least the number of mandatory nodes
affinedSlots = Math.max(affinedSlots, numRequiredNodes);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Stats.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Stats.java
index cbdb014..421d33c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Stats.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Stats.java
@@ -24,7 +24,7 @@ import java.util.List;
public class Stats {
private final ParallelizationInfoCollector collector = new ParallelizationInfoCollector();
- private double maxCost = 0.0;
+ private double maxCost;
private DistributionAffinity distributionAffinity = DistributionAffinity.NONE;
public void addParallelizationInfo(ParallelizationInfo parallelizationInfo) {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Wrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Wrapper.java
index ffa577e..687f024 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Wrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/Wrapper.java
@@ -39,28 +39,30 @@ import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
/**
- * A wrapping class that allows us to add additional information to each fragment node for planning purposes.
+ * Wrapper class that allows us to add additional information to each fragment
+ * node for planning purposes.
*/
public class Wrapper {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Wrapper.class);
private final Fragment node;
private final int majorFragmentId;
private int width = -1;
private final Stats stats;
private boolean endpointsAssigned;
- private long initialAllocation = 0;
- private long maxAllocation = 0;
+ private long initialAllocation;
+ private long maxAllocation;
// Resources (i.e memory and cpu) are stored per drillbit in this map.
// A Drillbit can have n number of minor fragments then the NodeResource
// contains cumulative resources required for all the minor fragments
// for that major fragment on that Drillbit.
private Map<DrillbitEndpoint, NodeResource> nodeResourceMap;
- // List of fragments this particular fragment depends on for determining its parallelization and endpoint assignments.
+ // List of fragments this particular fragment depends on for determining its
+ // parallelization and endpoint assignments.
private final List<Wrapper> fragmentDependencies = Lists.newArrayList();
- // a list of assigned endpoints. Technically, there could repeated endpoints in this list if we'd like to assign the
+ // List of assigned endpoints. Technically, there could repeated endpoints
+ // in this list if we'd like to assign the
// same fragment multiple times to the same endpoint.
private final List<DrillbitEndpoint> endpoints = Lists.newLinkedList();
@@ -68,7 +70,6 @@ public class Wrapper {
this.majorFragmentId = majorFragmentId;
this.node = node;
this.stats = new Stats();
- nodeResourceMap = null;
}
public Stats getStats() {
@@ -113,11 +114,13 @@ public class Wrapper {
maxAllocation += memory;
}
- private class AssignEndpointsToScanAndStore extends AbstractPhysicalVisitor<Void, List<DrillbitEndpoint>, PhysicalOperatorSetupException>{
+ private class AssignEndpointsToScanAndStore extends
+ AbstractPhysicalVisitor<Void, List<DrillbitEndpoint>, PhysicalOperatorSetupException> {
@Override
- public Void visitExchange(Exchange exchange, List<DrillbitEndpoint> value) throws PhysicalOperatorSetupException {
- if(exchange == node.getSendingExchange()){
+ public Void visitExchange(Exchange exchange, List<DrillbitEndpoint> value)
+ throws PhysicalOperatorSetupException {
+ if (exchange == node.getSendingExchange()) {
return visitOp(exchange, value);
}
// stop on receiver exchange.
@@ -125,32 +128,36 @@ public class Wrapper {
}
@Override
- public Void visitGroupScan(GroupScan groupScan, List<DrillbitEndpoint> value) throws PhysicalOperatorSetupException {
+ public Void visitGroupScan(GroupScan groupScan,
+ List<DrillbitEndpoint> value) throws PhysicalOperatorSetupException {
groupScan.applyAssignments(value);
return super.visitGroupScan(groupScan, value);
}
@Override
- public Void visitSubScan(SubScan subScan, List<DrillbitEndpoint> value) throws PhysicalOperatorSetupException {
+ public Void visitSubScan(SubScan subScan, List<DrillbitEndpoint> value)
+ throws PhysicalOperatorSetupException {
// TODO - implement this
return visitOp(subScan, value);
}
@Override
- public Void visitStore(Store store, List<DrillbitEndpoint> value) throws PhysicalOperatorSetupException {
+ public Void visitStore(Store store, List<DrillbitEndpoint> value)
+ throws PhysicalOperatorSetupException {
store.applyAssignments(value);
return super.visitStore(store, value);
}
@Override
- public Void visitOp(PhysicalOperator op, List<DrillbitEndpoint> value) throws PhysicalOperatorSetupException {
+ public Void visitOp(PhysicalOperator op, List<DrillbitEndpoint> value)
+ throws PhysicalOperatorSetupException {
return visitChildren(op, value);
}
}
- public void assignEndpoints(List<DrillbitEndpoint> assignedEndpoints) throws
- PhysicalOperatorSetupException {
+ public void assignEndpoints(List<DrillbitEndpoint> assignedEndpoints)
+ throws PhysicalOperatorSetupException {
Preconditions.checkState(!endpointsAssigned);
endpointsAssigned = true;
@@ -173,7 +180,8 @@ public class Wrapper {
@Override
public String toString() {
- return "FragmentWrapper [majorFragmentId=" + majorFragmentId + ", width=" + width + ", stats=" + stats + "]";
+ return "FragmentWrapper [majorFragmentId=" + majorFragmentId + ", width="
+ + width + ", stats=" + stats + "]";
}
public List<DrillbitEndpoint> getAssignedEndpoints() {
@@ -197,7 +205,9 @@ public class Wrapper {
/**
* Is the endpoints assignment done for this fragment?
- * @return True if the endpoints assignment done for this fragment. False otherwise.
+ *
+ * @return True if the endpoints assignment done for this fragment. False
+ * otherwise.
*/
public boolean isEndpointsAssignmentDone() {
return endpointsAssigned;
@@ -205,6 +215,7 @@ public class Wrapper {
/**
* Get the list of fragements this particular fragment depends on.
+ *
* @return The list of fragements this particular fragment depends on.
*/
public List<Wrapper> getFragmentDependencies() {
@@ -212,9 +223,9 @@ public class Wrapper {
}
/**
- * Compute the cpu resources required for all the minor fragments of this major fragment.
- * This information is stored per DrillbitEndpoint. It is assumed that this function is
- * called only once.
+ * Compute the cpu resources required for all the minor fragments of this
+ * major fragment. This information is stored per DrillbitEndpoint. It is
+ * assumed that this function is called only once.
*/
public void computeCpuResources() {
Preconditions.checkArgument(nodeResourceMap == null);
@@ -225,12 +236,12 @@ public class Wrapper {
return result;
};
- Function<DrillbitEndpoint, NodeResource> cpuPerEndpoint = (endpoint) -> new NodeResource(1, 0);
+ Function<DrillbitEndpoint, NodeResource> cpuPerEndpoint =
+ endpoint -> new NodeResource(1, 0);
nodeResourceMap = endpoints.stream()
- .collect(Collectors.groupingBy(Function.identity(),
- Collectors.reducing(NodeResource.create(),
- cpuPerEndpoint, merge)));
+ .collect(Collectors.groupingBy(Function.identity(),
+ Collectors.reducing(NodeResource.create(), cpuPerEndpoint, merge)));
}
public Map<DrillbitEndpoint, NodeResource> getResourceMap() {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/contrib/SplittingParallelizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/contrib/SplittingParallelizer.java
index 25d32fc..cd11281 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/contrib/SplittingParallelizer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/contrib/SplittingParallelizer.java
@@ -48,20 +48,21 @@ import org.apache.drill.exec.work.foreman.ForemanSetupException;
import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
- * SimpleParallelizerMultiPlans class is an extension to SimpleParallelizer
- * to help with getting PlanFragments for split plan.
- * Split plan is essentially ability to create multiple Physical Operator plans from original Physical Operator plan
- * to be able to run plans separately.
- * Moving functionality specific to splitting the plan to this class
- * allows not to pollute parent class with non-authentic functionality
- *
+ * SimpleParallelizerMultiPlans class is an extension to SimpleParallelizer to
+ * help with getting PlanFragments for split plan. Split plan is essentially
+ * ability to create multiple Physical Operator plans from original Physical
+ * Operator plan to be able to run plans separately. Moving functionality
+ * specific to splitting the plan to this class allows not to pollute parent
+ * class with non-authentic functionality.
*/
public class SplittingParallelizer extends DefaultQueryParallelizer {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SplittingParallelizer.class);
private boolean enableDynamicFC;
+ private static final Logger logger = LoggerFactory.getLogger(SplittingParallelizer.class);
public SplittingParallelizer(boolean doMemoryPlanning, QueryContext context) {
super(doMemoryPlanning, context);
@@ -81,6 +82,7 @@ public class SplittingParallelizer extends DefaultQueryParallelizer {
* @return
* @throws ExecutionSetupException
*/
+ @Override
public List<QueryWorkUnit> getSplitFragments(OptionList options, DrillbitEndpoint foremanNode, QueryId queryId,
Collection<DrillbitEndpoint> activeEndpoints, PhysicalPlanReader reader, Fragment rootFragment,
UserSession session, QueryContextInformation queryContextInfo) throws ExecutionSetupException {
@@ -118,8 +120,8 @@ public class SplittingParallelizer extends DefaultQueryParallelizer {
PhysicalPlanReader reader, Fragment rootNode, PlanningSet planningSet,
UserSession session, QueryContextInformation queryContextInfo) throws ExecutionSetupException {
- // now we generate all the individual plan fragments and associated assignments. Note, we need all endpoints
- // assigned before we can materialize, so we start a new loop here rather than utilizing the previous one.
+ // Generate all the individual plan fragments and associated assignments. Note, we need all endpoints
+ // assigned before we can materialize.
List<QueryWorkUnit> workUnits = Lists.newArrayList();
int plansCount = 0;
@@ -133,13 +135,13 @@ public class SplittingParallelizer extends DefaultQueryParallelizer {
boolean isLeafFragment = node.getReceivingExchangePairs().size() == 0;
final PhysicalOperator physicalOperatorRoot = node.getRoot();
// get all the needed info from leaf fragment
- if ( (physicalOperatorRoot instanceof Exchange) && isLeafFragment) {
+ if ((physicalOperatorRoot instanceof Exchange) && isLeafFragment) {
// need to get info about
// number of minor fragments
// assignedEndPoints
// allocation
plansCount = wrapper.getWidth();
- initialAllocation = (wrapper.getInitialAllocation() != 0 ) ? wrapper.getInitialAllocation()/plansCount : 0;
+ initialAllocation = (wrapper.getInitialAllocation() != 0) ? wrapper.getInitialAllocation()/plansCount : 0;
leafFragEndpoints = new DrillbitEndpoint[plansCount];
for (int mfId = 0; mfId < plansCount; mfId++) {
leafFragEndpoints[mfId] = wrapper.getAssignedEndpoint(mfId);
@@ -148,7 +150,7 @@ public class SplittingParallelizer extends DefaultQueryParallelizer {
}
DrillbitEndpoint[] endPoints = leafFragEndpoints;
- if ( plansCount == 0 ) {
+ if (plansCount == 0) {
// no exchange, return list of single QueryWorkUnit
workUnits.add(generateWorkUnit(options, foremanNode, queryId, rootNode, planningSet, session, queryContextInfo));
return workUnits;
@@ -157,7 +159,7 @@ public class SplittingParallelizer extends DefaultQueryParallelizer {
for (Wrapper wrapper : planningSet) {
Fragment node = wrapper.getNode();
final PhysicalOperator physicalOperatorRoot = node.getRoot();
- if ( physicalOperatorRoot instanceof Exchange ) {
+ if (physicalOperatorRoot instanceof Exchange) {
// get to 0 MajorFragment
continue;
}
@@ -171,11 +173,11 @@ public class SplittingParallelizer extends DefaultQueryParallelizer {
// this fragment is always leaf, as we are removing all the exchanges
boolean isLeafFragment = true;
- FragmentHandle handle = FragmentHandle //
- .newBuilder() //
- .setMajorFragmentId(wrapper.getMajorFragmentId()) //
+ FragmentHandle handle = FragmentHandle
+ .newBuilder()
+ .setMajorFragmentId(wrapper.getMajorFragmentId())
.setMinorFragmentId(0) // minor fragment ID is going to be always 0, as plan will be split
- .setQueryId(queryId) //
+ .setQueryId(queryId)
.build();
// Create a minorFragment for each major fragment.
@@ -197,14 +199,13 @@ public class SplittingParallelizer extends DefaultQueryParallelizer {
Preconditions.checkArgument(op instanceof FragmentRoot);
FragmentRoot root = (FragmentRoot) op;
-
- PlanFragment fragment = PlanFragment.newBuilder() //
- .setForeman(endPoints[minorFragmentId]) //
- .setHandle(handle) //
- .setAssignment(endPoints[minorFragmentId]) //
- .setLeafFragment(isLeafFragment) //
+ PlanFragment fragment = PlanFragment.newBuilder()
+ .setForeman(endPoints[minorFragmentId])
+ .setHandle(handle)
+ .setAssignment(endPoints[minorFragmentId])
+ .setLeafFragment(isLeafFragment)
.setContext(queryContextInfo)
- .setMemInitial(initialAllocation)//
+ .setMemInitial(initialAllocation)
.setMemMax(wrapper.getMaxAllocation()) // TODO - for some reason OOM is using leaf fragment max allocation divided by width
.setCredentials(session.getCredentials())
.addAllCollector(CountRequiredFragments.getCollectors(root, enableDynamicFC))
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/PathInExpr.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/PathInExpr.java
index 124fece..4cb4eeb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/PathInExpr.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/PathInExpr.java
@@ -31,14 +31,19 @@ import java.util.Map;
import java.util.Set;
/**
- * Class PathInExpr is to recursively analyze a expression trees with a map of indexed expression collected from indexDescriptor,
- * e.g. Map 'cast(a.q as int)' -> '$0' means the expression 'cast(a.q as int)' is named as '$0' in index table.
- *
- * for project expressions: cast(a.q as int), a.q, b + c, PathInExpr will get remainderPath {a.q, b, c}, which is
- * helpful to determine if q query is covering. remainderPathsInFunction will be {a.q}, which will be used to
- * decide if the {a.q} in scan column and project rowtype should be removed or not.
- *
- * This class could be more generic to support any expression, for now it works for only 'cast(schemapath as type)'.
+ * Class PathInExpr is to recursively analyze a expression trees with a map of
+ * indexed expression collected from indexDescriptor, e.g. Map 'cast(a.q as
+ * int)' -> '$0' means the expression 'cast(a.q as int)' is named as '$0' in
+ * index table.
+ * <p>
+ * for project expressions: cast(a.q as int), a.q, b + c, PathInExpr will get
+ * remainderPath {a.q, b, c}, which is helpful to determine if q query is
+ * covering. remainderPathsInFunction will be {a.q}, which will be used to
+ * decide if the {a.q} in scan column and project rowtype should be removed or
+ * not.
+ * <p>
+ * This class could be more generic to support any expression, for now it works
+ * for only 'cast(schemapath as type)'.
*/
public class PathInExpr extends AbstractExprVisitor<Boolean,Void,RuntimeException> {
@@ -46,13 +51,13 @@ public class PathInExpr extends AbstractExprVisitor<Boolean,Void,RuntimeExceptio
final private Map<LogicalExpression, Set<SchemaPath>> pathsInExpr;
//collection of paths in all functional indexes
- private Set<SchemaPath> allPaths;
+ private final Set<SchemaPath> allPaths;
//the paths were found out of functional index expressions in query.
- private Set<LogicalExpression> remainderPaths;
+ private final Set<LogicalExpression> remainderPaths;
//the paths in functional index fields but were found out of index functions expression in query
- private Set<LogicalExpression> remainderPathsInFunctions;
+ private final Set<LogicalExpression> remainderPathsInFunctions;
//constructor is provided a map of all functional expressions to the paths involved in the expressions.
public PathInExpr(Map<LogicalExpression, Set<SchemaPath>> pathsInExpr) {
@@ -88,7 +93,7 @@ public class PathInExpr extends AbstractExprVisitor<Boolean,Void,RuntimeExceptio
}
boolean bret = true;
- for (LogicalExpression arg : call.args) {
+ for (LogicalExpression arg : call.args()) {
bret &= arg.accept(this, null);
if(bret == false) {
break;
@@ -109,6 +114,7 @@ public class PathInExpr extends AbstractExprVisitor<Boolean,Void,RuntimeExceptio
return null;
}
+ @Override
public Boolean visitCastExpression(CastExpression castExpr, Void value) throws RuntimeException {
if (preProcess(castExpr)) {
//when it is true, we know this is exactly the indexed expression, no more deep search
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/generators/AbstractIndexPlanGenerator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/generators/AbstractIndexPlanGenerator.java
index 456542b..88e569a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/generators/AbstractIndexPlanGenerator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/generators/AbstractIndexPlanGenerator.java
@@ -23,7 +23,8 @@ import java.util.List;
import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.calcite.plan.RelTrait;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.RelNode;
@@ -64,7 +65,7 @@ import org.apache.calcite.rex.RexNode;
public abstract class AbstractIndexPlanGenerator extends SubsetTransformer<RelNode, InvalidRelException>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractIndexPlanGenerator.class);
+ private static final Logger logger = LoggerFactory.getLogger(AbstractIndexPlanGenerator.class);
final protected DrillProjectRelBase origProject;
final protected DrillScanRelBase origScan;
@@ -92,8 +93,8 @@ public abstract class AbstractIndexPlanGenerator extends SubsetTransformer<RelNo
this.settings = settings;
}
- //This class provides the utility functions that don't rely on index(one or multiple) or final plan (covering or not),
- //but those helper functions that focus on serving building index plan (project-filter-indexscan)
+ // Provides the utility functions that don't rely on index(one or multiple) or final plan (covering or not),
+ // but those helper functions that focus on serving building index plan (project-filter-indexscan)
public static int getRowKeyIndex(RelDataType rowType, DrillScanRelBase origScan) {
List<String> fieldNames = rowType.getFieldNames();
@@ -158,10 +159,11 @@ public abstract class AbstractIndexPlanGenerator extends SubsetTransformer<RelNo
return convertedRight;
}
+ @Override
public RelTraitSet newTraitSet(RelTrait... traits) {
RelTraitSet set = indexContext.getCall().getPlanner().emptyTraitSet();
for (RelTrait t : traits) {
- if(t != null) {
+ if (t != null) {
set = set.plus(t);
}
}
@@ -169,10 +171,7 @@ public abstract class AbstractIndexPlanGenerator extends SubsetTransformer<RelNo
}
protected static boolean toRemoveSort(RelCollation sortCollation, RelCollation inputCollation) {
- if ( (inputCollation != null) && inputCollation.satisfies(sortCollation)) {
- return true;
- }
- return false;
+ return (inputCollation != null) && inputCollation.satisfies(sortCollation);
}
public static RelNode getExchange(RelOptCluster cluster, boolean isSingleton, boolean isExchangeRequired,
@@ -240,8 +239,10 @@ public abstract class AbstractIndexPlanGenerator extends SubsetTransformer<RelNo
return newRel;
}
+ @Override
public abstract RelNode convertChild(RelNode current, RelNode child) throws InvalidRelException;
+ @Override
public boolean forceConvert(){
return true;
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
index ba3e72b..df93ea4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
@@ -67,6 +67,8 @@ import org.apache.calcite.util.NlsString;
import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.work.ExecErrorConstants;
@@ -77,7 +79,7 @@ import static org.apache.drill.exec.planner.physical.PlannerSettings.ENABLE_DECI
*/
public class DrillOptiq {
public static final String UNSUPPORTED_REX_NODE_ERROR = "Cannot convert RexNode to equivalent Drill expression. ";
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillOptiq.class);
+ private static final Logger logger = LoggerFactory.getLogger(DrillOptiq.class);
/**
* Converts a tree of {@link RexNode} operators into a scalar expression in Drill syntax using one input.
@@ -110,7 +112,6 @@ public class DrillOptiq {
}
public static class RexToDrill extends RexVisitorImpl<LogicalExpression> {
- private final List<RelNode> inputs;
private final DrillParseContext context;
private final List<RelDataTypeField> fieldList;
private final RelDataType rowType;
@@ -119,8 +120,7 @@ public class DrillOptiq {
RexToDrill(DrillParseContext context, List<RelNode> inputs) {
super(true);
this.context = context;
- this.inputs = inputs;
- this.fieldList = Lists.newArrayList();
+ this.fieldList = new ArrayList<>();
if (inputs.size() > 0 && inputs.get(0)!=null) {
this.rowType = inputs.get(0).getRowType();
this.builder = inputs.get(0).getCluster().getRexBuilder();
@@ -157,7 +157,6 @@ public class DrillOptiq {
this.context = context;
this.rowType = rowType;
this.builder = builder;
- this.inputs = Lists.newArrayList();
this.fieldList = rowType.getFieldList();
}
@@ -192,17 +191,18 @@ public class DrillOptiq {
case POSTFIX:
logger.debug("Postfix");
switch(call.getKind()){
- case IS_NOT_NULL:
- case IS_NOT_TRUE:
- case IS_NOT_FALSE:
- case IS_NULL:
- case IS_TRUE:
- case IS_FALSE:
- case OTHER:
- return FunctionCallFactory.createExpression(call.getOperator().getName().toLowerCase(),
- ExpressionPosition.UNKNOWN, call.getOperands().get(0).accept(this));
+ case IS_NOT_NULL:
+ case IS_NOT_TRUE:
+ case IS_NOT_FALSE:
+ case IS_NULL:
+ case IS_TRUE:
+ case IS_FALSE:
+ case OTHER:
+ return FunctionCallFactory.createExpression(call.getOperator().getName().toLowerCase(),
+ ExpressionPosition.UNKNOWN, call.getOperands().get(0).accept(this));
+ default:
+ throw notImplementedException(syntax, call);
}
- throw notImplementedException(syntax, call);
case PREFIX:
LogicalExpression arg = call.getOperands().get(0).accept(this);
switch(call.getKind()){
@@ -210,11 +210,12 @@ public class DrillOptiq {
return FunctionCallFactory.createExpression(call.getOperator().getName().toLowerCase(),
ExpressionPosition.UNKNOWN, arg);
case MINUS_PREFIX:
- List<LogicalExpression> operands = Lists.newArrayList();
+ List<LogicalExpression> operands = new ArrayList<>();
operands.add(call.getOperands().get(0).accept(this));
return FunctionCallFactory.createExpression("u-", operands);
+ default:
+ throw notImplementedException(syntax, call);
}
- throw notImplementedException(syntax, call);
case SPECIAL:
switch(call.getKind()){
case CAST:
@@ -234,7 +235,7 @@ public class DrillOptiq {
case SIMILAR:
return getDrillFunctionFromOptiqCall(call);
case CASE:
- List<LogicalExpression> caseArgs = Lists.newArrayList();
+ List<LogicalExpression> caseArgs = new ArrayList<>();
for(RexNode r : call.getOperands()){
caseArgs.add(r.accept(this));
}
@@ -250,6 +251,8 @@ public class DrillOptiq {
.setIfCondition(new IfCondition(caseArgs.get(i + 1), caseArgs.get(i))).build();
}
return elseExpression;
+
+ default:
}
if (call.getOperator() == SqlStdOperatorTable.ITEM) {
@@ -409,7 +412,7 @@ public class DrillOptiq {
}
private LogicalExpression doFunction(RexCall call, String funcName) {
- List<LogicalExpression> args = Lists.newArrayList();
+ List<LogicalExpression> args = new ArrayList<>();
for(RexNode r : call.getOperands()){
args.add(r.accept(this));
}
@@ -723,13 +726,12 @@ public class DrillOptiq {
case MILLENNIUM:
final String functionPostfix = StringUtils.capitalize(timeUnitStr.toLowerCase());
return FunctionCallFactory.createExpression("date_trunc_" + functionPostfix, args.subList(1, 2));
+ default:
+ throw new UnsupportedOperationException("date_trunc function supports the following time units: " +
+ "YEAR, MONTH, DAY, HOUR, MINUTE, SECOND, WEEK, QUARTER, DECADE, CENTURY, MILLENNIUM");
}
-
- throw new UnsupportedOperationException("date_trunc function supports the following time units: " +
- "YEAR, MONTH, DAY, HOUR, MINUTE, SECOND, WEEK, QUARTER, DECADE, CENTURY, MILLENNIUM");
}
-
@Override
public LogicalExpression visitLiteral(RexLiteral literal) {
switch(literal.getType().getSqlTypeName()){
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushLimitToScanRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushLimitToScanRule.java
index 01379cd..e046563 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushLimitToScanRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushLimitToScanRule.java
@@ -18,6 +18,8 @@
package org.apache.drill.exec.planner.logical;
import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.plan.RelOptRuleOperand;
@@ -26,7 +28,7 @@ import org.apache.drill.exec.physical.base.GroupScan;
import org.apache.drill.exec.planner.common.DrillRelOptUtil;
public abstract class DrillPushLimitToScanRule extends RelOptRule {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillPushLimitToScanRule.class);
+ private static final Logger logger = LoggerFactory.getLogger(DrillPushLimitToScanRule.class);
private DrillPushLimitToScanRule(RelOptRuleOperand operand, String description) {
super(operand, DrillRelFactories.LOGICAL_BUILDER, description);
@@ -39,7 +41,7 @@ public abstract class DrillPushLimitToScanRule extends RelOptRule {
public boolean matches(RelOptRuleCall call) {
DrillLimitRel limitRel = call.rel(0);
DrillScanRel scanRel = call.rel(1);
- // For now only applies to Parquet. And pushdown only apply limit but not offset,
+ // For now only applies to Parquet. And pushdown only applies to LIMIT but not OFFSET,
// so if getFetch() return null no need to run this rule.
if (scanRel.getGroupScan().supportsLimitPushdown() && (limitRel.getFetch() != null)) {
return true;
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ExprHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ExprHelper.java
index e1c6f5f..de48d3e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ExprHelper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/ExprHelper.java
@@ -23,12 +23,11 @@ import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.LogicalExpression;
public class ExprHelper {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExprHelper.class);
-
- private final static String COMPOUND_FAIL_MESSAGE = "The current Optiq based logical plan interpreter does not complicated expressions. For Order By and Filter";
+ private final static String COMPOUND_FAIL_MESSAGE =
+ "The Calcite-based logical plan interpreter does not handle complicated expressions. For Order By and Filter";
public static String getAggregateFieldName(FunctionCall c) {
- List<LogicalExpression> exprs = c.args;
+ List<LogicalExpression> exprs = c.args();
if (exprs.size() != 1) {
throw new UnsupportedOperationException(COMPOUND_FAIL_MESSAGE);
}
@@ -39,5 +38,4 @@ public class ExprHelper {
//if(e instanceof SchemaPath) return ((SchemaPath) e).getPath().toString();
throw new UnsupportedOperationException(COMPOUND_FAIL_MESSAGE);
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillDistributionTrait.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillDistributionTrait.java
index 6673c63..212434b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillDistributionTrait.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillDistributionTrait.java
@@ -35,7 +35,7 @@ public class DrillDistributionTrait implements RelTrait {
private final DistributionType type;
private final List<DistributionField> fields;
- private PartitionFunction partitionFunction = null;
+ private PartitionFunction partitionFunction;
public DrillDistributionTrait(DistributionType type) {
assert (type == DistributionType.SINGLETON || type == DistributionType.RANDOM_DISTRIBUTED || type == DistributionType.ANY
@@ -97,6 +97,7 @@ public class DrillDistributionTrait implements RelTrait {
return this.equals(trait);
}
+ @Override
public RelTraitDef<DrillDistributionTrait> getTraitDef() {
return DrillDistributionTraitDef.INSTANCE;
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillDistributionTraitDef.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillDistributionTraitDef.java
index 6d52184..a98a562 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillDistributionTraitDef.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillDistributionTraitDef.java
@@ -77,7 +77,7 @@ public class DrillDistributionTraitDef extends RelTraitDef<DrillDistributionTrai
return null;
}
- switch(toDist.getType()){
+ switch (toDist.getType()) {
// UnionExchange, HashToRandomExchange, OrderedPartitionExchange and BroadcastExchange destroy the ordering property,
// therefore RelCollation is set to default, which is EMPTY.
case SINGLETON:
@@ -105,5 +105,4 @@ public class DrillDistributionTraitDef extends RelTraitDef<DrillDistributionTrai
return null;
}
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ExchangePrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ExchangePrel.java
index cf93839..db832ed 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ExchangePrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ExchangePrel.java
@@ -25,7 +25,7 @@ import org.apache.drill.exec.server.options.OptionManager;
import java.util.Collections;
-public abstract class ExchangePrel extends SinglePrel{
+public abstract class ExchangePrel extends SinglePrel {
public ExchangePrel(RelOptCluster cluster, RelTraitSet traits, RelNode child) {
super(cluster, traits, child);
@@ -38,12 +38,11 @@ public abstract class ExchangePrel extends SinglePrel{
/**
* The derived classes can override this method to create relevant mux exchanges.
- * If this method is not overrided the default behaviour is to clone itself.
+ * If this method is not overridden then the default behavior is to clone itself.
* @param child input to the new muxPrel or new Exchange node.
* @param options options manager to check if mux is enabled.
*/
public Prel constructMuxPrel(Prel child, OptionManager options) {
return (Prel)copy(getTraitSet(), Collections.singletonList(((RelNode)child)));
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashPrelUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashPrelUtil.java
index a7e8cc0..2e3a253 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashPrelUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashPrelUtil.java
@@ -42,6 +42,7 @@ public class HashPrelUtil {
public static final String HASH_EXPR_NAME = "E_X_P_R_H_A_S_H_F_I_E_L_D";
public static final int DIST_SEED = 1301011; // distribution seed
+
/**
* Interface for creating different forms of hash expression types.
* @param <T>
@@ -63,6 +64,7 @@ public class HashPrelUtil {
this.rexBuilder = rexBuilder;
}
+ @SuppressWarnings("deprecation")
@Override
public RexNode createCall(String funcName, List<RexNode> inputFields) {
final DrillSqlOperator op =
@@ -172,11 +174,10 @@ public class HashPrelUtil {
return createHashExpression(ImmutableList.of(field), seed, HASH_HELPER_LOGICAL_EXPRESSION, hashAsDouble);
}
-
/**
* Create a distribution hash expression.
*
- * @param fields Distribution fields
+ * @param fields Distribution fields
* @param rowType Row type
* @return distribution hash expression
*/
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrel.java
index 3e1d4c6..f6e18b9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/JoinPrel.java
@@ -37,20 +37,21 @@ import org.apache.calcite.plan.RelOptCluster;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.util.Pair;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
- *
* Base class for MergeJoinPrel and HashJoinPrel
- *
*/
public abstract class JoinPrel extends DrillJoinRelBase implements Prel {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JoinPrel.class);
+ private static final Logger logger = LoggerFactory.getLogger(JoinPrel.class);
protected final boolean isSemiJoin;
protected JoinUtils.JoinCategory joincategory;
@@ -158,14 +159,16 @@ public abstract class JoinPrel extends DrillJoinRelBase implements Prel {
}
}
+ @Override
public boolean isSemiJoin() {
return isSemiJoin;
}
- /* A Drill physical rel which is semi join will have output row type with fields from only
- left side of the join. Calcite's join rel expects to have the output row type from
- left and right side of the join. This function is overloaded to not throw exceptions for
- a Drill semi join physical rel.
+ /**
+ * A Drill physical rel which is semi join will have output row type with fields from only
+ * left side of the join. Calcite's join rel expects to have the output row type from
+ * left and right side of the join. This function is overloaded to not throw exceptions for
+ * a Drill semi join physical rel.
*/
@Override public boolean isValid(Litmus litmus, Context context) {
if (!this.isSemiJoin && !super.isValid(litmus, context)) {
@@ -188,7 +191,7 @@ public abstract class JoinPrel extends DrillJoinRelBase implements Prel {
// due to outer joins.
RexChecker checker =
new RexChecker(
- getCluster().getTypeFactory().builder()
+ new RelDataTypeFactory.Builder(getCluster().getTypeFactory())
.addAll(getSystemFieldList())
.addAll(getLeft().getRowType().getFieldList())
.addAll(getRight().getRowType().getFieldList())
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MetadataControllerPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MetadataControllerPrel.java
index fb78a60..b5691e3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MetadataControllerPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MetadataControllerPrel.java
@@ -22,6 +22,7 @@ import org.apache.calcite.plan.RelTraitSet;
import org.apache.calcite.rel.BiRel;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.drill.exec.metastore.analyze.MetastoreAnalyzeConstants;
import org.apache.drill.exec.physical.base.PhysicalOperator;
@@ -87,7 +88,7 @@ public class MetadataControllerPrel extends BiRel implements DrillRelNode, Prel
@Override
protected RelDataType deriveRowType() {
- return getCluster().getTypeFactory().builder()
+ return new RelDataTypeFactory.Builder(getCluster().getTypeFactory())
.add(MetastoreAnalyzeConstants.OK_FIELD_NAME, SqlTypeName.BOOLEAN)
.add(MetastoreAnalyzeConstants.SUMMARY_FIELD_NAME, SqlTypeName.VARCHAR)
.build();
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/NestedLoopJoinPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/NestedLoopJoinPrule.java
index 5eadab2..d7b3df5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/NestedLoopJoinPrule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/NestedLoopJoinPrule.java
@@ -79,7 +79,6 @@ public class NestedLoopJoinPrule extends JoinPruleBase {
if (!settings.isNestedLoopJoinEnabled()) {
return;
}
- int[] joinFields = new int[2];
DrillJoinRel join = (DrillJoinRel) call.rel(0);
final RelNode left = join.getLeft();
final RelNode right = join.getRight();
@@ -99,5 +98,4 @@ public class NestedLoopJoinPrule extends JoinPruleBase {
tracer.warn(e.toString());
}
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PhysicalPlanCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PhysicalPlanCreator.java
index 7d950c1..1a7d18c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PhysicalPlanCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PhysicalPlanCreator.java
@@ -38,13 +38,11 @@ import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
public class PhysicalPlanCreator {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PhysicalPlanCreator.class);
private final Map<Prel, OpId> opIdMap;
-
- private List<PhysicalOperator> popList;
+ private final List<PhysicalOperator> popList;
private final QueryContext context;
- PhysicalPlan plan = null;
+ private PhysicalPlan plan;
public PhysicalPlanCreator(QueryContext context, Map<Prel, OpId> opIdMap) {
this.context = context;
@@ -88,7 +86,6 @@ public class PhysicalPlanCreator {
propsBuilder.resultMode(ResultMode.EXEC);
propsBuilder.generator(PhysicalPlanCreator.class.getName(), "");
-
try {
// invoke getPhysicalOperator on the root Prel which will recursively invoke it
// on the descendants and we should have a well-formed physical operator tree
@@ -105,5 +102,4 @@ public class PhysicalPlanCreator {
return plan;
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java
index 01d8e9c..897f4de 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/Prel.java
@@ -28,13 +28,14 @@ import org.apache.drill.exec.planner.physical.visitor.PrelVisitor;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
public interface Prel extends DrillRelNode, Iterable<Prel> {
- org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Prel.class);
Convention DRILL_PHYSICAL = new Convention.Impl("PHYSICAL", Prel.class) {
+ @Override
public boolean canConvertConvention(Convention toConvention) {
return true;
}
+ @Override
public boolean useAbstractConvertersForConversion(RelTraitSet fromTraits,
RelTraitSet toTraits) {
return true;
@@ -50,6 +51,7 @@ public interface Prel extends DrillRelNode, Iterable<Prel> {
* of its child Prel
*/
SelectionVectorMode[] getSupportedEncodings();
+
/**
* A Prel's own SelectionVector mode - i.e whether it generates an SV2, SV4 or None
*/
@@ -66,5 +68,4 @@ public interface Prel extends DrillRelNode, Iterable<Prel> {
throw new UnsupportedOperationException("Adding Implicit RowID column is not supported for " +
this.getClass().getSimpleName() + " operator ");
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java
index a07d3ed..69cd603 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrel.java
@@ -40,10 +40,11 @@ import org.apache.drill.exec.planner.cost.DrillCostBase.DrillCostFactory;
import org.apache.drill.exec.planner.fragment.DistributionAffinity;
import org.apache.drill.exec.planner.physical.visitor.PrelVisitor;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class ScanPrel extends DrillScanRelBase implements Prel, HasDistributionAffinity {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory
- .getLogger(ScanPrel.class);
+ private static final Logger logger = LoggerFactory.getLogger(ScanPrel.class);
private final RelDataType rowType;
@@ -65,6 +66,7 @@ public class ScanPrel extends DrillScanRelBase implements Prel, HasDistributionA
this.rowType, this.getTable());
}
+ @SuppressWarnings("unchecked")
private static GroupScan getCopy(GroupScan scan){
try {
return (GroupScan) scan.getNewWithChildren((List<PhysicalOperator>) (Object) Collections.emptyList());
@@ -108,19 +110,21 @@ public class ScanPrel extends DrillScanRelBase implements Prel, HasDistributionA
@Override
public RelOptCost computeSelfCost(final RelOptPlanner planner, RelMetadataQuery mq) {
final PlannerSettings settings = PrelUtil.getPlannerSettings(planner);
- final ScanStats stats = this.getGroupScan().getScanStats(settings);
- final int columnCount = this.getRowType().getFieldCount();
+ final ScanStats stats = getGroupScan().getScanStats(settings);
+ final int columnCount = getRowType().getFieldCount();
if (PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
return planner.getCostFactory().makeCost(stats.getRecordCount() * columnCount, stats.getCpuCost(), stats.getDiskCost());
}
double rowCount = mq.getRowCount(this);
- //double rowCount = stats.getRecordCount();
// As DRILL-4083 points out, when columnCount == 0, cpuCost becomes zero,
// which makes the costs of HiveScan and HiveDrillNativeParquetScan the same
- double cpuCost = rowCount * Math.max(columnCount, 1); // For now, assume cpu cost is proportional to row count.
+ // For now, assume cpu cost is proportional to row count.
+ // Note that this ignores the disk cost estimate (which should be a proxy for
+ // row count * row width.)
+ double cpuCost = rowCount * Math.max(columnCount, 1);
// If a positive value for CPU cost is given multiply the default CPU cost by given CPU cost.
if (stats.getCpuCost() > 0) {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrule.java
index 4d702a8..67baa8e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/ScanPrule.java
@@ -25,7 +25,7 @@ import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.plan.RelTraitSet;
-public class ScanPrule extends Prule{
+public class ScanPrule extends Prule {
public static final RelOptRule INSTANCE = new ScanPrule();
public ScanPrule() {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SingleMergeExchangePrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SingleMergeExchangePrel.java
index 223882c..ce2e511 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SingleMergeExchangePrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SingleMergeExchangePrel.java
@@ -42,7 +42,8 @@ public class SingleMergeExchangePrel extends ExchangePrel {
private final RelCollation collation;
- public SingleMergeExchangePrel(RelOptCluster cluster, RelTraitSet traitSet, RelNode input, RelCollation collation) {
+ public SingleMergeExchangePrel(RelOptCluster cluster, RelTraitSet traitSet,
+ RelNode input, RelCollation collation) {
super(cluster, traitSet, input);
this.collation = collation;
assert input.getConvention() == Prel.DRILL_PHYSICAL;
@@ -81,6 +82,7 @@ public class SingleMergeExchangePrel extends ExchangePrel {
return new SingleMergeExchangePrel(getCluster(), traitSet, sole(inputs), collation);
}
+ @Override
public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException {
Prel child = (Prel) this.getInput();
@@ -131,5 +133,4 @@ public class SingleMergeExchangePrel extends ExchangePrel {
public SelectionVectorMode getEncoding() {
return SelectionVectorMode.NONE;
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java
index 8064c42..7e415b2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrel.java
@@ -143,6 +143,7 @@ public class SortPrel extends org.apache.calcite.rel.core.Sort implements Ordere
fieldCollation.direction, fieldCollation.nullDirection));
}
+ @SuppressWarnings("deprecation")
RelCollation collationTrait = RelCollationImpl.of(relFieldCollations);
RelTraitSet traits = RelTraitSet.createEmpty()
.replace(this.getTraitSet().getTrait(DrillDistributionTraitDef.INSTANCE))
@@ -166,5 +167,4 @@ public class SortPrel extends org.apache.calcite.rel.core.Sort implements Ordere
public boolean canBeDropped() {
return isRemovable;
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrule.java
index bec1b6a..d211d32 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SortPrule.java
@@ -38,7 +38,7 @@ import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
* For Logical Sort, it requires one single data stream as the output.
*
*/
-public class SortPrule extends Prule{
+public class SortPrule extends Prule {
public static final RelOptRule INSTANCE = new SortPrule();
private SortPrule() {
@@ -48,7 +48,6 @@ public class SortPrule extends Prule{
@Override
public void onMatch(RelOptRuleCall call) {
final DrillSortRel sort = call.rel(0);
- final RelNode input = sort.getInput();
// Keep the collation in logical sort. Convert input into a RelNode with 1) this collation, 2) Physical, 3) hash distributed on
@@ -59,12 +58,12 @@ public class SortPrule extends Prule{
SortPrel child = new SortPrel(sort.getCluster(), traits.plus(sort.getCollation()),
convert(sort.getInput(), traits), sort.getCollation(), false);
- if(isSingleMode(call)){
+ if (isSingleMode(call)) {
call.transformTo(child);
- }else{
- RelNode exch = new SingleMergeExchangePrel(sort.getCluster(), sort.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON), child, sort.getCollation());
+ } else {
+ RelNode exch = new SingleMergeExchangePrel(sort.getCluster(), sort.getTraitSet().plus(
+ Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON), child, sort.getCollation());
call.transformTo(exch); // transform logical "sort" into "SingleMergeExchange".
-
}
}
@@ -77,5 +76,4 @@ public class SortPrule extends Prule{
}
return distFields;
}
-
-}
\ No newline at end of file
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SubsetTransformer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SubsetTransformer.java
index 1bc309d..043ce4e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SubsetTransformer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/SubsetTransformer.java
@@ -18,6 +18,8 @@
package org.apache.drill.exec.planner.physical;
import org.apache.drill.shaded.guava.com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.plan.ConventionTraitDef;
import org.apache.calcite.plan.RelOptRule;
@@ -29,13 +31,7 @@ import org.apache.calcite.plan.volcano.RelSubset;
import java.util.Set;
public abstract class SubsetTransformer<T extends RelNode, E extends Exception> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SubsetTransformer.class);
-
- public abstract RelNode convertChild(T current, RelNode child) throws E;
-
- public boolean forceConvert() {
- return false;
- }
+ private static final Logger logger = LoggerFactory.getLogger(SubsetTransformer.class);
private final RelOptRuleCall call;
@@ -43,13 +39,18 @@ public abstract class SubsetTransformer<T extends RelNode, E extends Exception>
this.call = call;
}
+ public abstract RelNode convertChild(T current, RelNode child) throws E;
+
+ public boolean forceConvert() {
+ return false;
+ }
+
public RelTraitSet newTraitSet(RelTrait... traits) {
RelTraitSet set = call.getPlanner().emptyTraitSet();
for (RelTrait t : traits) {
set = set.plus(t);
}
return set;
-
}
public boolean go(T n, RelNode candidateSet) throws E {
@@ -105,9 +106,4 @@ public abstract class SubsetTransformer<T extends RelNode, E extends Exception>
private boolean isPhysical(RelNode n){
return n.getTraitSet().getTrait(ConventionTraitDef.INSTANCE).equals(Prel.DRILL_PHYSICAL);
}
-
- private boolean isDefaultDist(RelNode n) {
- return n.getTraitSet().getTrait(DrillDistributionTraitDef.INSTANCE).equals(DrillDistributionTrait.DEFAULT);
- }
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/TopNPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/TopNPrel.java
index ad9a2d5..b608e3d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/TopNPrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/TopNPrel.java
@@ -22,7 +22,7 @@ import java.math.BigDecimal;
import java.util.List;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
-import org.apache.calcite.rel.RelCollationImpl;
+import org.apache.calcite.rel.RelCollations;
import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.type.SqlTypeName;
@@ -140,7 +140,7 @@ public class TopNPrel extends SinglePrel implements OrderedRel,Prel {
fieldCollation.direction, fieldCollation.nullDirection));
}
- RelCollation collationTrait = RelCollationImpl.of(relFieldCollations);
+ RelCollation collationTrait = RelCollations.of(relFieldCollations);
RelTraitSet traits = RelTraitSet.createEmpty()
.replace(this.getTraitSet().getTrait(DrillDistributionTraitDef.INSTANCE))
.replace(collationTrait)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionExchangePrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionExchangePrel.java
index b4efa0e..21f259c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionExchangePrel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/UnionExchangePrel.java
@@ -70,6 +70,7 @@ public class UnionExchangePrel extends ExchangePrel {
return new UnionExchangePrel(getCluster(), traitSet, sole(inputs));
}
+ @Override
public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException {
Prel child = (Prel) this.getInput();
@@ -87,5 +88,4 @@ public class UnionExchangePrel extends ExchangePrel {
public SelectionVectorMode getEncoding() {
return SelectionVectorMode.NONE;
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ExcessiveExchangeIdentifier.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ExcessiveExchangeIdentifier.java
index 267cecd..3660868 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ExcessiveExchangeIdentifier.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/ExcessiveExchangeIdentifier.java
@@ -37,7 +37,7 @@ import org.apache.drill.exec.planner.physical.UnnestPrel;
public class ExcessiveExchangeIdentifier extends BasePrelVisitor<Prel, ExcessiveExchangeIdentifier.MajorFragmentStat, RuntimeException> {
private final long targetSliceSize;
- private LateralJoinPrel topMostLateralJoin = null;
+ private LateralJoinPrel topMostLateralJoin;
public ExcessiveExchangeIdentifier(long targetSliceSize) {
this.targetSliceSize = targetSliceSize;
@@ -78,11 +78,7 @@ public class ExcessiveExchangeIdentifier extends BasePrelVisitor<Prel, Excessive
return true;
}
- if (parentFrag.isRightSideOfLateral()) {
- return true;
- }
-
- return false;
+ return parentFrag.isRightSideOfLateral();
}
@Override
@@ -147,7 +143,7 @@ public class ExcessiveExchangeIdentifier extends BasePrelVisitor<Prel, Excessive
s.setHashDistribution(prel);
- for(Prel p : prel) {
+ for (Prel p : prel) {
children.add(p.accept(this, s));
}
return (Prel) prel.copy(prel.getTraitSet(), children);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/MemoryEstimationVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/MemoryEstimationVisitor.java
index 05d0920..6fee054 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/MemoryEstimationVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/MemoryEstimationVisitor.java
@@ -21,11 +21,13 @@ import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.planner.cost.DrillCostBase;
import org.apache.drill.exec.planner.physical.Prel;
import org.apache.drill.exec.server.options.OptionManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
public class MemoryEstimationVisitor extends BasePrelVisitor<Double, Void, RuntimeException> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MemoryEstimationVisitor.class);
+ private static final Logger logger = LoggerFactory.getLogger(MemoryEstimationVisitor.class);
public static boolean enoughMemory(Prel prel, OptionManager options, int numDrillbits) {
long allottedMemory = options.getOption(ExecConstants.MAX_QUERY_MEMORY_PER_NODE_KEY).num_val * numDrillbits;
@@ -44,8 +46,7 @@ public class MemoryEstimationVisitor extends BasePrelVisitor<Double, Void, Runti
return prel.accept(new MemoryEstimationVisitor(), null);
}
- public MemoryEstimationVisitor() {
- }
+ public MemoryEstimationVisitor() { }
@Override
public Double visitPrel(Prel prel, Void value) throws RuntimeException {
@@ -54,6 +55,7 @@ public class MemoryEstimationVisitor extends BasePrelVisitor<Double, Void, Runti
// return findCost(prel, mq);
}
+ @SuppressWarnings("unused")
private double findCost(Prel prel, RelMetadataQuery mq) {
DrillCostBase cost = (DrillCostBase) mq.getNonCumulativeCost(prel);
double memory = cost.getMemory();
@@ -63,5 +65,4 @@ public class MemoryEstimationVisitor extends BasePrelVisitor<Double, Void, Runti
}
return memory;
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/TypeInferenceUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/TypeInferenceUtils.java
index 8ffab70..7e68c35 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/TypeInferenceUtils.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/TypeInferenceUtils.java
@@ -21,7 +21,8 @@ import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableMap;
import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableSet;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.calcite.avatica.util.TimeUnit;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
@@ -59,7 +60,7 @@ import java.util.Set;
@SuppressWarnings("WeakerAccess")
public class TypeInferenceUtils {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TypeInferenceUtils.class);
+ private static final Logger logger = LoggerFactory.getLogger(TypeInferenceUtils.class);
public static final TypeProtos.MajorType UNKNOWN_TYPE = TypeProtos.MajorType.getDefaultInstance();
private static final ImmutableMap<TypeProtos.MinorType, SqlTypeName> DRILL_TO_CALCITE_TYPE_MAPPING
@@ -351,7 +352,7 @@ public class TypeInferenceUtils {
final DrillFuncHolder func = resolveDrillFuncHolder(opBinding, functions, functionCall);
final RelDataType returnType = getReturnType(opBinding,
- func.getReturnType(functionCall.args), func.getNullHandling());
+ func.getReturnType(functionCall.args()), func.getNullHandling());
return returnType.getSqlTypeName() == SqlTypeName.VARBINARY
? createCalciteTypeWithNullability(factory, SqlTypeName.ANY, returnType.isNullable())
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
index a59d8a4..2a8e272 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
@@ -180,8 +180,8 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
}
/**
- * Rewrite the parse tree. Used before validating the parse tree. Useful if a particular statement needs to converted
- * into another statement.
+ * Rewrite the parse tree. Used before validating the parse tree. Useful if a
+ * particular statement needs to converted into another statement.
*
* @param node sql parse tree to be rewritten
* @return Rewritten sql parse tree
@@ -312,7 +312,7 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
}
/**
- * A shuttle designed to finalize all RelNodes.
+ * Finalize all RelNodes.
*/
private static class PrelFinalizer extends RelShuttleImpl {
@@ -425,7 +425,8 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
* @throws RelConversionException
* @throws SqlUnsupportedException
*/
- protected Prel convertToPrel(RelNode drel, RelDataType validatedRowType) throws RelConversionException, SqlUnsupportedException {
+ protected Prel convertToPrel(RelNode drel, RelDataType validatedRowType)
+ throws RelConversionException, SqlUnsupportedException {
Preconditions.checkArgument(drel.getConvention() == DrillRel.DRILL_LOGICAL);
final RelTraitSet traits = drel.getTraitSet().plus(Prel.DRILL_PHYSICAL).plus(DrillDistributionTrait.SINGLETON);
@@ -469,6 +470,8 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
}
}
}
+ // Handy way to visualize the plan while debugging
+ //ExplainHandler.printPlan(phyRelNode, context);
/* The order of the following transformations is important */
@@ -530,7 +533,7 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
/*
* 3.)
- * Since our operators work via names rather than indices, we have to make to reorder any
+ * Since our operators work via names rather than indices, we have to reorder any
* output before we return data to the user as we may have accidentally shuffled things.
* This adds a trivial project to reorder columns prior to output.
*/
@@ -557,17 +560,15 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
}
*/
-
/* 6.)
* if the client does not support complex types (Map, Repeated)
- * insert a project which which would convert
+ * insert a project which would convert
*/
if (!context.getSession().isSupportComplexTypes()) {
logger.debug("Client does not support complex types, add ComplexToJson operator.");
phyRelNode = ComplexToJsonPrelVisitor.addComplexToJsonPrel(phyRelNode);
}
-
/* 7.)
* Insert LocalExchange (mux and/or demux) nodes
*/
@@ -581,7 +582,6 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
phyRelNode = RuntimeFilterVisitor.addRuntimeFilter(phyRelNode, context);
}
-
/* 9.)
* Next, we add any required selection vector removers given the supported encodings of each
* operator. This will ultimately move to a new trait but we're managing here for now to avoid
@@ -589,7 +589,6 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
*/
phyRelNode = SelectionVectorPrelVisitor.addSelectionRemoversWhereNecessary(phyRelNode);
-
/* 10.)
* Finally, Make sure that the no rels are repeats.
* This could happen in the case of querying the same table twice as Optiq may canonicalize these.
@@ -635,6 +634,7 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
}
return null;
}
+
}
protected Pair<SqlNode, RelDataType> validateNode(SqlNode sqlNode) throws ValidationException, RelConversionException, ForemanSetupException {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ExplainHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ExplainHandler.java
index d62fb4a..91631e3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ExplainHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/ExplainHandler.java
@@ -41,9 +41,12 @@ import org.apache.drill.exec.planner.physical.explain.PrelSequencer;
import org.apache.drill.exec.planner.sql.DirectPlan;
import org.apache.drill.exec.util.Pointer;
import org.apache.drill.exec.work.foreman.ForemanSetupException;
+import org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class ExplainHandler extends DefaultSqlHandler {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExplainHandler.class);
+ private static final Logger logger = LoggerFactory.getLogger(ExplainHandler.class);
private ResultMode mode;
private SqlExplainLevel level = SqlExplainLevel.ALL_ATTRIBUTES;
@@ -96,23 +99,22 @@ public class ExplainHandler extends DefaultSqlHandler {
return node.operand(0);
}
-
- public static class LogicalExplain{
- public String text;
- public String json;
+ public static class LogicalExplain {
+ public final String text;
+ public final String json;
public LogicalExplain(RelNode node, SqlExplainLevel level, QueryContext context) {
this.text = RelOptUtil.toString(node, level);
DrillImplementor implementor = new DrillImplementor(new DrillParseContext(context.getPlannerSettings()), ResultMode.LOGICAL);
- implementor.go( (DrillRel) node);
+ implementor.go((DrillRel) node);
LogicalPlan plan = implementor.getPlan();
this.json = plan.unparse(context.getLpPersistence());
}
}
- public static class PhysicalExplain{
- public String text;
- public String json;
+ public static class PhysicalExplain {
+ public final String text;
+ public final String json;
public PhysicalExplain(RelNode node, PhysicalPlan plan, SqlExplainLevel level, QueryContext context) {
this.text = PrelSequencer.printWithIds((Prel) node, level);
@@ -120,4 +122,15 @@ public class ExplainHandler extends DefaultSqlHandler {
}
}
+ // Debug-only tool to dump a plan to stdout for inspection
+
+ @VisibleForTesting
+ public static void printPlan(Prel node, QueryContext context) {
+ System.out.println(PrelSequencer.printWithIds(node, SqlExplainLevel.ALL_ATTRIBUTES));
+ }
+
+ @VisibleForTesting
+ public static void printPlan(RelNode node) {
+ System.out.println(RelOptUtil.toString(node, SqlExplainLevel.ALL_ATTRIBUTES));
+ }
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/DefaultFunctionResolver.java b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/DefaultFunctionResolver.java
index c7e51ce..b768792 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/DefaultFunctionResolver.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/DefaultFunctionResolver.java
@@ -41,7 +41,7 @@ public class DefaultFunctionResolver implements FunctionResolver {
for (DrillFuncHolder h : methods) {
final List<TypeProtos.MajorType> argumentTypes = Lists.newArrayList();
- for (LogicalExpression expression : call.args) {
+ for (LogicalExpression expression : call.args()) {
argumentTypes.add(expression.getMajorType());
}
currcost = TypeCastRules.getCost(argumentTypes, h);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ExactFunctionResolver.java b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ExactFunctionResolver.java
index 73bdab8..59c2488 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ExactFunctionResolver.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ExactFunctionResolver.java
@@ -39,7 +39,7 @@ public class ExactFunctionResolver implements FunctionResolver {
int currCost;
final List<TypeProtos.MajorType> argumentTypes = Lists.newArrayList();
- for (LogicalExpression expression : call.args) {
+ for (LogicalExpression expression : call.args()) {
argumentTypes.add(expression.getMajorType());
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
index 4360571..e7c4128 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemPlugin.java
@@ -54,9 +54,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * A Storage engine associated with a Hadoop FileSystem Implementation. Examples include HDFS, MapRFS, QuantacastFileSystem,
- * LocalFileSystem, as well Apache Drill specific CachedFileSystem, ClassPathFileSystem and LocalSyncableFileSystem.
- * Tables are file names, directories and path patterns. This storage engine delegates to FSFormatEngines but shares
+ * A Storage engine associated with a Hadoop FileSystem Implementation. Examples
+ * include HDFS, MapRFS, QuantacastFileSystem, LocalFileSystem, as well Apache
+ * Drill specific CachedFileSystem, ClassPathFileSystem and
+ * LocalSyncableFileSystem. Tables are file names, directories and path
+ * patterns. This storage engine delegates to FSFormatEngines but shares
* references to the FileSystem configuration and path management.
*/
public class FileSystemPlugin extends AbstractStoragePlugin {
@@ -118,11 +120,12 @@ public class FileSystemPlugin extends AbstractStoragePlugin {
// if the "default" workspace is not given add one.
if (noWorkspace || !config.getWorkspaces().containsKey(DEFAULT_WS_NAME)) {
- factories.add(new WorkspaceSchemaFactory(
- this, DEFAULT_WS_NAME, name, WorkspaceConfig.DEFAULT, matchers, context.getLpPersistence(), context.getClasspathScan()));
+ factories.add(new WorkspaceSchemaFactory(this, DEFAULT_WS_NAME, name,
+ WorkspaceConfig.DEFAULT, matchers,
+ context.getLpPersistence(), context.getClasspathScan()));
}
- schemaFactory = new FileSystemSchemaFactory(name, factories);
+ this.schemaFactory = new FileSystemSchemaFactory(name, factories);
} catch (IOException e) {
throw new ExecutionSetupException("Failure setting up file system plugin.", e);
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java
index 404a5d0..b5d5b07 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FormatPlugin.java
@@ -53,21 +53,27 @@ public interface FormatPlugin {
FormatMatcher getMatcher();
- AbstractWriter getWriter(PhysicalOperator child, String location, List<String> partitionColumns) throws IOException;
+ AbstractWriter getWriter(PhysicalOperator child, String location,
+ List<String> partitionColumns) throws IOException;
Set<StoragePluginOptimizerRule> getOptimizerRules();
- AbstractGroupScan getGroupScan(String userName, FileSelection selection, List<SchemaPath> columns) throws IOException;
+ AbstractGroupScan getGroupScan(String userName, FileSelection selection,
+ List<SchemaPath> columns) throws IOException;
- default AbstractGroupScan getGroupScan(String userName, FileSelection selection, List<SchemaPath> columns, OptionManager options) throws IOException {
+ default AbstractGroupScan getGroupScan(String userName, FileSelection selection,
+ List<SchemaPath> columns, OptionManager options) throws IOException {
return getGroupScan(userName, selection, columns);
}
- default AbstractGroupScan getGroupScan(String userName, FileSelection selection, List<SchemaPath> columns, MetadataProviderManager metadataProviderManager) throws IOException {
+ default AbstractGroupScan getGroupScan(String userName, FileSelection selection,
+ List<SchemaPath> columns, MetadataProviderManager metadataProviderManager) throws IOException {
return getGroupScan(userName, selection, columns);
}
- default AbstractGroupScan getGroupScan(String userName, FileSelection selection, List<SchemaPath> columns, OptionManager options, MetadataProviderManager metadataProvider) throws IOException {
+ default AbstractGroupScan getGroupScan(String userName, FileSelection selection,
+ List<SchemaPath> columns, OptionManager options,
+ MetadataProviderManager metadataProvider) throws IOException {
return getGroupScan(userName, selection, columns, metadataProvider);
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
index 1caa55d..e6c875c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
@@ -119,7 +119,7 @@ public abstract class EasyFormatPlugin<T extends FormatPluginConfig> implements
}
/**
- * Builds the readers for the V3 text scan operator.
+ * Builds the readers row-set based scan operator.
*/
private static class EasyReaderFactory extends FileReaderFactory {
@@ -128,7 +128,7 @@ public abstract class EasyFormatPlugin<T extends FormatPluginConfig> implements
private final FragmentContext context;
public EasyReaderFactory(EasyFormatPlugin<? extends FormatPluginConfig> plugin,
- final EasySubScan scan, FragmentContext context) {
+ EasySubScan scan, FragmentContext context) {
this.plugin = plugin;
this.scan = scan;
this.context = context;
@@ -226,7 +226,7 @@ public abstract class EasyFormatPlugin<T extends FormatPluginConfig> implements
* boundaries. If not, the simple format engine will only split on file
* boundaries.
*
- * @return {@code true} if splittable.
+ * @return {@code true} if splitable.
*/
public boolean isBlockSplittable() { return easyConfig.blockSplittable; }
@@ -256,8 +256,8 @@ public abstract class EasyFormatPlugin<T extends FormatPluginConfig> implements
throw new ExecutionSetupException("Must implement getRecordReader() if using the legacy scanner.");
}
- protected CloseableRecordBatch getReaderBatch(final FragmentContext context,
- final EasySubScan scan) throws ExecutionSetupException {
+ protected CloseableRecordBatch getReaderBatch(FragmentContext context,
+ EasySubScan scan) throws ExecutionSetupException {
if (useEnhancedScan(context.getOptions())) {
return buildScan(context, scan);
} else {
@@ -448,9 +448,9 @@ public abstract class EasyFormatPlugin<T extends FormatPluginConfig> implements
}
}
- protected ScanStats getScanStats(final PlannerSettings settings, final EasyGroupScan scan) {
+ protected ScanStats getScanStats(PlannerSettings settings, EasyGroupScan scan) {
long data = 0;
- for (final CompleteFileWork work : scan.getWorkIterable()) {
+ for (CompleteFileWork work : scan.getWorkIterable()) {
data += work.getTotalBytes();
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
index 4b4e029..3c032c3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
@@ -77,7 +77,7 @@ public class EasyGroupScan extends AbstractFileGroupScan {
private ListMultimap<Integer, CompleteFileWork> mappings;
private List<CompleteFileWork> chunks;
private List<EndpointAffinity> endpointAffinities;
- private Path selectionRoot;
+ private final Path selectionRoot;
private TableMetadata tableMetadata;
@JsonCreator
@@ -185,6 +185,7 @@ public class EasyGroupScan extends AbstractFileGroupScan {
endpointAffinities = AffinityCreator.getAffinityMap(chunks);
}
+ @Override
public Path getSelectionRoot() {
return selectionRoot;
}
@@ -257,13 +258,13 @@ public class EasyGroupScan extends AbstractFileGroupScan {
for (EndpointAffinity e : affinities) {
endpoints.add(e.getEndpoint());
}
- this.applyAssignments(endpoints);
+ applyAssignments(endpoints);
}
@Override
public EasySubScan getSpecificScan(int minorFragmentId) {
if (mappings == null) {
- createMappings(this.endpointAffinities);
+ createMappings(endpointAffinities);
}
assert minorFragmentId < mappings.size() : String.format(
"Mappings length [%d] should be longer than minor fragment id [%d] but it isn't.", mappings.size(),
@@ -276,7 +277,7 @@ public class EasyGroupScan extends AbstractFileGroupScan {
EasySubScan subScan = new EasySubScan(getUserName(), convert(filesForMinor), formatPlugin,
columns, selectionRoot, partitionDepth, getSchema());
- subScan.setOperatorId(this.getOperatorId());
+ subScan.setOperatorId(getOperatorId());
return subScan;
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilter.java
index b1f1d25..8b31c07 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilter.java
@@ -22,6 +22,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.fasterxml.jackson.annotation.JsonTypeName;
+
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.exec.store.ischema.InfoSchemaFilter.ExprNode.Type;
import org.apache.drill.shaded.guava.com.google.common.base.Joiner;
@@ -149,7 +151,7 @@ public class InfoSchemaFilter {
private Result evaluateHelperFunction(Map<String, String> recordValues, FunctionExprNode exprNode) {
switch (exprNode.function) {
- case "like": {
+ case FunctionNames.LIKE: {
FieldExprNode col = (FieldExprNode) exprNode.args.get(0);
ConstantExprNode pattern = (ConstantExprNode) exprNode.args.get(1);
ConstantExprNode escape = exprNode.args.size() > 2 ? (ConstantExprNode) exprNode.args.get(2) : null;
@@ -166,16 +168,16 @@ public class InfoSchemaFilter {
return Result.INCONCLUSIVE;
}
- case "equal":
- case "not equal":
- case "notequal":
- case "not_equal": {
+ case FunctionNames.EQ:
+ case "not equal": // TODO: Is this name correct?
+ case "notequal": // TODO: Is this name correct?
+ case FunctionNames.NE: {
FieldExprNode arg0 = (FieldExprNode) exprNode.args.get(0);
ConstantExprNode arg1 = (ConstantExprNode) exprNode.args.get(1);
final String value = recordValues.get(arg0.field);
if (value != null) {
- if (exprNode.function.equals("equal")) {
+ if (exprNode.function.equals(FunctionNames.EQ)) {
return arg1.value.equals(value) ? Result.TRUE : Result.FALSE;
} else {
return arg1.value.equals(value) ? Result.FALSE : Result.TRUE;
@@ -185,7 +187,8 @@ public class InfoSchemaFilter {
return Result.INCONCLUSIVE;
}
- case "booleanor": {
+ case FunctionNames.OR:
+ case "booleanor": { // TODO: Is this name correct?
// If at least one arg returns TRUE, then the OR function value is TRUE
// If all args return FALSE, then OR function value is FALSE
// For all other cases, return INCONCLUSIVE
@@ -202,7 +205,8 @@ public class InfoSchemaFilter {
return result;
}
- case "booleanand": {
+ case FunctionNames.AND:
+ case "booleanand": { // TODO: Is this name correct?
// If at least one arg returns FALSE, then the AND function value is FALSE
// If at least one arg returns INCONCLUSIVE, then the AND function value is INCONCLUSIVE
// If all args return TRUE, then the AND function value is TRUE
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilterBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilterBuilder.java
index 7624788..7cdbbd7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilterBuilder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaFilterBuilder.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.exec.store.ischema;
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.common.expression.BooleanOperator;
import org.apache.drill.common.expression.CastExpression;
import org.apache.drill.common.expression.FieldReference;
@@ -30,8 +31,8 @@ import org.apache.drill.exec.store.ischema.InfoSchemaFilter.ExprNode;
import org.apache.drill.exec.store.ischema.InfoSchemaFilter.FieldExprNode;
import org.apache.drill.exec.store.ischema.InfoSchemaFilter.FunctionExprNode;
import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList;
-import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
+import java.util.ArrayList;
import java.util.List;
import static org.apache.drill.exec.store.ischema.InfoSchemaConstants.CATS_COL_CATALOG_NAME;
@@ -73,12 +74,12 @@ public class InfoSchemaFilterBuilder extends AbstractExprVisitor<ExprNode, Void,
public ExprNode visitFunctionCall(FunctionCall call, Void value) throws RuntimeException {
final String funcName = call.getName().toLowerCase();
switch (funcName) {
- case "equal":
- case "not equal":
- case "notequal":
- case "not_equal": {
- final ExprNode col = call.args.get(0).accept(this, value);
- final ExprNode constant = call.args.get(1).accept(this, value);
+ case FunctionNames.EQ:
+ case "not equal": // TODO: Is this name correct?
+ case "notequal": // TODO: Is this name correct?
+ case FunctionNames.NE: {
+ final ExprNode col = call.arg(0).accept(this, value);
+ final ExprNode constant = call.arg(1).accept(this, value);
if (col instanceof FieldExprNode && constant instanceof ConstantExprNode) {
return new FunctionExprNode(funcName, ImmutableList.of(col, constant));
@@ -86,10 +87,10 @@ public class InfoSchemaFilterBuilder extends AbstractExprVisitor<ExprNode, Void,
break;
}
- case "like": {
- final ExprNode col = call.args.get(0).accept(this, value);
- final ExprNode pattern = call.args.get(1).accept(this, value);
- final ExprNode escape = call.args.size() > 2 ? call.args.get(2).accept(this, value) : null;
+ case FunctionNames.LIKE: {
+ final ExprNode col = call.arg(0).accept(this, value);
+ final ExprNode pattern = call.arg(1).accept(this, value);
+ final ExprNode escape = call.argCount() > 2 ? call.arg(2).accept(this, value) : null;
if (col instanceof FieldExprNode && pattern instanceof ConstantExprNode &&
(escape == null || escape instanceof ConstantExprNode)) {
@@ -99,9 +100,10 @@ public class InfoSchemaFilterBuilder extends AbstractExprVisitor<ExprNode, Void,
break;
}
- case "booleanand": {
- List<ExprNode> args = Lists.newArrayList();
- for(LogicalExpression arg : call.args) {
+ case FunctionNames.AND:
+ case "booleanand": { // TODO: Is this name correct?
+ List<ExprNode> args = new ArrayList<>();
+ for(LogicalExpression arg : call.args()) {
ExprNode exprNode = arg.accept(this, value);
if (exprNode instanceof FunctionExprNode) {
args.add(exprNode);
@@ -114,9 +116,10 @@ public class InfoSchemaFilterBuilder extends AbstractExprVisitor<ExprNode, Void,
return visitUnknown(call, value);
}
- case "booleanor": {
- List<ExprNode> args = Lists.newArrayList();
- for(LogicalExpression arg : call.args) {
+ case FunctionNames.OR:
+ case "booleanor": { // TODO: Is this name correct?
+ List<ExprNode> args = new ArrayList<>();
+ for(LogicalExpression arg : call.args()) {
ExprNode exprNode = arg.accept(this, value);
if (exprNode instanceof FunctionExprNode) {
args.add(exprNode);
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FilterEvaluatorUtils.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FilterEvaluatorUtils.java
index 80acade..8d35b1d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FilterEvaluatorUtils.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/FilterEvaluatorUtils.java
@@ -107,12 +107,12 @@ public class FilterEvaluatorUtils {
long rowCount,
TupleMetadata fileMetadata,
Set<SchemaPath> schemaPathsInExpr) {
- RowsMatch rowsMatch = RowsMatch.SOME;
- if (parquetPredicate != null) {
- @SuppressWarnings("rawtypes")
- StatisticsProvider<T> rangeExprEvaluator = new StatisticsProvider(columnsStatistics, rowCount);
- rowsMatch = parquetPredicate.matches(rangeExprEvaluator);
+ if (parquetPredicate == null) {
+ return RowsMatch.SOME;
}
+ @SuppressWarnings("rawtypes")
+ StatisticsProvider<T> rangeExprEvaluator = new StatisticsProvider(columnsStatistics, rowCount);
+ RowsMatch rowsMatch = parquetPredicate.matches(rangeExprEvaluator);
if (rowsMatch == RowsMatch.ALL && isMetaNotApplicable(schemaPathsInExpr, fileMetadata)) {
rowsMatch = RowsMatch.SOME;
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java
index af5e421..a22c4f7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/AssignmentCreator.java
@@ -36,13 +36,14 @@ import org.apache.drill.shaded.guava.com.google.common.collect.Iterators;
import org.apache.drill.shaded.guava.com.google.common.collect.ListMultimap;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
import org.apache.drill.shaded.guava.com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
- * The AssignmentCreator is responsible for assigning a set of work units to the available slices.
+ * Responsible for assigning a set of work units to the available slices.
*/
public class AssignmentCreator<T extends CompleteWork> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AssignmentCreator.class);
-
+ static final Logger logger = LoggerFactory.getLogger(AssignmentCreator.class);
/**
* Comparator used to sort in order of decreasing affinity
@@ -56,24 +57,24 @@ public class AssignmentCreator<T extends CompleteWork> {
};
/**
- * the maximum number of work units to assign to any minor fragment
+ * The maximum number of work units to assign to any minor fragment
*/
private int maxWork;
/**
* The units of work to be assigned
*/
- private List<T> units;
+ private final List<T> units;
/**
* Mappings
*/
- private ArrayListMultimap<Integer, T> mappings = ArrayListMultimap.create();
+ private final ArrayListMultimap<Integer, T> mappings = ArrayListMultimap.create();
/**
* A list of DrillbitEndpoints, where the index in the list corresponds to the minor fragment id
*/
- private List<DrillbitEndpoint> incomingEndpoints;
+ private final List<DrillbitEndpoint> incomingEndpoints;
private AssignmentCreator(List<DrillbitEndpoint> incomingEndpoints, List<T> units) {
this.incomingEndpoints = incomingEndpoints;
@@ -106,16 +107,16 @@ public class AssignmentCreator<T extends CompleteWork> {
LinkedList<WorkEndpointListPair<T>> unassignedWorkList;
Map<DrillbitEndpoint,FragIteratorWrapper> endpointIterators = getEndpointIterators();
- // Assign upto maxCount per node based on locality.
+ // Assign up to maxCount per node based on locality.
unassignedWorkList = assign(workList, endpointIterators, false);
- // Assign upto minCount per node in a round robin fashion.
+ // Assign up to minCount per node in a round robin fashion.
assignLeftovers(unassignedWorkList, endpointIterators, true);
- // Assign upto maxCount + leftovers per node based on locality.
+ // Assign up to maxCount + leftovers per node based on locality.
unassignedWorkList = assign(unassignedWorkList, endpointIterators, true);
- // Assign upto maxCount + leftovers per node in a round robin fashion.
+ // Assign up to maxCount + leftovers per node in a round robin fashion.
assignLeftovers(unassignedWorkList, endpointIterators, false);
if (unassignedWorkList.size() != 0) {
@@ -130,7 +131,7 @@ public class AssignmentCreator<T extends CompleteWork> {
*
* @param workList the list of work units to assign
* @param endpointIterators the endpointIterators to assign to
- * @param assignMaxLeftOvers whether to assign upto maximum including leftovers
+ * @param assignMaxLeftOvers whether to assign up to maximum including leftovers
* @return a list of unassigned work units
*/
private LinkedList<WorkEndpointListPair<T>> assign(List<WorkEndpointListPair<T>> workList,
@@ -160,7 +161,7 @@ public class AssignmentCreator<T extends CompleteWork> {
*
* @param unassignedWorkList the work units to assign
* @param endpointIterators the endpointIterators to assign to
- * @param assignMinimum wheterh to assign the minimum amount
+ * @param assignMinimum whether to assign the minimum amount
*/
private void assignLeftovers(LinkedList<WorkEndpointListPair<T>> unassignedWorkList,
Map<DrillbitEndpoint,FragIteratorWrapper> endpointIterators,
@@ -183,7 +184,6 @@ public class AssignmentCreator<T extends CompleteWork> {
* @return the list of WorkEndpointListPairs
*/
private LinkedList<WorkEndpointListPair<T>> getWorkList() {
- Stopwatch watch = Stopwatch.createStarted();
LinkedList<WorkEndpointListPair<T>> workList = Lists.newLinkedList();
for (T work : units) {
List<Map.Entry<DrillbitEndpoint,Long>> entries = Lists.newArrayList();
@@ -220,7 +220,8 @@ public class AssignmentCreator<T extends CompleteWork> {
}
/**
- * A wrapper class around a work unit and its associated sort list of Endpoints (sorted by affinity in decreasing order)
+ * Wrapper class around a work unit and its associated sort list of Endpoints
+ * (sorted by affinity in decreasing order)
*/
private static class WorkEndpointListPair<T> {
T work;
@@ -239,7 +240,6 @@ public class AssignmentCreator<T extends CompleteWork> {
* @return
*/
private Map<DrillbitEndpoint,FragIteratorWrapper> getEndpointIterators() {
- Stopwatch watch = Stopwatch.createStarted();
Map<DrillbitEndpoint,FragIteratorWrapper> map = Maps.newLinkedHashMap();
Map<DrillbitEndpoint,List<Integer>> mmap = Maps.newLinkedHashMap();
for (int i = 0; i < incomingEndpoints.size(); i++) {
@@ -280,7 +280,7 @@ public class AssignmentCreator<T extends CompleteWork> {
}
/**
- * A struct that holds a fragment iterator and keeps track of how many units have been assigned, as well as the maximum
+ * Holds a fragment iterator and keeps track of how many units have been assigned, as well as the maximum
* number of assignment it will accept
*/
private static class FragIteratorWrapper {
@@ -290,5 +290,4 @@ public class AssignmentCreator<T extends CompleteWork> {
int minCount;
Iterator<Integer> iter;
}
-
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/metadata/MetadataProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/metadata/MetadataProvider.java
index ff23958..6ff2383 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/metadata/MetadataProvider.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/metadata/MetadataProvider.java
@@ -34,6 +34,7 @@ import java.util.List;
import java.util.UUID;
import org.apache.calcite.schema.SchemaPlus;
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.ErrorHelper;
import org.apache.drill.exec.ops.ViewExpansionContext;
@@ -78,17 +79,19 @@ import org.apache.drill.shaded.guava.com.google.common.base.Preconditions;
import org.apache.drill.shaded.guava.com.google.common.collect.ComparisonChain;
import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList;
import org.apache.drill.shaded.guava.com.google.common.collect.Ordering;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Contains worker {@link Runnable} classes for providing the metadata and related helper methods.
*/
public class MetadataProvider {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MetadataProvider.class);
+ private static final Logger logger = LoggerFactory.getLogger(MetadataProvider.class);
private static final String IN_FUNCTION = "in";
- private static final String LIKE_FUNCTION = "like";
- private static final String AND_FUNCTION = "booleanand";
- private static final String OR_FUNCTION = "booleanor";
+ // Is this intended? Function name here is lower case of actual name?
+ private static final String AND_FUNCTION = FunctionNames.AND.toLowerCase();
+ private static final String OR_FUNCTION = FunctionNames.OR.toLowerCase();
/**
* @return Runnable that fetches the catalog metadata for given {@link GetCatalogsReq} and sends response at the end.
@@ -506,7 +509,7 @@ public class MetadataProvider {
return null;
}
- return new FunctionExprNode(LIKE_FUNCTION,
+ return new FunctionExprNode(FunctionNames.LIKE,
likeFilter.hasEscape() ?
ImmutableList.of(
new FieldExprNode(fieldName),
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/resultSet/impl/TestResultSetLoaderMapArray.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/resultSet/impl/TestResultSetLoaderMapArray.java
index 1cfd653..5e607f6 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/resultSet/impl/TestResultSetLoaderMapArray.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/resultSet/impl/TestResultSetLoaderMapArray.java
@@ -204,7 +204,6 @@ public class TestResultSetLoaderMapArray extends SubOperatorTest {
mapValue(330, strArray("d3.3.1", "d1.2.2"))));
// Verify the batch
-
RowSet actual = fixture.wrap(rsLoader.harvest());
SingleRowSet expected = fixture.rowSetBuilder(schema)
.addRow(10, mapArray(
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/resultSet/impl/TestResultSetLoaderRepeatedList.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/resultSet/impl/TestResultSetLoaderRepeatedList.java
index f3d25f2..b4c1fea 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/resultSet/impl/TestResultSetLoaderRepeatedList.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/resultSet/impl/TestResultSetLoaderRepeatedList.java
@@ -64,7 +64,6 @@ import static org.junit.Assert.assertTrue;
* actually, since the different "slices" need not have the same length...)
* Repeated lists appear to be used only by JSON.
*/
-
@Category(RowSetTests.class)
public class TestResultSetLoaderRepeatedList extends SubOperatorTest {
@@ -90,7 +89,6 @@ public class TestResultSetLoaderRepeatedList extends SubOperatorTest {
final RowSetLoader writer = rsLoader.writer();
// Sanity check of writer structure
-
assertEquals(2, writer.size());
final ObjectWriter listObj = writer.column("list2");
assertEquals(ObjectType.ARRAY, listObj.type());
@@ -102,7 +100,6 @@ public class TestResultSetLoaderRepeatedList extends SubOperatorTest {
assertEquals(ValueType.STRING, strWriter.valueType());
// Sanity test of schema
-
final TupleMetadata rowSchema = writer.tupleSchema();
assertEquals(2, rowSchema.size());
final ColumnMetadata listSchema = rowSchema.metadata(1);
@@ -118,7 +115,6 @@ public class TestResultSetLoaderRepeatedList extends SubOperatorTest {
assertEquals(DataMode.REPEATED, elementSchema.mode());
// Write values
-
rsLoader.startBatch();
writer
.addRow(1, objArray(strArray("a", "b"), strArray("c", "d")))
@@ -128,7 +124,6 @@ public class TestResultSetLoaderRepeatedList extends SubOperatorTest {
// Verify the values.
// (Relies on the row set level repeated list tests having passed.)
-
final RowSet expected = fixture.rowSetBuilder(schema)
.addRow(1, objArray(strArray("a", "b"), strArray("c", "d")))
.addRow(2, objArray(strArray("e"), strArray(), strArray("f", "g", "h")))
@@ -154,15 +149,13 @@ public class TestResultSetLoaderRepeatedList extends SubOperatorTest {
final RowSetLoader writer = rsLoader.writer();
// Add columns dynamically
-
writer.addColumn(schema.metadata(0));
writer.addColumn(schema.metadata(1).cloneEmpty());
// Yes, this is ugly. The whole repeated array idea is awkward.
// The only place it is used at present is in JSON where the
- // awkwardness is mixed in with a logs of JSON complexity.
+ // awkwardness is mixed in with JSON complexity.
// Consider improving this API in the future.
-
((RepeatedListWriter) writer.array(1)).defineElement(schema.metadata(1).childSchema());
do2DTest(schema, rsLoader);
@@ -187,27 +180,22 @@ public class TestResultSetLoaderRepeatedList extends SubOperatorTest {
final RowSetLoader writer = rsLoader.writer();
// Add columns dynamically
-
writer.addColumn(schema.metadata(0));
// Write a row without the array.
-
rsLoader.startBatch();
writer.addRow(1);
// Add the repeated list, but without contents.
-
writer.addColumn(schema.metadata(1).cloneEmpty());
// Sanity check of writer structure
-
assertEquals(2, writer.size());
final ObjectWriter listObj = writer.column("list1");
assertEquals(ObjectType.ARRAY, listObj.type());
final ArrayWriter listWriter = listObj.array();
// No child defined yet. A dummy child is inserted instead.
-
assertEquals(MinorType.NULL, listWriter.entry().schema().type());
assertEquals(ObjectType.ARRAY, listWriter.entryType());
assertEquals(ObjectType.SCALAR, listWriter.array().entryType());
@@ -215,19 +203,16 @@ public class TestResultSetLoaderRepeatedList extends SubOperatorTest {
// Although we don't know the type of the inner, we can still
// create null (empty) elements in the outer array.
-
writer
.addRow(2, null)
.addRow(3, objArray())
.addRow(4, objArray(objArray(), null));
// Define the inner type.
-
final RepeatedListWriter listWriterImpl = (RepeatedListWriter) listWriter;
listWriterImpl.defineElement(MaterializedField.create("list1", Types.repeated(MinorType.VARCHAR)));
// Sanity check of completed structure
-
assertEquals(ObjectType.ARRAY, listWriter.entryType());
final ArrayWriter innerWriter = listWriter.array();
assertEquals(ObjectType.SCALAR, innerWriter.entryType());
@@ -235,16 +220,13 @@ public class TestResultSetLoaderRepeatedList extends SubOperatorTest {
assertEquals(ValueType.STRING, strWriter.valueType());
// Write values
-
writer
.addRow(5, objArray(strArray("a1", "b1"), strArray("c1", "d1")));
// Add the second list, with a complete type
-
writer.addColumn(schema.metadata(2));
// Sanity check of writer structure
-
assertEquals(3, writer.size());
final ObjectWriter list2Obj = writer.column("list2");
assertEquals(ObjectType.ARRAY, list2Obj.type());
@@ -256,7 +238,6 @@ public class TestResultSetLoaderRepeatedList extends SubOperatorTest {
assertEquals(ValueType.STRING, str2Writer.valueType());
// Write values
-
writer
.addRow(6,
objArray(strArray("a2", "b2"), strArray("c2", "d2")),
@@ -266,7 +247,6 @@ public class TestResultSetLoaderRepeatedList extends SubOperatorTest {
// Verify the values.
// (Relies on the row set level repeated list tests having passed.)
-
final RowSet expected = fixture.rowSetBuilder(schema)
.addRow(1, objArray(), objArray())
.addRow(2, objArray(), objArray())
@@ -301,7 +281,6 @@ public class TestResultSetLoaderRepeatedList extends SubOperatorTest {
// Fill the batch with enough data to cause overflow.
// Data must be large enough to cause overflow before 64K rows
// Make a bit bigger to overflow early.
-
final int outerSize = 7;
final int innerSize = 5;
final int strLength = ValueVector.MAX_BUFFER_SIZE / ValueVector.MAX_ROW_COUNT / outerSize / innerSize + 20;
@@ -331,25 +310,20 @@ public class TestResultSetLoaderRepeatedList extends SubOperatorTest {
// Number of rows should be driven by vector size.
// Our row count should include the overflow row
-
final int expectedCount = ValueVector.MAX_BUFFER_SIZE / (strLength * innerSize * outerSize);
assertEquals(expectedCount + 1, rowCount);
// Loader's row count should include only "visible" rows
-
assertEquals(expectedCount, writer.rowCount());
// Total count should include invisible and look-ahead rows.
-
assertEquals(expectedCount + 1, rsLoader.totalRowCount());
// Result should exclude the overflow row
-
RowSet result = fixture.wrap(rsLoader.harvest());
assertEquals(expectedCount, result.rowCount());
// Verify the data.
-
RowSetReader reader = result.reader();
ArrayReader outerReader = reader.array(1);
ArrayReader innerReader = outerReader.array();
@@ -375,7 +349,6 @@ public class TestResultSetLoaderRepeatedList extends SubOperatorTest {
result.clear();
// Write a few more rows to verify the overflow row.
-
rsLoader.startBatch();
for (int i = 0; i < 1000; i++) {
writer.start();
@@ -424,7 +397,6 @@ public class TestResultSetLoaderRepeatedList extends SubOperatorTest {
// That test exercises the low-level schema and writer mechanisms.
// Here we simply ensure that the 3D case continues to work when
// wrapped in the Result Set Loader
-
@Test
public void test3DEarlySchema() {
final TupleMetadata schema = new SchemaBuilder()
@@ -432,7 +404,6 @@ public class TestResultSetLoaderRepeatedList extends SubOperatorTest {
// Uses a short-hand method to avoid mucking with actual
// nested lists.
-
.addArray("cube", MinorType.VARCHAR, 3)
.buildSchema();
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsvWithHeaders.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsvWithHeaders.java
index 0bbdf78..108c09a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsvWithHeaders.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/text/compliant/TestCsvWithHeaders.java
@@ -406,13 +406,13 @@ public class TestCsvWithHeaders extends BaseCsvTest {
}
/**
- * Test partition expansion .
+ * Test partition expansion.
* <p>
* This test is tricky because it will return two data batches
* (preceded by an empty schema batch.) File read order is random
* so we have to expect the files in either order.
* <p>
- * V3, as in V2 before Drill 1.12, puts partition columns after
+ * V3 puts partition columns after
* data columns (so that data columns don't shift positions if
* files are nested to another level.)
*/
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
index e51e311..e71845e 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDown.java
@@ -108,7 +108,7 @@ public class TestParquetFilterPushDown extends PlanTestBase {
};
@Test
- // Test filter evaluation directly without go through SQL queries.
+ // Test filter evaluation directly without going through SQL queries.
public void testIntPredicateWithEval() throws Exception {
// intTbl.parquet has only one int column
// intCol : [0, 100].
@@ -150,7 +150,6 @@ public class TestParquetFilterPushDown extends PlanTestBase {
testParquetRowGroupFilterEval(footer, "intCol = 50 or intCol = 160", RowsMatch.SOME);
//"nonExistCol" does not exist in the table. "AND" with a filter on exist column
-
testParquetRowGroupFilterEval(footer, "intCol > 100 and nonExistCol = 100", RowsMatch.NONE);
testParquetRowGroupFilterEval(footer, "intCol > 50 and nonExistCol = 100", RowsMatch.NONE); // since nonExistCol = 100 -> Unknown -> could drop.
testParquetRowGroupFilterEval(footer, "nonExistCol = 100 and intCol > 50", RowsMatch.NONE); // since nonExistCol = 100 -> Unknown -> could drop.
@@ -162,7 +161,6 @@ public class TestParquetFilterPushDown extends PlanTestBase {
// is ignored.
//"nonExistCol" does not exist in the table. "OR" with a filter on exist column
-
testParquetRowGroupFilterEval(footer, "intCol > 100 or nonExistCol = 100", RowsMatch.NONE); // nonExistCol = 100 -> could drop.
testParquetRowGroupFilterEval(footer, "nonExistCol = 100 or intCol > 100", RowsMatch.NONE); // nonExistCol = 100 -> could drop.
@@ -726,6 +724,7 @@ public class TestParquetFilterPushDown extends PlanTestBase {
runAndCheckResults(client, sql, expectedRows, numPartitions, numPruned);
}
}
+
/**
* Test runtime pruning
*
@@ -757,5 +756,4 @@ public class TestParquetFilterPushDown extends PlanTestBase {
long resultNumPruned = parquestScan0.getMetric(ParquetRecordReader.Metric.ROWGROUPS_PRUNED.ordinal());
assertEquals(numPruned,resultNumPruned);
}
-
}
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java b/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java
index 4e7b298..e1b5d99 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java
@@ -99,14 +99,14 @@ public class MaterializedField {
* The type is immutable. But, it contains subtypes, used or lists
* and unions. To add a subtype, we must create a whole new major type.
* <p>
- * It appears that the <tt>MaterializedField</tt> class was also meant
+ * It appears that the {@code MaterializedField} class was also meant
* to be immutable. But, it holds the children for a map, and contains
* methods to add children. So, it is not immutable.
* <p>
* This method allows evolving a list or union without the need to create
- * a new <tt>MaterializedField</tt>. Doing so is problematic for nested
+ * a new {@code MaterializedField}. Doing so is problematic for nested
* maps because the map (or list, or union) holds onto the
- * <tt>MaterializedField</tt>'s of its children. There is no way for
+ * {@code MaterializedField}'s of its children. There is no way for
* an inner map to reach out and change the child of its parent.
* <p>
* By allowing the non-critical metadata to change, we preserve the
@@ -227,7 +227,7 @@ public class MaterializedField {
* tricky issue. The rules here:
* <ul>
* <li>The other schema is assumed to be non-null (unlike
- * <tt>equals()</tt>).</li>
+ * {@code equals()}).</li>
* <li>Names must be identical, ignoring case. (Drill, like SQL, is case
* insensitive.)
* <li>Type, mode, precision and scale must be identical.</li>
@@ -235,7 +235,7 @@ public class MaterializedField {
* "$bits" and "$offsets" vector columns are not compared, as one schema may
* be an "original" (without these hidden columns) while the other may come
* from a vector (which has the hidden columns added. The standard
- * <tt>equals()</tt> comparison does consider hidden columns.</li>
+ * {@code equals()} comparison does consider hidden columns.</li>
* <li>For maps, the child columns are compared recursively. This version
* requires that the two sets of columns appear in the same order. (It assumes
* it is being used in a context where column indexes make sense.) Operators
@@ -274,10 +274,9 @@ public class MaterializedField {
*
* @param other
* another field
- * @return <tt>true</tt> if the columns are identical according to the above
- * rules, <tt>false</tt> if they differ
+ * @return {@code true} if the columns are identical according to the above
+ * rules, {@code false} if they differ
*/
-
public boolean isEquivalent(MaterializedField other) {
if (! name.equalsIgnoreCase(other.name)) {
return false;
@@ -286,14 +285,12 @@ public class MaterializedField {
// Requires full type equality, including fields such as precision and scale.
// But, unset fields are equivalent to 0. Can't use the protobuf-provided
// isEquals(), that treats set and unset fields as different.
-
if (! Types.isEquivalent(type, other.type)) {
return false;
}
// Compare children -- but only for maps, not the internal children
// for Varchar, repeated or nullable types.
-
if (type.getMinorType() != MinorType.MAP) {
return true;
}
@@ -307,7 +304,6 @@ public class MaterializedField {
// Maps are name-based, not position. But, for our
// purposes, we insist on identical ordering.
-
final Iterator<MaterializedField> thisIter = children.iterator();
final Iterator<MaterializedField> otherIter = other.children.iterator();
while (thisIter.hasNext()) {
@@ -329,7 +325,6 @@ public class MaterializedField {
* @param other the field to which this one is to be promoted
* @return true if promotion is possible, false otherwise
*/
-
public boolean isPromotableTo(MaterializedField other, boolean allowModeChange) {
if (! name.equalsIgnoreCase(other.name)) {
return false;
@@ -338,7 +333,6 @@ public class MaterializedField {
// Requires full type equality, including fields such as precision and scale.
// But, unset fields are equivalent to 0. Can't use the protobuf-provided
// isEquals(), that treats set and unset fields as different.
-
if (type.getMinorType() != other.type.getMinorType()) {
return false;
}
@@ -346,7 +340,6 @@ public class MaterializedField {
// Modes differ, but type can be promoted from required to
// nullable
-
if (! allowModeChange) {
return false;
}
@@ -363,7 +356,6 @@ public class MaterializedField {
// Compare children -- but only for maps, not the internal children
// for Varchar, repeated or nullable types.
-
if (type.getMinorType() != MinorType.MAP) {
return true;
}
@@ -377,7 +369,6 @@ public class MaterializedField {
// Maps are name-based, not position. But, for our
// purposes, we insist on identical ordering.
-
final Iterator<MaterializedField> thisIter = children.iterator();
final Iterator<MaterializedField> otherIter = other.children.iterator();
while (thisIter.hasNext()) {
@@ -400,7 +391,6 @@ public class MaterializedField {
*
* @return materialized field string representation
*/
-
public String toString(boolean includeChildren) {
final int maxLen = 10;
final StringBuilder builder = new StringBuilder();
@@ -451,6 +441,14 @@ public class MaterializedField {
return toString(true);
}
+ /**
+ * Return true if two fields have identical MinorType and Mode.
+ */
+ public boolean hasSameTypeAndMode(MaterializedField that) {
+ return (getType().getMinorType() == that.getType().getMinorType())
+ && (getType().getMode() == that.getType().getMode());
+ }
+
private String toString(Collection<?> collection, int maxLen) {
final StringBuilder builder = new StringBuilder();
int i = 0;
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/AbstractColumnMetadata.java b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/AbstractColumnMetadata.java
index 0c8a8bf..2735d62 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/AbstractColumnMetadata.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/AbstractColumnMetadata.java
@@ -172,7 +172,6 @@ public abstract class AbstractColumnMetadata extends AbstractPropertied implemen
// TODO: This converts each column to a MaterializedField in
// order to do the comparison. This is done to avoid duplicating
// the checks. Consider doing checks here at some point.
-
return schema().isEquivalent(other.schema());
}
@@ -279,6 +278,10 @@ public abstract class AbstractColumnMetadata extends AbstractPropertied implemen
buf.append(", variant: ")
.append(variantSchema().toString());
}
+ if (childSchema() != null) {
+ buf.append(", child: ")
+ .append(childSchema().toString());
+ }
if (tupleSchema() != null) {
buf.append(", schema: ")
.append(tupleSchema().toString());
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/PropertyAccessor.java b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/PropertyAccessor.java
index 0cf0a32..5d92dcd 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/PropertyAccessor.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/record/metadata/PropertyAccessor.java
@@ -18,7 +18,7 @@
package org.apache.drill.exec.record.metadata;
/**
- * Helper utilities to get/set typed values within a propertied object
+ * Utilities to get/set typed values within a propertied object
*/
public class PropertyAccessor {
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractScalarReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractScalarReader.java
index 2e729e8..1fe9c69 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractScalarReader.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/reader/AbstractScalarReader.java
@@ -253,7 +253,7 @@ public abstract class AbstractScalarReader implements ScalarReader, ReaderEvents
case PERIOD:
return getPeriod().normalizedStandard().toString();
default:
- return getObject().toString();
+ return getValue().toString();
}
}
}
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractTupleWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractTupleWriter.java
index 7b5309c..4401b6c 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractTupleWriter.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/AbstractTupleWriter.java
@@ -156,7 +156,7 @@ public abstract class AbstractTupleWriter implements TupleWriter, WriterEvents {
*/
static class MemberWriterIndex implements ColumnWriterIndex {
- private ColumnWriterIndex baseIndex;
+ private final ColumnWriterIndex baseIndex;
MemberWriterIndex(ColumnWriterIndex baseIndex) {
this.baseIndex = baseIndex;
@@ -243,15 +243,15 @@ public abstract class AbstractTupleWriter implements TupleWriter, WriterEvents {
@Override
public int addColumn(ColumnMetadata column) {
verifyAddColumn(column.name());
- final AbstractObjectWriter colWriter = (AbstractObjectWriter) listener.addColumn(this, column);
- return addColumnWriter(colWriter);
+ return addColumnWriter(
+ (AbstractObjectWriter) listener.addColumn(this, column));
}
@Override
public int addColumn(MaterializedField field) {
verifyAddColumn(field.getName());
- final AbstractObjectWriter colWriter = (AbstractObjectWriter) listener.addColumn(this, field);
- return addColumnWriter(colWriter);
+ return addColumnWriter(
+ (AbstractObjectWriter) listener.addColumn(this, field));
}
private void verifyAddColumn(String colName) {
@@ -262,7 +262,7 @@ public abstract class AbstractTupleWriter implements TupleWriter, WriterEvents {
if (tupleSchema().column(colName) != null) {
throw UserException
.validationError()
- .message("Duplicate column name: ", colName)
+ .message("Duplicate column name: %s", colName)
.build(logger);
}
}
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ObjectArrayWriter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ObjectArrayWriter.java
index e2a1fc3..a1f724c 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ObjectArrayWriter.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/accessor/writer/ObjectArrayWriter.java
@@ -104,7 +104,6 @@ import org.apache.drill.exec.vector.accessor.writer.AbstractArrayWriter.BaseArra
* vector can never be in the Behind state, when we have an array of
* maps then each vector can be in any of the scalar writer states.
*/
-
public class ObjectArrayWriter extends BaseArrayWriter {
protected ObjectArrayWriter(ColumnMetadata schema,
@@ -123,7 +122,6 @@ public class ObjectArrayWriter extends BaseArrayWriter {
public void setObject(Object array) {
// Null array = 0-length array
-
if (array == null) {
return;
}
diff --git a/logical/src/main/java/org/apache/drill/common/FunctionNames.java b/logical/src/main/java/org/apache/drill/common/FunctionNames.java
new file mode 100644
index 0000000..8db4ecd
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/FunctionNames.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common;
+
+public interface FunctionNames {
+
+ String NEGATE = "negative";
+ String ADD = "add";
+ String SUBTRACT = "subtract";
+ String MULTIPLY = "multiply";
+ String DIVIDE = "divide";
+ String MODULO = "modulo";
+
+ String NOT = "not";
+ String OR = "booleanOr";
+ String AND = "booleanAnd";
+ String XOR = "xor";
+
+ String COMPARE_TO_NULLS_HIGH = "compare_to_nulls_high";
+ String COMPARE_TO_NULLS_LOW = "compare_to_nulls_low";
+
+ String EQ = "equal";
+ String NE = "not_equal";
+ String GT = "greater_than";
+ String GE = "greater_than_or_equal_to";
+ String LT = "less_than";
+ String LE = "less_than_or_equal_to";
+
+ String IS_NULL = "isnull";
+ String IS_NOT_NULL = "isnotnull";
+ String IS_TRUE = "istrue";
+ String IS_NOT_TRUE = "isnottrue";
+ String IS_FALSE = "isfalse";
+ String IS_NOT_FALSE = "isnotfalse";
+
+ String CONCAT = "concatOperator";
+ String SIMILAR_TO = "similar_to";
+ String LIKE = "like";
+}
diff --git a/logical/src/main/java/org/apache/drill/common/expression/BooleanOperator.java b/logical/src/main/java/org/apache/drill/common/expression/BooleanOperator.java
index 831821e..81fd29c 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/BooleanOperator.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/BooleanOperator.java
@@ -24,20 +24,20 @@ import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.Types;
-public class BooleanOperator extends FunctionCall{
+public class BooleanOperator extends FunctionCall {
public BooleanOperator(String name, List<LogicalExpression> args, ExpressionPosition pos) {
super(name, args, pos);
}
@Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E{
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
return visitor.visitBooleanOperator(this, value);
}
@Override
public MajorType getMajorType() {
- // If any of argumgnet of a boolean "and"/"or" is nullable, the result is nullable bit.
+ // If any of argument of a boolean "and"/"or" is nullable, the result is nullable bit.
// Otherwise, it's non-nullable bit.
for (LogicalExpression e : args) {
if (e.getMajorType().getMode() == DataMode.OPTIONAL) {
@@ -45,7 +45,6 @@ public class BooleanOperator extends FunctionCall{
}
}
return Types.REQUIRED_BIT;
-
}
@Override
@@ -61,5 +60,4 @@ public class BooleanOperator extends FunctionCall{
return cost / i;
}
-
}
diff --git a/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java b/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
index 303639d..de345ee 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
@@ -18,6 +18,7 @@
package org.apache.drill.common.expression;
import java.math.BigDecimal;
+import java.util.List;
import org.apache.drill.common.expression.IfExpression.IfCondition;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
@@ -43,7 +44,6 @@ import org.joda.time.Period;
import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList;
public class ExpressionStringBuilder extends AbstractExprVisitor<Void, StringBuilder, RuntimeException>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpressionStringBuilder.class);
static final ExpressionStringBuilder INSTANCE = new ExpressionStringBuilder();
@@ -67,7 +67,7 @@ public class ExpressionStringBuilder extends AbstractExprVisitor<Void, StringBui
@Override
public Void visitFunctionCall(FunctionCall call, StringBuilder sb) throws RuntimeException {
- ImmutableList<LogicalExpression> args = call.args;
+ List<LogicalExpression> args = call.args();
sb.append(call.getName());
sb.append("(");
for (int i = 0; i < args.size(); i++) {
diff --git a/logical/src/main/java/org/apache/drill/common/expression/FunctionCall.java b/logical/src/main/java/org/apache/drill/common/expression/FunctionCall.java
index b8df465..6be3fed 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/FunctionCall.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/FunctionCall.java
@@ -27,8 +27,12 @@ import org.apache.drill.shaded.guava.com.google.common.collect.ImmutableList;
import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
public class FunctionCall extends LogicalExpressionBase implements Iterable<LogicalExpression> {
+
+ // See FunctionNames for a list of well-known built-in functions
+ // often used in filter push-down rules
+
private final String name;
- public final ImmutableList<LogicalExpression> args;
+ protected final ImmutableList<LogicalExpression> args;
private final ExpressionPosition pos;
public FunctionCall(String name, List<LogicalExpression> args, ExpressionPosition pos) {
@@ -46,17 +50,17 @@ public class FunctionCall extends LogicalExpressionBase implements Iterable<Logi
this.pos = pos;
}
- public String getName() {
- return name;
- }
+ public String getName() { return name; }
@Override
- public ExpressionPosition getPosition() {
- return pos;
- }
+ public ExpressionPosition getPosition() { return pos; }
+
+ public List<LogicalExpression> args() { return args; }
+ public int argCount() { return args.size(); }
+ public LogicalExpression arg(int i) { return args.get(i); }
@Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E{
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
return visitor.visitFunctionCall(this, value);
}
@@ -77,5 +81,4 @@ public class FunctionCall extends LogicalExpressionBase implements Iterable<Logi
return "FunctionCall [func=" + name + ", args="
+ (args != null ? args.subList(0, Math.min(args.size(), maxLen)) : null) + ", pos=" + pos + "]";
}
-
}
diff --git a/logical/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java b/logical/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java
index a2c70f1..e0aa4b0 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java
@@ -21,6 +21,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.drill.common.FunctionNames;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.types.TypeProtos.MajorType;
@@ -30,32 +31,32 @@ import org.apache.drill.shaded.guava.com.google.common.collect.Lists;
public class FunctionCallFactory {
private static final Map<String, String> OP_TO_FUNC_NAME = ImmutableMap.<String, String>builder()
- .put("+", "add")
- .put("-", "subtract")
- .put("/", "divide")
- .put("*", "multiply")
- .put("%", "modulo")
- .put("^", "xor")
- .put("||", "concatOperator")
- .put("or", "booleanOr")
- .put("and", "booleanAnd")
- .put(">", "greater_than")
- .put("<", "less_than")
- .put("==", "equal")
- .put("=", "equal")
- .put("!=", "not_equal")
- .put("<>", "not_equal")
- .put(">=", "greater_than_or_equal_to")
- .put("<=", "less_than_or_equal_to")
- .put("is null", "isnull")
- .put("is not null", "isnotnull")
- .put("is true", "istrue")
- .put("is not true", "isnottrue")
- .put("is false", "isfalse")
- .put("is not false", "isnotfalse")
- .put("similar to", "similar_to")
- .put("!", "not")
- .put("u-", "negative")
+ .put("+", FunctionNames.ADD)
+ .put("-", FunctionNames.SUBTRACT)
+ .put("/", FunctionNames.DIVIDE)
+ .put("*", FunctionNames.MULTIPLY)
+ .put("%", FunctionNames.MODULO)
+ .put("^", FunctionNames.XOR)
+ .put("||", FunctionNames.CONCAT)
+ .put("or", FunctionNames.OR)
+ .put("and", FunctionNames.AND)
+ .put(">", FunctionNames.GT)
+ .put("<", FunctionNames.LT)
+ .put("==", FunctionNames.EQ)
+ .put("=", FunctionNames.EQ)
+ .put("!=", FunctionNames.NE)
+ .put("<>", FunctionNames.NE)
+ .put(">=", FunctionNames.GE)
+ .put("<=", FunctionNames.LE)
+ .put("is null", FunctionNames.IS_NULL)
+ .put("is not null", FunctionNames.IS_NOT_NULL)
+ .put("is true", FunctionNames.IS_TRUE)
+ .put("is not true", FunctionNames.IS_NOT_TRUE)
+ .put("is false", FunctionNames.IS_FALSE)
+ .put("is not false", FunctionNames.IS_NOT_FALSE)
+ .put("similar to", FunctionNames.SIMILAR_TO)
+ .put("!", FunctionNames.NOT)
+ .put("u-", FunctionNames.NEGATE)
.build();
public static String convertToDrillFunctionName(String op) {
@@ -64,7 +65,7 @@ public class FunctionCallFactory {
public static boolean isBooleanOperator(String funcName) {
String drillFuncName = convertToDrillFunctionName(funcName);
- return drillFuncName.equals("booleanAnd") || drillFuncName.equals("booleanOr");
+ return drillFuncName.equals(FunctionNames.AND) || drillFuncName.equals(FunctionNames.OR);
}
/*
@@ -132,5 +133,4 @@ public class FunctionCallFactory {
}
return first;
}
-
}
diff --git a/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java b/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
index 8ec81ea..0b5292d 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
@@ -156,8 +156,7 @@ public class SchemaPath extends LogicalExpressionBase {
* @return un-indexed schema path
*/
public SchemaPath getUnIndexed() {
- NameSegment nameSegment = getUnIndexedNameSegment(rootSegment, null);
- return new SchemaPath(nameSegment);
+ return new SchemaPath(getUnIndexedNameSegment(rootSegment, null));
}
/**
diff --git a/logical/src/main/java/org/apache/drill/common/expression/TypedFieldExpr.java b/logical/src/main/java/org/apache/drill/common/expression/TypedFieldExpr.java
index 93c7a3c..13a5a91 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/TypedFieldExpr.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/TypedFieldExpr.java
@@ -45,16 +45,15 @@ public class TypedFieldExpr extends LogicalExpressionBase {
@Override
public TypeProtos.MajorType getMajorType() {
- return this.type;
+ return type;
}
@Override
public String toString() {
- return this.path.getRootSegment().getPath() + "(" + type.getMinorType() + "_" + type.getMode() +")";
+ return path.getRootSegment().getPath() + "(" + type.getMinorType() + "_" + type.getMode() +")";
}
public SchemaPath getPath() {
- return this.path;
+ return path;
}
-
}
diff --git a/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java b/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
index 012aff4..0d4587c 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
@@ -63,12 +63,12 @@ public class ValueExpressions {
}
public static LogicalExpression getTime(GregorianCalendar time) {
- int millis = time.get(GregorianCalendar.HOUR_OF_DAY) * 60 * 60 * 1000 +
- time.get(GregorianCalendar.MINUTE) * 60 * 1000 +
- time.get(GregorianCalendar.SECOND) * 1000 +
- time.get(GregorianCalendar.MILLISECOND);
+ int millis = time.get(GregorianCalendar.HOUR_OF_DAY) * 60 * 60 * 1000 +
+ time.get(GregorianCalendar.MINUTE) * 60 * 1000 +
+ time.get(GregorianCalendar.SECOND) * 1000 +
+ time.get(GregorianCalendar.MILLISECOND);
- return new TimeExpression(millis);
+ return new TimeExpression(millis);
}
public static LogicalExpression getTime(int milliSeconds) {
@@ -98,28 +98,22 @@ public class ValueExpressions {
public static LogicalExpression getNumericExpression(String sign, String s, ExpressionPosition ep) {
String numStr = (sign == null) ? s : sign+s;
try {
- int a = Integer.parseInt(numStr);
- return new IntExpression(a, ep);
- } catch (Exception e) {
+ int a = Integer.parseInt(numStr);
+ return new IntExpression(a, ep);
+ } catch (Exception e) { }
- }
try {
long l = Long.parseLong(numStr);
return new LongExpression(l, ep);
- } catch (Exception e) {
-
- }
+ } catch (Exception e) { }
try {
double d = Double.parseDouble(numStr);
return new DoubleExpression(d, ep);
- } catch (Exception e) {
-
- }
+ } catch (Exception e) { }
throw new IllegalArgumentException(String.format("Unable to parse string %s as integer or floating point number.",
numStr));
-
}
public static LogicalExpression getParameterExpression(String name, MajorType type) {
@@ -151,7 +145,6 @@ public class ValueExpressions {
super(value, pos);
}
-
@Override
protected Boolean parseValue(String s) {
return Boolean.parseBoolean(s);
@@ -170,11 +163,10 @@ public class ValueExpressions {
public boolean getBoolean() {
return value;
}
-
}
public static class FloatExpression extends LogicalExpressionBase {
- private float f;
+ private final float f;
private static final MajorType FLOAT_CONSTANT = Types.required(MinorType.FLOAT4);
@@ -201,14 +193,13 @@ public class ValueExpressions {
public Iterator<LogicalExpression> iterator() {
return Collections.emptyIterator();
}
-
}
public static class IntExpression extends LogicalExpressionBase {
private static final MajorType INT_CONSTANT = Types.required(MinorType.INT);
- private int i;
+ private final int i;
public IntExpression(int i, ExpressionPosition pos) {
super(pos);
@@ -233,14 +224,13 @@ public class ValueExpressions {
public Iterator<LogicalExpression> iterator() {
return Collections.emptyIterator();
}
-
}
public static class Decimal9Expression extends LogicalExpressionBase {
- private int decimal;
- private int scale;
- private int precision;
+ private final int decimal;
+ private final int scale;
+ private final int precision;
public Decimal9Expression(BigDecimal input, ExpressionPosition pos) {
super(pos);
@@ -249,7 +239,6 @@ public class ValueExpressions {
this.decimal = input.setScale(scale, BigDecimal.ROUND_HALF_UP).intValue();
}
-
public int getIntFromDecimal() {
return decimal;
}
@@ -280,9 +269,9 @@ public class ValueExpressions {
public static class Decimal18Expression extends LogicalExpressionBase {
- private long decimal;
- private int scale;
- private int precision;
+ private final long decimal;
+ private final int scale;
+ private final int precision;
public Decimal18Expression(BigDecimal input, ExpressionPosition pos) {
super(pos);
@@ -291,7 +280,6 @@ public class ValueExpressions {
this.decimal = input.setScale(scale, BigDecimal.ROUND_HALF_UP).longValue();
}
-
public long getLongFromDecimal() {
return decimal;
}
@@ -318,19 +306,17 @@ public class ValueExpressions {
public Iterator<LogicalExpression> iterator() {
return Collections.emptyIterator();
}
-
}
public static class Decimal28Expression extends LogicalExpressionBase {
- private BigDecimal bigDecimal;
+ private final BigDecimal bigDecimal;
public Decimal28Expression(BigDecimal input, ExpressionPosition pos) {
super(pos);
this.bigDecimal = input;
}
-
public BigDecimal getBigDecimal() {
return bigDecimal;
}
@@ -349,12 +335,11 @@ public class ValueExpressions {
public Iterator<LogicalExpression> iterator() {
return Collections.emptyIterator();
}
-
}
public static class Decimal38Expression extends LogicalExpressionBase {
- private BigDecimal bigDecimal;
+ private final BigDecimal bigDecimal;
public Decimal38Expression(BigDecimal input, ExpressionPosition pos) {
super(pos);
@@ -379,7 +364,6 @@ public class ValueExpressions {
public Iterator<LogicalExpression> iterator() {
return Collections.emptyIterator();
}
-
}
public static class VarDecimalExpression extends LogicalExpressionBase {
@@ -422,7 +406,7 @@ public class ValueExpressions {
}
public static class DoubleExpression extends LogicalExpressionBase {
- private double d;
+ private final double d;
private static final MajorType DOUBLE_CONSTANT = Types.required(MinorType.FLOAT8);
@@ -449,20 +433,19 @@ public class ValueExpressions {
public Iterator<LogicalExpression> iterator() {
return Collections.emptyIterator();
}
-
}
public static class LongExpression extends LogicalExpressionBase {
private static final MajorType LONG_CONSTANT = Types.required(MinorType.BIGINT);
- private long l;
+ private final long l;
public LongExpression(long l) {
this(l, ExpressionPosition.UNKNOWN);
}
- public LongExpression(long l, ExpressionPosition pos) {
+ public LongExpression(long l, ExpressionPosition pos) {
super(pos);
this.l = l;
}
@@ -485,21 +468,19 @@ public class ValueExpressions {
public Iterator<LogicalExpression> iterator() {
return Collections.emptyIterator();
}
-
}
-
public static class DateExpression extends LogicalExpressionBase {
private static final MajorType DATE_CONSTANT = Types.required(MinorType.DATE);
- private long dateInMillis;
+ private final long dateInMillis;
public DateExpression(long l) {
this(l, ExpressionPosition.UNKNOWN);
}
- public DateExpression(long dateInMillis, ExpressionPosition pos) {
+ public DateExpression(long dateInMillis, ExpressionPosition pos) {
super(pos);
this.dateInMillis = dateInMillis;
}
@@ -522,21 +503,19 @@ public class ValueExpressions {
public Iterator<LogicalExpression> iterator() {
return Collections.emptyIterator();
}
-
}
-
public static class TimeExpression extends LogicalExpressionBase {
private static final MajorType TIME_CONSTANT = Types.required(MinorType.TIME);
- private int timeInMillis;
+ private final int timeInMillis;
public TimeExpression(int timeInMillis) {
this(timeInMillis, ExpressionPosition.UNKNOWN);
}
- public TimeExpression(int timeInMillis, ExpressionPosition pos) {
+ public TimeExpression(int timeInMillis, ExpressionPosition pos) {
super(pos);
this.timeInMillis = timeInMillis;
}
@@ -559,20 +538,19 @@ public class ValueExpressions {
public Iterator<LogicalExpression> iterator() {
return Collections.emptyIterator();
}
-
}
public static class TimeStampExpression extends LogicalExpressionBase {
private static final MajorType TIMESTAMP_CONSTANT = Types.required(MinorType.TIMESTAMP);
- private long timeInMillis;
+ private final long timeInMillis;
public TimeStampExpression(long timeInMillis) {
this(timeInMillis, ExpressionPosition.UNKNOWN);
}
- public TimeStampExpression(long timeInMillis, ExpressionPosition pos) {
+ public TimeStampExpression(long timeInMillis, ExpressionPosition pos) {
super(pos);
this.timeInMillis = timeInMillis;
}
@@ -595,20 +573,19 @@ public class ValueExpressions {
public Iterator<LogicalExpression> iterator() {
return Collections.emptyIterator();
}
-
}
public static class IntervalYearExpression extends LogicalExpressionBase {
private static final MajorType INTERVALYEAR_CONSTANT = Types.required(MinorType.INTERVALYEAR);
- private int months;
+ private final int months;
public IntervalYearExpression(int months) {
this(months, ExpressionPosition.UNKNOWN);
}
- public IntervalYearExpression(int months, ExpressionPosition pos) {
+ public IntervalYearExpression(int months, ExpressionPosition pos) {
super(pos);
this.months = months;
}
@@ -631,7 +608,6 @@ public class ValueExpressions {
public Iterator<LogicalExpression> iterator() {
return Collections.emptyIterator();
}
-
}
public static class IntervalDayExpression extends LogicalExpressionBase {
@@ -639,14 +615,14 @@ public class ValueExpressions {
private static final MajorType INTERVALDAY_CONSTANT = Types.required(MinorType.INTERVALDAY);
private static final long MILLIS_IN_DAY = 1000 * 60 * 60 * 24;
- private int days;
- private int millis;
+ private final int days;
+ private final int millis;
public IntervalDayExpression(long intervalInMillis) {
this((int) (intervalInMillis / MILLIS_IN_DAY), (int) (intervalInMillis % MILLIS_IN_DAY), ExpressionPosition.UNKNOWN);
}
- public IntervalDayExpression(int days, int millis, ExpressionPosition pos) {
+ public IntervalDayExpression(int days, int millis, ExpressionPosition pos) {
super(pos);
this.days = days;
this.millis = millis;
@@ -674,7 +650,6 @@ public class ValueExpressions {
public Iterator<LogicalExpression> iterator() {
return Collections.emptyIterator();
}
-
}
public static class QuotedString extends ValueExpression<String> {
@@ -709,7 +684,7 @@ public class ValueExpressions {
}
/**
- * Is used to identify method parameter based on given name and type.
+ * Identifies method parameter based on given name and type.
*/
public static class ParameterExpression extends LogicalExpressionBase {
@@ -741,5 +716,4 @@ public class ValueExpressions {
return ImmutableList.<LogicalExpression>of().iterator();
}
}
-
}
diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
index ac46e42..23c1f40 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
@@ -87,7 +87,7 @@ public final class AggregateChecker implements ExprVisitor<Boolean, ErrorCollect
@Override
public Boolean visitBooleanOperator(BooleanOperator op, ErrorCollector errors) {
- for (LogicalExpression arg : op.args) {
+ for (LogicalExpression arg : op.args()) {
if (arg.accept(this, errors)) {
return true;
}
@@ -133,6 +133,7 @@ public final class AggregateChecker implements ExprVisitor<Boolean, ErrorCollect
public Boolean visitBooleanConstant(BooleanExpression e, ErrorCollector errors) {
return false;
}
+ @Override
public Boolean visitDecimal9Constant(Decimal9Expression decExpr, ErrorCollector errors) {
return false;
}
diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/ConditionalExprOptimizer.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/ConditionalExprOptimizer.java
index 330adf4..eb2177b 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/visitors/ConditionalExprOptimizer.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/ConditionalExprOptimizer.java
@@ -40,7 +40,7 @@ public class ConditionalExprOptimizer extends AbstractExprVisitor<LogicalExpress
@Override
public LogicalExpression visitBooleanOperator(BooleanOperator op, Void value) throws RuntimeException {
List<LogicalExpression> newArgs = Lists.newArrayList();
- newArgs.addAll(op.args);
+ newArgs.addAll(op.args());
Collections.sort(newArgs, costComparator);
return new BooleanOperator(op.getName(), newArgs, op.getPosition());
diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
index a7648ac..08c05bf 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
@@ -84,7 +84,7 @@ final class ConstantChecker implements ExprVisitor<Boolean, ErrorCollector, Runt
@Override
public Boolean visitBooleanOperator(BooleanOperator op, ErrorCollector errors) {
- for (LogicalExpression e : op.args) {
+ for (LogicalExpression e : op.args()) {
if (!e.accept(this, errors)) {
return false;
}
diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java
index df72a34..786c0ff 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java
@@ -51,7 +51,6 @@ import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
public class ExpressionValidator implements ExprVisitor<Void, ErrorCollector, RuntimeException> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpressionValidator.class);
@Override
public Void visitFunctionCall(FunctionCall call, ErrorCollector errors) throws RuntimeException {
@@ -77,7 +76,7 @@ public class ExpressionValidator implements ExprVisitor<Void, ErrorCollector, Ru
@Override
public Void visitBooleanOperator(BooleanOperator op, ErrorCollector errors) throws RuntimeException {
int i = 0;
- for (LogicalExpression arg : op.args) {
+ for (LogicalExpression arg : op.args()) {
if ( arg.getMajorType().getMinorType() != MinorType.BIT) {
errors
.addGeneralError(
diff --git a/logical/src/main/java/org/apache/drill/common/parser/LogicalExpressionParser.java b/logical/src/main/java/org/apache/drill/common/parser/LogicalExpressionParser.java
index ad30a20..e69ec11 100644
--- a/logical/src/main/java/org/apache/drill/common/parser/LogicalExpressionParser.java
+++ b/logical/src/main/java/org/apache/drill/common/parser/LogicalExpressionParser.java
@@ -22,13 +22,14 @@ import org.antlr.v4.runtime.CommonTokenStream;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.parser.ExprLexer;
import org.apache.drill.common.expression.parser.ExprParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Helper class for parsing logical expression.
*/
public class LogicalExpressionParser {
-
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(LogicalExpressionParser.class);
+ private static final Logger logger = LoggerFactory.getLogger(LogicalExpressionParser.class);
/**
* Initializes logical expression lexer and parser, add error listener that converts all
@@ -51,5 +52,4 @@ public class LogicalExpressionParser {
logger.trace("Tokens: [{}]. Parsing details: [{}].", tokens.getText(), parseContext.toInfoString(parser));
return parseContext.e;
}
-
}
[drill] 02/02: DRILL-7631: Updates to the Json Structure Parser
Posted by vo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
volodymyr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
commit cc51f2458c49ab786634a0a739652a4b7d454b44
Author: Paul Rogers <pa...@yahoo.com>
AuthorDate: Mon Mar 9 11:34:46 2020 -0700
DRILL-7631: Updates to the Json Structure Parser
Revised how "look-ahead" works. Added support for unknown
types.
closes #2016
---
.../easy/json/parser/AbstractElementParser.java | 3 +
.../exec/store/easy/json/parser/ArrayListener.java | 89 ++-----
.../exec/store/easy/json/parser/ArrayParser.java | 63 ++++-
.../store/easy/json/parser/DummyValueParser.java | 2 -
.../easy/json/parser/JsonStructureParser.java | 12 +
.../exec/store/easy/json/parser/JsonType.java | 25 --
.../store/easy/json/parser/JsonValueParser.java | 116 ++++++++
.../store/easy/json/parser/ObjectListener.java | 110 ++++----
.../exec/store/easy/json/parser/ObjectParser.java | 60 ++++-
.../exec/store/easy/json/parser/TokenIterator.java | 16 ++
.../exec/store/easy/json/parser/ValueDef.java | 85 ++++++
.../store/easy/json/parser/ValueDefFactory.java | 99 +++++++
.../exec/store/easy/json/parser/ValueFactory.java | 222 ----------------
.../exec/store/easy/json/parser/ValueListener.java | 37 +--
.../exec/store/easy/json/parser/ValueParser.java | 55 ++--
.../{ => easy}/json/parser/BaseTestJsonParser.java | 128 ++++-----
.../json/parser/TestJsonParserArrays.java | 86 +++---
.../json/parser/TestJsonParserBasics.java | 92 +++++--
.../json/parser/TestJsonParserErrors.java | 5 +-
.../json/parser/TestJsonParserObjects.java | 9 +-
.../easy/json/parser/TestJsonParserUnknowns.java | 294 +++++++++++++++++++++
21 files changed, 1051 insertions(+), 557 deletions(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/AbstractElementParser.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/AbstractElementParser.java
index ed811d5..a99dcec 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/AbstractElementParser.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/AbstractElementParser.java
@@ -18,6 +18,9 @@
package org.apache.drill.exec.store.easy.json.parser;
+/**
+ * Abstract base class for all JSON element parsers.
+ */
public abstract class AbstractElementParser implements ElementParser {
final JsonStructureParser structParser;
private final ElementParser parent;
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ArrayListener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ArrayListener.java
index c5d588d..06a67aa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ArrayListener.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ArrayListener.java
@@ -18,8 +18,8 @@
package org.apache.drill.exec.store.easy.json.parser;
/**
- * Represents one level within an array. The first time the parser sees
- * the array element, it will call one of the "Element" methods with the
+ * Represents one level within array. The first time the parser sees the array element,
+ * it will call the {@link #element(ValueDef)} method with the
* look-ahead values visible to the parser. Since JSON is flexible, later
* data shapes may not necessarily follow the first shape. The implementation
* must handle this or throw an error if not supported.
@@ -44,15 +44,16 @@ package org.apache.drill.exec.store.easy.json.parser;
* Three JSON-specific cases warrant attention:
* <ol>
* <li>The first occurrence of the array is empty: {@code [ ]}. In this case,
- * the structure parser will ask for an element child by providing the
- * {@link JsonType#EMPTY} type, which is not very useful, but is all that
- * the parser knows. The listener is responsible for implementing some kind of
- * "deferred type" logic to wait and see what kind of element appears
- * later..</li>
+ * the structure parser will defer asking for an element parser (and listener)
+ * until an actual value appears. The array listener is responsible for
+ * implementing some kind of "deferred type" logic to wait and see what
+ * kind of element appears later.</li>
* <li>The first occurrence of the array has, as its first element, a
* {@code null} value. The structure parser will ask this listener to create
* an array child for the {@code null} value, but the listener has no type
- * information. Again, the listener is responsible for type-deferal.</li>
+ * information. Since null values must be recorded (so we know how many
+ * appear in each array), the listener is forced to choose a type. Choose
+ * wisely as there is no way to know what type will appear in the future.</li>
* <li>A generalized form of the above is that the structure parser only
* knows what it sees on the first element when it asks for an element
* child. In a well-formed file, that first token will predict the type
@@ -81,8 +82,19 @@ package org.apache.drill.exec.store.easy.json.parser;
public interface ArrayListener {
/**
- * Called at the start of a set of values for an array. That is, called
- * when the structure parser accepts the {@code [} token.
+ * Provide an element listener for the first non-empty value
+ * seen for the array.
+ *
+ * @param valueDef description of the element (without the array
+ * dimensions)
+ * @return a listener to consume values of the array element
+ */
+ ValueListener element(ValueDef valueDef);
+
+ /**
+ * Called at the entrance to each level (dimension) of an array.
+ * That is, called when the structure parser accepts the {@code [}
+ * token.
*/
void onStart();
@@ -91,61 +103,16 @@ public interface ArrayListener {
* by its own listener which receives the value of the element (if
* scalar) or element events (if structured.)
*/
- void onElement();
-
- /**
- * Called at the end of a set of values for an array. That is, called
- * when the structure parser accepts the {@code ]} token.
- */
- void onEnd();
+ void onElementStart();
/**
- * The first element seen is a scalar, {@code null} or empty. That is,
- * {@code [ <scalar>}, {@code [ null} or {@code [ ]}.
- *
- * @param type the JSON type of the object as given by the token
- * which the Jackson parser returned for the value. The type can
- * be {@code null}, which means that the parser does not know what
- * actual type might occur later
- * @return a value listener for the scalar type, or if {@code null},
- * perhaps waiting for more information to commit to a type
+ * Called after each element of the array.
*/
- ValueListener scalarElement(JsonType type);
+ void onElementEnd();
/**
- * The first element an array or scalars (or {@code null}.That is,
- * {@code [ [+ <scalar>}.
- *
- * @param arrayDims the number of dimensions observed during the
- * first-element parse, not including the surrounding array
- * itself. As in all cases, there is no guarantee that
- * that this number will remain valid later, and may be wrong if the
- * first-seen element was empty: {@code []}.
- * @return a listener for the value of the top-level element (which
- * the listener can assume will turn out to be an array.)
- */
- ValueListener arrayElement(int arrayDims, JsonType type);
-
- /**
- * The first element seen for an array is an object. That is,
- * <code>[ {</code>.
- *
- * @return a listener for the value of the top-level element (which
- * the listener can assume will turn out to be an object.)
- */
- ValueListener objectElement();
-
- /**
- * The first element seen is an object array.That is,
- * <code>[ [* {</code>.
- *
- * @param arrayDims the number of dimensions observed during the
- * first-element parse, not including the surrounding array
- * itself. As in all cases, there is no guarantee that
- * that this number will remain valid later, and may be wrong if the
- * first-seen element was empty: {@code []}.
- * @return a listener for the value of the top-level element (which
- * the listener can assume will turn out to be an array.)
+ * Called at the end of a set of values for an array. That is, called
+ * when the structure parser accepts the {@code ]} token.
*/
- ValueListener objectArrayElement(int arrayDims);
+ void onEnd();
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ArrayParser.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ArrayParser.java
index 2d4431a..963b8c5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ArrayParser.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ArrayParser.java
@@ -17,6 +17,8 @@
*/
package org.apache.drill.exec.store.easy.json.parser;
+import org.apache.drill.exec.store.easy.json.parser.ObjectListener.FieldType;
+
import com.fasterxml.jackson.core.JsonToken;
/**
@@ -24,6 +26,10 @@ import com.fasterxml.jackson.core.JsonToken;
* represented by a {@code ValueListener}. There is a single listener
* for all the elements, which are presumed to be of the same type.
* <p>
+ * The element is created when first encountered, either as part of field
+ * creation (<code>{a: [10]}</code>) or when later encountered in parsing
+ * (<code{a: []} {a: [10]}</code>).
+ * <p>
* This parser <i>does not</i> attempt to parse an array as a poor-man's
* tuple: {@code [ 101, "fred", 23.45 ]}. The listener could handle this
* case. But, if we need to handle such a case, it would be better to
@@ -32,13 +38,12 @@ import com.fasterxml.jackson.core.JsonToken;
*/
public class ArrayParser extends AbstractElementParser {
- private final ArrayListener arrayListener;
- private final ValueParser elementParser;
+ private ValueParser elementParser;
+ private ArrayListener arrayListener;
- public ArrayParser(ValueParser parent, ArrayListener arrayListener, ValueListener elementListener) {
+ public ArrayParser(ValueParser parent, ArrayListener arrayListener) {
super(parent);
this.arrayListener = arrayListener;
- this.elementParser = new ValueParser(this, "[]", elementListener);
}
public ValueParser elementParser() { return elementParser; }
@@ -51,18 +56,58 @@ public class ArrayParser extends AbstractElementParser {
arrayListener.onStart();
top: for (;;) {
// Position: [ (value, )* ^ ?
- JsonToken token = tokenizer.requireNext();
+ JsonToken token = tokenizer.requireNext();
switch (token) {
case END_ARRAY:
break top;
-
default:
tokenizer.unget(token);
- arrayListener.onElement();
- elementParser.parse(tokenizer);
- break;
+ parseElement(tokenizer);
}
}
arrayListener.onEnd();
}
+
+ private void parseElement(TokenIterator tokenizer) {
+ if (elementParser == null) {
+ detectElement(tokenizer);
+ }
+ arrayListener.onElementStart();
+ elementParser.parse(tokenizer);
+ arrayListener.onElementEnd();
+ }
+
+ private void detectElement(TokenIterator tokenizer) {
+ addElement(ValueDefFactory.lookAhead(tokenizer));
+ }
+
+ public void addElement(ValueDef valueDef) {
+ bindElement(arrayListener.element(valueDef));
+ }
+
+ public void bindElement(ValueListener elementListener) {
+ elementParser = new ValueParser(this, "[]", FieldType.TYPED);
+ elementParser.bindListener(elementListener);
+ }
+
+ public void bindListener(ArrayListener newListener) {
+ arrayListener = newListener;
+ if (elementParser != null) {
+ elementParser.bindListener(arrayListener.element(ValueDef.UNKNOWN));
+ }
+ }
+
+ /**
+ * Expand the structure of this array given a description of the
+ * look-ahead value. Skip if this is a 1D array of unknown type.
+ * If 2D or greater, then we must create the child array of one
+ * less dimension.
+ */
+ public void expandStructure(ValueDef valueDef) {
+ if (valueDef.dimensions() > 1 || !valueDef.type().isUnknown()) {
+ ValueDef elementDef = new ValueDef(valueDef.type(), valueDef.dimensions() - 1);
+ addElement(elementDef);
+ elementParser.expandStructure(elementDef);
+ }
+ }
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/DummyValueParser.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/DummyValueParser.java
index 86b345c..7d5131b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/DummyValueParser.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/DummyValueParser.java
@@ -55,14 +55,12 @@ class DummyValueParser extends AbstractElementParser {
public void parseTail(TokenIterator tokenizer) {
// Parse (field: value)* }
-
for (;;) {
JsonToken token = tokenizer.requireNext();
switch (token) {
// Not exactly precise, but the JSON parser handles the
// details.
-
case END_OBJECT:
case END_ARRAY:
return;
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/JsonStructureParser.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/JsonStructureParser.java
index 23693ca..14016a8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/JsonStructureParser.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/JsonStructureParser.java
@@ -204,6 +204,18 @@ public class JsonStructureParser {
public int recoverableErrorCount() { return errorRecoveryCount; }
+ public int lineNumber() {
+ return tokenizer.lineNumber();
+ }
+
+ public int columnNumber() {
+ return tokenizer.columnNumber();
+ }
+
+ public String token() {
+ return tokenizer.token();
+ }
+
public void close() {
if (errorRecoveryCount > 0) {
logger.warn("Read JSON input with {} recoverable error(s).",
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/JsonType.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/JsonType.java
deleted file mode 100644
index 9481a4a..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/JsonType.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.easy.json.parser;
-
-/**
- * Description of JSON types as derived from JSON tokens.
- */
-public enum JsonType {
- ARRAY, OBJECT, NULL, EMPTY, BOOLEAN, INTEGER, FLOAT, STRING
-}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/JsonValueParser.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/JsonValueParser.java
new file mode 100644
index 0000000..15fc128
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/JsonValueParser.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.easy.json.parser;
+
+import com.fasterxml.jackson.core.JsonToken;
+
+/**
+ * Parses an arbitrary JSON value (which can be a subtree of any
+ * complexity) into a JSON string. That is, converts the parsed
+ * JSON tokens back into the original JSON text.
+ */
+public class JsonValueParser extends AbstractElementParser {
+
+ private final ValueListener listener;
+ private final StringBuilder json = new StringBuilder();
+
+ protected JsonValueParser(ElementParser parent, String key,
+ ValueListener listener) {
+ super(parent);
+ this.listener = listener;
+ }
+
+ @Override
+ public void parse(TokenIterator tokenizer) {
+ JsonToken token = tokenizer.requireNext();
+ json.setLength(0);
+ parseValue(tokenizer, token);
+ listener.onString(json.toString());
+ json.setLength(0);
+ }
+
+ private void parseValue(TokenIterator tokenizer, JsonToken token) {
+ String textValue = tokenizer.textValue();
+ switch (token) {
+ case START_ARRAY:
+ json.append(textValue);
+ parseArrayTail(tokenizer);
+ break;
+
+ case START_OBJECT:
+ json.append(textValue);
+ parseObjectTail(tokenizer);
+ break;
+
+ case VALUE_STRING:
+ json.append("\"");
+ json.append(textValue);
+ json.append("\"");
+ break;
+
+ default:
+ json.append(textValue);
+ break;
+ }
+ }
+
+ public void parseArrayTail(TokenIterator tokenizer) {
+
+ // Accept value* ]
+
+ boolean first = true;
+ for (;;) {
+ JsonToken token = tokenizer.requireNext();
+ if (token == JsonToken.END_ARRAY) {
+ json.append(tokenizer.textValue());
+ return;
+ }
+ if (! first) {
+ json.append(", ");
+ }
+ first = false;
+ parseValue(tokenizer, token);
+ }
+ }
+
+ public void parseObjectTail(TokenIterator tokenizer) {
+
+ // Accept (field: value)* }
+
+ boolean first = true;
+ for (;;) {
+ JsonToken token = tokenizer.requireNext();
+ if (token == JsonToken.END_OBJECT) {
+ json.append(tokenizer.textValue());
+ return;
+ }
+ if (! first) {
+ json.append(", ");
+ }
+ first = false;
+ if (token != JsonToken.FIELD_NAME) {
+ throw errorFactory().syntaxError(token);
+ }
+
+ json.append("\"");
+ json.append(tokenizer.textValue());
+ json.append("\": ");
+ parseValue(tokenizer, tokenizer.requireNext());
+ }
+ }
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ObjectListener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ObjectListener.java
index ec40b0e..40b8617 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ObjectListener.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ObjectListener.java
@@ -17,6 +17,8 @@
*/
package org.apache.drill.exec.store.easy.json.parser;
+import org.apache.calcite.model.JsonType;
+
/**
* Represents events on a object value. The object value may be a top-level
* field or may be the element of an array. The listener gets an event when
@@ -52,6 +54,35 @@ package org.apache.drill.exec.store.easy.json.parser;
*/
public interface ObjectListener {
+ enum FieldType {
+
+ /**
+ * The field is unprojected, ignore its content. No value listener
+ * is created.
+ */
+ IGNORE,
+
+ /**
+ * Parse the JSON object according to its type.
+ */
+ TYPED,
+
+ /**
+ * The field is to be treated as "all-text". Used when the parser-level
+ * setting for {@code allTextMode} is {@code false}; allows per-field
+ * overrides to, perhaps, ride over inconsistent scalar types for a
+ * single field. The listener will receive only strings.
+ */
+ TEXT,
+
+ /**
+ * Parse the value, and all its children, as JSON.
+ * That is, converts the parsed JSON back into a
+ * JSON string. The listener will receive only strings.
+ */
+ JSON
+ }
+
/**
* Called at the start of a set of values for an object. That is, called
* when the structure parser accepts the <code>{</code> token.
@@ -59,73 +90,46 @@ public interface ObjectListener {
void onStart();
/**
- * Called at the end of a set of values for an object. That is, called
- * when the structure parser accepts the <code>}</code> token.
- */
- void onEnd();
-
- /**
* Called by the structure parser when it first sees a new field for
- * and object to determine if that field is to be projected (is needed
- * by the listener.) If not projected, the structure parser will not
+ * and object to determine how to parse the field.
+ * If not projected, the structure parser will not
* ask for a value listener and will insert a "dummy" parser that will
* free-wheel over any value of that field. As a result, unprojected
* fields can not cause type errors: they are invisible as long as
* they are syntactically valid.
+ * <p>
+ * The {@link FieldType#JSON} type says to parse the entire field, and
+ * its children, as a JSON string. The parser will ask for a value
+ * listener to accept the JSON text.
*
* @param key the object field name
- * @return {@code true} if this listener wants to provide a listener
- * for the field, {@code false} if the field should be ignored
+ * @return how the field should be parsed
*/
- boolean isProjected(String key);
+ FieldType fieldType(String key);
/**
- * A new field has appeared with a scalar (or {@code null}) value.
- * That is: {@code key: <scalar>}.
+ * The structure parser has just encountered a new field for this
+ * object. The {@link #fieldType(String)} indicated that the field is
+ * to be projected. This method performs any setup needed to handle the
+ * field, then returns a value listener to receive events for the
+ * field value. The value listener may be asked to create additional
+ * structure, such as arrays or nested objects.
*
* @param key the field name
- * @param type the type as given by the JSON token for the value
- * @return a value listener for the scalar value
+ * @param valueDef a description of the field as inferred by looking
+ * ahead some number of tokens in the input JSON. Provides both a data
+ * type and array depth (dimensions.) If the type is
+ * {@link JsonType#NONE EMPTY}, then the field is an empty array.
+ * If the type is {@link JsonType#NULL NULL}, then the value is null. In these
+ * cases, the listener can replace itself when an actual value appears
+ * later
+ * @return a listener to receive events for the newly-created field
*/
- ValueListener addScalar(String key, JsonType type);
+ ValueListener addField(String key, ValueDef valueDef);
/**
- * A new field has appeared with a scalar, {@code null} or empty array
- * value. That is, one of:
- * <ul>
- * <li><code>key: [+ <scalar></code></li>
- * <li><code>key: [+ null</code></li>
- * <li><code>key: [+ ]</code></li>
- * </ul>
- * Where "[+" means one or more opening array elements.
- *
- * @param key the field name
- * @param arrayDims number of dimensions observed in the first appearance
- * of the array (more may appear later)
- * @param type the observed type of the first element of the array, or
- * {@link JsonType.NULL} if {@code null} was see, or
- * {@link JsonType.EMPTY} if an empty array was seen
- * @return a listener for the field itself which is prepared to
- * return an array listener
- */
- ValueListener addArray(String key, int arrayDims, JsonType type);
-
- /**
- * A new field has appeared with an object value.
- * That is: {@code key: <scalar>}.
- *
- * @param key the field name
- * @return a value listener which assumes the value is an object
- */
- ValueListener addObject(String key);
-
- /**
- * A new field has appeared with an object array value.
- * That is: <code>key: ]+ {</code>.
- *
- * @param key the field name
- * @return a value listener which assumes the value is an object
- * array
+ * Called at the end of a set of values for an object. That is, called
+ * when the structure parser accepts the <code>}</code> token.
*/
- ValueListener addObjectArray(String key, int dims);
+ void onEnd();
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ObjectParser.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ObjectParser.java
index cfc0c76..226cc29 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ObjectParser.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ObjectParser.java
@@ -20,6 +20,8 @@ package org.apache.drill.exec.store.easy.json.parser;
import java.util.Map;
import org.apache.drill.common.map.CaseInsensitiveMap;
+import org.apache.drill.exec.store.easy.json.parser.ObjectListener.FieldType;
+import org.apache.drill.exec.store.easy.json.parser.ValueDef.JsonType;
import com.fasterxml.jackson.core.JsonToken;
@@ -159,9 +161,61 @@ public class ObjectParser extends AbstractElementParser {
throw errorFactory().structureError(
"Drill does not allow empty keys in JSON key/value pairs");
}
- if (!listener.isProjected(key)) {
- return new DummyValueParser(this);
+ FieldType type = listener.fieldType(key);
+ switch (type) {
+ case IGNORE:
+ return new DummyValueParser(this);
+ case JSON:
+ return new JsonValueParser(this, key,
+ listener.addField(key, new ValueDef(JsonType.STRING, 0)));
+ default:
+ return createFieldParser(key, type, tokenizer);
}
- return ValueFactory.createFieldParser(this, key, tokenizer);
+ }
+
+ /**
+ * Parse position: <code>{ ... field : ^ ?</code> for a newly-seen field.
+ * Constructs a value parser and its listeners by looking ahead
+ * some number of tokens to "sniff" the type of the value. For
+ * example:
+ * <ul>
+ * <li>{@code foo: <value>} - Field value</li>
+ * <li>{@code foo: [ <value> ]} - 1D array value</li>
+ * <li>{@code foo: [ [<value> ] ]} - 2D array value</li>
+ * <li>Etc.</li>
+ * </ul>
+ * <p>
+ * There are two cases in which no type estimation is possible:
+ * <ul>
+ * <li>The value is {@code null}, indicated by
+ * {@link JsonType#NULL}.</code>
+ * <li>The value is an array, and the array is empty, indicated
+ * by {@link JsonType#EMPTY}.</li>
+ * </ul>
+ * {@link ValueDefFactory} handles syntactic type inference. The associated
+ * listener enforces semantic rules. For example, if a schema is
+ * available, and we know that field "x" must be an Integer, but
+ * this class reports that it is an object, then the listener should
+ * raise an exception.
+ * <p>
+ * Also, the parser cannot enforce type consistency. This method
+ * looks only at the first appearance of a value: a sample size of
+ * one. JSON allows anything.
+ * The listener must enforce semantic rules that say whether a different
+ * type is allowed for later values.
+ *
+ * @param key the name of the field
+ * @param type the kind of field parser to create
+ * @param tokenizer the token parser
+ * @return the value parser for the element, which may contain additional
+ * structure for objects or arrays
+ */
+ public ElementParser createFieldParser(String key, FieldType type,
+ TokenIterator tokenizer) {
+ ValueParser fp = new ValueParser(this, key, type);
+ ValueDef valueDef = ValueDefFactory.lookAhead(tokenizer);
+ fp.bindListener(listener.addField(key, valueDef));
+ fp.expandStructure(valueDef);
+ return fp;
}
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/TokenIterator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/TokenIterator.java
index cfb7440..ecd5b29 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/TokenIterator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/TokenIterator.java
@@ -87,6 +87,22 @@ public class TokenIterator {
.toString();
}
+ public int lineNumber() {
+ return parser.getCurrentLocation().getLineNr();
+ }
+
+ public int columnNumber() {
+ return parser.getCurrentLocation().getColumnNr();
+ }
+
+ public String token() {
+ try {
+ return parser.getText();
+ } catch (IOException e) {
+ return null;
+ }
+ }
+
public JsonToken requireNext() {
JsonToken token = next();
if (token == null) {
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ValueDef.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ValueDef.java
new file mode 100644
index 0000000..8896129
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ValueDef.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.easy.json.parser;
+
+/**
+ * Description of a JSON value as inferred from looking ahead in
+ * the JSON stream. Includes a type (which can be empty for an empty
+ * array, or null), and an array size (which is 0 for simple values.)
+ * <p>
+ * To be clear, this is the JSON parser's best guess at a field type
+ * from the input token stream. This is <i>not</i> a description of the
+ * desired data type as JSON can only react to what it sees on input.
+ */
+public class ValueDef {
+
+ /**
+ * Description of JSON types as derived from JSON tokens.
+ */
+ public enum JsonType {
+ OBJECT, NULL, BOOLEAN,
+ INTEGER, FLOAT, STRING, EMBEDDED_OBJECT,
+
+ /**
+ * Indicates an empty array.
+ */
+ EMPTY,
+
+ /**
+ * Indicates an unknown array, appears when replacing the
+ * value listener for an array.
+ */
+ UNKNOWN;
+
+ public boolean isObject() { return this == OBJECT; }
+
+ public boolean isUnknown() {
+ return this == NULL || this == EMPTY ||
+ this == UNKNOWN;
+ }
+
+ public boolean isScalar() {
+ return !isObject() && !isUnknown();
+ }
+ }
+
+ public static final ValueDef UNKNOWN_ARRAY = new ValueDef(JsonType.UNKNOWN, 1);
+ public static final ValueDef UNKNOWN = new ValueDef(JsonType.UNKNOWN, 0);
+
+ private final int arrayDims;
+ private final JsonType type;
+
+ public ValueDef(JsonType type, int dims) {
+ this.type = type;
+ this.arrayDims = dims;
+ }
+
+ public JsonType type() { return type; }
+ public int dimensions() { return arrayDims; }
+ public boolean isArray() { return arrayDims > 0; }
+
+ @Override
+ public String toString() {
+ StringBuilder buf = new StringBuilder()
+ .append(type.name());
+ for (int i = 0; i < arrayDims; i++) {
+ buf.append("[]");
+ }
+ return buf.toString();
+ }
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ValueDefFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ValueDefFactory.java
new file mode 100644
index 0000000..9013dd2
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ValueDefFactory.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.easy.json.parser;
+
+import org.apache.drill.exec.store.easy.json.parser.ValueDef.JsonType;
+
+import com.fasterxml.jackson.core.JsonToken;
+
+/**
+ * Constructs a {@link ValueDef} by looking ahead on the input stream.
+ * Looking ahead is safe because this class only looks at syntactic
+ * tokens such as <code>{</code>, {@code [} or the first value token.
+ * The underlying JSON parser is left with the first value token
+ * as its current token. Pushes other tokens back on the token stack
+ * so they can be re-consumed by the actual parser.
+ */
+public class ValueDefFactory {
+
+ private int arrayDims;
+ private JsonType jsonType = JsonType.EMPTY;
+
+ public ValueDefFactory(TokenIterator tokenizer) {
+ inferValueType(tokenizer);
+ }
+
+ public static ValueDef lookAhead(TokenIterator tokenizer) {
+ ValueDefFactory factory = new ValueDefFactory(tokenizer);
+ return new ValueDef(factory.jsonType, factory.arrayDims);
+ }
+
+ public static ValueDef arrayLookAhead(TokenIterator tokenizer) {
+ ValueDefFactory factory = new ValueDefFactory(tokenizer);
+ // Already in an array (saw [), so add one to dimensions
+ return new ValueDef(factory.jsonType, factory.arrayDims + 1);
+ }
+
+ private void inferValueType(TokenIterator tokenizer) {
+ JsonToken token = tokenizer.requireNext();
+ switch (token) {
+ case START_ARRAY:
+ // Position: key: [ ^
+ arrayDims++;
+ inferValueType(tokenizer);
+ break;
+
+ case END_ARRAY:
+ break;
+
+ case START_OBJECT:
+ // Position: key: { ^
+ jsonType = JsonType.OBJECT;
+ break;
+
+ case VALUE_NULL:
+
+ // Position: key: null ^
+ jsonType = JsonType.NULL;
+ break;
+
+ case VALUE_FALSE:
+ case VALUE_TRUE:
+ jsonType = JsonType.BOOLEAN;
+ break;
+
+ case VALUE_NUMBER_INT:
+ jsonType = JsonType.INTEGER;
+ break;
+
+ case VALUE_NUMBER_FLOAT:
+ jsonType = JsonType.FLOAT;
+ break;
+
+ case VALUE_STRING:
+ jsonType = JsonType.STRING;
+ break;
+
+ default:
+ // Won't get here: the Jackson parser catches
+ // errors.
+ throw tokenizer.errorFactory().syntaxError(token);
+ }
+ tokenizer.unget(token);
+ }
+}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ValueFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ValueFactory.java
deleted file mode 100644
index 97ca0ca..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ValueFactory.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.store.easy.json.parser;
-
-import com.fasterxml.jackson.core.JsonToken;
-
-/**
- * Constructs a value parser and its listeners by looking ahead
- * some number of tokens to "sniff" the type of the value. For
- * example:
- * <ul>
- * <li>{@code foo: <value>} - Field value</li>
- * <li>{@code foo: [ <value> ]} - 1D array value</li>
- * <li>{@code foo: [ [<value> ] ]} - 2D array value</li>
- * <li>Etc.</li>
- * </ul>
- * <p>
- * There are two cases in which no type estimation is possible:
- * <ul>
- * <li>The value is {@code null}, indicated by
- * {@link JsonType#NULL}.</code>
- * <li>The value is an array, and the array is empty, indicated
- * by {@link JsonType#EMPTY}.</li>
- * </ul>
- * This class handles syntactic type inference. The associated
- * listener enforces semantic rules. For example, if a schema is
- * available, and we know that field "x" must be an Integer, but
- * this class reports that it is an object, then the listener should
- * raise an exception.
- * <p>
- * Also, the parser cannot enforce type consistency. This class
- * looks only at the first appearance of a value. JSON allows anything.
- * The listener must enforce semantic rules that say whether a different
- * type is allowed for later values.
- */
-public class ValueFactory {
-
- public static class FieldDescrip {
- protected int arrayDims;
- protected JsonType type;
-
- public boolean isArray() { return arrayDims > 0; }
-
- public boolean isObject() { return type == JsonType.OBJECT; }
- }
-
- private ValueFactory() { }
-
- /**
- * Parse position: <code>{ ... field : ^ ?</code> for a newly-seen field.
- * Look ahead to guess the field type, then declare the field.
- *
- * @param parent the object parser declaring the field
- * @param key the name of the field
- * @param tokenizer the token parser
- * @return the value parser for the element, which may contain additional
- * structure for objects or arrays
- */
- public static ElementParser createFieldParser(ObjectParser parent, String key, TokenIterator tokenizer) {
- FieldDescrip descrip = new FieldDescrip();
- inferFieldType(descrip, tokenizer);
- ObjectListener objListener = parent.listener();
- ValueListener fieldListener;
- if (descrip.isObject()) {
- if (descrip.isArray()) {
- // Object array field
- fieldListener = objListener.addObjectArray(key, descrip.arrayDims);
- } else {
- // Object field
- fieldListener = objListener.addObject(key);
- }
- } else {
- if (descrip.isArray()) {
- // Scalar (or unknown) array field
- fieldListener = objListener.addArray(key, descrip.arrayDims, descrip.type);
- } else {
- // Scalar field
- fieldListener = objListener.addScalar(key, descrip.type);
- }
- }
- ValueParser fp = new ValueParser(parent, key, fieldListener);
- createStructureParser(fp, descrip);
- return fp;
- }
-
- /**
- * Add the object or array parser, if the structured type is known.
- */
- private static void createStructureParser(ValueParser valueParser,
- FieldDescrip descrip) {
- if (descrip.isArray()) {
- valueParser.bindArrayParser(createArrayParser(valueParser, descrip));
- } else if (descrip.isObject()) {
- valueParser.bindObjectParser(objectParser(valueParser));
- }
- }
-
- /**
- * Parse position: <code>... [ ?</code> for a field or array element not previously
- * known to be an array. Look ahead to determine if the array is nested and its
- * element types.
- *
- * @param parent the parser for the value that has been found to contain an
- * array
- * @param tokenizer the JSON token parser
- * @return an array parser to bind to the parent value parser to parse the
- * array
- */
- public static ArrayParser createArrayParser(ValueParser parent, TokenIterator tokenizer) {
- FieldDescrip descrip = new FieldDescrip();
- // Already in an array, so add the outer dimension.
- descrip.arrayDims++;
- inferFieldType(descrip, tokenizer);
- return createArrayParser(parent, descrip);
- }
-
- public static ArrayParser createArrayParser(ValueParser parent, FieldDescrip descrip) {
- ValueListener fieldListener = parent.listener();
- ArrayListener arrayListener;
- if (descrip.isObject()) {
- // Object array elements
- arrayListener = fieldListener.objectArray(descrip.arrayDims);
- } else {
- arrayListener = fieldListener.array(descrip.arrayDims, descrip.type);
- }
- descrip.arrayDims--;
- ValueListener elementListener;
- if (descrip.isObject()) {
- if (descrip.isArray()) {
- // Object array elements
- elementListener = arrayListener.objectArrayElement(descrip.arrayDims);
- } else {
- // Object elements
- elementListener = arrayListener.objectElement();
- }
- } else {
- if (descrip.isArray()) {
- // Scalar (or unknown) array elements
- elementListener = arrayListener.arrayElement(descrip.arrayDims, descrip.type);
- } else {
- // Scalar elements
- elementListener = arrayListener.scalarElement(descrip.type);
- }
- }
- ArrayParser arrayParser = new ArrayParser(parent, arrayListener, elementListener);
- createStructureParser(arrayParser.elementParser(), descrip);
- return arrayParser;
- }
-
- public static ObjectParser objectParser(ValueParser parent) {
- ValueListener valueListener = parent.listener();
- ObjectListener objListener = valueListener.object();
- return new ObjectParser(parent, objListener);
- }
-
- protected static void inferFieldType(FieldDescrip descrip, TokenIterator tokenizer) {
- JsonToken token = tokenizer.requireNext();
- switch (token) {
- case START_ARRAY:
- // Position: key: [ ^
- descrip.arrayDims++;
- inferFieldType(descrip, tokenizer);
- break;
-
- case END_ARRAY:
- if (descrip.arrayDims == 0) {
- throw tokenizer.errorFactory().syntaxError(token);
- }
- descrip.type = JsonType.EMPTY;
- break;
-
- case START_OBJECT:
- // Position: key: { ^
- descrip.type = JsonType.OBJECT;
- break;
-
- case VALUE_NULL:
-
- // Position: key: null ^
- descrip.type = JsonType.NULL;
- break;
-
- case VALUE_FALSE:
- case VALUE_TRUE:
- descrip.type = JsonType.BOOLEAN;
- break;
-
- case VALUE_NUMBER_INT:
- descrip.type = JsonType.INTEGER;
- break;
-
- case VALUE_NUMBER_FLOAT:
- descrip.type = JsonType.FLOAT;
- break;
-
- case VALUE_STRING:
- descrip.type = JsonType.STRING;
- break;
-
- default:
- // Won't get here: the Jackson parser catches
- // errors.
- throw tokenizer.errorFactory().syntaxError(token);
- }
- tokenizer.unget(token);
- }
-}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ValueListener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ValueListener.java
index 9625f55..6037069 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ValueListener.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ValueListener.java
@@ -62,16 +62,15 @@ package org.apache.drill.exec.store.easy.json.parser;
public interface ValueListener {
/**
- * The field is to be treated as "all-text". Used when the parser-level
- * setting for {@code allTextMode} is {@code false}; allows per-field
- * overrides to, perhaps, ride over inconsistent scalar types for a
- * single field.
- *
- * @return {@code true} if the field is to be read in "all-text mode" even
- * if the global setting is off, {@code false} to read the field as
- * typed values.
+ * Allows the object listener to revise the listener for a field,
+ * such as when a field starts null and resolves to some concrete
+ * type.
*/
- boolean isText();
+ interface ValueHost {
+ void bindListener(ValueListener listener);
+ }
+
+ void bind(ValueHost host);
/**
* Called on parsing a {@code null} value for the field. Called whether
@@ -125,28 +124,18 @@ public interface ValueListener {
/**
* The parser has encountered a object value for the field for the first
- * time. That is: {@code foo: {</code}.
+ * time. That is: <code>foo: {</code>.
*
* @return an object listener for the object
*/
ObjectListener object();
/**
- * The parser has encountered a array value for the first
- * time, and that array is scalar, null or empty.
- *
- * @param arrayDims the number of observed array dimensions
- * @param type the observed JSON token type for the array element
- * @return an array listener for the array
- */
- ArrayListener array(int arrayDims, JsonType type);
-
- /**
- * The parser has encountered a array value for the first
- * time, and that array contains an object.
+ * The parser has encountered a array value for the first time.
*
- * @param arrayDims the number of observed array dimensions
+ * @param valueDef description of the array dimensions (if
+ * a multi-dimensional array) and type (if known)
* @return an array listener for the array
*/
- ArrayListener objectArray(int arrayDims);
+ ArrayListener array(ValueDef valueDef);
}
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ValueParser.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ValueParser.java
index ac7ee31..8a7fd77 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ValueParser.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/parser/ValueParser.java
@@ -17,6 +17,9 @@
*/
package org.apache.drill.exec.store.easy.json.parser;
+import org.apache.drill.exec.store.easy.json.parser.ObjectListener.FieldType;
+import org.apache.drill.exec.store.easy.json.parser.ValueListener.ValueHost;
+
import com.fasterxml.jackson.core.JsonToken;
/**
@@ -36,7 +39,7 @@ import com.fasterxml.jackson.core.JsonToken;
* Listeners can enforce one type only, or can be more flexible and
* allow multiple types.
*/
-public class ValueParser extends AbstractElementParser {
+public class ValueParser extends AbstractElementParser implements ValueHost {
private interface ValueHandler {
void accept(TokenIterator tokenizer, JsonToken token);
@@ -107,34 +110,34 @@ public class ValueParser extends AbstractElementParser {
}
private final String key;
- private final ValueListener listener;
- private final ValueHandler valueHandler;
+ protected final ValueHandler valueHandler;
+ private ValueListener listener;
private ObjectParser objectParser;
private ArrayParser arrayParser;
- public ValueParser(ElementParser parent, String key, ValueListener listener) {
+ public ValueParser(ElementParser parent, String key, FieldType type) {
super(parent);
this.key = key;
- this.listener = listener;
- if (listener.isText() || structParser().options().allTextMode) {
+ if (type == FieldType.TEXT || structParser().options().allTextMode) {
valueHandler = new TextValueHandler();
} else {
valueHandler = new TypedValueHandler();
}
}
+ @Override
+ public void bindListener(ValueListener listener) {
+ this.listener = listener;
+ listener.bind(this);
+ if (arrayParser != null) {
+ arrayParser.bindListener(listener.array(ValueDef.UNKNOWN_ARRAY));
+ }
+ }
+
public String key() { return key; }
public ValueListener listener() { return listener; }
- public void bindObjectParser(ObjectParser parser) {
- objectParser = parser;
- }
-
- public void bindArrayParser(ArrayParser parser) {
- arrayParser = parser;
- }
-
/**
* Parses <code>true | false | null | integer | float | string|
* embedded-object | { ... } | [ ... ]</code>
@@ -148,17 +151,17 @@ public class ValueParser extends AbstractElementParser {
if (objectParser == null) {
// No object parser yet. May be that the value was null,
// or may be that it changed types.
- objectParser = ValueFactory.objectParser(this);
+ addObjectParser();
}
objectParser.parse(tokenizer);
break;
case START_ARRAY:
- // Position: { ^
+ // Position: [ ^
if (arrayParser == null) {
// No array parser yet. May be that the value was null,
// or may be that it changed types.
- arrayParser = ValueFactory.createArrayParser(this, tokenizer);
+ addArrayParser(ValueDefFactory.arrayLookAhead(tokenizer));
}
arrayParser.parse(tokenizer);
break;
@@ -171,4 +174,22 @@ public class ValueParser extends AbstractElementParser {
valueHandler.accept(tokenizer, token);
}
}
+
+ public void addObjectParser() {
+ objectParser = new ObjectParser(this, listener().object());
+ }
+
+ private void addArrayParser(ValueDef valueDef) {
+ ArrayListener arrayListener = listener().array(valueDef);
+ arrayParser = new ArrayParser(this, arrayListener);
+ arrayParser.expandStructure(valueDef);
+ }
+
+ public void expandStructure(ValueDef valueDef) {
+ if (valueDef.isArray()) {
+ addArrayParser(valueDef);
+ } else if (valueDef.type().isObject()) {
+ addObjectParser();
+ }
+ }
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/parser/BaseTestJsonParser.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/parser/BaseTestJsonParser.java
similarity index 71%
rename from exec/java-exec/src/test/java/org/apache/drill/exec/store/json/parser/BaseTestJsonParser.java
rename to exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/parser/BaseTestJsonParser.java
index 3f9b715..a3c3e08 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/parser/BaseTestJsonParser.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/parser/BaseTestJsonParser.java
@@ -15,12 +15,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.drill.exec.store.json.parser;
+package org.apache.drill.exec.store.easy.json.parser;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
@@ -31,13 +32,6 @@ import java.util.Map;
import java.util.Set;
import org.apache.commons.io.input.ReaderInputStream;
-import org.apache.drill.exec.store.easy.json.parser.ArrayListener;
-import org.apache.drill.exec.store.easy.json.parser.ErrorFactory;
-import org.apache.drill.exec.store.easy.json.parser.JsonStructureOptions;
-import org.apache.drill.exec.store.easy.json.parser.JsonStructureParser;
-import org.apache.drill.exec.store.easy.json.parser.JsonType;
-import org.apache.drill.exec.store.easy.json.parser.ObjectListener;
-import org.apache.drill.exec.store.easy.json.parser.ValueListener;
import org.apache.drill.exec.vector.accessor.UnsupportedConversionError;
import com.fasterxml.jackson.core.JsonParseException;
@@ -105,23 +99,19 @@ public class BaseTestJsonParser {
protected static class ValueListenerFixture implements ValueListener {
- final int dimCount;
- final JsonType type;
+ final ValueDef valueDef;
int nullCount;
int valueCount;
Object value;
+ ValueHost host;
ObjectListenerFixture objectValue;
ArrayListenerFixture arrayValue;
- public ValueListenerFixture(int dimCount, JsonType type) {
- this.dimCount = dimCount;
- this.type = type;
+ public ValueListenerFixture(ValueDef valueDef) {
+ this.valueDef = valueDef;
}
@Override
- public boolean isText() { return false; }
-
- @Override
public void onNull() {
nullCount++;
}
@@ -164,32 +154,29 @@ public class BaseTestJsonParser {
}
@Override
- public ArrayListener array(int arrayDims, JsonType type) {
- assertNull(arrayValue);
- arrayValue = new ArrayListenerFixture(arrayDims, type);
+ public ArrayListener array(ValueDef valueDef) {
+ if (arrayValue == null) {
+ arrayValue = new ArrayListenerFixture(valueDef);
+ }
return arrayValue;
}
@Override
- public ArrayListener objectArray(int arrayDims) {
- assertNull(arrayValue);
- arrayValue = new ArrayListenerFixture(arrayDims, JsonType.OBJECT);
- return arrayValue;
+ public void bind(ValueHost host) {
+ this.host = host;
}
}
protected static class ArrayListenerFixture implements ArrayListener {
- final int dimCount;
- final JsonType type;
+ final ValueDef valueDef;
int startCount;
int endCount;
int elementCount;
ValueListenerFixture element;
- public ArrayListenerFixture(int dimCount, JsonType type) {
- this.dimCount = dimCount;
- this.type = type;
+ public ArrayListenerFixture(ValueDef valueDef) {
+ this.valueDef = valueDef;
}
@Override
@@ -198,38 +185,23 @@ public class BaseTestJsonParser {
}
@Override
- public void onElement() {
+ public void onElementStart() {
elementCount++;
}
@Override
- public void onEnd() {
- endCount++;
- }
-
- @Override
- public ValueListener objectArrayElement(int arrayDims) {
- return element(arrayDims, JsonType.OBJECT);
- }
+ public void onElementEnd() { }
@Override
- public ValueListener objectElement() {
- return element(0, JsonType.OBJECT);
- }
-
- @Override
- public ValueListener arrayElement(int arrayDims, JsonType type) {
- return element(arrayDims, type);
+ public void onEnd() {
+ endCount++;
}
@Override
- public ValueListener scalarElement(JsonType type) {
- return element(0, type);
- }
-
- private ValueListener element(int arrayDims, JsonType type) {
- assertNull(element);
- element = new ValueListenerFixture(arrayDims, type);
+ public ValueListener element(ValueDef valueDef) {
+ if (element == null) {
+ element = new ValueListenerFixture(valueDef);
+ }
return element;
}
}
@@ -238,6 +210,7 @@ public class BaseTestJsonParser {
final Map<String, ValueListenerFixture> fields = new HashMap<>();
Set<String> projectFilter;
+ FieldType fieldType = FieldType.TYPED;
int startCount;
int endCount;
@@ -252,37 +225,25 @@ public class BaseTestJsonParser {
}
@Override
- public boolean isProjected(String key) {
- return projectFilter == null || projectFilter.contains(key);
- }
-
- @Override
- public ValueListener addScalar(String key, JsonType type) {
- return field(key, 0, type);
- }
-
- @Override
- public ValueListener addArray(String key, int dims, JsonType type) {
- return field(key, dims, type);
- }
-
- @Override
- public ValueListener addObject(String key) {
- return field(key, 0, JsonType.OBJECT);
+ public FieldType fieldType(String key) {
+ if (projectFilter != null && !projectFilter.contains(key)) {
+ return FieldType.IGNORE;
+ }
+ return fieldType;
}
@Override
- public ValueListener addObjectArray(String key, int dims) {
- return field(key, dims, JsonType.OBJECT);
- }
-
- private ValueListener field(String key, int dims, JsonType type) {
+ public ValueListener addField(String key, ValueDef valueDef) {
assertFalse(fields.containsKey(key));
- ValueListenerFixture field = new ValueListenerFixture(dims, type);
+ ValueListenerFixture field = makeField(key, valueDef);
fields.put(key, field);
return field;
}
+ public ValueListenerFixture makeField(String key, ValueDef valueDef) {
+ return new ValueListenerFixture(valueDef);
+ }
+
public ValueListenerFixture field(String key) {
ValueListenerFixture field = fields.get(key);
assertNotNull(field);
@@ -320,6 +281,25 @@ public class BaseTestJsonParser {
return rootObject.field(key);
}
+ public void expect(String key, Object[] values) {
+ ValueListenerFixture valueListener = null;
+ int expectedNullCount = 0;
+ for (int i = 0; i < values.length; i++) {
+ assertTrue(next());
+ if (valueListener == null) {
+ valueListener = field(key);
+ expectedNullCount = valueListener.nullCount;
+ }
+ Object value = values[i];
+ if (value == null) {
+ expectedNullCount++;
+ } else {
+ assertEquals(value, valueListener.value);
+ }
+ assertEquals(expectedNullCount, valueListener.nullCount);
+ }
+ }
+
public void close() {
if (parser != null) {
parser.close();
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/parser/TestJsonParserArrays.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/parser/TestJsonParserArrays.java
similarity index 82%
rename from exec/java-exec/src/test/java/org/apache/drill/exec/store/json/parser/TestJsonParserArrays.java
rename to exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/parser/TestJsonParserArrays.java
index 97ccfc8..7275733 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/parser/TestJsonParserArrays.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/parser/TestJsonParserArrays.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.drill.exec.store.json.parser;
+package org.apache.drill.exec.store.easy.json.parser;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -23,13 +23,16 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import org.apache.drill.exec.store.easy.json.parser.JsonType;
+import org.apache.drill.categories.RowSetTests;
+import org.apache.drill.exec.store.easy.json.parser.ValueDef.JsonType;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
/**
* Tests array (including multi-dimensional and object) support
* for the JSON structure parser.
*/
+@Category(RowSetTests.class)
public class TestJsonParserArrays extends BaseTestJsonParser {
@Test
@@ -45,21 +48,21 @@ public class TestJsonParserArrays extends BaseTestJsonParser {
// Value of object.a
ValueListenerFixture a = fixture.field("a");
- assertEquals(JsonType.INTEGER, a.type);
- assertEquals(1, a.dimCount);
+ assertEquals(JsonType.INTEGER, a.valueDef.type());
+ assertEquals(1, a.valueDef.dimensions());
// Array aspect of a
assertNotNull(a.arrayValue);
ArrayListenerFixture aArray = a.arrayValue;
assertEquals(1, aArray.startCount);
assertEquals(aArray.startCount, aArray.endCount);
- assertEquals(1, aArray.dimCount);
+ assertEquals(1, aArray.valueDef.dimensions());
// Value of each element of array aspect of a
assertNotNull(aArray.element);
ValueListenerFixture aElement = aArray.element;
- assertEquals(JsonType.INTEGER, aElement.type);
- assertEquals(0, aElement.dimCount);
+ assertEquals(JsonType.INTEGER, aElement.valueDef.type());
+ assertEquals(0, aElement.valueDef.dimensions());
assertNull(aElement.arrayValue);
assertEquals(2, aElement.valueCount);
assertEquals(100L, aElement.value);
@@ -105,8 +108,8 @@ public class TestJsonParserArrays extends BaseTestJsonParser {
// {a: null}
assertTrue(fixture.next());
ValueListenerFixture a = fixture.field("a");
- assertEquals(JsonType.NULL, a.type);
- assertEquals(0, a.dimCount);
+ assertEquals(JsonType.NULL, a.valueDef.type());
+ assertEquals(0, a.valueDef.dimensions());
assertNull(a.arrayValue);
// See an array, can revise estimate of field type
@@ -114,7 +117,7 @@ public class TestJsonParserArrays extends BaseTestJsonParser {
assertTrue(fixture.next());
assertNotNull(a.arrayValue);
ArrayListenerFixture aArray = a.arrayValue;
- assertEquals(1, aArray.dimCount);
+ assertEquals(1, aArray.valueDef.dimensions());
ValueListenerFixture aElement = aArray.element;
assertEquals(2, aElement.valueCount);
assertEquals(100L, aElement.value);
@@ -130,21 +133,22 @@ public class TestJsonParserArrays extends BaseTestJsonParser {
JsonParserFixture fixture = new JsonParserFixture();
fixture.open(json);
- // Can't predict the future, all we know is a is an array.
+ // Can't predict the future, all we know is `a` is an array.
// "{a: []}
assertTrue(fixture.next());
ValueListenerFixture a = fixture.field("a");
- assertEquals(JsonType.EMPTY, a.type);
- assertEquals(1, a.dimCount);
+ assertEquals(JsonType.EMPTY, a.valueDef.type());
+ assertEquals(1, a.valueDef.dimensions());
assertNotNull(a.arrayValue);
ArrayListenerFixture aArray = a.arrayValue;
- assertEquals(1, aArray.dimCount);
- ValueListenerFixture aElement = aArray.element;
- assertEquals(JsonType.EMPTY, aElement.type);
+ assertEquals(1, aArray.valueDef.dimensions());
+ assertNull(aArray.element); // Saw no element yet
// See elements, can revise estimate of element type
// {a: [1, 100]}
assertTrue(fixture.next());
+ ValueListenerFixture aElement = aArray.element;
+ assertNotNull(aElement);
assertEquals(2, aElement.valueCount);
assertEquals(100L, aElement.value);
@@ -155,43 +159,43 @@ public class TestJsonParserArrays extends BaseTestJsonParser {
@Test
public void test2DArray() {
final String json =
- "{a: [ [10, 1], [20, 2]]}\n" +
+ "{a: [[10, 1], [20, 2]]}\n" +
"{a: [[null]]} {a: [[]]} {a: [null]} {a: null}";
JsonParserFixture fixture = new JsonParserFixture();
fixture.open(json);
- // {a: [ [10, 1], [20, 2]]}
+ // {a: [[10, 1], [20, 2]]}
assertTrue(fixture.next());
// Value of a
ValueListenerFixture a = fixture.field("a");
- assertEquals(JsonType.INTEGER, a.type);
- assertEquals(2, a.dimCount);
+ assertEquals(JsonType.INTEGER, a.valueDef.type());
+ assertEquals(2, a.valueDef.dimensions());
// Array for a[]
assertNotNull(a.arrayValue);
ArrayListenerFixture outerArr = a.arrayValue;
- assertEquals(2, outerArr.dimCount);
+ assertEquals(2, outerArr.valueDef.dimensions());
assertEquals(1, outerArr.startCount);
assertEquals(outerArr.startCount, outerArr.endCount);
// Value of a[] elements
ValueListenerFixture outerElement = outerArr.element;
- assertEquals(JsonType.INTEGER, outerElement.type);
- assertEquals(1, outerElement.dimCount);
+ assertEquals(JsonType.INTEGER, outerElement.valueDef.type());
+ assertEquals(1, outerElement.valueDef.dimensions());
assertNotNull(outerElement.arrayValue);
// Array for a[][]
assertNotNull(outerElement.arrayValue);
ArrayListenerFixture innerArr = outerElement.arrayValue;
- assertEquals(1, innerArr.dimCount);
+ assertEquals(1, innerArr.valueDef.dimensions());
assertEquals(2, innerArr.startCount);
assertEquals(innerArr.startCount, innerArr.endCount);
// Value of a[][] elements
ValueListenerFixture innerElement = innerArr.element;
- assertEquals(JsonType.INTEGER, innerElement.type);
- assertEquals(0, innerElement.dimCount);
+ assertEquals(JsonType.INTEGER, innerElement.valueDef.type());
+ assertEquals(0, innerElement.valueDef.dimensions());
assertEquals(4, innerElement.valueCount);
assertEquals(0, innerElement.nullCount);
assertEquals(2L, innerElement.value);
@@ -271,21 +275,21 @@ public class TestJsonParserArrays extends BaseTestJsonParser {
// Value of object.a
ValueListenerFixture a = fixture.field("a");
- assertEquals(JsonType.OBJECT, a.type);
- assertEquals(1, a.dimCount);
+ assertEquals(JsonType.OBJECT, a.valueDef.type());
+ assertEquals(1, a.valueDef.dimensions());
// a[]
assertNotNull(a.arrayValue);
ArrayListenerFixture aArray = a.arrayValue;
assertEquals(1, aArray.startCount);
assertEquals(aArray.startCount, aArray.endCount);
- assertEquals(1, aArray.dimCount);
+ assertEquals(1, aArray.valueDef.dimensions());
// Value of each element of a[]
assertNotNull(aArray.element);
ValueListenerFixture aElement = aArray.element;
- assertEquals(JsonType.OBJECT, aElement.type);
- assertEquals(0, aElement.dimCount);
+ assertEquals(JsonType.OBJECT, aElement.valueDef.type());
+ assertEquals(0, aElement.valueDef.dimensions());
assertNull(aElement.arrayValue);
assertEquals(0, aElement.valueCount);
assertEquals(0, aElement.nullCount);
@@ -317,21 +321,21 @@ public class TestJsonParserArrays extends BaseTestJsonParser {
// Value of object.a
ValueListenerFixture a = fixture.field("a");
- assertEquals(JsonType.OBJECT, a.type);
- assertEquals(2, a.dimCount);
+ assertEquals(JsonType.OBJECT, a.valueDef.type());
+ assertEquals(2, a.valueDef.dimensions());
// a[]
assertNotNull(a.arrayValue);
ArrayListenerFixture outerArray = a.arrayValue;
assertEquals(1, outerArray.startCount);
assertEquals(outerArray.startCount, outerArray.endCount);
- assertEquals(2, outerArray.dimCount);
+ assertEquals(2, outerArray.valueDef.dimensions());
// Value of each element of a[]
assertNotNull(outerArray.element);
ValueListenerFixture outerElement = outerArray.element;
- assertEquals(JsonType.OBJECT, outerElement.type);
- assertEquals(1, outerElement.dimCount);
+ assertEquals(JsonType.OBJECT, outerElement.valueDef.type());
+ assertEquals(1, outerElement.valueDef.dimensions());
assertEquals(0, outerElement.valueCount);
assertEquals(0, outerElement.nullCount);
@@ -340,13 +344,13 @@ public class TestJsonParserArrays extends BaseTestJsonParser {
ArrayListenerFixture innerArray = outerElement.arrayValue;
assertEquals(2, innerArray.startCount);
assertEquals(innerArray.startCount, innerArray.endCount);
- assertEquals(1, innerArray.dimCount);
+ assertEquals(1, innerArray.valueDef.dimensions());
// Value of each element of a[][]
assertNotNull(innerArray.element);
ValueListenerFixture innerElement = innerArray.element;
- assertEquals(JsonType.OBJECT, innerElement.type);
- assertEquals(0, innerElement.dimCount);
+ assertEquals(JsonType.OBJECT, innerElement.valueDef.type());
+ assertEquals(0, innerElement.valueDef.dimensions());
assertEquals(0, innerElement.valueCount);
assertEquals(0, innerElement.nullCount);
@@ -380,8 +384,8 @@ public class TestJsonParserArrays extends BaseTestJsonParser {
assertEquals(4, fixture.read());
ValueListenerFixture a = fixture.field("a");
// Type first seen
- assertEquals(JsonType.INTEGER, a.type);
- assertEquals(1, a.dimCount);
+ assertEquals(JsonType.INTEGER, a.valueDef.type());
+ assertEquals(1, a.valueDef.dimensions());
// Everything populated
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/parser/TestJsonParserBasics.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/parser/TestJsonParserBasics.java
similarity index 74%
rename from exec/java-exec/src/test/java/org/apache/drill/exec/store/json/parser/TestJsonParserBasics.java
rename to exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/parser/TestJsonParserBasics.java
index 4215ff5..d51e1a8 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/parser/TestJsonParserBasics.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/parser/TestJsonParserBasics.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.drill.exec.store.json.parser;
+package org.apache.drill.exec.store.easy.json.parser;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -23,13 +23,17 @@ import static org.junit.Assert.assertTrue;
import java.util.HashSet;
-import org.apache.drill.exec.store.easy.json.parser.JsonType;
+import org.apache.drill.categories.RowSetTests;
+import org.apache.drill.exec.store.easy.json.parser.ObjectListener.FieldType;
+import org.apache.drill.exec.store.easy.json.parser.ValueDef.JsonType;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
/**
* Tests JSON structure parser functionality excluding nested objects
* and arrays. Tests the "happy path."
*/
+@Category(RowSetTests.class)
public class TestJsonParserBasics extends BaseTestJsonParser {
@Test
@@ -63,8 +67,8 @@ public class TestJsonParserBasics extends BaseTestJsonParser {
assertEquals(1, fixture.rootObject.startCount);
assertEquals(1, fixture.rootObject.fields.size());
ValueListenerFixture a = fixture.field("a");
- assertEquals(JsonType.BOOLEAN, a.type);
- assertEquals(0, a.dimCount);
+ assertEquals(JsonType.BOOLEAN, a.valueDef.type());
+ assertEquals(0, a.valueDef.dimensions());
assertEquals(0, a.nullCount);
assertEquals(Boolean.TRUE, a.value);
assertEquals(2, fixture.read());
@@ -80,7 +84,7 @@ public class TestJsonParserBasics extends BaseTestJsonParser {
fixture.open(json);
assertTrue(fixture.next());
ValueListenerFixture a = fixture.field("a");
- assertEquals(JsonType.INTEGER, a.type);
+ assertEquals(JsonType.INTEGER, a.valueDef.type());
assertEquals(2, fixture.read());
assertEquals(1, a.nullCount);
assertEquals(100L, a.value);
@@ -94,7 +98,7 @@ public class TestJsonParserBasics extends BaseTestJsonParser {
fixture.open(json);
assertTrue(fixture.next());
ValueListenerFixture a = fixture.field("a");
- assertEquals(JsonType.FLOAT, a.type);
+ assertEquals(JsonType.FLOAT, a.valueDef.type());
assertEquals(2, fixture.read());
assertEquals(1, a.nullCount);
assertEquals(2, a.valueCount);
@@ -111,7 +115,7 @@ public class TestJsonParserBasics extends BaseTestJsonParser {
fixture.open(json);
assertTrue(fixture.next());
ValueListenerFixture a = fixture.field("a");
- assertEquals(JsonType.FLOAT, a.type);
+ assertEquals(JsonType.FLOAT, a.valueDef.type());
assertEquals(2, fixture.read());
assertEquals(3, a.valueCount);
assertEquals(Double.NEGATIVE_INFINITY, a.value);
@@ -125,7 +129,7 @@ public class TestJsonParserBasics extends BaseTestJsonParser {
fixture.open(json);
assertTrue(fixture.next());
ValueListenerFixture a = fixture.field("a");
- assertEquals(JsonType.STRING, a.type);
+ assertEquals(JsonType.STRING, a.valueDef.type());
assertEquals(2, fixture.read());
assertEquals(1, a.nullCount);
assertEquals(2, a.valueCount);
@@ -140,7 +144,7 @@ public class TestJsonParserBasics extends BaseTestJsonParser {
fixture.open(json);
assertTrue(fixture.next());
ValueListenerFixture a = fixture.field("a");
- assertEquals(JsonType.STRING, a.type);
+ assertEquals(JsonType.STRING, a.valueDef.type());
assertEquals("hi", a.value);
assertTrue(fixture.next());
assertEquals(10L, a.value);
@@ -255,24 +259,68 @@ public class TestJsonParserBasics extends BaseTestJsonParser {
fixture.options.allTextMode = true;
fixture.open(json);
- assertTrue(fixture.next());
- ValueListenerFixture a = fixture.field("a");
- assertEquals("1", a.value);
+ fixture.expect("a",
+ new Object[] {"1", "foo", "true", "20.5", null});
+ assertFalse(fixture.next());
+ fixture.close();
+ }
- assertTrue(fixture.next());
- assertEquals("foo", a.value);
+ @Test
+ public void testColumnTextMode() {
+ final String json =
+ "{a: 1} {a: \"foo\"} {a: true} {a: 20.5} {a: null}";
+ JsonParserFixture fixture = new JsonParserFixture();
+ fixture.rootObject.fieldType = FieldType.TEXT;
+ fixture.open(json);
- assertTrue(fixture.next());
- assertEquals("true", a.value);
+ fixture.expect("a",
+ new Object[] {"1", "foo", "true", "20.5", null});
+ assertFalse(fixture.next());
+ fixture.close();
+ }
- assertTrue(fixture.next());
- assertEquals("20.5", a.value);
- assertEquals(0, a.nullCount);
+ @Test
+ public void testJsonModeScalars() {
+ final String json =
+ "{a: 1} {a: \"foo\"} {a: true} {a: 20.5} {a: null}";
+ JsonParserFixture fixture = new JsonParserFixture();
+ fixture.rootObject.fieldType = FieldType.JSON;
+ fixture.open(json);
- assertTrue(fixture.next());
- assertEquals("20.5", a.value);
- assertEquals(1, a.nullCount);
+ fixture.expect("a",
+ new Object[] {"1", "\"foo\"", "true", "20.5", "null"});
+ assertFalse(fixture.next());
+ fixture.close();
+ }
+
+ @Test
+ public void testJsonModeArrays() {
+ final String json =
+ "{a: []} {a: [null]} {a: [null, null]} {a: [[]]}\n" +
+ "{a: [1, \"foo\", true]} {a: [[1, 2], [3, 4]]}\n";
+ JsonParserFixture fixture = new JsonParserFixture();
+ fixture.rootObject.fieldType = FieldType.JSON;
+ fixture.open(json);
+
+ fixture.expect("a",
+ new Object[] {"[]", "[null]", "[null, null]", "[[]]",
+ "[1, \"foo\", true]", "[[1, 2], [3, 4]]"});
+ assertFalse(fixture.next());
+ fixture.close();
+ }
+
+ @Test
+ public void testJsonModeObjects() {
+ final String json =
+ "{a: {}} {a: {b: null}} {a: {b: null, b: null}}\n" +
+ "{a: {b: {c: {d: [{e: 10}, null, 20], f: \"foo\"}, g:30}, h: 40}}\n";
+ JsonParserFixture fixture = new JsonParserFixture();
+ fixture.rootObject.fieldType = FieldType.JSON;
+ fixture.open(json);
+ fixture.expect("a",
+ new Object[] {"{}", "{\"b\": null}", "{\"b\": null, \"b\": null}",
+ "{\"b\": {\"c\": {\"d\": [{\"e\": 10}, null, 20], \"f\": \"foo\"}, \"g\": 30}, \"h\": 40}"});
assertFalse(fixture.next());
fixture.close();
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/parser/TestJsonParserErrors.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/parser/TestJsonParserErrors.java
similarity index 94%
rename from exec/java-exec/src/test/java/org/apache/drill/exec/store/json/parser/TestJsonParserErrors.java
rename to exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/parser/TestJsonParserErrors.java
index 1b99ca2..2693635 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/parser/TestJsonParserErrors.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/parser/TestJsonParserErrors.java
@@ -15,19 +15,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.drill.exec.store.json.parser;
+package org.apache.drill.exec.store.easy.json.parser;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import org.apache.drill.categories.RowSetTests;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
/**
* Tests the un-happy path cases in the JSON structure parser. Some
* error cases can't occur because the Jackson parser catches them
* first.
*/
+@Category(RowSetTests.class)
public class TestJsonParserErrors extends BaseTestJsonParser {
@Test
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/parser/TestJsonParserObjects.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/parser/TestJsonParserObjects.java
similarity index 94%
rename from exec/java-exec/src/test/java/org/apache/drill/exec/store/json/parser/TestJsonParserObjects.java
rename to exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/parser/TestJsonParserObjects.java
index 9a350b6..d2cca9c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/parser/TestJsonParserObjects.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/parser/TestJsonParserObjects.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.drill.exec.store.json.parser;
+package org.apache.drill.exec.store.easy.json.parser;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -23,12 +23,15 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import org.apache.drill.exec.store.easy.json.parser.JsonType;
+import org.apache.drill.categories.RowSetTests;
+import org.apache.drill.exec.store.easy.json.parser.ValueDef.JsonType;
import org.junit.Test;
+import org.junit.experimental.categories.Category;
/**
* Tests nested object support in the JSON structure parser.
*/
+@Category(RowSetTests.class)
public class TestJsonParserObjects extends BaseTestJsonParser {
@Test
@@ -48,7 +51,7 @@ public class TestJsonParserObjects extends BaseTestJsonParser {
assertEquals(1, custObj.startCount);
assertEquals(custObj.startCount, custObj.endCount);
ValueListenerFixture name = custObj.field("name");
- assertEquals(JsonType.STRING, name.type);
+ assertEquals(JsonType.STRING, name.valueDef.type());
assertEquals("fred", name.value);
assertTrue(fixture.next());
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/parser/TestJsonParserUnknowns.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/parser/TestJsonParserUnknowns.java
new file mode 100644
index 0000000..8fa37b8
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/easy/json/parser/TestJsonParserUnknowns.java
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.easy.json.parser;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.drill.categories.RowSetTests;
+import org.apache.drill.exec.store.easy.json.parser.ValueDef.JsonType;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(RowSetTests.class)
+public class TestJsonParserUnknowns extends BaseTestJsonParser {
+
+ /**
+ * Test replacing a value lister "in-flight". Handles the
+ * (simulated) case where the initial value is ambiguous (here, null),
+ * and a later token resolves the value to something concrete.
+ */
+ @Test
+ public void testReplaceListener() {
+ final String json = "{a: null} {a: true} {a: false}";
+ JsonParserFixture fixture = new JsonParserFixture();
+ fixture.open(json);
+ assertTrue(fixture.next());
+ assertEquals(1, fixture.rootObject.startCount);
+ assertEquals(1, fixture.rootObject.fields.size());
+ ValueListenerFixture a = fixture.field("a");
+ assertEquals(JsonType.NULL, a.valueDef.type());
+ assertEquals(0, a.valueDef.dimensions());
+ assertEquals(1, a.nullCount);
+
+ // Replace the listener with a new one
+ ValueListenerFixture a2 = new ValueListenerFixture(new ValueDef(JsonType.BOOLEAN, a.valueDef.dimensions()));
+ a.host.bindListener(a2);
+ assertSame(a.host, a2.host);
+
+ assertTrue(fixture.next());
+ assertEquals(Boolean.TRUE, a2.value);
+ assertTrue(fixture.next());
+ assertEquals(0, a2.nullCount);
+ assertEquals(Boolean.FALSE, a2.value);
+ fixture.close();
+ }
+
+ private static class SwapperElementFixture extends ValueListenerFixture {
+
+ private final SwapperListenerFixture parent;
+
+ public SwapperElementFixture(SwapperListenerFixture parent, ValueDef valueDef) {
+ super(valueDef);
+ this.parent = parent;
+ }
+
+ @Override
+ public void onInt(long value) {
+ ValueListenerFixture newParent = parent.replace();
+ newParent.array(ValueDef.UNKNOWN_ARRAY);
+ newParent.arrayValue.element(ValueDef.UNKNOWN);
+ newParent.arrayValue.element.onInt(value);
+ }
+
+ }
+ private static class SwapperListenerFixture extends ValueListenerFixture {
+
+ private final ObjectListenerFixture parent;
+ private final String key;
+
+ public SwapperListenerFixture(ObjectListenerFixture parent,
+ String key, ValueDef valueDef) {
+ super(valueDef);
+ this.parent = parent;
+ this.key = key;
+ }
+
+ @Override
+ public void onInt(long value) {
+ replace().onInt(value);
+ }
+
+ @Override
+ public ObjectListener object() {
+ return replace().object();
+ }
+
+ @Override
+ public ArrayListener array(ValueDef valueDef) {
+ if (valueDef.type() == JsonType.EMPTY) {
+ super.array(valueDef);
+ arrayValue.element = new SwapperElementFixture(this, ValueDef.UNKNOWN);
+ return arrayValue;
+ } else {
+ return replace().array(valueDef);
+ }
+ }
+
+ private ValueListenerFixture replace() {
+ return replaceWith(new ValueListenerFixture(valueDef));
+ }
+
+ private ValueListenerFixture replaceWith(ValueListenerFixture newListener) {
+ parent.fields.put(key, newListener);
+ host.bindListener(newListener);
+ return newListener;
+ }
+ }
+
+ @Test
+ public void testNullToScalar() {
+ final String json =
+ "{a: null} {a: 2} {a: 3}";
+ JsonParserFixture fixture = new JsonParserFixture();
+ fixture.rootObject = new ObjectListenerFixture() {
+ @Override
+ public ValueListenerFixture makeField(String key, ValueDef valueDef) {
+ return new SwapperListenerFixture(this, key, valueDef);
+ }
+ };
+ fixture.open(json);
+
+ // {a: null}
+ assertTrue(fixture.next());
+ ValueListenerFixture a = fixture.field("a");
+ assertEquals(JsonType.NULL, a.valueDef.type());
+ assertEquals(0, a.valueDef.dimensions());
+ assertNull(a.arrayValue);
+
+ // See a scalar, can revise estimate of field type
+ // {a: 2}
+ assertTrue(fixture.next());
+ ValueListenerFixture a2 = fixture.field("a");
+ assertNotSame(a, a2);
+ assertEquals(2L, a2.value);
+ assertEquals(0, a2.nullCount);
+ assertEquals(1, a2.valueCount);
+
+ assertTrue(fixture.next());
+ assertEquals(3L, a2.value);
+
+ fixture.close();
+ }
+
+ @Test
+ public void testNullToObject() {
+ final String json =
+ "{a: null} {a: {}} {a: {b: 3}}";
+ JsonParserFixture fixture = new JsonParserFixture();
+ fixture.rootObject = new ObjectListenerFixture() {
+ @Override
+ public ValueListenerFixture makeField(String key, ValueDef valueDef) {
+ return new SwapperListenerFixture(this, key, valueDef);
+ }
+ };
+ fixture.open(json);
+
+ // {a: null}
+ assertTrue(fixture.next());
+ ValueListenerFixture a = fixture.field("a");
+ assertEquals(1, a.nullCount);
+
+ // See an object, can revise estimate of field type
+ // {a: {}}
+ assertTrue(fixture.next());
+ ValueListenerFixture a2 = fixture.field("a");
+ assertNotSame(a, a2);
+ assertNotNull(a2.objectValue);
+ assertTrue(a2.objectValue.fields.isEmpty());
+
+ assertTrue(fixture.next());
+ ValueListenerFixture b = a2.objectValue.field("b");
+ assertEquals(3L, b.value);
+
+ fixture.close();
+ }
+
+ @Test
+ public void testNullToEmptyArray() {
+ final String json =
+ "{a: null} {a: []} {a: []} {a: [10, 20]} {a: [30, 40]}";
+ JsonParserFixture fixture = new JsonParserFixture();
+ fixture.rootObject = new ObjectListenerFixture() {
+ @Override
+ public ValueListenerFixture makeField(String key, ValueDef valueDef) {
+ return new SwapperListenerFixture(this, key, valueDef);
+ }
+ };
+ fixture.open(json);
+
+ // {a: null}
+ assertTrue(fixture.next());
+ ValueListenerFixture a = fixture.field("a");
+ assertEquals(1, a.nullCount);
+
+ // See an empty array, can revise estimate of field type
+ // {a: []}
+ assertTrue(fixture.next());
+ assertSame(a, fixture.field("a"));
+ assertEquals(1, a.arrayValue.startCount);
+
+ // Ensure things are stable
+ // {a: []}
+ assertTrue(fixture.next());
+ assertSame(a, fixture.field("a"));
+ assertEquals(2, a.arrayValue.startCount);
+
+ // Revise again once we see element type
+ // {a: [10, 20]}
+ assertTrue(fixture.next());
+ ValueListenerFixture a2 = fixture.field("a");
+ assertNotSame(a, a2);
+ assertEquals(3, a.arrayValue.startCount); // Start on old listener
+ assertEquals(0, a2.arrayValue.startCount);
+ assertEquals(1, a2.arrayValue.endCount); // End on new one
+ assertEquals(2, a2.arrayValue.element.valueCount);
+ assertEquals(20L, a2.arrayValue.element.value);
+
+ // Check stability again
+ // {a: [30, 40]}
+ assertTrue(fixture.next());
+ assertSame(a2, fixture.field("a"));
+ assertEquals(1, a2.arrayValue.startCount);
+ assertEquals(4, a2.arrayValue.element.valueCount);
+ assertEquals(40L, a2.arrayValue.element.value);
+
+ fixture.close();
+ }
+
+ /**
+ * As above, but skips the intermediate empty array.
+ * The array, when it appears, has type info.
+ */
+ @Test
+ public void testNullToArray() {
+ final String json =
+ "{a: null} {a: [10, 20]} {a: [30, 40]}";
+ JsonParserFixture fixture = new JsonParserFixture();
+ fixture.rootObject = new ObjectListenerFixture() {
+ @Override
+ public ValueListenerFixture makeField(String key, ValueDef valueDef) {
+ return new SwapperListenerFixture(this, key, valueDef);
+ }
+ };
+ fixture.open(json);
+
+ // {a: null}
+ assertTrue(fixture.next());
+ ValueListenerFixture a = fixture.field("a");
+ assertEquals(1, a.nullCount);
+
+ // See a typed empty array, can revise estimate of field type
+ // {a: [10, 20]}
+ assertTrue(fixture.next());
+ ValueListenerFixture a2 = fixture.field("a");
+ assertNotSame(a, a2);
+ assertEquals(1, a2.arrayValue.startCount);
+ assertEquals(1, a2.arrayValue.startCount);
+ assertEquals(JsonType.INTEGER, a2.arrayValue.valueDef.type());
+ assertEquals(1, a2.arrayValue.valueDef.dimensions());
+ assertEquals(JsonType.INTEGER, a2.arrayValue.element.valueDef.type());
+ assertEquals(0, a2.arrayValue.element.valueDef.dimensions());
+ assertEquals(2, a2.arrayValue.element.valueCount);
+ assertEquals(20L, a2.arrayValue.element.value);
+
+ // Ensure things are stable
+ // {a: [30, 40]}
+ assertTrue(fixture.next());
+ assertSame(a2, fixture.field("a"));
+ assertEquals(2, a2.arrayValue.startCount);
+ assertEquals(4, a2.arrayValue.element.valueCount);
+ assertEquals(40L, a2.arrayValue.element.value);
+
+ fixture.close();
+ }
+}