You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2014/03/23 03:03:15 UTC
[01/12] DRILL-387: Support using Hive simple UDFs in Drill * As part
of this change FunctionDefinition (and related code) * are deleted,
instead the same information available in * Function Holders are used whenever
required * Freemarker/CodeModel codege
Repository: incubator-drill
Updated Branches:
refs/heads/master 2ce675679 -> 30ce7d710
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
index 0de64b4..679001c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
@@ -30,7 +30,6 @@ import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.FunctionRegistry;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.expression.parser.ExprLexer;
@@ -100,7 +99,6 @@ public class ExpressionTest {
ExprLexer lexer = new ExprLexer(new ANTLRStringStream(expr));
CommonTokenStream tokens = new CommonTokenStream(lexer);
ExprParser parser = new ExprParser(tokens);
- parser.setRegistry(new FunctionRegistry(DrillConfig.create()));
parse_return ret = parser.parse();
return ret.e;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/GeneratorFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/GeneratorFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/GeneratorFunctions.java
index 7ec7243..738d7f4 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/GeneratorFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/GeneratorFunctions.java
@@ -17,10 +17,6 @@
*/
package org.apache.drill.exec.fn.impl;
-import org.apache.drill.common.expression.ArgumentValidators;
-import org.apache.drill.common.expression.CallProvider;
-import org.apache.drill.common.expression.FunctionDefinition;
-import org.apache.drill.common.expression.OutputTypeDeterminer;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
@@ -35,25 +31,9 @@ import java.util.Random;
public class GeneratorFunctions {
public static final Random random = new Random(1234L);
- public static final FunctionDefinition RANDOM_BIG_INT = FunctionDefinition.simpleRandom("randomBigInt", new ArgumentValidators.NumericTypeAllowed(1,2, true),
- OutputTypeDeterminer.FIXED_BIGINT, "randomBigInt");
- public static final FunctionDefinition RANDOM_FLOAT8 = FunctionDefinition.simpleRandom("randomFloat8", new ArgumentValidators.NumericTypeAllowed(1,2, true),
- OutputTypeDeterminer.FIXED_FLOAT8, "randomFloat8");
- public static final FunctionDefinition INCREASING_BIGINT = FunctionDefinition.simpleRandom("increasingBigInt", new ArgumentValidators.NumericTypeAllowed(1, true),
- OutputTypeDeterminer.FIXED_BIGINT, "increasingBigInt");
-
- public static class Provider implements CallProvider {
-
- @Override
- public FunctionDefinition[] getFunctionDefintions() {
- return new FunctionDefinition[] { RANDOM_BIG_INT,
- RANDOM_FLOAT8,
- INCREASING_BIGINT };
- }
-
- }
- @FunctionTemplate(name = "increasingBigInt", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
+ @FunctionTemplate(name = "increasingBigInt", isRandom = true,
+ scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
public static class IncreasingBigInt implements DrillSimpleFunc {
@Param BigIntHolder start;
@@ -69,7 +49,8 @@ public class GeneratorFunctions {
}
}
- @FunctionTemplate(name = "randomBigInt", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
+ @FunctionTemplate(name = "randomBigInt", isRandom = true,
+ scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
public static class RandomBigIntGauss implements DrillSimpleFunc {
@Param BigIntHolder range;
@@ -83,7 +64,8 @@ public class GeneratorFunctions {
}
}
- @FunctionTemplate(name = "randomBigInt", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
+ @FunctionTemplate(name = "randomBigInt", isRandom = true,
+ scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
public static class RandomBigInt implements DrillSimpleFunc {
@Param BigIntHolder min;
@@ -98,7 +80,8 @@ public class GeneratorFunctions {
}
}
- @FunctionTemplate(name = "randomFloat8", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
+ @FunctionTemplate(name = "randomFloat8", isRandom = true,
+ scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
public static class RandomFloat8Gauss implements DrillSimpleFunc {
@Param BigIntHolder range;
@@ -113,7 +96,8 @@ public class GeneratorFunctions {
}
}
- @FunctionTemplate(name = "randomFloat8", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
+ @FunctionTemplate(name = "randomFloat8", isRandom = true,
+ scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
public static class RandomFloat8 implements DrillSimpleFunc {
@Param BigIntHolder min;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHiveUDFs.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHiveUDFs.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHiveUDFs.java
new file mode 100644
index 0000000..eb9bd0e
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestHiveUDFs.java
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import mockit.Injectable;
+import mockit.NonStrictExpectations;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.memory.TopLevelAllocator;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.PhysicalPlan;
+import org.apache.drill.exec.physical.base.FragmentRoot;
+import org.apache.drill.exec.planner.PhysicalPlanReader;
+import org.apache.drill.exec.proto.BitControl.PlanFragment;
+import org.apache.drill.exec.proto.CoordinationProtos;
+import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.vector.Float4Vector;
+import org.apache.drill.exec.vector.NullableBigIntVector;
+import org.apache.drill.exec.vector.NullableBitVector;
+import org.apache.drill.exec.vector.NullableFloat8Vector;
+import org.apache.drill.exec.vector.NullableIntVector;
+import org.apache.drill.exec.vector.NullableVar16CharVector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.Var16CharVector;
+import org.junit.Test;
+
+import com.codahale.metrics.MetricRegistry;
+import com.google.common.base.Charsets;
+import com.google.common.io.Resources;
+
+import java.util.Iterator;
+
+public class TestHiveUDFs {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestHiveUDFs.class);
+
+ DrillConfig c = DrillConfig.create();
+ PhysicalPlanReader reader;
+ FunctionImplementationRegistry registry;
+ FragmentContext context;
+
+ private void setup(final DrillbitContext bitContext, UserClientConnection connection) throws Throwable {
+ if(reader == null)
+ reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
+ if(registry == null)
+ registry = new FunctionImplementationRegistry(c);
+ if(context == null)
+ context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
+ }
+
+ @Test
+ public void testGenericUDF(@Injectable final DrillbitContext bitContext,
+ @Injectable UserClientConnection connection) throws Throwable {
+ new NonStrictExpectations(){{
+ bitContext.getMetrics(); result = new MetricRegistry();
+ bitContext.getAllocator(); result = new TopLevelAllocator();
+ bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
+ }};
+
+ String planString = Resources.toString(Resources.getResource("functions/hive/GenericUDF.json"), Charsets.UTF_8);
+
+ setup(bitContext, connection);
+ PhysicalPlan plan = reader.readPhysicalPlan(planString);
+ SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
+
+ int numRecords = 0;
+
+ while(exec.next()){
+ // Output columns and types
+ // 1. str1 : Var16Char
+ // 2. upperStr1 : NullableVar16Char
+ // 3. unix_timestamp : NullableBigInt
+ // 4. concat : NullableVarChar
+ // 5. flt1 : Float4
+ // 6. format_number : NullableFloat8
+ // 7. nullableStr1 : NullableVar16Char
+ // 8. upperNullableStr1 : NullableVar16Char
+ Iterator<ValueVector> vv = exec.iterator();
+ Var16CharVector str1V = (Var16CharVector) vv.next();
+ NullableVar16CharVector upperStr1V = (NullableVar16CharVector) vv.next();
+ NullableBigIntVector unix_timestampV = (NullableBigIntVector) vv.next();
+ NullableVar16CharVector concatV = (NullableVar16CharVector) vv.next();
+ Float4Vector flt1V = (Float4Vector) vv.next();
+ NullableVar16CharVector format_numberV = (NullableVar16CharVector) vv.next();
+
+ /* DRILL-425
+ NullableVar16CharVector nullableStr1V = ((NullableVar16CharVector) vv.next());
+ NullableVar16CharVector upperNullableStr1V = ((NullableVar16CharVector) vv.next()); */
+
+ for(int i=0; i<exec.getRecordCount(); i++) {
+
+
+ String in = new String(str1V.getAccessor().get(i));
+ String upper = new String(upperStr1V.getAccessor().get(i));
+ assertTrue(in.toUpperCase().equals(upper));
+
+ long unix_timestamp = unix_timestampV.getAccessor().get(i);
+
+ String concat = new String(concatV.getAccessor().get(i));
+ assertTrue(concat.equals(in+"-"+in));
+
+ float flt1 = flt1V.getAccessor().get(i);
+ String format_number = new String(format_numberV.getAccessor().get(i));
+
+
+ String nullableStr1 = null;
+ /* DRILL-425
+ if (!nullableStr1V.getAccessor().isNull(i))
+ nullableStr1 = new String(nullableStr1V.getAccessor().get(i));*/
+
+ String upperNullableStr1 = null;
+ /* DRILL-425
+ if (!upperNullableStr1V.getAccessor().isNull(i))
+ upperNullableStr1 = new String(upperNullableStr1V.getAccessor().get(i)); */
+
+ assertEquals(nullableStr1 != null, upperNullableStr1 != null);
+ if (nullableStr1 != null)
+ assertEquals(nullableStr1.toUpperCase(), upperNullableStr1);
+
+ System.out.println(in + ", " + upper + ", " + unix_timestamp + ", " + concat + ", " +
+ flt1 + ", " + format_number + ", " + nullableStr1 + ", " + upperNullableStr1);
+
+ numRecords++;
+ }
+ }
+
+ System.out.println("Processed " + numRecords + " records");
+
+ if(context.getFailureCause() != null){
+ throw context.getFailureCause();
+ }
+
+ assertTrue(!context.isFailed());
+ }
+
+ @Test
+ public void testUDF(@Injectable final DrillbitContext bitContext,
+ @Injectable UserClientConnection connection) throws Throwable {
+ new NonStrictExpectations(){{
+ bitContext.getMetrics(); result = new MetricRegistry();
+ bitContext.getAllocator(); result = new TopLevelAllocator();
+ bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
+ }};
+
+ String planString = Resources.toString(Resources.getResource("functions/hive/UDF.json"), Charsets.UTF_8);
+
+ setup(bitContext, connection);
+ PhysicalPlan plan = reader.readPhysicalPlan(planString);
+ SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
+
+ int numRecords = 0;
+ while(exec.next()){
+ // Output columns and types
+ // 1. str1 : Var16Char
+ // 2. str1Length : Int
+ // 3. str1Ascii : Int
+ // 4. flt1 : Float4
+ // 5. pow : Float8
+ Iterator<ValueVector> vv = exec.iterator();
+ Var16CharVector str1V = (Var16CharVector) vv.next();
+ NullableIntVector str1LengthV = (NullableIntVector) vv.next();
+ NullableIntVector str1AsciiV = (NullableIntVector) vv.next();
+ Float4Vector flt1V = (Float4Vector) vv.next();
+ NullableFloat8Vector powV = (NullableFloat8Vector) vv.next();
+
+ for(int i=0; i<exec.getRecordCount(); i++) {
+
+ String str1 = new String(str1V.getAccessor().get(i));
+ int str1Length = str1LengthV.getAccessor().get(i);
+ assertTrue(str1.length() == str1Length);
+
+ int str1Ascii = str1AsciiV.getAccessor().get(i);
+
+ float flt1 = flt1V.getAccessor().get(i);
+
+ double pow = 0;
+ if (!powV.getAccessor().isNull(i)) {
+ pow = powV.getAccessor().get(i);
+ assertTrue(Math.pow(flt1, 2.0) == pow);
+ }
+
+ System.out.println(str1 + ", " + str1Length + ", " + str1Ascii + ", " + flt1 + ", " + pow);
+ numRecords++;
+ }
+ }
+
+ System.out.println("Processed " + numRecords + " records");
+
+ if(context.getFailureCause() != null){
+ throw context.getFailureCause();
+ }
+
+ assertTrue(!context.isFailed());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
index 5bfe369..767b75e 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
@@ -143,15 +143,12 @@ public class TestSimpleFunctions {
args.add(arg);
String[] registeredNames = { "hash" };
FunctionCall call = new FunctionCall(
- FunctionDefinition.simple("hash",
- new BasicArgumentValidator(new Arg[]{ new Arg(expectedArg)}),
- new OutputTypeDeterminer.FixedType(expectedOut),
- registeredNames),
+ "hash",
args,
ExpressionPosition.UNKNOWN
);
FunctionResolver resolver = FunctionResolverFactory.getResolver(call);
- DrillFuncHolder matchedFuncHolder = resolver.getBestMatch(registry.getMethods().get(call.getDefinition().getName()), call);
+ DrillFuncHolder matchedFuncHolder = resolver.getBestMatch(registry.getDrillRegistry().getMethods().get(call.getName()), call);
assertEquals( expectedBestInputMode, matchedFuncHolder.getParmMajorType(0).getMode());
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java
index b3b7536..95bee87 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/ExpressionTreeMaterializerTest.java
@@ -21,22 +21,17 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-import java.util.List;
-
import mockit.Injectable;
import mockit.NonStrictExpectations;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.ArgumentValidator;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.FunctionCall;
-import org.apache.drill.common.expression.FunctionDefinition;
import org.apache.drill.common.expression.IfExpression;
import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.OutputTypeDeterminer;
import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
@@ -207,25 +202,11 @@ public class ExpressionTreeMaterializerTest {
};
- LogicalExpression functionCallExpr = new FunctionCall(FunctionDefinition.simple("testFunc",
- new ArgumentValidator() {
- @Override
- public void validateArguments(ExpressionPosition expr, List<LogicalExpression> expressions,
- ErrorCollector errors) {
- errors.addGeneralError(expr, "Error!");
- }
-
- @Override
- public String[] getArgumentNamesByPosition() {
- return new String[0];
- }
- }, OutputTypeDeterminer.FIXED_BIT), ImmutableList.of((LogicalExpression) //
- new FieldReference("test", ExpressionPosition.UNKNOWN) ), ExpressionPosition.UNKNOWN);
+ LogicalExpression functionCallExpr = new FunctionCall("testFunc",
+ ImmutableList.of((LogicalExpression) new FieldReference("test", ExpressionPosition.UNKNOWN) ),
+ ExpressionPosition.UNKNOWN);
LogicalExpression newExpr = ExpressionTreeMaterializer.materialize(functionCallExpr, batch, ec, registry);
- assertTrue(newExpr instanceof FunctionCall);
- FunctionCall funcExpr = (FunctionCall) newExpr;
- assertEquals(1, funcExpr.args.size());
- assertEquals(bigIntType, funcExpr.args.get(0).getMajorType());
+ assertEquals(newExpr, null);
assertEquals(1, ec.getErrorCount());
System.out.println(ec.toErrorString());
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/test/resources/functions/hive/GenericUDF.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/functions/hive/GenericUDF.json b/exec/java-exec/src/test/resources/functions/hive/GenericUDF.json
new file mode 100644
index 0000000..7b82570
--- /dev/null
+++ b/exec/java-exec/src/test/resources/functions/hive/GenericUDF.json
@@ -0,0 +1,45 @@
+{
+ head:{
+ type:"APACHE_DRILL_PHYSICAL",
+ version:"1",
+ generator:{
+ type:"manual"
+ }
+ },
+ graph:[
+ {
+ @id:1,
+ pop:"mock-sub-scan",
+ url: "http://apache.org",
+ entries:[
+ {records: 100, types: [
+ {name: "str1", type: "VAR16CHAR", mode: "REQUIRED"},
+ {name: "str2", type: "VAR16CHAR", mode: "REQUIRED"},
+ {name: "str3", type: "VAR16CHAR", mode: "REQUIRED"},
+ /* DRILL-425 {name: "nullableStr1", type: "VAR16CHAR", mode: "OPTIONAL"}, */
+ {name: "flt1", type: "FLOAT4", mode: "REQUIRED"}
+ ]}
+ ]
+ },
+ {
+ @id:2,
+ child: 1,
+ pop:"project",
+ exprs: [
+ { ref: "str1", expr: "str1" },
+ { ref: "upperStr1", expr: "ucase(str1)" },
+ { ref: "unix_timestamp", expr: "unix_timestamp()" },
+ { ref: "concat", expr: "concat_ws('-', str2, str3)" },
+ { ref: "flt1", expr: "flt1" },
+ { ref: "format_number", expr: "format_number(cast(flt1 as float8), cast(2 as int))" } /* DRILL-425,
+ { ref: "nullableStr1", expr: "nullableStr1" },
+ { ref: "upperNulableStr1", expr: "upper(nullableStr1)" } */
+ ]
+ },
+ {
+ @id: 3,
+ child: 2,
+ pop: "screen"
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/test/resources/functions/hive/UDF.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/functions/hive/UDF.json b/exec/java-exec/src/test/resources/functions/hive/UDF.json
new file mode 100644
index 0000000..cd4c7b1
--- /dev/null
+++ b/exec/java-exec/src/test/resources/functions/hive/UDF.json
@@ -0,0 +1,39 @@
+{
+ head:{
+ type:"APACHE_DRILL_PHYSICAL",
+ version:"1",
+ generator:{
+ type:"manual"
+ }
+ },
+ graph:[
+ {
+ @id:1,
+ pop:"mock-sub-scan",
+ url: "http://apache.org",
+ entries:[
+ {records: 100, types: [
+ {name: "str1", type: "VAR16CHAR", mode: "REQUIRED"},
+ {name: "flt1", type: "FLOAT4", mode: "REQUIRED"}
+ ]}
+ ]
+ },
+ {
+ @id:2,
+ child: 1,
+ pop:"project",
+ exprs: [
+ { ref: "str1", expr: "str1" },
+ { ref: "str1Length", expr: "length(str1)" },
+ { ref: "str1Ascii", expr: "ascii(str1)" },
+ { ref: "flt1", expr: "flt1" },
+ { ref: "pow", expr: "pow(cast(flt1 as float8), 2.0)" }
+ ]
+ },
+ {
+ @id: 3,
+ child: 2,
+ pop: "screen"
+ }
+ ]
+}
[03/12] DRILL-387: Support using Hive simple UDFs in Drill * As part
of this change FunctionDefinition (and related code) * are deleted,
instead the same information available in * Function Holders are used whenever
required * Freemarker/CodeModel codege
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java b/common/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java
index 71266f7..8d29719 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java
@@ -18,6 +18,7 @@
package org.apache.drill.common.expression.visitors;
import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.IfExpression;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
@@ -35,6 +36,11 @@ public abstract class SimpleExprVisitor<T> implements ExprVisitor<T, Void, Runti
}
@Override
+ public T visitFunctionHolderExpression(FunctionHolderExpression holder, Void value) throws RuntimeException {
+ return visitFunctionHolderExpression(holder);
+ }
+
+ @Override
public T visitIfExpression(IfExpression ifExpr, Void value) throws RuntimeException {
return visitIfExpression(ifExpr);
}
@@ -76,6 +82,7 @@ public abstract class SimpleExprVisitor<T> implements ExprVisitor<T, Void, Runti
public abstract T visitFunctionCall(FunctionCall call);
+ public abstract T visitFunctionHolderExpression(FunctionHolderExpression call);
public abstract T visitIfExpression(IfExpression ifExpr);
public abstract T visitSchemaPath(SchemaPath path);
public abstract T visitIntConstant(IntExpression intExpr);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java b/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
index 6833b69..f49b042 100644
--- a/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
+++ b/common/src/test/java/org/apache/drill/common/expression/parser/TreeTest.java
@@ -24,8 +24,6 @@ import org.antlr.runtime.CommonTokenStream;
import org.antlr.runtime.tree.CommonTree;
import org.antlr.runtime.tree.DOTTreeGenerator;
import org.antlr.stringtemplate.StringTemplate;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.FunctionRegistry;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.parser.ExprParser.parse_return;
@@ -47,7 +45,6 @@ public class TreeTest {
CommonTokenStream tokens = new CommonTokenStream(lexer);
ExprParser parser = new ExprParser(tokens);
- parser.setRegistry(new FunctionRegistry(DrillConfig.create()));
parse_return ret = parser.parse();
LogicalExpression e = ret.e;
ObjectMapper mapper = new ObjectMapper();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/contrib/storage-hive/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/src/main/resources/drill-module.conf b/contrib/storage-hive/src/main/resources/drill-module.conf
new file mode 100644
index 0000000..271395e
--- /dev/null
+++ b/contrib/storage-hive/src/main/resources/drill-module.conf
@@ -0,0 +1,5 @@
+// This file tells Drill to consider this module when class path scanning.
+// This file can also include any supplementary configuration information.
+// This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information.
+
+drill.logical.function.packages += "org.apache.hadoop.hive"
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/codegen/config.fmpp
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/config.fmpp b/exec/java-exec/src/main/codegen/config.fmpp
index fcf1f4d..f2e2c69 100644
--- a/exec/java-exec/src/main/codegen/config.fmpp
+++ b/exec/java-exec/src/main/codegen/config.fmpp
@@ -20,6 +20,7 @@ data: {
cast: tdd(../data/Casts.tdd),
MathFunctionTypes: tdd(../data/MathFunctionTypes.tdd)
mathFunc:tdd(../data/MathFunc.tdd)
+ drillOI:tdd(../data/HiveTypes.tdd)
}
freemarkerLinks: {
includes: includes/
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/codegen/data/HiveTypes.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/HiveTypes.tdd b/exec/java-exec/src/main/codegen/data/HiveTypes.tdd
new file mode 100644
index 0000000..ddc8041
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/data/HiveTypes.tdd
@@ -0,0 +1,92 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+ map: [
+ {
+ hiveType: "BOOLEAN",
+ hiveOI: "BooleanObjectInspector",
+ serdeConstant: "BOOLEAN_TYPE_NAME",
+ javaType: "boolean",
+ minorType: "BIT",
+ holder: "Bit"
+ },
+ {
+ hiveType: "BYTE",
+ hiveOI: "ByteObjectInspector",
+ serdeConstant: "TINYINT_TYPE_NAME",
+ javaType: "byte",
+ minorType: "TINYINT",
+ holder: "TinyInt"
+ },
+ {
+ hiveType: "SHORT",
+ hiveOI: "ShortObjectInspector",
+ serdeConstant: "SMALLINT_TYPE_NAME",
+ javaType: "short",
+ minorType: "SMALLINT",
+ holder: "SmallInt"
+ },
+ {
+ hiveType: "INT",
+ hiveOI: "IntObjectInspector",
+ serdeConstant: "INT_TYPE_NAME",
+ javaType: "int",
+ minorType: "INT",
+ holder: "Int"
+ },
+ {
+ hiveType: "LONG",
+ hiveOI: "LongObjectInspector",
+ serdeConstant: "BIGINT_TYPE_NAME",
+ javaType: "long",
+ minorType: "BIGINT",
+ holder: "BigInt"
+ },
+ {
+ hiveType: "FLOAT",
+ hiveOI: "FloatObjectInspector",
+ serdeConstant: "FLOAT_TYPE_NAME",
+ javaType: "float",
+ minorType: "FLOAT4",
+ holder: "Float4"
+ },
+ {
+ hiveType: "DOUBLE",
+ hiveOI: "DoubleObjectInspector",
+ serdeConstant: "DOUBLE_TYPE_NAME",
+ javaType: "double",
+ minorType: "FLOAT8",
+ holder: "Float8"
+ },
+ {
+ hiveType: "VARCHAR",
+ hiveOI: "HiveVarcharObjectInspector",
+ serdeConstant: "VARCHAR_TYPE_NAME",
+ javaType: "",
+ minorType: "VARCHAR",
+ holder: "VarChar"
+ },
+ {
+ hiveType: "STRING",
+ hiveOI: "StringObjectInspector",
+ serdeConstant: "STRING_TYPE_NAME",
+ javaType: "",
+ minorType: "VAR16CHAR",
+ holder: "Var16Char"
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java b/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
index ba9da07..f5ccc6f 100644
--- a/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/ComparisonFunctions.java
@@ -118,7 +118,7 @@ public class GCompare${left}${right}{
}
}
- @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ @FunctionTemplate(names = {"less than", "<"}, scope = FunctionTemplate.FunctionScope.SIMPLE)
public static class LessThan${left}${right} implements DrillSimpleFunc {
@Param ${left}Holder left;
@@ -149,7 +149,7 @@ public class GCompare${left}${right}{
}
}
- @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ @FunctionTemplate(names = {"less than or equal to", "<="}, scope = FunctionTemplate.FunctionScope.SIMPLE)
public static class LessThanE${left}${right} implements DrillSimpleFunc {
@Param ${left}Holder left;
@@ -180,7 +180,7 @@ public class GCompare${left}${right}{
}
}
- @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ @FunctionTemplate(names = {"greater than", ">"}, scope = FunctionTemplate.FunctionScope.SIMPLE)
public static class GreaterThan${left}${right} implements DrillSimpleFunc {
@Param ${left}Holder left;
@@ -211,7 +211,7 @@ public class GCompare${left}${right}{
}
}
- @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ @FunctionTemplate(names = {"greater than or equal to", ">="}, scope = FunctionTemplate.FunctionScope.SIMPLE)
public static class GreaterThanE${left}${right} implements DrillSimpleFunc {
@Param ${left}Holder left;
@@ -242,7 +242,7 @@ public class GCompare${left}${right}{
}
}
- @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ @FunctionTemplate(names = {"equal","==","="}, scope = FunctionTemplate.FunctionScope.SIMPLE)
public static class Equals${left}${right} implements DrillSimpleFunc {
@Param ${left}Holder left;
@@ -273,7 +273,7 @@ public class GCompare${left}${right}{
}
}
- @FunctionTemplate(name = "not equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ @FunctionTemplate(names = {"not equal","<>","!="}, scope = FunctionTemplate.FunctionScope.SIMPLE)
public static class NotEquals${left}${right} implements DrillSimpleFunc {
@Param ${left}Holder left;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/codegen/templates/MathFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/MathFunctions.java b/exec/java-exec/src/main/codegen/templates/MathFunctions.java
index c683261..66373f5 100644
--- a/exec/java-exec/src/main/codegen/templates/MathFunctions.java
+++ b/exec/java-exec/src/main/codegen/templates/MathFunctions.java
@@ -29,13 +29,6 @@
package org.apache.drill.exec.expr.fn.impl;
-import org.apache.drill.common.expression.Arg;
-import org.apache.drill.common.expression.ArgumentValidators;
-import org.apache.drill.common.expression.BasicArgumentValidator;
-import org.apache.drill.common.expression.CallProvider;
-import org.apache.drill.common.expression.FunctionDefinition;
-import org.apache.drill.common.expression.OutputTypeDeterminer;
-import org.apache.drill.common.expression.ArgumentValidators.AllowedTypeList;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
import org.apache.drill.common.types.TypeProtos.MinorType;
@@ -79,21 +72,6 @@ public class GMathFunctions{
</#list>
</#list>
-
- /*
- * Function Definitions
- */
- public static class UnaryMathFuncProvider implements CallProvider {
-
- @Override
- public FunctionDefinition[] getFunctionDefintions() {
- return new FunctionDefinition[] {
- <#list mathFunc.unaryMathFunctions as func>
- FunctionDefinition.simple("${func.funcName}", new ArgumentValidators.NumericTypeAllowed(1, 2, false), new OutputTypeDeterminer.SameAsAnySoft()),
- </#list>
- };
- }
- }
//////////////////////////////////////////////////////////////////////////////////////////////////
//Math functions which take two arguments (of same type).
@@ -118,21 +96,4 @@ public class GMathFunctions{
}
</#list>
</#list>
-
- /*
- * Function Definitions
- */
- public static class BinaryMathFuncProvider implements CallProvider {
-
- @Override
- public FunctionDefinition[] getFunctionDefintions() {
- return new FunctionDefinition[] {
- <#list mathFunc.binaryMathFunctions as func>
- FunctionDefinition.simple("${func.funcName}", new ArgumentValidators.NumericTypeAllowed(1, 3, false), new OutputTypeDeterminer.SameAsAnySoft()),
- </#list>
- };
- }
- }
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/codegen/templates/ObjectInspectorHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/ObjectInspectorHelper.java b/exec/java-exec/src/main/codegen/templates/ObjectInspectorHelper.java
new file mode 100644
index 0000000..61078c7
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/templates/ObjectInspectorHelper.java
@@ -0,0 +1,182 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/hive/ObjectInspectorHelper.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.expr.fn.impl.hive;
+
+import com.sun.codemodel.*;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.expr.holders.*;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.*;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class ObjectInspectorHelper {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ObjectInspectorHelper.class);
+
+ private static Map<MinorType, Class> OIMAP = new HashMap<>();
+ static {
+<#list drillOI.map as entry>
+ OIMAP.put(MinorType.${entry.minorType}, Drill${entry.holder}ObjectInspector.class);
+</#list>
+ }
+
+ public static ObjectInspector getDrillObjectInspector(MinorType drillType) {
+ if (OIMAP.containsKey(drillType)) {
+ try {
+ return (ObjectInspector)OIMAP.get(drillType).newInstance();
+ } catch(InstantiationException | IllegalAccessException e) {
+ throw new RuntimeException("Failed to instantiate ObjectInspector", e);
+ }
+ }
+
+ throw new UnsupportedOperationException(drillType.toString());
+ }
+
+ public static JBlock initReturnValueHolder(JCodeModel m, JVar returnValueHolder, ObjectInspector oi, MinorType returnType) {
+ JBlock block = new JBlock(false, false);
+ switch(oi.getCategory()) {
+ case PRIMITIVE: {
+ PrimitiveObjectInspector poi = (PrimitiveObjectInspector)oi;
+ switch(poi.getPrimitiveCategory()) {
+<#list drillOI.map as entry>
+ case ${entry.hiveType}:{
+ JType holderClass = TypeHelper.getHolderType(m, returnType, TypeProtos.DataMode.OPTIONAL);
+ block.assign(returnValueHolder, JExpr._new(holderClass));
+
+ <#if entry.hiveType == "VARCHAR" || entry.hiveType == "STRING">
+ block.assign(returnValueHolder.ref("buffer"),
+ m.directClass(io.netty.buffer.Unpooled.class.getCanonicalName())
+ .staticInvoke("wrappedBuffer")
+ .arg(JExpr.newArray(m._ref(byte.class), JExpr.lit(1000))));
+ </#if>
+ return block;
+ }
+</#list>
+ default:
+ throw new UnsupportedOperationException(String.format("Received unknown/unsupported type '%s'", poi.getPrimitiveCategory().toString()));
+ }
+ }
+
+ case MAP:
+ case LIST:
+ case STRUCT:
+ default:
+ throw new UnsupportedOperationException(String.format("Received unknown/unsupported type '%s'", oi.getCategory().toString()));
+ }
+ }
+
+ private static Map<PrimitiveCategory, MinorType> TYPE_HIVE2DRILL = new HashMap<>();
+ static {
+<#list drillOI.map as entry>
+ TYPE_HIVE2DRILL.put(PrimitiveCategory.${entry.hiveType}, MinorType.${entry.minorType});
+</#list>
+ }
+
+ public static MinorType getDrillType(ObjectInspector oi) {
+ switch(oi.getCategory()) {
+ case PRIMITIVE: {
+ PrimitiveObjectInspector poi = (PrimitiveObjectInspector)oi;
+ if (TYPE_HIVE2DRILL.containsKey(poi.getPrimitiveCategory())) {
+ return TYPE_HIVE2DRILL.get(poi.getPrimitiveCategory());
+ }
+ throw new UnsupportedOperationException();
+ }
+
+ case MAP:
+ case LIST:
+ case STRUCT:
+ default:
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ public static JBlock getDrillObject(JCodeModel m, ObjectInspector oi,
+ JVar returnOI, JVar returnValueHolder, JVar returnValue) {
+ JBlock block = new JBlock(false, false);
+ switch(oi.getCategory()) {
+ case PRIMITIVE: {
+ PrimitiveObjectInspector poi = (PrimitiveObjectInspector)oi;
+ switch(poi.getPrimitiveCategory()) {
+<#list drillOI.map as entry>
+ case ${entry.hiveType}:{
+ JConditional jc = block._if(returnValue.eq(JExpr._null()));
+ jc._then().assign(returnValueHolder.ref("isSet"), JExpr.lit(0));
+ jc._else().assign(returnValueHolder.ref("isSet"), JExpr.lit(1));
+ JVar castedOI = jc._else().decl(
+ m.directClass(${entry.hiveOI}.class.getCanonicalName()), "castOI", JExpr._null());
+ jc._else().assign(castedOI,
+ JExpr.cast(m.directClass(${entry.hiveOI}.class.getCanonicalName()), returnOI));
+
+ <#if entry.hiveType == "BOOLEAN">
+ JConditional booleanJC = jc._else()._if(castedOI.invoke("get").arg(returnValue));
+ booleanJC._then().assign(returnValueHolder.ref("value"), JExpr.lit(1));
+ booleanJC._else().assign(returnValueHolder.ref("value"), JExpr.lit(0));
+
+ <#elseif entry.hiveType == "VARCHAR">
+ JVar data = jc._else().decl(m.directClass(byte[].class.getCanonicalName()), "data",
+ castedOI.invoke("getPrimitiveJavaObject").arg(returnValue)
+ .invoke("getValue")
+ .invoke("getBytes"));
+
+ jc._else().add(returnValueHolder.ref("buffer")
+ .invoke("setBytes").arg(JExpr.lit(0)).arg(data));
+
+
+ jc._else().assign(returnValueHolder.ref("start"), JExpr.lit(0));
+ jc._else().assign(returnValueHolder.ref("end"), data.ref("length"));
+
+ <#elseif entry.hiveType == "STRING">
+ JVar data = jc._else().decl(m.directClass(byte[].class.getCanonicalName()), "data",
+ castedOI.invoke("getPrimitiveJavaObject").arg(returnValue)
+ .invoke("getBytes"));
+ jc._else().add(returnValueHolder.ref("buffer")
+ .invoke("setBytes").arg(JExpr.lit(0)).arg(data));
+ jc._else().assign(returnValueHolder.ref("start"), JExpr.lit(0));
+ jc._else().assign(returnValueHolder.ref("end"), data.ref("length"));
+
+ <#else>
+ jc._else().assign(returnValueHolder.ref("value"),
+ castedOI.invoke("get").arg(returnValue));
+ </#if>
+ return block;
+ }
+
+</#list>
+ default:
+ throw new UnsupportedOperationException(String.format("Received unknown/unsupported type '%s'", poi.getPrimitiveCategory().toString()));
+ }
+ }
+
+ case MAP:
+ case LIST:
+ case STRUCT:
+ default:
+ throw new UnsupportedOperationException(String.format("Received unknown/unsupported type '%s'", oi.getCategory().toString()));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/codegen/templates/ObjectInspectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/ObjectInspectors.java b/exec/java-exec/src/main/codegen/templates/ObjectInspectors.java
new file mode 100644
index 0000000..57a7bda
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/templates/ObjectInspectors.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+<#list drillOI.map as entry>
+
+
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/hive/Drill${entry.holder}ObjectInspector.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.drill.exec.expr.fn.impl.hive;
+
+import org.apache.drill.exec.expr.holders.*;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.*;
+import org.apache.hadoop.io.Text;
+
+public class Drill${entry.holder}ObjectInspector extends AbstractPrimitiveObjectInspector
+ implements ${entry.hiveOI} {
+
+ @Override
+ public String getTypeName() {
+ return serdeConstants.${entry.serdeConstant};
+ }
+
+<#if entry.minorType == "VARCHAR">
+ @Override
+ public HiveVarcharWritable getPrimitiveWritableObject(Object o) {
+ HiveVarcharWritable valW = new HiveVarcharWritable();
+ valW.set(getPrimitiveJavaObject(o));
+ return valW;
+ }
+
+ @Override
+ public HiveVarchar getPrimitiveJavaObject(Object o) {
+ String val = ((VarCharHolder)o).toString();
+ return new HiveVarchar(val, HiveVarchar.MAX_VARCHAR_LENGTH);
+ }
+<#elseif entry.minorType == "VAR16CHAR">
+@Override
+ public Text getPrimitiveWritableObject(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String getPrimitiveJavaObject(Object o) {
+ if (o instanceof Var16CharHolder)
+ return ((Var16CharHolder)o).toString();
+ else
+ return ((NullableVar16CharHolder)o).toString();
+ }
+<#elseif entry.minorType == "BIT">
+ @Override
+ public boolean get(Object o) {
+ if (o instanceof BitHolder)
+ return ((BitHolder)o).value == 0 ? false : true;
+ else
+ return ((NullableBitHolder)o).value == 0 ? false : true;
+ }
+<#else>
+ @Override
+ public ${entry.javaType} get(Object o) {
+ if (o instanceof ${entry.holder}Holder)
+ return ((${entry.holder}Holder)o).value;
+ else
+ return ((Nullable${entry.holder}Holder)o).value;
+ }
+</#if>
+
+ @Override
+ public PrimitiveCategory getPrimitiveCategory() {
+ return PrimitiveCategory.${entry.hiveType};
+ }
+}
+
+</#list>
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
index 83a5e6f..fbb7cc8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Set;
import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.IfExpression;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
@@ -86,9 +87,14 @@ public class ConstantExpressionIdentifier implements ExprVisitor<Boolean, Identi
@Override
public Boolean visitFunctionCall(FunctionCall call, IdentityHashMap<LogicalExpression, Object> value){
- return checkChildren(call, value, !call.getDefinition().isAggregating() && !call.getDefinition().isRandom());
+ throw new UnsupportedOperationException("FunctionCall is not expected here. "+
+ "It should have been converted to FunctionHolderExpression in materialization");
}
+ @Override
+ public Boolean visitFunctionHolderExpression(FunctionHolderExpression holder, IdentityHashMap<LogicalExpression, Object> value) throws RuntimeException {
+ return checkChildren(holder, value, !holder.isAggregating() && !holder.isRandom());
+ }
@Override
public Boolean visitIfExpression(IfExpression ifExpr, IdentityHashMap<LogicalExpression, Object> value){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFuncHolderExpr.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFuncHolderExpr.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFuncHolderExpr.java
new file mode 100644
index 0000000..e6370d8
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/DrillFuncHolderExpr.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr;
+
+import java.util.Iterator;
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FunctionHolderExpression;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.visitors.ExprVisitor;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+
+import org.apache.drill.exec.expr.fn.DrillFuncHolder;
+
+public class DrillFuncHolderExpr extends FunctionHolderExpression implements Iterable<LogicalExpression>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillFuncHolderExpr.class);
+ private DrillFuncHolder holder;
+
+ public DrillFuncHolderExpr(String nameUsed, DrillFuncHolder holder, List<LogicalExpression> args, ExpressionPosition pos) {
+ super(nameUsed, pos, args);
+ this.holder = holder;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return holder.getReturnType(args);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return args.iterator();
+ }
+
+ public DrillFuncHolder getHolder() {
+ return holder;
+ }
+
+ @Override
+ public boolean isAggregating() {
+ return holder.isAggregating();
+ }
+
+ @Override
+ public boolean isRandom() {
+ return holder.isRandom();
+ }
+
+ public boolean argConstantOnly(int i) {
+ return holder.isConstant(i);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
index cac47b3..c7f7336 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
@@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.Set;
import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.IfExpression;
import org.apache.drill.common.expression.IfExpression.IfCondition;
import org.apache.drill.common.expression.LogicalExpression;
@@ -40,6 +41,7 @@ import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
import org.apache.drill.exec.compile.sig.ConstantExpressionIdentifier;
import org.apache.drill.exec.expr.fn.DrillFuncHolder;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.expr.fn.HiveFuncHolder;
import org.apache.drill.exec.physical.impl.filter.ReturnValueExpression;
import org.apache.drill.exec.record.NullExpression;
import org.apache.drill.exec.vector.ValueHolderHelper;
@@ -56,7 +58,7 @@ import com.sun.codemodel.JVar;
public class EvaluationVisitor {
private final FunctionImplementationRegistry registry;
-
+
public EvaluationVisitor(FunctionImplementationRegistry registry) {
super();
this.registry = registry;
@@ -74,18 +76,47 @@ public class EvaluationVisitor {
@Override
public HoldingContainer visitFunctionCall(FunctionCall call, ClassGenerator<?> generator) throws RuntimeException {
- DrillFuncHolder holder = registry.getFunction(call);
- JVar[] workspaceVars = holder.renderStart(generator, null);
+ throw new UnsupportedOperationException("FunctionCall is not expected here. "+
+ "It should have been converted to FunctionHolderExpression in materialization");
+ }
-
- if(holder.isNested()) generator.getMappingSet().enterChild();
- HoldingContainer[] args = new HoldingContainer[call.args.size()];
- for (int i = 0; i < call.args.size(); i++) {
- args[i] = call.args.get(i).accept(this, generator);
+ @Override
+ public HoldingContainer visitFunctionHolderExpression(
+ FunctionHolderExpression holderExpr, ClassGenerator<?> generator) throws RuntimeException {
+ // TODO: hack: (Drill/Hive)FuncHolderExpr reference classes in exec so
+ // code generate methods can't be superclass FunctionHolderExpression
+ // which is defined in common
+
+ if (holderExpr instanceof DrillFuncHolderExpr) {
+ DrillFuncHolder holder = ((DrillFuncHolderExpr)holderExpr).getHolder();
+ JVar[] workspaceVars = holder.renderStart(generator, null);
+
+ if(holder.isNested()) generator.getMappingSet().enterChild();
+
+ HoldingContainer[] args = new HoldingContainer[holderExpr.args.size()];
+ for (int i = 0; i < holderExpr.args.size(); i++) {
+ args[i] = holderExpr.args.get(i).accept(this, generator);
+ }
+
+ holder.renderMiddle(generator, args, workspaceVars);
+
+ if(holder.isNested()) generator.getMappingSet().exitChild();
+
+ return holder.renderEnd(generator, args, workspaceVars);
+
+ } else if (holderExpr instanceof HiveFuncHolderExpr) {
+
+ HiveFuncHolder holder = ((HiveFuncHolderExpr)holderExpr).getHolder();
+
+ HoldingContainer[] args = new HoldingContainer[holderExpr.args.size()];
+ for (int i = 0; i < holderExpr.args.size(); i++) {
+ args[i] = holderExpr.args.get(i).accept(this, generator);
+ }
+
+ return holder.renderEnd(generator, args, holder.renderStart(generator, null));
}
- holder.renderMiddle(generator, args, workspaceVars);
- if(holder.isNested()) generator.getMappingSet().exitChild();
- return holder.renderEnd(generator, args, workspaceVars);
+
+ throw new UnsupportedOperationException(String.format("Unknown expression '%s'", holderExpr.getClass().getCanonicalName()));
}
@Override
@@ -297,19 +328,25 @@ public class EvaluationVisitor {
super();
this.constantBoundaries = constantBoundaries;
}
-
+
@Override
public HoldingContainer visitFunctionCall(FunctionCall e, ClassGenerator<?> generator) throws RuntimeException {
+ throw new UnsupportedOperationException("FunctionCall is not expected here. "+
+ "It should have been converted to FunctionHolderExpression in materialization");
+ }
+
+ @Override
+ public HoldingContainer visitFunctionHolderExpression(FunctionHolderExpression e, ClassGenerator<?> generator) throws RuntimeException {
if (constantBoundaries.contains(e)) {
generator.getMappingSet().enterConstant();
- HoldingContainer c = super.visitFunctionCall(e, generator);
+ HoldingContainer c = super.visitFunctionHolderExpression(e, generator);
//generator.getMappingSet().exitConstant();
//return c;
return renderConstantExpression(generator, c);
} else if (generator.getMappingSet().isWithinConstant()) {
- return super.visitFunctionCall(e, generator).setConstant(true);
+ return super.visitFunctionHolderExpression(e, generator).setConstant(true);
} else {
- return super.visitFunctionCall(e, generator);
+ return super.visitFunctionHolderExpression(e, generator);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
index c0f34f4..82f45c0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
@@ -20,19 +20,34 @@ package org.apache.drill.exec.expr;
import java.util.List;
import org.apache.drill.common.expression.ErrorCollector;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.ExpressionValidator;
import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.IfExpression;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.expression.TypedNullConstant;
-import org.apache.drill.common.expression.ValueExpressions;
-import org.apache.drill.common.expression.visitors.SimpleExprVisitor;
+import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
+import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
+import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
+import org.apache.drill.common.expression.ValueExpressions.LongExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
+import org.apache.drill.common.expression.ValueExpressions.QuotedString;
+import org.apache.drill.common.expression.fn.CastFunctions;
+import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
+import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.fn.DrillFuncHolder;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.expr.fn.HiveFuncHolder;
import org.apache.drill.exec.record.NullExpression;
import org.apache.drill.exec.record.TypedFieldId;
import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.resolver.FunctionResolver;
+import org.apache.drill.exec.resolver.FunctionResolverFactory;
import com.google.common.collect.Lists;
@@ -40,14 +55,11 @@ public class ExpressionTreeMaterializer {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpressionTreeMaterializer.class);
-
-
private ExpressionTreeMaterializer() {
};
public static LogicalExpression materialize(LogicalExpression expr, VectorAccessible batch, ErrorCollector errorCollector, FunctionImplementationRegistry registry) {
- LogicalExpression materializedExpr = expr.accept(new MaterializeVisitor(batch, errorCollector), null);
- LogicalExpression out = ImplicitCastBuilder.injectImplicitCast(materializedExpr, errorCollector, registry);
+ LogicalExpression out = expr.accept(new MaterializeVisitor(batch, errorCollector), registry);
if(out instanceof NullExpression){
return new TypedNullConstant(Types.optional(MinorType.INT));
}else{
@@ -55,47 +67,142 @@ public class ExpressionTreeMaterializer {
}
}
- private static class MaterializeVisitor extends SimpleExprVisitor<LogicalExpression> {
+ private static class MaterializeVisitor extends AbstractExprVisitor<LogicalExpression, FunctionImplementationRegistry, RuntimeException> {
+ private ExpressionValidator validator = new ExpressionValidator();
+ private final ErrorCollector errorCollector;
private final VectorAccessible batch;
public MaterializeVisitor(VectorAccessible batch, ErrorCollector errorCollector) {
this.batch = batch;
+ this.errorCollector = errorCollector;
+ }
+
+ private LogicalExpression validateNewExpr(LogicalExpression newExpr) {
+ newExpr.accept(validator, errorCollector);
+ return newExpr;
+ }
+
+ @Override
+ public LogicalExpression visitUnknown(LogicalExpression e, FunctionImplementationRegistry registry)
+ throws RuntimeException {
+ return e;
}
@Override
- public LogicalExpression visitUnknown(LogicalExpression e, Void value) throws RuntimeException {
- throw new UnsupportedOperationException(String.format("Expression tree materializer does not currently support materializing nodes of type %s.", e.getClass().getCanonicalName()));
+ public LogicalExpression visitFunctionHolderExpression(FunctionHolderExpression holder, FunctionImplementationRegistry value) throws RuntimeException {
+ throw new UnsupportedOperationException(
+ String.format("Can't support materialize %s", holder.getClass().getCanonicalName()));
}
@Override
- public LogicalExpression visitFunctionCall(FunctionCall call) {
+ public LogicalExpression visitFunctionCall(FunctionCall call, FunctionImplementationRegistry registry) {
List<LogicalExpression> args = Lists.newArrayList();
for (int i = 0; i < call.args.size(); ++i) {
- LogicalExpression newExpr = call.args.get(i).accept(this, null);
+ LogicalExpression newExpr = call.args.get(i).accept(this, registry);
args.add(newExpr);
}
- return new FunctionCall(call.getDefinition(), args, call.getPosition());
+ //replace with a new function call, since its argument could be changed.
+ call = new FunctionCall(call.getName(), args, call.getPosition());
+
+ // First try to resolve as Drill function, if that fails try resolving as hive function
+ // TODO: Need to refactor the resolver to have generic interface for Drill and Hive functions
+
+ FunctionResolver resolver = FunctionResolverFactory.getResolver(call);
+ DrillFuncHolder matchedFuncHolder =
+ resolver.getBestMatch(registry.getDrillRegistry().getMethods().get(call.getName()), call);
+
+ //new arg lists, possible with implicit cast inserted.
+ List<LogicalExpression> argsWithCast = Lists.newArrayList();
+
+ if (matchedFuncHolder!=null) {
+ //Compare parm type against arg type. Insert cast on top of arg, whenever necessary.
+ for (int i = 0; i < call.args.size(); ++i) {
+ TypeProtos.MajorType parmType = matchedFuncHolder.getParmMajorType(i);
+
+ //Case 1: If 1) the argument is NullExpression
+ // 2) the parameter of matchedFuncHolder allows null input, or func's null_handling is NULL_IF_NULL (means null and non-null are exchangable).
+ // then replace NullExpression with a TypedNullConstant
+ if (call.args.get(i).equals(NullExpression.INSTANCE) &&
+ ( parmType.getMode().equals(TypeProtos.DataMode.OPTIONAL) ||
+ matchedFuncHolder.getNullHandling() == FunctionTemplate.NullHandling.NULL_IF_NULL)) {
+ argsWithCast.add(new TypedNullConstant(parmType));
+ } else if (Types.softEquals(parmType, call.args.get(i).getMajorType(), matchedFuncHolder.getNullHandling() ==
+ FunctionTemplate.NullHandling.NULL_IF_NULL)) {
+ //Case 2: argument and parameter matches. Do nothing.
+ argsWithCast.add(call.args.get(i));
+ } else {
+ //Case 3: insert cast if param type is different from arg type.
+ String castFuncName = CastFunctions.getCastFunc(parmType.getMinorType());
+ List<LogicalExpression> castArgs = Lists.newArrayList();
+ castArgs.add(call.args.get(i)); //input_expr
+ FunctionCall castCall = new FunctionCall(castFuncName, castArgs, ExpressionPosition.UNKNOWN);
+ DrillFuncHolder matchedCastFuncHolder = resolver.getBestMatch(
+ registry.getDrillRegistry().getMethods().get(castFuncName), castCall);
+
+ if (matchedCastFuncHolder == null) {
+ logFunctionResolutionError(errorCollector, castCall);
+ return null;
+ }
+
+ argsWithCast.add(new DrillFuncHolderExpr(call.getName(), matchedCastFuncHolder, castArgs, ExpressionPosition.UNKNOWN));
+ }
+ }
+ return new DrillFuncHolderExpr(call.getName(), matchedFuncHolder, argsWithCast, call.getPosition());
+ }
+
+ // as no drill func is found, search for the function in hive
+ HiveFuncHolder matchedHiveHolder = registry.getHiveRegistry().getFunction(call);
+ if (matchedHiveHolder != null)
+ return new HiveFuncHolderExpr(call.getName(), matchedHiveHolder, call.args, call.getPosition());
+
+ logFunctionResolutionError(errorCollector, call);
+ return null;
+ }
+
+ private void logFunctionResolutionError(ErrorCollector errorCollector, FunctionCall call) {
+ // add error to collector
+ StringBuilder sb = new StringBuilder();
+ sb.append("Missing function implementation: ");
+ sb.append("[");
+ sb.append(call.getName());
+ sb.append("(");
+ boolean first = true;
+ for(LogicalExpression e : call.args) {
+ TypeProtos.MajorType mt = e.getMajorType();
+ if(first){
+ first = false;
+ } else {
+ sb.append(", ");
+ }
+ sb.append(mt.getMinorType().name());
+ sb.append("-");
+ sb.append(mt.getMode().name());
+ }
+ sb.append(")");
+ sb.append("]");
+
+ errorCollector.addGeneralError(call.getPosition(), sb.toString());
}
@Override
- public LogicalExpression visitIfExpression(IfExpression ifExpr) {
+ public LogicalExpression visitIfExpression(IfExpression ifExpr, FunctionImplementationRegistry registry) {
List<IfExpression.IfCondition> conditions = Lists.newArrayList(ifExpr.conditions);
- LogicalExpression newElseExpr = ifExpr.elseExpression.accept(this, null);
+ LogicalExpression newElseExpr = ifExpr.elseExpression.accept(this, registry);
for (int i = 0; i < conditions.size(); ++i) {
IfExpression.IfCondition condition = conditions.get(i);
- LogicalExpression newCondition = condition.condition.accept(this, null);
- LogicalExpression newExpr = condition.expression.accept(this, null);
+ LogicalExpression newCondition = condition.condition.accept(this, registry);
+ LogicalExpression newExpr = condition.expression.accept(this, registry);
conditions.set(i, new IfExpression.IfCondition(newCondition, newExpr));
}
- return IfExpression.newBuilder().setElse(newElseExpr).addConditions(conditions).build();
+ return validateNewExpr(IfExpression.newBuilder().setElse(newElseExpr).addConditions(conditions).build());
}
@Override
- public LogicalExpression visitSchemaPath(SchemaPath path) {
+ public LogicalExpression visitSchemaPath(SchemaPath path, FunctionImplementationRegistry value) {
// logger.debug("Visiting schema path {}", path);
TypedFieldId tfId = batch.getValueVectorId(path);
if (tfId == null) {
@@ -107,32 +214,32 @@ public class ExpressionTreeMaterializer {
}
@Override
- public LogicalExpression visitIntConstant(ValueExpressions.IntExpression intExpr) {
+ public LogicalExpression visitIntConstant(IntExpression intExpr, FunctionImplementationRegistry value) {
return intExpr;
}
@Override
- public LogicalExpression visitFloatConstant(ValueExpressions.FloatExpression fExpr) {
+ public LogicalExpression visitFloatConstant(FloatExpression fExpr, FunctionImplementationRegistry value) {
return fExpr;
}
@Override
- public LogicalExpression visitLongConstant(ValueExpressions.LongExpression intExpr) {
+ public LogicalExpression visitLongConstant(LongExpression intExpr, FunctionImplementationRegistry registry) {
return intExpr;
}
@Override
- public LogicalExpression visitDoubleConstant(ValueExpressions.DoubleExpression dExpr) {
+ public LogicalExpression visitDoubleConstant(DoubleExpression dExpr, FunctionImplementationRegistry registry) {
return dExpr;
}
@Override
- public LogicalExpression visitBooleanConstant(ValueExpressions.BooleanExpression e) {
+ public LogicalExpression visitBooleanConstant(BooleanExpression e, FunctionImplementationRegistry registry) {
return e;
}
@Override
- public LogicalExpression visitQuotedStringConstant(ValueExpressions.QuotedString e) {
+ public LogicalExpression visitQuotedStringConstant(QuotedString e, FunctionImplementationRegistry registry) {
return e;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/HiveFuncHolderExpr.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/HiveFuncHolderExpr.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/HiveFuncHolderExpr.java
new file mode 100644
index 0000000..afcd9cd
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/HiveFuncHolderExpr.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FunctionHolderExpression;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.visitors.ExprVisitor;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+
+import org.apache.drill.exec.expr.fn.HiveFuncHolder;
+
+public class HiveFuncHolderExpr extends FunctionHolderExpression implements Iterable<LogicalExpression>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillFuncHolderExpr.class);
+ private HiveFuncHolder holder;
+
+ public HiveFuncHolderExpr(String nameUsed, HiveFuncHolder holder, List<LogicalExpression> args, ExpressionPosition pos) {
+ super(nameUsed, pos, args);
+ this.holder = holder;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return holder.getReturnType();
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return args.iterator();
+ }
+
+ public HiveFuncHolder getHolder() {
+ return holder;
+ }
+
+ @Override
+ public boolean isAggregating() {
+ return holder.isAggregating();
+ }
+
+ @Override
+ public boolean argConstantOnly(int i) {
+ // looks like hive UDF has no notion of constant argument input
+ return false;
+ }
+
+ @Override
+ public boolean isRandom() {
+ return holder.isRandom();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ImplicitCastBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ImplicitCastBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ImplicitCastBuilder.java
deleted file mode 100644
index 99c26f3..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ImplicitCastBuilder.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.drill.exec.expr;
-
-import java.util.List;
-
-import org.apache.drill.common.expression.ErrorCollector;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.ExpressionValidator;
-import org.apache.drill.common.expression.FunctionCall;
-import org.apache.drill.common.expression.FunctionDefinition;
-import org.apache.drill.common.expression.IfExpression;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.expression.TypedNullConstant;
-import org.apache.drill.common.expression.ValueExpressions;
-import org.apache.drill.common.expression.fn.CastFunctionDefs;
-import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
-import org.apache.drill.exec.expr.fn.DrillFuncHolder;
-import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
-import org.apache.drill.exec.record.NullExpression;
-import org.apache.drill.exec.resolver.FunctionResolver;
-import org.apache.drill.exec.resolver.FunctionResolverFactory;
-
-import com.google.common.collect.Lists;
-
-public class ImplicitCastBuilder {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ImplicitCastBuilder.class);
-
- private ImplicitCastBuilder() {
- }
-
- public static LogicalExpression injectImplicitCast(LogicalExpression expr, ErrorCollector errorCollector, FunctionImplementationRegistry registry) {
- return expr.accept(new ImplicitCastVisitor(errorCollector), registry);
- }
-
-
- private static class ImplicitCastVisitor extends AbstractExprVisitor<LogicalExpression, FunctionImplementationRegistry, RuntimeException> {
- private final ErrorCollector errorCollector;
- private ExpressionValidator validator = new ExpressionValidator();
-
- public ImplicitCastVisitor(ErrorCollector errorCollector) {
- this.errorCollector = errorCollector;
- }
-
-
- private LogicalExpression validateNewExpr(LogicalExpression newExpr) {
- newExpr.accept(validator, errorCollector);
- return newExpr;
- }
-
-
- @Override
- public LogicalExpression visitFunctionCall(FunctionCall call, FunctionImplementationRegistry registry) {
- List<LogicalExpression> args = Lists.newArrayList();
- for (int i = 0; i < call.args.size(); ++i) {
- LogicalExpression newExpr = call.args.get(i).accept(this, registry);
- args.add(newExpr);
- }
-
- //replace with a new function call, since its argument could be changed.
- call = new FunctionCall(call.getDefinition(), args, call.getPosition());
-
- //call function resolver, get the best match.
- FunctionResolver resolver = FunctionResolverFactory.getResolver(call);
- DrillFuncHolder matchedFuncHolder = resolver.getBestMatch(registry.getMethods().get(call.getDefinition().getName()), call);
-
- //new arg lists, possible with implicit cast inserted.
- List<LogicalExpression> argsWithCast = Lists.newArrayList();
-
- if (matchedFuncHolder==null) {
- //TODO: found no matched funcholder. Raise exception here?
- return validateNewExpr(call);
- } else {
- //Compare parm type against arg type. Insert cast on top of arg, whenever necessary.
- for (int i = 0; i < call.args.size(); ++i) {
- MajorType parmType = matchedFuncHolder.getParmMajorType(i);
-
- //Case 1: If 1) the argument is NullExpression
- // 2) the parameter of matchedFuncHolder allows null input, or func's null_handling is NULL_IF_NULL (means null and non-null are exchangable).
- // then replace NullExpression with a TypedNullConstant
- if (call.args.get(i).equals(NullExpression.INSTANCE) &&
- ( parmType.getMode().equals(DataMode.OPTIONAL) || matchedFuncHolder.getNullHandling() == NullHandling.NULL_IF_NULL)) {
- argsWithCast.add(new TypedNullConstant(parmType));
- } else if (Types.softEquals(parmType, call.args.get(i).getMajorType(), matchedFuncHolder.getNullHandling() == NullHandling.NULL_IF_NULL)) {
- //Case 2: argument and parameter matches. Do nothing.
- argsWithCast.add(call.args.get(i));
- } else {
- //Case 3: insert cast if param type is different from arg type.
- FunctionDefinition castFuncDef = CastFunctionDefs.getCastFuncDef(parmType.getMinorType());
- List<LogicalExpression> castArgs = Lists.newArrayList();
- castArgs.add(call.args.get(i)); //input_expr
- argsWithCast.add(new FunctionCall(castFuncDef, castArgs, ExpressionPosition.UNKNOWN));
- }
- }
- }
-
- return validateNewExpr(new FunctionCall(call.getDefinition(), argsWithCast, call.getPosition()));
- }
-
- @Override
- public LogicalExpression visitIfExpression(IfExpression ifExpr, FunctionImplementationRegistry registry) {
- List<IfExpression.IfCondition> conditions = Lists.newArrayList(ifExpr.conditions);
- LogicalExpression newElseExpr = ifExpr.elseExpression.accept(this, registry);
-
- for (int i = 0; i < conditions.size(); ++i) {
- IfExpression.IfCondition condition = conditions.get(i);
-
- LogicalExpression newCondition = condition.condition.accept(this, registry);
- LogicalExpression newExpr = condition.expression.accept(this, registry);
- conditions.set(i, new IfExpression.IfCondition(newCondition, newExpr));
- }
-
- return validateNewExpr(IfExpression.newBuilder().setElse(newElseExpr).addConditions(conditions).build());
- }
-
- @Override
- public LogicalExpression visitSchemaPath(SchemaPath path, FunctionImplementationRegistry registry) {
- return path;
- }
-
- @Override
- public LogicalExpression visitLongConstant(ValueExpressions.LongExpression intExpr, FunctionImplementationRegistry registry) {
- return intExpr;
- }
-
- @Override
- public LogicalExpression visitDoubleConstant(ValueExpressions.DoubleExpression dExpr, FunctionImplementationRegistry registry) {
- return dExpr;
- }
-
- @Override
- public LogicalExpression visitBooleanConstant(ValueExpressions.BooleanExpression e, FunctionImplementationRegistry registry) {
- return e;
- }
-
- @Override
- public LogicalExpression visitQuotedStringConstant(ValueExpressions.QuotedString e, FunctionImplementationRegistry registry) {
- return e;
- }
-
- @Override
- public LogicalExpression visitUnknown(LogicalExpression e, FunctionImplementationRegistry registry) throws RuntimeException {
- return e;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java
index 2208e53..d91b282 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/annotations/FunctionTemplate.java
@@ -26,11 +26,25 @@ import java.lang.annotation.Target;
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.TYPE})
public @interface FunctionTemplate {
-
- String name();
+
+ /**
+ * Use this annotation if there is only one name for function
+ * Note: If you use this annotation don't use {@link #names()}
+ * @return
+ */
+ String name() default "";
+
+ /**
+ * Use this annotation if there are multiple names for function
+ * Note: If you use this annotation don't use {@link #name()}
+ * @return
+ */
+ String[] names() default {};
+
FunctionScope scope();
NullHandling nulls() default NullHandling.INTERNAL;
boolean isBinaryCommutative() default false;
+ boolean isRandom() default false;
public static enum NullHandling {
INTERNAL, NULL_IF_NULL;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ComparatorFunctionHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ComparatorFunctionHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ComparatorFunctionHelper.java
new file mode 100644
index 0000000..8bceefd
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ComparatorFunctionHelper.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.fn;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FunctionHolderExpression;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
+import org.apache.drill.exec.expr.DrillFuncHolderExpr;
+import org.apache.drill.exec.expr.HoldingContainerExpression;
+
+public class ComparatorFunctionHelper {
+ public static final String COMPARE_TO = "compare_to";
+
+ /**
+ * Given materialized arguments find the "compare_to" FunctionHolderExpression
+ * @param left
+ * @param right
+ * @param registry
+ * @return FunctionHolderExpression containing the function implementation
+ */
+ public static FunctionHolderExpression get(HoldingContainer left,
+ HoldingContainer right,
+ FunctionImplementationRegistry registry) {
+
+ ImmutableList<TypeProtos.MajorType> args = ImmutableList.of(left.getMajorType(), right.getMajorType()) ;
+ TypeProtos.MajorType returnType = Types.required(TypeProtos.MinorType.INT);
+
+ for (DrillFuncHolder h : registry.getDrillRegistry().getMethods().get(COMPARE_TO)) {
+ if (h.matches(returnType, args)) {
+ return new DrillFuncHolderExpr(COMPARE_TO, h,
+ ImmutableList.of((LogicalExpression)
+ new HoldingContainerExpression(left),
+ new HoldingContainerExpression(right)),
+ ExpressionPosition.UNKNOWN);
+ }
+ }
+
+ return null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
index 5f1e358..90d4521 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
@@ -41,10 +41,11 @@ class DrillAggFuncHolder extends DrillFuncHolder{
private final String output;
private final String cleanup;
- public DrillAggFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative,
- String functionName, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars,
+ public DrillAggFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom,
+ String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars,
Map<String, String> methods, List<String> imports) {
- super(scope, nullHandling, isBinaryCommutative, functionName, parameters, returnValue, workspaceVars, methods, imports);
+ super(scope, nullHandling, isBinaryCommutative, isRandom,
+ registeredNames, parameters, returnValue, workspaceVars, methods, imports);
Preconditions.checkArgument(nullHandling == NullHandling.INTERNAL, "An aggregation function is required to do its own null handling.");
setup = methods.get("setup");
reset = methods.get("reset");
@@ -59,7 +60,11 @@ class DrillAggFuncHolder extends DrillFuncHolder{
public boolean isNested(){
return true;
}
-
+
+ public boolean isAggregating() {
+ return true;
+ }
+
@Override
public JVar[] renderStart(ClassGenerator<?> g, HoldingContainer[] inputVariables) {
JVar[] workspaceJVars = declareWorkspaceVariables(g);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
index d1619b2..5868421 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
@@ -17,11 +17,13 @@
*/
package org.apache.drill.exec.expr.fn;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.ExecConstants;
@@ -51,27 +53,30 @@ public abstract class DrillFuncHolder {
protected final FunctionTemplate.FunctionScope scope;
protected final FunctionTemplate.NullHandling nullHandling;
protected final boolean isBinaryCommutative;
- protected final String functionName;
+ protected final boolean isRandom;
+ protected final String[] registeredNames;
protected final ImmutableList<String> imports;
protected final WorkspaceReference[] workspaceVars;
protected final ValueReference[] parameters;
protected final ValueReference returnValue;
- protected final ImmutableMap<String, String> methodMap;
+ protected final ImmutableMap<String, String> methodMap;
- public DrillFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, String functionName, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars, Map<String, String> methods, List<String> imports) {
+ public DrillFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom,
+ String[] registeredNames, ValueReference[] parameters, ValueReference returnValue,
+ WorkspaceReference[] workspaceVars, Map<String, String> methods, List<String> imports) {
super();
this.scope = scope;
this.nullHandling = nullHandling;
this.workspaceVars = workspaceVars;
this.isBinaryCommutative = isBinaryCommutative;
- this.functionName = functionName;
+ this.isRandom = isRandom;
+ this.registeredNames = registeredNames;
this.methodMap = ImmutableMap.copyOf(methods);
this.parameters = parameters;
this.returnValue = returnValue;
this.imports = ImmutableList.copyOf(imports);
-
}
-
+
public List<String> getImports() {
return imports;
}
@@ -83,7 +88,15 @@ public abstract class DrillFuncHolder {
public void renderMiddle(ClassGenerator<?> g, HoldingContainer[] inputVariables, JVar[] workspaceJVars){};
public abstract HoldingContainer renderEnd(ClassGenerator<?> g, HoldingContainer[] inputVariables, JVar[] workspaceJVars);
public abstract boolean isNested();
-
+
+ public boolean isAggregating() {
+ return false;
+ }
+
+ public boolean isRandom() {
+ return isRandom;
+ }
+
protected JVar[] declareWorkspaceVariables(ClassGenerator<?> g){
JVar[] workspaceJVars = new JVar[workspaceVars.length];
for(int i =0 ; i < workspaceVars.length; i++){
@@ -100,9 +113,9 @@ public abstract class DrillFuncHolder {
} else {
addProtectedBlock(g, sub, body, null, workspaceJVars, false);
}
- g.getBlock(bt).directStatement(String.format("/** start %s for function %s **/ ", bt.name(), functionName));
+ g.getBlock(bt).directStatement(String.format("/** start %s for function %s **/ ", bt.name(), registeredNames[0]));
g.getBlock(bt).add(sub);
- g.getBlock(bt).directStatement(String.format("/** end %s for function %s **/ ", bt.name(), functionName));
+ g.getBlock(bt).directStatement(String.format("/** end %s for function %s **/ ", bt.name(), registeredNames[0]));
}
}
@@ -132,24 +145,20 @@ public abstract class DrillFuncHolder {
}
}
-
-
-
- public boolean matches(FunctionCall call){
+ public boolean matches(MajorType returnType, List<MajorType> argTypes){
- if(!softCompare(call.getMajorType(), returnValue.type)){
+ if(!softCompare(returnType, returnValue.type)){
// logger.debug(String.format("Call [%s] didn't match as return type [%s] was different than expected [%s]. ", call.getDefinition().getName(), returnValue.type, call.getMajorType()));
return false;
}
- if(call.args.size() != parameters.length){
+ if(argTypes.size() != parameters.length){
// logger.debug(String.format("Call [%s] didn't match as the number of arguments provided [%d] were different than expected [%d]. ", call.getDefinition().getName(), parameters.length, call.args.size()));
return false;
}
+
for(int i =0; i < parameters.length; i++){
- ValueReference param = parameters[i];
- LogicalExpression arg = call.args.get(i);
- if(!softCompare(param.type, arg.getMajorType())){
+ if(!softCompare(parameters[i].type, argTypes.get(i))){
// logger.debug(String.format("Call [%s] didn't match as the argument [%s] didn't match the expected type [%s]. ", call.getDefinition().getName(), arg.getMajorType(), param.type));
return false;
}
@@ -165,6 +174,23 @@ public abstract class DrillFuncHolder {
public int getParmSize(){
return this.parameters.length;
}
+
+ public boolean isConstant(int i) {
+ return this.parameters[i].isConstant;
+ }
+
+ public MajorType getReturnType(List<LogicalExpression> args) {
+ if (nullHandling == NullHandling.NULL_IF_NULL) {
+ // if any one of the input types is nullable, then return nullable return type
+ for (LogicalExpression e : args) {
+ if (e.getMajorType().getMode() == TypeProtos.DataMode.OPTIONAL) {
+ return Types.optional(returnValue.type.getMinorType());
+ }
+ }
+ }
+
+ return returnValue.type;
+ }
public NullHandling getNullHandling() {
return this.nullHandling ;
@@ -174,8 +200,8 @@ public abstract class DrillFuncHolder {
return Types.softEquals(a, b, nullHandling == NullHandling.NULL_IF_NULL);
}
- public String getFunctionName() {
- return functionName;
+ public String[] getRegisteredNames() {
+ return registeredNames;
}
public static class ValueReference{
@@ -215,9 +241,5 @@ public abstract class DrillFuncHolder {
this.type = type;
this.name = name;
}
-
}
-
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionImplementationRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionImplementationRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionImplementationRegistry.java
new file mode 100644
index 0000000..c78895e
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionImplementationRegistry.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.fn;
+
+import java.util.Set;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.util.PathScanner;
+import org.apache.drill.exec.ExecConstants;
+import org.apache.drill.exec.expr.DrillFunc;
+
+import com.google.common.collect.ArrayListMultimap;
+
+public class DrillFunctionImplementationRegistry {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillFunctionImplementationRegistry.class);
+
+ private ArrayListMultimap<String, DrillFuncHolder> methods = ArrayListMultimap.create();
+
+ public DrillFunctionImplementationRegistry(DrillConfig config){
+ FunctionConverter converter = new FunctionConverter();
+ Set<Class<? extends DrillFunc>> providerClasses = PathScanner.scanForImplementations(DrillFunc.class, config.getStringList(ExecConstants.FUNCTION_PACKAGES));
+ for (Class<? extends DrillFunc> clazz : providerClasses) {
+ DrillFuncHolder holder = converter.getHolder(clazz);
+ if(holder != null){
+ // register handle for each name the function can be referred to
+ String[] names = holder.getRegisteredNames();
+ for(String name : names)
+ methods.put(name, holder);
+ }else{
+ logger.warn("Unable to initialize function for class {}", clazz.getName());
+ }
+ }
+ }
+
+ public ArrayListMultimap<String, DrillFuncHolder> getMethods() {
+ return this.methods;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java
index de847df..33265de 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillSimpleFuncHolder.java
@@ -49,10 +49,10 @@ class DrillSimpleFuncHolder extends DrillFuncHolder{
private final String cleanupBody;
- public DrillSimpleFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative,
- String functionName, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars,
+ public DrillSimpleFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom,
+ String[] registeredNames, ValueReference[] parameters, ValueReference returnValue, WorkspaceReference[] workspaceVars,
Map<String, String> methods, List<String> imports) {
- super(scope, nullHandling, isBinaryCommutative, functionName, parameters, returnValue, workspaceVars, methods, imports);
+ super(scope, nullHandling, isBinaryCommutative, isRandom, registeredNames, parameters, returnValue, workspaceVars, methods, imports);
setupBody = methods.get("setup");
evalBody = methods.get("eval");
resetBody = methods.get("reset");
@@ -70,7 +70,7 @@ class DrillSimpleFuncHolder extends DrillFuncHolder{
//for the argument is not, then raise exception.
for(int i =0; i < inputVariables.length; i++){
if (parameters[i].isConstant && !inputVariables[i].isConstant()) {
- throw new DrillRuntimeException(String.format("The argument '%s' of Function '%s' has to be constant!", parameters[i].name, this.getFunctionName()));
+ throw new DrillRuntimeException(String.format("The argument '%s' of Function '%s' has to be constant!", parameters[i].name, this.getRegisteredNames()[0]));
}
}
generateBody(g, BlockType.SETUP, setupBody, inputVariables, workspaceJVars, true);
@@ -130,7 +130,7 @@ class DrillSimpleFuncHolder extends DrillFuncHolder{
@Override
public String toString() {
final int maxLen = 10;
- return "DrillSimpleFuncHolder [, functionName=" + functionName + ", nullHandling=" + nullHandling + "parameters="
+ return "DrillSimpleFuncHolder [, functionName=" + Arrays.toString(registeredNames) + ", nullHandling=" + nullHandling + "parameters="
+ (parameters != null ? Arrays.asList(parameters).subList(0, Math.min(parameters.length, maxLen)) : null) + "]";
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
index 3145240..4dff94d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
@@ -17,7 +17,6 @@
*/
package org.apache.drill.exec.expr.fn;
-import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
@@ -35,7 +34,6 @@ import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.annotations.Workspace;
import org.apache.drill.exec.expr.fn.DrillFuncHolder.ValueReference;
import org.apache.drill.exec.expr.fn.DrillFuncHolder.WorkspaceReference;
-import org.apache.drill.exec.expr.fn.impl.GCompareBigIntNullableBigInt;
import org.apache.drill.exec.expr.holders.ValueHolder;
import org.codehaus.commons.compiler.CompileException;
import org.codehaus.janino.Java.CompilationUnit;
@@ -92,7 +90,12 @@ public class FunctionConverter {
if(template == null){
return failure("Class does not declare FunctionTemplate annotation.", clazz);
}
-
+
+ if ((template.name().isEmpty() && template.names().length == 0) || // none set
+ (!template.name().isEmpty() && template.names().length != 0)) { // both are set
+ return failure("Must use only one annotations 'name' or 'names', not both", clazz);
+ }
+
// start by getting field information.
List<ValueReference> params = Lists.newArrayList();
List<WorkspaceReference> workspaceFields = Lists.newArrayList();
@@ -116,8 +119,6 @@ public class FunctionConverter {
return failure("The field must be only one of @Param, @Output or @Workspace. It currently has more than one of these annotations.", clazz, field);
}
-
-
if(param != null || output != null){
// check that param and output are value holders.
@@ -180,13 +181,15 @@ public class FunctionConverter {
ValueReference[] ps = params.toArray(new ValueReference[params.size()]);
WorkspaceReference[] works = workspaceFields.toArray(new WorkspaceReference[workspaceFields.size()]);
-
+
+ String[] registeredNames = ((template.name().isEmpty()) ? template.names() : new String[] {template.name()} );
switch(template.scope()){
case POINT_AGGREGATE:
- return new DrillAggFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(), template.name(), ps, outputField, works, methods, imports);
+ return new DrillAggFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(),
+ template.isRandom(), registeredNames, ps, outputField, works, methods, imports);
case SIMPLE:
- DrillFuncHolder fh = new DrillSimpleFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(), template.name(), ps, outputField, works, methods, imports);
- return fh;
+ return new DrillSimpleFuncHolder(template.scope(), template.nulls(), template.isBinaryCommutative(),
+ template.isRandom(), registeredNames, ps, outputField, works, methods, imports);
case HOLISTIC_AGGREGATE:
case RANGE_AGGREGATE:
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
index 93f1992..7a28a2f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionImplementationRegistry.java
@@ -17,77 +17,22 @@
*/
package org.apache.drill.exec.expr.fn;
-import java.util.List;
-import java.util.Set;
-
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.FunctionCall;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.util.PathScanner;
-import org.apache.drill.exec.ExecConstants;
-import org.apache.drill.exec.expr.DrillFunc;
-import org.apache.drill.exec.resolver.FunctionResolver;
-import org.apache.drill.exec.resolver.FunctionResolverFactory;
-
-import com.beust.jcommander.internal.Lists;
-import com.google.common.collect.ArrayListMultimap;
public class FunctionImplementationRegistry {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionImplementationRegistry.class);
-
- private ArrayListMultimap<String, DrillFuncHolder> methods = ArrayListMultimap.create();
-
+ private DrillFunctionImplementationRegistry drillFuncRegistry;
+ private HiveFunctionImplementationRegistry hiveFuncRegistry;
+
public FunctionImplementationRegistry(DrillConfig config){
- FunctionConverter converter = new FunctionConverter();
- Set<Class<? extends DrillFunc>> providerClasses = PathScanner.scanForImplementations(DrillFunc.class, config.getStringList(ExecConstants.FUNCTION_PACKAGES));
- for (Class<? extends DrillFunc> clazz : providerClasses) {
- DrillFuncHolder holder = converter.getHolder(clazz);
- if(holder != null){
- methods.put(holder.getFunctionName(), holder);
-// logger.debug("Registering function {}", holder);
- }else{
- logger.warn("Unable to initialize function for class {}", clazz.getName());
- }
- }
- }
-
- public ArrayListMultimap<String, DrillFuncHolder> getMethods() {
- return this.methods;
- }
-
- public DrillFuncHolder getFunction(FunctionCall call){
- for(DrillFuncHolder h : methods.get(call.getDefinition().getName())){
- if(h.matches(call)){
- return h;
- }
- }
-
- List<MajorType> types = Lists.newArrayList();
- for(LogicalExpression e : call.args){
- types.add(e.getMajorType());
- }
- StringBuilder sb = new StringBuilder();
- sb.append("Missing function implementation: ");
- sb.append("[");
- sb.append(call.getDefinition().getName());
- sb.append("(");
- boolean first = true;
- for(MajorType mt : types){
- if(first){
- first = false;
- }else{
- sb.append(", ");
- }
- sb.append(mt.getMinorType().name());
- sb.append("-");
- sb.append(mt.getMode().name());
- }
- sb.append(")");
- sb.append("]");
- throw new UnsupportedOperationException(sb.toString());
+ drillFuncRegistry = new DrillFunctionImplementationRegistry(config);
+ hiveFuncRegistry = new HiveFunctionImplementationRegistry(config);
}
-
+ public DrillFunctionImplementationRegistry getDrillRegistry() {
+ return drillFuncRegistry;
+ }
+ public HiveFunctionImplementationRegistry getHiveRegistry() {
+ return hiveFuncRegistry;
+ }
}
[10/12] git commit: DRILL-411: Fix Optiq to Drill conversion of
varchar literals.
Posted by ja...@apache.org.
DRILL-411: Fix Optiq to Drill conversion of varchar literals.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/6804b525
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/6804b525
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/6804b525
Branch: refs/heads/master
Commit: 6804b525535c3cfe39d1319c9c7377595402f333
Parents: 5a94bf1
Author: vkorukanti <ve...@gmail.com>
Authored: Sat Mar 22 10:02:01 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sat Mar 22 10:02:01 2014 -0700
----------------------------------------------------------------------
.../apache/drill/exec/planner/logical/DrillOptiq.java | 5 +++--
.../java/org/apache/drill/jdbc/test/TestJdbcQuery.java | 13 ++++++++++++-
2 files changed, 15 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6804b525/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
index 4fb9a01..b39b230 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
@@ -47,6 +47,7 @@ import org.eigenbase.sql.SqlSyntax;
import org.eigenbase.sql.fun.SqlStdOperatorTable;
import com.google.common.collect.Lists;
+import org.eigenbase.util.NlsString;
/**
* Utilities for Drill's planner.
@@ -190,7 +191,7 @@ public class DrillOptiq {
case BOOLEAN:
return ValueExpressions.getBit(((Boolean) literal.getValue()));
case CHAR:
- return ValueExpressions.getChar(((String) literal.getValue()));
+ return ValueExpressions.getChar(((NlsString)literal.getValue()).getValue());
case DOUBLE:
double d = ((BigDecimal) literal.getValue()).doubleValue();
return ValueExpressions.getFloat8(d);
@@ -202,7 +203,7 @@ public class DrillOptiq {
int i = ((BigDecimal) literal.getValue()).intValue();
return ValueExpressions.getInt(i);
case VARCHAR:
- return ValueExpressions.getChar(((String) literal.getValue()));
+ return ValueExpressions.getChar(((NlsString)literal.getValue()).getValue());
default:
throw new UnsupportedOperationException(String.format("Unable to convert the value of %s and type %s to a Drill constant expression.", literal, literal.getTypeName()));
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6804b525/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
index 1d678cc..8ea758d 100644
--- a/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
+++ b/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
@@ -94,7 +94,18 @@ public class TestJdbcQuery {
public void testWildcard() throws Exception{
testQuery(String.format("select * from dfs.`%s/../sample-data/region.parquet`", WORKING_PATH));
}
-
+
+ @Test
+ public void testCharLiteral() throws Exception {
+ testQuery(String.format("select 'test literal' from INFORMATION_SCHEMA.TABLES LIMIT 1"));
+ }
+
+ @Test
+ @Ignore // failing due to a bug in cast to varchar function
+ public void testVarCharLiteral() throws Exception {
+ testQuery(String.format("select cast('test literal' as VARCHAR) from INFORMATION_SCHEMA.TABLES LIMIT 1"));
+ }
+
@Test
public void testLogicalExplain() throws Exception{
testQuery(String.format("EXPLAIN PLAN WITHOUT IMPLEMENTATION FOR select * from dfs.`%s/../sample-data/region.parquet`", WORKING_PATH));
[09/12] git commit: DRILL-425 mock-sub-scan is not setting 'lastSet'
in NullableVarCharVector or NullableVar16CharVector
Posted by ja...@apache.org.
DRILL-425 mock-sub-scan is not setting 'lastSet' in NullableVarCharVector or NullableVar16CharVector
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/5a94bf1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/5a94bf1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/5a94bf1d
Branch: refs/heads/master
Commit: 5a94bf1dacefb02350a5d49c4156db2183689b51
Parents: 8d725f6
Author: Steven Phillips <sp...@maprtech.com>
Authored: Wed Mar 19 18:57:26 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sat Mar 22 09:59:26 2014 -0700
----------------------------------------------------------------------
exec/java-exec/src/main/codegen/templates/NullableValueVectors.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5a94bf1d/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
index 1439ee2..7878aed 100644
--- a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
@@ -392,6 +392,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
public void generateTestData(){
bits.getMutator().generateTestData();
values.getMutator().generateTestData();
+ <#if type.major = "VarLen">lastSet = valueCount;</#if>
}
public void reset(){
[06/12] DRILL-335: Implement Hash Aggregation
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggTemplate.java
new file mode 100644
index 0000000..f0cb0d4
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggTemplate.java
@@ -0,0 +1,304 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.aggregate;
+
+import javax.inject.Named;
+
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.RecordBatch.IterOutcome;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.vector.allocator.VectorAllocator;
+
+public abstract class StreamingAggTemplate implements StreamingAggregator {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StreamingAggregator.class);
+ private static final boolean EXTRA_DEBUG = false;
+ private static final String TOO_BIG_ERROR = "Couldn't add value to an empty batch. This likely means that a single value is too long for a varlen field.";
+ private IterOutcome lastOutcome = null;
+ private boolean first = true;
+ private boolean newSchema = false;
+ private int previousIndex = 0;
+ private int underlyingIndex = 0;
+ private int currentIndex;
+ private int addedRecordCount = 0;
+ private boolean pendingOutput = false;
+ private IterOutcome outcome;
+ private int outputCount = 0;
+ private RecordBatch incoming;
+ private BatchSchema schema;
+ private RecordBatch outgoing;
+ private VectorAllocator[] allocators;
+ private FragmentContext context;
+ private InternalBatch remainderBatch;
+
+
+ @Override
+ public void setup(FragmentContext context, RecordBatch incoming, RecordBatch outgoing, VectorAllocator[] allocators) throws SchemaChangeException {
+ this.allocators = allocators;
+ this.context = context;
+ this.incoming = incoming;
+ this.schema = incoming.getSchema();
+ this.allocators = allocators;
+ this.outgoing = outgoing;
+ setupInterior(incoming, outgoing);
+ this.currentIndex = this.getVectorIndex(underlyingIndex);
+ }
+
+
+ private void allocateOutgoing() {
+ for (VectorAllocator a : allocators) {
+ if(EXTRA_DEBUG) logger.debug("Allocating {} with {} records.", a, 20000);
+ a.alloc(20000);
+ }
+ }
+
+ @Override
+ public IterOutcome getOutcome() {
+ return outcome;
+ }
+
+ @Override
+ public int getOutputCount() {
+ return outputCount;
+ }
+
+ private AggOutcome tooBigFailure(){
+ context.fail(new Exception(TOO_BIG_ERROR));
+ this.outcome = IterOutcome.STOP;
+ return AggOutcome.CLEANUP_AND_RETURN;
+ }
+
+ @Override
+ public AggOutcome doWork() {
+ try{ // outside loop to ensure that first is set to false after the first run.
+
+ // if we're in the first state, allocate outgoing.
+ if(first){
+ allocateOutgoing();
+ }
+
+ // pick up a remainder batch if we have one.
+ if(remainderBatch != null){
+ if (!outputToBatch( previousIndex )) return tooBigFailure();
+ remainderBatch.clear();
+ remainderBatch = null;
+ return setOkAndReturn();
+ }
+
+
+ // setup for new output and pick any remainder.
+ if (pendingOutput) {
+ allocateOutgoing();
+ pendingOutput = false;
+ if (!outputToBatch( previousIndex)) return tooBigFailure();
+ }
+
+ if(newSchema){
+ return AggOutcome.UPDATE_AGGREGATOR;
+ }
+
+ if(lastOutcome != null){
+ outcome = lastOutcome;
+ return AggOutcome.CLEANUP_AND_RETURN;
+ }
+
+ outside: while(true){
+ // loop through existing records, adding as necessary.
+ for (; underlyingIndex < incoming.getRecordCount(); incIndex()) {
+ if(EXTRA_DEBUG) logger.debug("Doing loop with values underlying {}, current {}", underlyingIndex, currentIndex);
+ if (isSame( previousIndex, currentIndex )) {
+ if(EXTRA_DEBUG) logger.debug("Values were found the same, adding.");
+ addRecordInc(currentIndex);
+ } else {
+ if(EXTRA_DEBUG) logger.debug("Values were different, outputting previous batch.");
+ if (outputToBatch(previousIndex)) {
+ if(EXTRA_DEBUG) logger.debug("Output successful.");
+ addRecordInc(currentIndex);
+ } else {
+ if(EXTRA_DEBUG) logger.debug("Output failed.");
+ if(outputCount == 0) return tooBigFailure();
+
+ // mark the pending output but move forward for the next cycle.
+ pendingOutput = true;
+ previousIndex = currentIndex;
+ incIndex();
+ return setOkAndReturn();
+
+ }
+ }
+ previousIndex = currentIndex;
+ }
+
+
+ InternalBatch previous = null;
+
+ try{
+ while(true){
+ previous = new InternalBatch(incoming);
+ IterOutcome out = incoming.next();
+ if(EXTRA_DEBUG) logger.debug("Received IterOutcome of {}", out);
+ switch(out){
+ case NONE:
+ lastOutcome = out;
+ if(addedRecordCount > 0){
+ if( !outputToBatchPrev( previous, previousIndex, outputCount) ) remainderBatch = previous;
+ if(EXTRA_DEBUG) logger.debug("Received no more batches, returning.");
+ return setOkAndReturn();
+ }else{
+ outcome = out;
+ return AggOutcome.CLEANUP_AND_RETURN;
+ }
+
+
+
+ case NOT_YET:
+ this.outcome = out;
+ return AggOutcome.RETURN_OUTCOME;
+
+ case OK_NEW_SCHEMA:
+ if(EXTRA_DEBUG) logger.debug("Received new schema. Batch has {} records.", incoming.getRecordCount());
+ if(addedRecordCount > 0){
+ if( !outputToBatchPrev( previous, previousIndex, outputCount) ) remainderBatch = previous;
+ if(EXTRA_DEBUG) logger.debug("Wrote out end of previous batch, returning.");
+ newSchema = true;
+ return setOkAndReturn();
+ }
+ cleanup();
+ return AggOutcome.UPDATE_AGGREGATOR;
+ case OK:
+ resetIndex();
+ if(incoming.getRecordCount() == 0){
+ continue;
+ }else{
+ if(isSamePrev(previousIndex , previous, currentIndex)){
+ if(EXTRA_DEBUG) logger.debug("New value was same as last value of previous batch, adding.");
+ addRecordInc(currentIndex);
+ previousIndex = currentIndex;
+ incIndex();
+ if(EXTRA_DEBUG) logger.debug("Continuing outside");
+ continue outside;
+ }else{ // not the same
+ if(EXTRA_DEBUG) logger.debug("This is not the same as the previous, add record and continue outside.");
+ previousIndex = currentIndex;
+ if(addedRecordCount > 0){
+ if( !outputToBatchPrev( previous, previousIndex, outputCount) ){
+ remainderBatch = previous;
+ return setOkAndReturn();
+ }
+ continue outside;
+ }
+ }
+ }
+ case STOP:
+ default:
+ lastOutcome = out;
+ outcome = out;
+ return AggOutcome.CLEANUP_AND_RETURN;
+ }
+
+
+ }
+ }finally{
+ // make sure to clear previous if we haven't saved it.
+ if(remainderBatch == null && previous != null){
+ previous.clear();
+ }
+ }
+ }
+ }finally{
+ if(first) first = !first;
+ }
+
+ }
+
+
+ private final void incIndex(){
+ underlyingIndex++;
+ if(underlyingIndex >= incoming.getRecordCount()){
+ currentIndex = Integer.MAX_VALUE;
+ return;
+ }
+ currentIndex = getVectorIndex(underlyingIndex);
+ }
+
+ private final void resetIndex(){
+ underlyingIndex = -1;
+ incIndex();
+ }
+
+ private final AggOutcome setOkAndReturn(){
+ if(first){
+ this.outcome = IterOutcome.OK_NEW_SCHEMA;
+ }else{
+ this.outcome = IterOutcome.OK;
+ }
+ for(VectorWrapper<?> v : outgoing){
+ v.getValueVector().getMutator().setValueCount(outputCount);
+ }
+ return AggOutcome.RETURN_OUTCOME;
+ }
+
+ private final boolean outputToBatch(int inIndex){
+ boolean success = outputRecordKeys(inIndex, outputCount) //
+ && outputRecordValues(outputCount) //
+ && resetValues();
+ if(success){
+ if(EXTRA_DEBUG) logger.debug("Outputting values to {}", outputCount);
+ outputCount++;
+ addedRecordCount = 0;
+ }
+
+ return success;
+ }
+
+ private final boolean outputToBatchPrev(InternalBatch b1, int inIndex, int outIndex){
+ boolean success = outputRecordKeysPrev(b1, inIndex, outIndex) //
+ && outputRecordValues(outIndex) //
+ && resetValues();
+ if(success){
+ outputCount++;
+ addedRecordCount = 0;
+ }
+
+ return success;
+ }
+
+ private void addRecordInc(int index){
+ addRecord(index);
+ this.addedRecordCount++;
+ }
+
+ @Override
+ public void cleanup(){
+ if(remainderBatch != null) remainderBatch.clear();
+ }
+
+
+ public abstract void setupInterior(@Named("incoming") RecordBatch incoming, @Named("outgoing") RecordBatch outgoing) throws SchemaChangeException;
+ public abstract boolean isSame(@Named("index1") int index1, @Named("index2") int index2);
+ public abstract boolean isSamePrev(@Named("b1Index") int b1Index, @Named("b1") InternalBatch b1, @Named("b2Index") int b2Index);
+ public abstract void addRecord(@Named("index") int index);
+ public abstract boolean outputRecordKeys(@Named("inIndex") int inIndex, @Named("outIndex") int outIndex);
+ public abstract boolean outputRecordKeysPrev(@Named("previous") InternalBatch previous, @Named("previousIndex") int previousIndex, @Named("outIndex") int outIndex);
+ public abstract boolean outputRecordValues(@Named("outIndex") int outIndex);
+ public abstract int getVectorIndex(@Named("recordIndex") int recordIndex);
+ public abstract boolean resetValues();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggregator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggregator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggregator.java
new file mode 100644
index 0000000..7c10e9d
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggregator.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.aggregate;
+
+import org.apache.drill.exec.compile.TemplateClassDefinition;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.RecordBatch.IterOutcome;
+import org.apache.drill.exec.vector.allocator.VectorAllocator;
+
+public interface StreamingAggregator {
+
+ public static TemplateClassDefinition<StreamingAggregator> TEMPLATE_DEFINITION = new TemplateClassDefinition<StreamingAggregator>(StreamingAggregator.class, StreamingAggTemplate.class);
+
+ public static enum AggOutcome {
+ RETURN_OUTCOME, CLEANUP_AND_RETURN, UPDATE_AGGREGATOR;
+ }
+
+ public abstract void setup(FragmentContext context, RecordBatch incoming, RecordBatch outgoing,
+ VectorAllocator[] allocators) throws SchemaChangeException;
+
+ public abstract IterOutcome getOutcome();
+
+ public abstract int getOutputCount();
+
+ public abstract AggOutcome doWork();
+
+ public abstract void cleanup();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
new file mode 100644
index 0000000..507be33
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
@@ -0,0 +1,280 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed
+ * with this work for additional information regarding copyright
+ * ownership. The ASF licenses this file to you under the Apache
+ * License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.common;
+
+import java.io.IOException;
+
+import org.apache.drill.common.expression.ErrorCollector;
+import org.apache.drill.common.expression.ErrorCollectorImpl;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.NamedExpression;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.compile.sig.GeneratorMapping;
+import org.apache.drill.exec.compile.sig.MappingSet;
+import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.expr.ClassGenerator;
+import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
+import org.apache.drill.exec.expr.CodeGenerator;
+import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
+import org.apache.drill.exec.expr.HoldingContainerExpression;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.expr.ValueVectorReadExpression;
+import org.apache.drill.exec.expr.ValueVectorWriteExpression;
+import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
+import org.apache.drill.exec.expr.fn.impl.BitFunctions;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.allocator.VectorAllocator;
+
+import com.google.common.collect.ImmutableList;
+import com.sun.codemodel.JConditional;
+import com.sun.codemodel.JExpr;
+
+
+public class ChainedHashTable {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ChainedHashTable.class);
+
+ private static final GeneratorMapping KEY_MATCH_BUILD =
+ GeneratorMapping.create("setupInterior" /* setup method */, "isKeyMatchInternalBuild" /* eval method */,
+ null /* reset */, null /* cleanup */);
+
+ private static final GeneratorMapping KEY_MATCH_PROBE =
+ GeneratorMapping.create("setupInterior" /* setup method */, "isKeyMatchInternalProbe" /* eval method */,
+ null /* reset */, null /* cleanup */);
+
+ private static final GeneratorMapping GET_HASH_BUILD =
+ GeneratorMapping.create("doSetup" /* setup method */, "getHashBuild" /* eval method */,
+ null /* reset */, null /* cleanup */);
+
+ private static final GeneratorMapping GET_HASH_PROBE =
+ GeneratorMapping.create("doSetup" /* setup method */, "getHashProbe" /* eval method */,
+ null /* reset */, null /* cleanup */);
+
+ private static final GeneratorMapping SET_VALUE =
+ GeneratorMapping.create("setupInterior" /* setup method */, "setValue" /* eval method */,
+ null /* reset */, null /* cleanup */);
+
+ private static final GeneratorMapping OUTPUT_KEYS =
+ GeneratorMapping.create("setupInterior" /* setup method */, "outputRecordKeys" /* eval method */,
+ null /* reset */, null /* cleanup */) ;
+
+ private final MappingSet KeyMatchIncomingBuildMapping = new MappingSet("incomingRowIdx", null, "incomingBuild", null, KEY_MATCH_BUILD, KEY_MATCH_BUILD);
+ private final MappingSet KeyMatchIncomingProbeMapping = new MappingSet("incomingRowIdx", null, "incomingProbe", null, KEY_MATCH_PROBE, KEY_MATCH_PROBE);
+ private final MappingSet KeyMatchHtableMapping = new MappingSet("htRowIdx", null, "htContainer", null, KEY_MATCH_BUILD, KEY_MATCH_BUILD);
+ private final MappingSet GetHashIncomingBuildMapping = new MappingSet("incomingRowIdx", null, "incomingBuild", null, GET_HASH_BUILD, GET_HASH_BUILD);
+ private final MappingSet GetHashIncomingProbeMapping = new MappingSet("incomingRowIdx", null, "incomingProbe", null, GET_HASH_PROBE, GET_HASH_PROBE);
+ private final MappingSet SetValueMapping = new MappingSet("incomingRowIdx" /* read index */, "htRowIdx" /* write index */, "incomingBuild" /* read container */, "htContainer" /* write container */, SET_VALUE, SET_VALUE);
+
+ private final MappingSet OutputRecordKeysMapping = new MappingSet("htRowIdx" /* read index */, "outRowIdx" /* write index */, "htContainer" /* read container */, "outgoing" /* write container */, OUTPUT_KEYS, OUTPUT_KEYS);
+
+ private HashTableConfig htConfig;
+ private final FragmentContext context;
+ private final RecordBatch incomingBuild;
+ private final RecordBatch incomingProbe;
+ private final RecordBatch outgoing;
+
+ public ChainedHashTable(HashTableConfig htConfig,
+ FragmentContext context,
+ RecordBatch incomingBuild,
+ RecordBatch incomingProbe,
+ RecordBatch outgoing) {
+
+ this.htConfig = htConfig;
+ this.context = context;
+ this.incomingBuild = incomingBuild;
+ this.incomingProbe = incomingProbe;
+ this.outgoing = outgoing;
+ }
+
+ public HashTable createAndSetupHashTable (TypedFieldId[] outKeyFieldIds) throws ClassTransformationException, IOException, SchemaChangeException {
+ CodeGenerator<HashTable> top = CodeGenerator.get(HashTable.TEMPLATE_DEFINITION, context.getFunctionRegistry());
+ ClassGenerator<HashTable> cg = top.getRoot();
+ ClassGenerator<HashTable> cgInner = cg.getInnerGenerator("BatchHolder");
+
+ if (outKeyFieldIds.length > htConfig.getKeyExprsBuild().length) {
+ throw new IllegalArgumentException("Mismatched number of output key fields.");
+ }
+
+ LogicalExpression[] keyExprsBuild = new LogicalExpression[htConfig.getKeyExprsBuild().length];
+ LogicalExpression[] keyExprsProbe = null;
+ boolean isProbe = (htConfig.getKeyExprsProbe() != null) ;
+ if (isProbe) {
+ keyExprsProbe = new LogicalExpression[htConfig.getKeyExprsProbe().length];
+ }
+
+ ErrorCollector collector = new ErrorCollectorImpl();
+ VectorContainer htContainerOrig = new VectorContainer(); // original ht container from which others may be cloned
+ LogicalExpression[] htKeyExprs = new LogicalExpression[htConfig.getKeyExprsBuild().length];
+ TypedFieldId[] htKeyFieldIds = new TypedFieldId[htConfig.getKeyExprsBuild().length];
+
+ int i = 0;
+ for (NamedExpression ne : htConfig.getKeyExprsBuild()) {
+ final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incomingBuild, collector, context.getFunctionRegistry());
+ if(collector.hasErrors()) throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
+ if (expr == null) continue;
+ keyExprsBuild[i] = expr;
+
+ final MaterializedField outputField = MaterializedField.create(ne.getRef(), expr.getMajorType());
+ // create a type-specific ValueVector for this key
+ ValueVector vv = TypeHelper.getNewVector(outputField, context.getAllocator());
+ VectorAllocator.getAllocator(vv, 50 /* avg width */).alloc(HashTable.BATCH_SIZE);
+ htKeyFieldIds[i] = htContainerOrig.add(vv);
+
+ i++;
+ }
+
+ if (isProbe) {
+ for (NamedExpression ne : htConfig.getKeyExprsProbe()) {
+ final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incomingProbe, collector, context.getFunctionRegistry());
+ if(collector.hasErrors()) throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
+ if (expr == null) continue;
+ keyExprsProbe[i] = expr;
+ }
+ }
+
+ // generate code for isKeyMatch(), setValue(), getHash() and outputRecordKeys()
+
+ setupIsKeyMatchInternal(cgInner, KeyMatchIncomingBuildMapping, KeyMatchHtableMapping, keyExprsBuild, htKeyFieldIds);
+ setupIsKeyMatchInternal(cgInner, KeyMatchIncomingProbeMapping, KeyMatchHtableMapping, keyExprsProbe, htKeyFieldIds) ;
+
+ setupSetValue(cgInner, keyExprsBuild, htKeyFieldIds);
+ setupOutputRecordKeys(cgInner, htKeyFieldIds, outKeyFieldIds);
+
+ setupGetHash(cg /* use top level code generator for getHash */, GetHashIncomingBuildMapping, keyExprsBuild);
+ setupGetHash(cg /* use top level code generator for getHash */, GetHashIncomingProbeMapping, keyExprsProbe);
+
+ HashTable ht = context.getImplementationClass(top);
+ ht.setup(htConfig, context, incomingBuild, incomingProbe, outgoing, htContainerOrig);
+
+ return ht;
+ }
+
+
+ private void setupIsKeyMatchInternal(ClassGenerator<HashTable> cg, MappingSet incomingMapping, MappingSet htableMapping,
+ LogicalExpression[] keyExprs, TypedFieldId[] htKeyFieldIds)
+ throws SchemaChangeException {
+ cg.setMappingSet(incomingMapping);
+
+ if (keyExprs == null || keyExprs.length == 0) {
+ cg.getEvalBlock()._return(JExpr.FALSE);
+ return;
+ }
+
+ int i = 0;
+ for (LogicalExpression expr : keyExprs) {
+ cg.setMappingSet(incomingMapping);
+ HoldingContainer left = cg.addExpr(expr, false);
+
+ cg.setMappingSet(htableMapping);
+ ValueVectorReadExpression vvrExpr = new ValueVectorReadExpression(htKeyFieldIds[i++]);
+ HoldingContainer right = cg.addExpr(vvrExpr, false);
+
+ // next we wrap the two comparison sides and add the expression block for the comparison.
+ LogicalExpression f = FunctionGenerationHelper.getComparator(left, right, context.getFunctionRegistry());
+ HoldingContainer out = cg.addExpr(f, false);
+
+ // check if two values are not equal (comparator result != 0)
+ JConditional jc = cg.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)));
+
+ jc._then()._return(JExpr.FALSE);
+ }
+
+ // All key expressions compared equal, so return TRUE
+ cg.getEvalBlock()._return(JExpr.TRUE);
+ }
+
+ private void setupSetValue(ClassGenerator<HashTable> cg, LogicalExpression[] keyExprs, TypedFieldId[] htKeyFieldIds)
+ throws SchemaChangeException {
+
+ cg.setMappingSet(SetValueMapping);
+
+ int i = 0;
+ for (LogicalExpression expr : keyExprs) {
+ ValueVectorWriteExpression vvwExpr = new ValueVectorWriteExpression(htKeyFieldIds[i++], expr, true) ;
+
+ HoldingContainer hc = cg.addExpr(vvwExpr, false); // this will write to the htContainer at htRowIdx
+ cg.getEvalBlock()._if(hc.getValue().eq(JExpr.lit(0)))._then()._return(JExpr.FALSE);
+ }
+
+ cg.getEvalBlock()._return(JExpr.TRUE);
+
+ }
+
+ private void setupOutputRecordKeys(ClassGenerator<HashTable> cg, TypedFieldId[] htKeyFieldIds, TypedFieldId[] outKeyFieldIds) {
+
+ cg.setMappingSet(OutputRecordKeysMapping);
+
+ for (int i = 0; i < outKeyFieldIds.length; i++) {
+ ValueVectorReadExpression vvrExpr = new ValueVectorReadExpression(htKeyFieldIds[i]);
+ ValueVectorWriteExpression vvwExpr = new ValueVectorWriteExpression(outKeyFieldIds[i], vvrExpr, true);
+ HoldingContainer hc = cg.addExpr(vvwExpr);
+ cg.getEvalBlock()._if(hc.getValue().eq(JExpr.lit(0)))._then()._return(JExpr.FALSE);
+ }
+
+ cg.getEvalBlock()._return(JExpr.TRUE);
+ }
+
+ private void setupGetHash(ClassGenerator<HashTable> cg, MappingSet incomingMapping, LogicalExpression[] keyExprs) throws SchemaChangeException {
+
+ cg.setMappingSet(incomingMapping);
+
+ if (keyExprs == null || keyExprs.length == 0) {
+ cg.getEvalBlock()._return(JExpr.lit(0));
+ return;
+ }
+
+ HoldingContainer combinedHashValue = null;
+
+ for (int i = 0; i < keyExprs.length; i++) {
+ LogicalExpression expr = keyExprs[i];
+
+ cg.setMappingSet(incomingMapping);
+ HoldingContainer input = cg.addExpr(expr, false);
+
+ // compute the hash(expr)
+ LogicalExpression hashfunc = FunctionGenerationHelper.getFunctionExpression("hash", Types.required(MinorType.INT), context.getFunctionRegistry(), input);
+ HoldingContainer hashValue = cg.addExpr(hashfunc, false);
+
+ if (i == 0) {
+ combinedHashValue = hashValue; // first expression..just use the hash value
+ }
+ else {
+
+ // compute the combined hash value using XOR
+ LogicalExpression xorfunc = FunctionGenerationHelper.getFunctionExpression("xor", Types.required(MinorType.INT), context.getFunctionRegistry(), hashValue, combinedHashValue);
+ combinedHashValue = cg.addExpr(xorfunc, false);
+ }
+ }
+
+ if (combinedHashValue != null) {
+ cg.getEvalBlock()._return(combinedHashValue.getValue()) ;
+ }
+ else {
+ cg.getEvalBlock()._return(JExpr.lit(0));
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
new file mode 100644
index 0000000..d9321b9
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTable.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.common;
+
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.exec.compile.TemplateClassDefinition;
+import org.apache.drill.exec.expr.holders.IntHolder;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.VectorContainer;
+
+public interface HashTable {
+
+ public static TemplateClassDefinition<HashTable> TEMPLATE_DEFINITION = new TemplateClassDefinition<HashTable>(HashTable.class, HashTableTemplate.class);
+
+ /** The initial default capacity of the hash table (in terms of number of buckets). */
+ static final public int DEFAULT_INITIAL_CAPACITY = 1 << 8;
+
+ /** The maximum capacity of the hash table (in terms of number of buckets). */
+ static final public int MAXIMUM_CAPACITY = 1 << 30;
+
+ /** The default load factor of a hash table. */
+ static final public float DEFAULT_LOAD_FACTOR = 0.75f;
+
+ static public enum PutStatus {KEY_PRESENT, KEY_ADDED, PUT_FAILED ;}
+
+ /** The batch size used for internal batch holders */
+ static final public int BATCH_SIZE = Character.MAX_VALUE+1;
+ static final public int BATCH_MASK = 0x0000FFFF;
+
+ public void setup(HashTableConfig htConfig, FragmentContext context,
+ RecordBatch incomingBuild, RecordBatch incomingProbe,
+ RecordBatch outgoing, VectorContainer htContainerOrig);
+
+ public PutStatus put(int incomingRowIdx, IntHolder htIdxHolder);
+
+ public int containsKey(int incomingRowIdx);
+
+ public int size();
+
+ public boolean isEmpty();
+
+ public void clear();
+
+ public boolean outputKeys();
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableConfig.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableConfig.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableConfig.java
new file mode 100644
index 0000000..98892c0
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableConfig.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.common;
+
+import org.apache.drill.common.logical.data.NamedExpression;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+@JsonTypeName("hashtable-config")
+public class HashTableConfig {
+
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HashTableConfig.class);
+
+ private final int initialCapacity;
+ private final float loadFactor;
+ private final NamedExpression[] keyExprsBuild;
+ private final NamedExpression[] keyExprsProbe;
+
+ @JsonCreator
+ public HashTableConfig(@JsonProperty("initialCapacity") int initialCapacity, @JsonProperty("loadFactor") float loadFactor,
+ @JsonProperty("keyExprsBuild") NamedExpression[] keyExprsBuild,
+ @JsonProperty("keyExprsProbe") NamedExpression[] keyExprsProbe) {
+ this.initialCapacity = initialCapacity;
+ this.loadFactor = loadFactor;
+ this.keyExprsBuild = keyExprsBuild;
+ this.keyExprsProbe = keyExprsProbe;
+ }
+
+ public int getInitialCapacity() {
+ return initialCapacity;
+ }
+
+ public float getLoadFactor() {
+ return loadFactor;
+ }
+
+ public NamedExpression[] getKeyExprsBuild() {
+ return keyExprsBuild;
+ }
+
+ public NamedExpression[] getKeyExprsProbe() {
+ return keyExprsProbe;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
new file mode 100644
index 0000000..a4400f0
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/HashTableTemplate.java
@@ -0,0 +1,616 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.common;
+
+import java.util.ArrayList;
+
+import javax.inject.Named;
+
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.logical.data.NamedExpression;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.compile.sig.RuntimeOverridden;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.expr.holders.IntHolder;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.vector.IntVector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.allocator.VectorAllocator;
+import org.apache.drill.exec.vector.BigIntVector;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.vector.allocator.FixedVectorAllocator;
+
+public abstract class HashTableTemplate implements HashTable {
+
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HashTable.class);
+ private static final boolean EXTRA_DEBUG = false;
+
+ private static final int EMPTY_SLOT = -1;
+
+ // A hash 'bucket' consists of the start index to indicate start of a hash chain
+
+ // Array of start indexes. start index is a global index across all batch holders
+ private IntVector startIndices;
+
+ // Array of batch holders..each batch holder can hold up to BATCH_SIZE entries
+ private ArrayList<BatchHolder> batchHolders;
+
+ // Size of the hash table in terms of number of buckets
+ private int tableSize = 0;
+
+ // Threshold after which we rehash; It must be the tableSize * loadFactor
+ private int threshold;
+
+ // Actual number of entries in the hash table
+ private int numEntries = 0;
+
+ // current available (free) slot globally across all batch holders
+ private int freeIndex = 0;
+
+ // Placeholder for the current index while probing the hash table
+ private IntHolder currentIdxHolder;
+
+ private FragmentContext context;
+
+ // The incoming build side record batch
+ private RecordBatch incomingBuild;
+
+ // The incoming probe side record batch (may be null)
+ private RecordBatch incomingProbe;
+
+ // The outgoing record batch
+ private RecordBatch outgoing;
+
+ // Hash table configuration parameters
+ private HashTableConfig htConfig;
+
+ // The original container from which others may be cloned
+ private VectorContainer htContainerOrig;
+
+ private MaterializedField dummyIntField;
+
+ private int outputCount = 0;
+
+ // This class encapsulates the links, keys and values for up to BATCH_SIZE
+ // *unique* records. Thus, suppose there are N incoming record batches, each
+ // of size BATCH_SIZE..but they have M unique keys altogether, the number of
+ // BatchHolders will be (M/BATCH_SIZE) + 1
+ public class BatchHolder {
+
+ // Container of vectors to hold type-specific keys
+ private VectorContainer htContainer;
+
+ // Array of 'link' values
+ private IntVector links;
+
+ // Array of hash values - this is useful when resizing the hash table
+ private IntVector hashValues;
+
+ int maxOccupiedIdx = 0;
+
+ private BatchHolder(int idx) {
+
+ if (idx == 0) { // first batch holder can use the original htContainer
+ htContainer = htContainerOrig;
+ } else { // otherwise create a new one using the original's fields
+ htContainer = new VectorContainer();
+ for (VectorWrapper<?> w : htContainerOrig) {
+ ValueVector vv = TypeHelper.getNewVector(w.getField(), context.getAllocator());
+ VectorAllocator.getAllocator(vv, 50 /* avg width */).alloc(HashTable.BATCH_SIZE);
+ htContainer.add(vv);
+ }
+ }
+
+ links = allocMetadataVector(HashTable.BATCH_SIZE, EMPTY_SLOT);
+ hashValues = allocMetadataVector(HashTable.BATCH_SIZE, 0);
+ }
+
+ private void init(IntVector links, IntVector hashValues, int size) {
+ for (int i=0; i < size; i++) {
+ links.getMutator().set(i, EMPTY_SLOT);
+ }
+ for (int i=0; i < size; i++) {
+ hashValues.getMutator().set(i, 0);
+ }
+ links.getMutator().setValueCount(size);
+ hashValues.getMutator().setValueCount(size);
+ }
+
+ private void setup() {
+ setupInterior(incomingBuild, incomingProbe, outgoing, htContainer);
+ }
+
+ // Check if the key at the currentIdx position in hash table matches the key
+ // at the incomingRowIdx. if the key does not match, update the
+ // currentIdxHolder with the index of the next link.
+ private boolean isKeyMatch(int incomingRowIdx,
+ IntHolder currentIdxHolder,
+ boolean isProbe) {
+
+ int currentIdxWithinBatch = currentIdxHolder.value & BATCH_MASK;
+ boolean match = false;
+
+ if (isProbe)
+ match = isKeyMatchInternalProbe(incomingRowIdx, currentIdxWithinBatch);
+ else
+ match = isKeyMatchInternalBuild(incomingRowIdx, currentIdxWithinBatch);
+
+ if (! match) {
+ currentIdxHolder.value = links.getAccessor().get(currentIdxWithinBatch);
+ }
+ return match;
+ }
+
+ // Insert a new <key1, key2...keyN> entry coming from the incoming batch into the hash table
+ // container at the specified index
+ private boolean insertEntry(int incomingRowIdx, int currentIdx, int hashValue, BatchHolder lastEntryBatch, int lastEntryIdxWithinBatch) {
+ int currentIdxWithinBatch = currentIdx & BATCH_MASK;
+
+ if (! setValue(incomingRowIdx, currentIdxWithinBatch)) {
+ return false;
+ }
+
+ // the previous entry in this hash chain should now point to the entry in this currentIdx
+ if (lastEntryBatch != null) {
+ lastEntryBatch.updateLinks(lastEntryIdxWithinBatch, currentIdx);
+ }
+
+ // since this is the last entry in the hash chain, the links array at position currentIdx
+ // will point to a null (empty) slot
+ links.getMutator().set(currentIdxWithinBatch, EMPTY_SLOT);
+ hashValues.getMutator().set(currentIdxWithinBatch, hashValue);
+
+ maxOccupiedIdx = Math.max(maxOccupiedIdx, currentIdxWithinBatch);
+
+ if (EXTRA_DEBUG) logger.debug("BatchHolder: inserted key at incomingRowIdx = {}, currentIdx = {}, hash value = {}.", incomingRowIdx, currentIdx, hashValue);
+
+ return true;
+ }
+
+ private void updateLinks(int lastEntryIdxWithinBatch, int currentIdx) {
+ links.getMutator().set(lastEntryIdxWithinBatch, currentIdx);
+ }
+
+ private void rehash(int numbuckets, IntVector newStartIndices, int batchStartIdx) {
+
+ if (EXTRA_DEBUG) logger.debug("Resizing and rehashing table...");
+
+ int size = links.getAccessor().getValueCount();
+ IntVector newLinks = allocMetadataVector(size, EMPTY_SLOT);
+ IntVector newHashValues = allocMetadataVector(size, 0);
+
+ for (int i = 0; i <= maxOccupiedIdx; i++) {
+ int entryIdxWithinBatch = i;
+ int entryIdx = entryIdxWithinBatch + batchStartIdx;
+ int hash = hashValues.getAccessor().get(entryIdxWithinBatch); // get the already saved hash value
+ int bucketIdx = getBucketIndex(hash, numbuckets);
+ int newStartIdx = newStartIndices.getAccessor().get(bucketIdx);
+
+ if (newStartIdx == EMPTY_SLOT) { // new bucket was empty
+ newStartIndices.getMutator().set(bucketIdx, entryIdx); // update the start index to point to entry
+ newLinks.getMutator().set(entryIdxWithinBatch, EMPTY_SLOT);
+ newHashValues.getMutator().set(entryIdxWithinBatch, hash);
+
+ if (EXTRA_DEBUG) logger.debug("New bucket was empty. bucketIdx = {}, newStartIndices[ {} ] = {}, newLinks[ {} ] = {}, hash value = {}.", bucketIdx, bucketIdx, newStartIndices.getAccessor().get(bucketIdx), entryIdxWithinBatch, newLinks.getAccessor().get(entryIdxWithinBatch), newHashValues.getAccessor().get(entryIdxWithinBatch));
+
+ } else {
+ // follow the new table's hash chain until we encounter empty slot
+ int idx = newStartIdx;
+ int idxWithinBatch = 0;
+ while (true) {
+ idxWithinBatch = idx & BATCH_MASK;
+ if (newLinks.getAccessor().get(idxWithinBatch) == EMPTY_SLOT) {
+ newLinks.getMutator().set(idxWithinBatch, entryIdx);
+ newLinks.getMutator().set(entryIdxWithinBatch, EMPTY_SLOT);
+ newHashValues.getMutator().set(entryIdxWithinBatch, hash);
+
+ if (EXTRA_DEBUG) logger.debug("Followed hash chain in new bucket. bucketIdx = {}, newLinks[ {} ] = {}, newLinks[ {} ] = {}, hash value = {}.", bucketIdx, idxWithinBatch, newLinks.getAccessor().get(idxWithinBatch), entryIdxWithinBatch, newLinks.getAccessor().get(entryIdxWithinBatch), newHashValues.getAccessor().get(entryIdxWithinBatch));
+
+ break;
+ }
+ idx = newLinks.getAccessor().get(idxWithinBatch);
+ }
+
+ }
+
+ }
+
+ links = newLinks;
+ hashValues = newHashValues;
+ }
+
+ private boolean outputKeys() {
+
+ /** for debugging
+ Object tmp = (htContainer).getValueAccessorById(0, BigIntVector.class).getValueVector();
+ BigIntVector vv0 = null;
+ BigIntHolder holder = null;
+
+ if (tmp != null) {
+ vv0 = ((BigIntVector) tmp);
+ holder = new BigIntHolder();
+ }
+ */
+
+ for (int i = 0; i <= maxOccupiedIdx; i++) {
+ if (outputRecordKeys(i, outputCount) ) {
+ if (EXTRA_DEBUG) logger.debug("Outputting keys to {}", outputCount) ;
+
+ // debugging
+ // holder.value = vv0.getAccessor().get(i);
+ // if (holder.value == 100018 || holder.value == 100021) {
+ // logger.debug("Outputting key = {} at index - {} to outgoing index = {}.", holder.value, i, outputCount);
+ // }
+
+ outputCount++;
+ } else {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private void dump(int idx) {
+ while (true) {
+ int idxWithinBatch = idx & BATCH_MASK;
+ if (idxWithinBatch == EMPTY_SLOT) {
+ break;
+ } else {
+ logger.debug("links[ {} ] = {}, hashValues[ {} ] = {}.", idxWithinBatch, links.getAccessor().get(idxWithinBatch), idxWithinBatch, hashValues.getAccessor().get(idxWithinBatch));
+ idx = links.getAccessor().get(idxWithinBatch);
+ }
+ }
+ }
+
+ private void clear() {
+ htContainer = null;
+ links.clear();
+ links = null;
+ hashValues.clear();
+ hashValues = null;
+ }
+
+ // These methods will be code-generated
+
+ @RuntimeOverridden
+ protected void setupInterior(@Named("incomingBuild") RecordBatch incomingBuild,
+ @Named("incomingProbe") RecordBatch incomingProbe,
+ @Named("outgoing") RecordBatch outgoing,
+ @Named("htContainer") VectorContainer htContainer) {}
+
+ @RuntimeOverridden
+ protected boolean isKeyMatchInternalBuild(@Named("incomingRowIdx") int incomingRowIdx, @Named("htRowIdx") int htRowIdx) {return false;}
+
+ @RuntimeOverridden
+ protected boolean isKeyMatchInternalProbe(@Named("incomingRowIdx") int incomingRowIdx, @Named("htRowIdx") int htRowIdx) {return false;}
+
+ @RuntimeOverridden
+ protected boolean setValue(@Named("incomingRowIdx") int incomingRowIdx, @Named("htRowIdx") int htRowIdx) {return false;}
+
+ @RuntimeOverridden
+ protected boolean outputRecordKeys(@Named("htRowIdx") int htRowIdx, @Named("outRowIdx") int outRowIdx) {return false;}
+
+ } // class BatchHolder
+
+
+ @Override
+ public void setup(HashTableConfig htConfig, FragmentContext context,
+ RecordBatch incomingBuild, RecordBatch incomingProbe,
+ RecordBatch outgoing, VectorContainer htContainerOrig) {
+ float loadf = htConfig.getLoadFactor();
+ int initialCap = htConfig.getInitialCapacity();
+
+ if (loadf <= 0 || Float.isNaN(loadf)) throw new IllegalArgumentException("Load factor must be a valid number greater than 0");
+ if (initialCap <= 0) throw new IllegalArgumentException("The initial capacity must be greater than 0");
+ if (initialCap > MAXIMUM_CAPACITY) throw new IllegalArgumentException("The initial capacity must be less than maximum capacity allowed");
+
+ if (htConfig.getKeyExprsBuild() == null || htConfig.getKeyExprsBuild().length == 0) throw new IllegalArgumentException("Hash table must have at least 1 key expression");
+
+ this.htConfig = htConfig;
+ this.context = context;
+ this.incomingBuild = incomingBuild;
+ this.incomingProbe = incomingProbe;
+ this.outgoing = outgoing;
+ this.htContainerOrig = htContainerOrig;
+
+ // round up the initial capacity to nearest highest power of 2
+ tableSize = roundUpToPowerOf2(initialCap);
+ if (tableSize > MAXIMUM_CAPACITY)
+ tableSize = MAXIMUM_CAPACITY;
+
+ threshold = (int) Math.ceil(tableSize * loadf);
+
+ dummyIntField = MaterializedField.create(new SchemaPath("dummy", ExpressionPosition.UNKNOWN),
+ Types.required(MinorType.INT));
+
+ startIndices = allocMetadataVector(tableSize, EMPTY_SLOT);
+
+ // Create the first batch holder
+ batchHolders = new ArrayList<BatchHolder>();
+ addBatchHolder();
+
+ doSetup(incomingBuild, incomingProbe);
+
+ currentIdxHolder = new IntHolder();
+ }
+
+ public int numBuckets() {
+ return startIndices.getAccessor().getValueCount();
+ }
+
+ public int size() {
+ return numEntries;
+ }
+
+ public boolean isEmpty() {
+ return numEntries == 0;
+ }
+
+ public void clear() {
+ for (BatchHolder bh : batchHolders) {
+ bh.clear();
+ }
+ batchHolders.clear();
+ batchHolders = null;
+ startIndices.clear();
+ startIndices = null;
+ currentIdxHolder = null;
+ numEntries = 0;
+ }
+
+ private int getBucketIndex(int hash, int numBuckets) {
+ return hash & (numBuckets - 1);
+ }
+
+ private static int roundUpToPowerOf2(int number) {
+ int rounded = number >= MAXIMUM_CAPACITY
+ ? MAXIMUM_CAPACITY
+ : (rounded = Integer.highestOneBit(number)) != 0
+ ? (Integer.bitCount(number) > 1) ? rounded << 1 : rounded
+ : 1;
+
+ return rounded;
+ }
+
+ public PutStatus put(int incomingRowIdx, IntHolder htIdxHolder) {
+
+ int hash = getHashBuild(incomingRowIdx);
+ int i = getBucketIndex(hash, numBuckets());
+ int startIdx = startIndices.getAccessor().get(i);
+ int currentIdx;
+ int currentIdxWithinBatch;
+ BatchHolder bh;
+ BatchHolder lastEntryBatch = null;
+ int lastEntryIdxWithinBatch = EMPTY_SLOT;
+
+
+ if (startIdx == EMPTY_SLOT) {
+ // this is the first entry in this bucket; find the first available slot in the
+ // container of keys and values
+ currentIdx = freeIndex++;
+ addBatchIfNeeded(currentIdx);
+
+ if (EXTRA_DEBUG) logger.debug("Empty bucket index = {}. incomingRowIdx = {}; inserting new entry at currentIdx = {}.", i, incomingRowIdx, currentIdx);
+
+ if (insertEntry(incomingRowIdx, currentIdx, hash, lastEntryBatch, lastEntryIdxWithinBatch)) {
+ // update the start index array
+ startIndices.getMutator().set(getBucketIndex(hash, numBuckets()), currentIdx);
+ htIdxHolder.value = currentIdx;
+ return PutStatus.KEY_ADDED;
+ }
+ return PutStatus.PUT_FAILED;
+ }
+
+ currentIdx = startIdx;
+ boolean found = false;
+
+ bh = batchHolders.get( (currentIdx >>> 16) & BATCH_MASK);
+ currentIdxHolder.value = currentIdx;
+
+ // if startIdx is non-empty, follow the hash chain links until we find a matching
+ // key or reach the end of the chain
+ while (true) {
+ currentIdxWithinBatch = currentIdxHolder.value & BATCH_MASK;
+
+ if (bh.isKeyMatch(incomingRowIdx, currentIdxHolder, false)) {
+ htIdxHolder.value = currentIdxHolder.value;
+ found = true;
+ break;
+ }
+ else if (currentIdxHolder.value == EMPTY_SLOT) {
+ lastEntryBatch = bh;
+ lastEntryIdxWithinBatch = currentIdxWithinBatch;
+ break;
+ } else {
+ bh = batchHolders.get( (currentIdxHolder.value >>> 16) & HashTable.BATCH_MASK);
+ lastEntryBatch = bh;
+ }
+ }
+
+ if (!found) {
+ // no match was found, so insert a new entry
+ currentIdx = freeIndex++;
+ addBatchIfNeeded(currentIdx);
+
+ if (EXTRA_DEBUG) logger.debug("No match was found for incomingRowIdx = {}; inserting new entry at currentIdx = {}.", incomingRowIdx, currentIdx);
+
+ if (insertEntry(incomingRowIdx, currentIdx, hash, lastEntryBatch, lastEntryIdxWithinBatch)) {
+ htIdxHolder.value = currentIdx;
+ return PutStatus.KEY_ADDED;
+ }
+ else
+ return PutStatus.PUT_FAILED;
+ }
+
+ return found ? PutStatus.KEY_PRESENT : PutStatus.KEY_ADDED ;
+ }
+
+ private boolean insertEntry(int incomingRowIdx, int currentIdx, int hashValue, BatchHolder lastEntryBatch, int lastEntryIdx) {
+
+ // resize hash table if needed and transfer the metadata
+ resizeAndRehashIfNeeded(currentIdx);
+
+ addBatchIfNeeded(currentIdx);
+
+ BatchHolder bh = batchHolders.get( (currentIdx >>> 16) & BATCH_MASK);
+
+ if (bh.insertEntry(incomingRowIdx, currentIdx, hashValue, lastEntryBatch, lastEntryIdx)) {
+ numEntries++ ;
+ return true;
+ }
+
+ return false;
+ }
+
+ // Return -1 if key is not found in the hash table. Otherwise, return the global index of the key
+ public int containsKey(int incomingRowIdx, boolean isProbe) {
+ int hash = isProbe ? getHashProbe(incomingRowIdx) : getHashBuild(incomingRowIdx);
+ int i = getBucketIndex(hash, numBuckets());
+
+ int currentIdx = startIndices.getAccessor().get(i);
+
+ if (currentIdx == EMPTY_SLOT)
+ return -1;
+
+ BatchHolder bh = batchHolders.get( (currentIdx >>> 16) & BATCH_MASK);
+ currentIdxHolder.value = currentIdx;
+
+ boolean found = false;
+
+ while (true) {
+ if (bh.isKeyMatch(incomingRowIdx, currentIdxHolder, isProbe)) {
+ found = true;
+ break;
+ } else if (currentIdxHolder.value == EMPTY_SLOT) {
+ break;
+ } else {
+ bh = batchHolders.get( (currentIdxHolder.value >>> 16) & BATCH_MASK);
+ }
+ }
+
+ return found ? currentIdxHolder.value : -1;
+ }
+
+ // Add a new BatchHolder to the list of batch holders if needed. This is based on the supplied
+ // currentIdx; since each BatchHolder can hold up to BATCH_SIZE entries, if the currentIdx exceeds
+ // the capacity, we will add a new BatchHolder.
+ private BatchHolder addBatchIfNeeded(int currentIdx) {
+ int totalBatchSize = batchHolders.size() * BATCH_SIZE;
+
+ if (currentIdx >= totalBatchSize) {
+ BatchHolder bh = addBatchHolder();
+ if (EXTRA_DEBUG) logger.debug("HashTable: Added new batch. Num batches = {}.", batchHolders.size());
+ return bh;
+ }
+ else {
+ return batchHolders.get(batchHolders.size() - 1);
+ }
+ }
+
+ private BatchHolder addBatchHolder() {
+ BatchHolder bh = new BatchHolder(batchHolders.size());
+ batchHolders.add(bh);
+ bh.setup();
+ return bh;
+ }
+
+ // Resize the hash table if needed by creating a new one with double the number of buckets.
+ // For each entry in the old hash table, re-hash it to the new table and update the metadata
+ // in the new table.. the metadata consists of the startIndices, links and hashValues.
+ // Note that the keys stored in the BatchHolders are not moved around.
+ private void resizeAndRehashIfNeeded(int currentIdx) {
+ if (numEntries < threshold)
+ return;
+
+ if (EXTRA_DEBUG) logger.debug("Hash table numEntries = {}, threshold = {}; resizing the table...", numEntries, threshold);
+
+ // If the table size is already MAXIMUM_CAPACITY, don't resize
+ // the table, but set the threshold to Integer.MAX_VALUE such that
+ // future attempts to resize will return immediately.
+ if (tableSize == MAXIMUM_CAPACITY) {
+ threshold = Integer.MAX_VALUE;
+ return;
+ }
+
+ int newSize = 2 * tableSize;
+
+ tableSize = roundUpToPowerOf2(newSize);
+ if (tableSize > MAXIMUM_CAPACITY)
+ tableSize = MAXIMUM_CAPACITY;
+
+ // set the new threshold based on the new table size and load factor
+ threshold = (int) Math.ceil(tableSize * htConfig.getLoadFactor());
+
+ IntVector newStartIndices = allocMetadataVector(tableSize, EMPTY_SLOT);
+
+ for (int i = 0; i < batchHolders.size(); i++) {
+ BatchHolder bh = batchHolders.get(i) ;
+ int batchStartIdx = i * BATCH_SIZE;
+ bh.rehash(tableSize, newStartIndices, batchStartIdx);
+ }
+
+ startIndices = newStartIndices;
+
+ if (EXTRA_DEBUG) {
+ logger.debug("After resizing and rehashing, dumping the hash table...");
+ logger.debug("Number of buckets = {}.", startIndices.getAccessor().getValueCount());
+ for (int i = 0; i < startIndices.getAccessor().getValueCount(); i++) {
+ logger.debug("Bucket: {}, startIdx[ {} ] = {}.", i, i, startIndices.getAccessor().get(i));
+ int idx = startIndices.getAccessor().get(i);
+ BatchHolder bh = batchHolders.get( (idx >>> 16) & BATCH_MASK);
+ bh.dump(idx);
+ }
+ }
+ }
+
+ public boolean outputKeys() {
+ for (BatchHolder bh : batchHolders) {
+ if ( ! bh.outputKeys()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private IntVector allocMetadataVector(int size, int initialValue) {
+ IntVector vector = (IntVector) TypeHelper.getNewVector(dummyIntField, context.getAllocator());
+ vector.allocateNew(size);
+ for (int i=0; i < size; i++) {
+ vector.getMutator().set(i, initialValue);
+ }
+ vector.getMutator().setValueCount(size);
+ return vector;
+ }
+
+ // These methods will be code-generated in the context of the outer class
+ protected abstract void doSetup(@Named("incomingBuild") RecordBatch incomingBuild, @Named("incomingProbe") RecordBatch incomingProbe);
+ protected abstract int getHashBuild(@Named("incomingRowIdx") int incomingRowIdx) ;
+ protected abstract int getHashProbe(@Named("incomingRowIdx") int incomingRowIdx) ;
+
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
index 1fa34b6..0d27a1f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
@@ -36,7 +36,7 @@ import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.HoldingContainerExpression;
import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.expr.fn.ComparatorFunctionHelper;
+import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.MergeJoinPOP;
import org.apache.drill.exec.physical.impl.filter.ReturnValueExpression;
@@ -296,7 +296,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
._return(JExpr.lit(1));
}
- LogicalExpression fh = ComparatorFunctionHelper.get(compareLeftExprHolder,
+ LogicalExpression fh = FunctionGenerationHelper.getComparator(compareLeftExprHolder,
compareRightExprHolder,
context.getFunctionRegistry());
cg.addExpr(new ReturnValueExpression(fh, false), false);
@@ -347,7 +347,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
// check value equality
- LogicalExpression gh = ComparatorFunctionHelper.get(compareThisLeftExprHolder,
+ LogicalExpression gh = FunctionGenerationHelper.getComparator(compareThisLeftExprHolder,
compareNextLeftExprHolder,
context.getFunctionRegistry());
cg.addExpr(new ReturnValueExpression(gh, false), false);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
index 84bc897..d6d42b6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
@@ -40,7 +40,7 @@ import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.ValueVectorReadExpression;
-import org.apache.drill.exec.expr.fn.ComparatorFunctionHelper;
+import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.MergingReceiverPOP;
import org.apache.drill.exec.proto.UserBitShared;
@@ -540,7 +540,7 @@ public class MergingRecordBatch implements RecordBatch {
rightVar.ref("isSet"));
// generate the comparison function
- LogicalExpression fh = ComparatorFunctionHelper.get(left, right, context.getFunctionRegistry());
+ LogicalExpression fh = FunctionGenerationHelper.getComparator(left, right, context.getFunctionRegistry());
ClassGenerator.HoldingContainer out = cg.addExpr(fh, false);
// generate less than/greater than checks (fixing results for ASCending vs. DESCending)
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
index 867200e..4eee6af 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
@@ -46,7 +46,7 @@ import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.ValueVectorReadExpression;
import org.apache.drill.exec.expr.ValueVectorWriteExpression;
-import org.apache.drill.exec.expr.fn.ComparatorFunctionHelper;
+import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.OrderedPartitionSender;
import org.apache.drill.exec.physical.impl.sort.SortBatch;
@@ -529,7 +529,7 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart
new ValueVectorReadExpression(new TypedFieldId(expr.getMajorType(), count++)), false);
cg.setMappingSet(mainMapping);
- LogicalExpression fh = ComparatorFunctionHelper.get(left, right, context.getFunctionRegistry());
+ LogicalExpression fh = FunctionGenerationHelper.getComparator(left, right, context.getFunctionRegistry());
ClassGenerator.HoldingContainer out = cg.addExpr(fh, false);
JConditional jc = cg.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
index 4384f0b..949b5f7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectRecordBatch.java
@@ -162,7 +162,7 @@ public class ProjectRecordBatch extends AbstractSingleRecordBatch<Project>{
container.buildSchema(incoming.getSchema().getSelectionVectorMode());
try {
- this.projector = context.getImplementationClass(cg.getClassGenerator());
+ this.projector = context.getImplementationClass(cg.getCodeGenerator());
projector.setup(context, incoming, this, transfers);
} catch (ClassTransformationException | IOException e) {
throw new SchemaChangeException("Failure while attempting to load generated class", e);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
index 9a1b5ab..e2ae823 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
@@ -31,7 +31,7 @@ import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
-import org.apache.drill.exec.expr.fn.ComparatorFunctionHelper;
+import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.Sort;
import org.apache.drill.exec.record.AbstractRecordBatch;
@@ -186,7 +186,7 @@ public class SortBatch extends AbstractRecordBatch<Sort> {
g.setMappingSet(mainMapping);
// next we wrap the two comparison sides and add the expression block for the comparison.
- LogicalExpression fh = ComparatorFunctionHelper.get(left, right, context.getFunctionRegistry());
+ LogicalExpression fh = FunctionGenerationHelper.getComparator(left, right, context.getFunctionRegistry());
HoldingContainer out = g.addExpr(fh, false);
JConditional jc = g.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
index 679001c..3491e7a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java
@@ -114,7 +114,7 @@ public class ExpressionTest {
ClassGenerator<Projector> cg = CodeGenerator.get(Projector.TEMPLATE_DEFINITION, new FunctionImplementationRegistry(DrillConfig.create())).getRoot();
cg.addExpr(new ValueVectorWriteExpression(new TypedFieldId(materializedExpr.getMajorType(), -1), materializedExpr));
- return cg.getClassGenerator().generate();
+ return cg.getCodeGenerator().generate();
}
@AfterClass
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java
index ccf7758..4d09b1d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/SimpleRootExec.java
@@ -95,5 +95,9 @@ public class SimpleRootExec implements RootExec, Iterable<ValueVector>{
return incoming.getRecordCount();
}
+ /// Temporary: for exposing the incoming batch to TestHashTable
+ public RecordBatch getIncoming() {
+ return incoming;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
index 767b75e..4a5f470 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
@@ -20,9 +20,8 @@ package org.apache.drill.exec.physical.impl;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import com.google.common.collect.Lists;
import com.sun.codemodel.JClassAlreadyExistsException;
-import com.sun.codemodel.JVar;
+
import mockit.Injectable;
import mockit.NonStrictExpectations;
@@ -31,24 +30,16 @@ import org.apache.drill.common.expression.*;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
import org.apache.drill.common.util.FileUtils;
-import org.apache.drill.exec.compile.sig.GeneratorMapping;
-import org.apache.drill.exec.compile.sig.MappingSet;
-import org.apache.drill.exec.expr.ClassGenerator;
-import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.fn.DrillFuncHolder;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.expr.holders.NullableVarBinaryHolder;
import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
-import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.memory.TopLevelAllocator;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.PhysicalPlan;
import org.apache.drill.exec.physical.base.FragmentRoot;
-import org.apache.drill.exec.physical.impl.aggregate.Aggregator;
-import org.apache.drill.exec.physical.impl.project.Projector;
import org.apache.drill.exec.planner.PhysicalPlanReader;
import org.apache.drill.exec.proto.CoordinationProtos;
-import org.apache.drill.exec.proto.ExecProtos;
import org.apache.drill.exec.proto.BitControl.PlanFragment;
import org.apache.drill.exec.resolver.FunctionResolver;
import org.apache.drill.exec.resolver.FunctionResolverFactory;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggr.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggr.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggr.java
new file mode 100644
index 0000000..1688bed
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/agg/TestHashAggr.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.physical.impl.agg;
+
+import static org.junit.Assert.*;
+
+import org.apache.drill.exec.client.QuerySubmitter;
+import org.junit.AfterClass;
+import org.junit.Test;
+
+
+public class TestHashAggr {
+
+ /*
+ @Test
+ public void test1() throws Exception {
+ String[] strArr = {"-f",
+ // "/Users/asinha/queries/hashaggr/q8.json",
+ // "/Users/asinha/queries/hashaggr/q8_1.json",
+ // "/Users/asinha/queries/hashaggr/q8_2.json",
+ // "/Users/asinha/queries/hashaggr/q8_3.json",
+ //"/Users/asinha/queries/streamaggr/q8_2.json",
+ // "/Users/asinha/queries/hashaggr/q5.json",
+ //"/Users/asinha/queries/hashaggr/q5_1.json",
+ // "/Users/asinha/queries/hashaggr/q5_2.json",
+ // "/Users/asinha/queries/streamaggr/q4_1.json",
+ // "/Users/asinha/queries/streamaggr/q4_3.json",
+ // "/Users/asinha/queries/hashaggr/q6.json",
+ // "/Users/asinha/queries/hashaggr/q9.json",
+ // "/Users/asinha/queries/hashaggr/q9_1.json",
+ "/Users/asinha/queries/hashaggr/q7_2.json",
+ "-t",
+ "physical",
+ "-local" };
+ QuerySubmitter.main(strArr);
+
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception{
+ // pause to get logger to catch up.
+ Thread.sleep(1000);
+ }
+ */
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java
new file mode 100644
index 0000000..02bb60c
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/common/TestHashTable.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.common;
+
+import static org.junit.Assert.*;
+import mockit.Injectable;
+import mockit.NonStrictExpectations;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.logical.data.NamedExpression;
+import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.expr.holders.IntHolder;
+import org.apache.drill.exec.memory.TopLevelAllocator;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.PhysicalPlan;
+import org.apache.drill.exec.physical.base.FragmentRoot;
+import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
+import org.apache.drill.exec.physical.impl.ImplCreator;
+import org.apache.drill.exec.physical.impl.SimpleRootExec;
+import org.apache.drill.exec.planner.PhysicalPlanReader;
+import org.apache.drill.exec.proto.CoordinationProtos;
+import org.apache.drill.exec.proto.BitControl.PlanFragment;
+import org.apache.drill.exec.rpc.user.UserServer.UserClientConnection;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.vector.BigIntVector;
+import org.apache.drill.exec.vector.IntVector;
+import org.junit.Test;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import com.codahale.metrics.MetricRegistry;
+
+public class TestHashTable {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestHashTable.class);
+ DrillConfig c = DrillConfig.create();
+
+ @SuppressWarnings("deprecation")
+private SimpleRootExec doTest(final DrillbitContext bitContext, UserClientConnection connection, String plan_path) throws Exception{
+ new NonStrictExpectations(){{
+ bitContext.getMetrics(); result = new MetricRegistry();
+ bitContext.getAllocator(); result = new TopLevelAllocator();
+ bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c);
+ }};
+
+
+
+ PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
+ PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile(plan_path), Charsets.UTF_8));
+ FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
+ FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), connection, registry);
+ SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
+ return exec;
+ }
+/*
+ @Test
+ public void testHashTable1(@Injectable final DrillbitContext bitContext, @Injectable UserClientConnection connection) throws Throwable {
+ String plan_path = "/common/test_hashtable1.json";
+ SimpleRootExec exec = doTest(bitContext, connection, plan_path);
+
+ SchemaPath regionkey = new SchemaPath("regionkey", ExpressionPosition.UNKNOWN);
+ SchemaPath nationkey = new SchemaPath("nationkey", ExpressionPosition.UNKNOWN);
+
+ NamedExpression[] keyExprs = new NamedExpression[1];
+ keyExprs[0] = new NamedExpression(regionkey, new FieldReference(regionkey));
+
+ HashTableConfig htConfig = new HashTableConfig(HashTable.DEFAULT_INITIAL_CAPACITY, HashTable.DEFAULT_LOAD_FACTOR, keyExprs);
+ ChainedHashTable cht = new ChainedHashTable(htConfig, exec.getContext(), exec.getIncoming(), null);
+ HashTable htable = cht.createAndSetupHashTable();
+ IntHolder htIdxHolder = new IntHolder();
+
+ for (int i = 0; i < exec.getRecordCount(); i++) {
+ HashTable.PutStatus putStatus = htable.put(i, htIdxHolder);
+ assertNotEquals(putStatus, HashTable.PutStatus.PUT_FAILED);
+ }
+ assertEquals(htable.size(), 5);
+ }
+ */
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/test/resources/common/test_hashtable1.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/common/test_hashtable1.json b/exec/java-exec/src/test/resources/common/test_hashtable1.json
new file mode 100644
index 0000000..7f0ccaa
--- /dev/null
+++ b/exec/java-exec/src/test/resources/common/test_hashtable1.json
@@ -0,0 +1,40 @@
+{
+ head:{
+ type:"APACHE_DRILL_PHYSICAL",
+ version:"1",
+ generator:{
+ type:"manual"
+ }
+ },
+ graph : [ {
+ pop : "parquet-scan",
+ @id : 1,
+ entries : [ {
+ path : "/tmp/parquet/nation.parquet"
+ } ],
+ storageengine : {
+ type : "parquet",
+ dfsName : "file:///"
+ },
+ ref : "_MAP",
+ fragmentPointer : 0
+ }, {
+ pop : "project",
+ @id : 2,
+ exprs : [ {
+ ref : "regionkey",
+ expr : "_MAP.N_REGIONKEY"
+ }, {
+ ref : "nationkey",
+ expr : "_MAP.N_NATIONKEY"
+ } ],
+ child : 1
+ }, {
+ @id: 3,
+ child: 2,
+ pop: "screen"
+ }
+ ]
+}
+
+
[11/12] git commit: DRILL-432: Can not run 'sqlline' on Windows with
CYGWIN * Clean up and unify various shell scripts.
Posted by ja...@apache.org.
DRILL-432: Can not run 'sqlline' on Windows with CYGWIN
* Clean up and unify various shell scripts.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/822cbb06
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/822cbb06
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/822cbb06
Branch: refs/heads/master
Commit: 822cbb0692a99969c19305096a5ba4a948e120c8
Parents: 6804b52
Author: Aditya Kishore <ad...@maprtech.com>
Authored: Fri Mar 21 16:39:42 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sat Mar 22 10:04:11 2014 -0700
----------------------------------------------------------------------
distribution/src/resources/drill-config.sh | 135 ++++++++++++++++--------
distribution/src/resources/runbit | 29 +----
distribution/src/resources/sqlline | 50 ++-------
distribution/src/resources/submit_plan | 33 +-----
4 files changed, 105 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822cbb06/distribution/src/resources/drill-config.sh
----------------------------------------------------------------------
diff --git a/distribution/src/resources/drill-config.sh b/distribution/src/resources/drill-config.sh
index 7be5bad..d5667f9 100644
--- a/distribution/src/resources/drill-config.sh
+++ b/distribution/src/resources/drill-config.sh
@@ -1,31 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
-#/**
-# * Copyright 2013 The Apache Software Foundation
-# *
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements. See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership. The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-
# included in all the drill scripts with source command
# should not be executable directly
# also should not be passed any arguments, since we need original $*
-# Modelled after $HADOOP_HOME/bin/hadoop-env.sh.
+# Modelled after $HADOOP_HOME/bin/hadoop-config.sh
+#
+# Environment Variables:
+#
+# JAVA_HOME The java implementation to use.
+#
+# DRILL_CLASSPATH Extra Java CLASSPATH entries.
+#
+# HADOOP_HOME Hadoop home
# resolve links - "${BASH_SOURCE-$0}" may be a softlink
-
this="${BASH_SOURCE-$0}"
while [ -h "$this" ]; do
ls=`ls -ld "$this"`
@@ -40,19 +41,17 @@ done
# convert relative path to absolute path
bin=`dirname "$this"`
script=`basename "$this"`
-bin=`cd "$bin">/dev/null; pwd`
-this="$bin/$script"
+home=`cd "$bin/..">/dev/null; pwd`
+this="$home/bin/$script"
# the root of the drill installation
if [ -z "$DRILL_HOME" ]; then
- export DRILL_HOME=`dirname "$this"`/..
+ DRILL_HOME="$home"
fi
#check to see if the conf dir or drill home are given as an optional arguments
-while [ $# -gt 1 ]
-do
- if [ "--config" = "$1" ]
- then
+while [ $# -gt 1 ]; do
+ if [ "--config" = "$1" ]; then
shift
confdir=$1
shift
@@ -62,35 +61,55 @@ do
break
fi
done
-
+
# Allow alternate drill conf dir location.
-export DRILL_CONF_DIR="${DRILL_CONF_DIR:-/etc/drill/conf}"
+DRILL_CONF_DIR="${DRILL_CONF_DIR:-/etc/drill/conf}"
-if [ ! -d $DRILL_CONF_DIR ]
-then
- export DRILL_CONF_DIR=$DRILL_HOME/conf
+if [ ! -d $DRILL_CONF_DIR ]; then
+ DRILL_CONF_DIR=$DRILL_HOME/conf
fi
+# Source drill-env.sh for any user configured values
. "${DRILL_CONF_DIR}/drill-env.sh"
-if [ "${HADOOP_HOME}x" != "x" ]
-then
+# get log directory
+if [ "$DRILL_LOG_DIR" = "" ]; then
+ DRILL_LOG_DIR=/var/log/drill
+fi
+
+# If HADOOP_HOME is specified, add all Hadoop jars except those
+# specifically excluded in $DRILL_HOME/bin/hadoop-excludes.txt
+if [ "${HADOOP_HOME}x" != "x" ] ; then
HADOOP_CLASSPATH=""
- for jar in `ls $HADOOP_HOME/lib/*jar`
- do
+ for jar in `ls $HADOOP_HOME/lib/*jar` ; do
echo $jar | grep -v -f $DRILL_HOME/bin/hadoop-excludes.txt >/dev/null
- if [ "$?" -eq "0" ]
- then
+ if [ "$?" -eq "0" ] ; then
HADOOP_CLASSPATH=$jar:$HADOOP_CLASSPATH
fi
done
- export HADOOP_CLASSPATH=$HADOOP_HOME/conf:$HADOOP_CLASSPATH
+ HADOOP_CLASSPATH=$HADOOP_HOME/conf:$HADOOP_CLASSPATH
+fi
+
+# Setup Drill classpath entries
+CP=$DRILL_HOME/jars/*:$DRILL_CLASSPATH
+CP=$DRILL_HOME/lib/*:$CP
+CP=$DRILL_HOME/contrib/*:$CP
+if [ "${HADOOP_CLASSPATH}x" != "x" ]; then
+ CP=$HADOOP_CLASSPATH:$CP
fi
+CP=$DRILL_CONF_DIR:$CP
# Newer versions of glibc use an arena memory allocator that causes virtual
# memory usage to explode. Tune the variable down to prevent vmem explosion.
export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4}
+# Test for cygwin
+is_cygwin=false
+case "`uname`" in
+CYGWIN*) is_cygwin=true;;
+esac
+
+# Test for or find JAVA_HOME
if [ -z "$JAVA_HOME" ]; then
if [ -e `which java` ]; then
SOURCE=`which java`
@@ -107,9 +126,41 @@ if [ -z "$JAVA_HOME" ]; then
+======================================================================+
| Error: JAVA_HOME is not set and Java could not be found |
+----------------------------------------------------------------------+
-| Drill requires Java 1.7 or later. |
+| Apache Drill requires Java 1.7 or later. |
+======================================================================+
EOF
exit 1
fi
fi
+# Now, verify that 'java' binary exists and is suitable for Drill.
+if $is_cygwin; then
+ JAVA_BIN="java.exe"
+else
+ JAVA_BIN="java"
+fi
+JAVA=`find -L "$JAVA_HOME" -name $JAVA_BIN | head -n 1`
+if [ ! -e "$JAVA" ]; then
+ echo "Java not found at JAVA_HOME=$JAVA_HOME."
+ exit 1
+fi
+# Ensure that Java version is at least 1.7
+"$JAVA" -version 2>&1 | grep "version" | egrep -e "1.4|1.5|1.6" > /dev/null
+if [ $? -eq 0 ]; then
+ echo "Java 1.7 or later is required to run Apache Drill."
+ exit 1
+fi
+
+# Adjust paths for CYGWIN
+if $is_cygwin; then
+ DRILL_HOME=`cygpath -w "$DRILL_HOME"`
+ DRILL_CONF_DIR=`cygpath -w "$DRILL_CONF_DIR"`
+ DRILL_LOG_DIR=`cygpath -w "$DRILL_LOG_DIR"`
+ CP=`cygpath -w -p "$CP"`
+fi
+
+# Variables exported form this script
+export is_cygwin
+export DRILL_HOME
+export DRILL_CONF_DIR
+export DRILL_LOG_DIR
+export CP
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822cbb06/distribution/src/resources/runbit
----------------------------------------------------------------------
diff --git a/distribution/src/resources/runbit b/distribution/src/resources/runbit
index 85daf87..a52c9e6 100755
--- a/distribution/src/resources/runbit
+++ b/distribution/src/resources/runbit
@@ -1,5 +1,5 @@
#!/bin/bash
-
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -15,30 +15,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+bin=`dirname "${BASH_SOURCE-$0}"`
+bin=`cd "$bin">/dev/null; pwd`
-if [ -z $JAVA_HOME ]
-then
- JAVA=`which java`
-else
- JAVA=`find -L $JAVA_HOME -name java | head -n 1`
-fi
-
-if [ -e $JAVA ]; then
- echo ""
-else
- echo "Java not found."
- exit 1
-fi
-
-$JAVA -version 2>&1 | grep "version" | egrep -e "1.7" > /dev/null
-if [ $? -ne 0 ]; then
- echo "Java 1.7 is required to run Apache Drill."
- exit 1
-fi
-
-CP=$DRILL_HOME/jars/*:$CP
-CP=$DRILL_HOME/lib/*:$CP
-CP=$CP:$DRILL_CONF_DIR
-CP=$HADOOP_CLASSPATH:$CP
+. "$bin"/drill-config.sh
exec $JAVA $DRILL_JAVA_OPTS -cp $CP org.apache.drill.exec.server.Drillbit
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822cbb06/distribution/src/resources/sqlline
----------------------------------------------------------------------
diff --git a/distribution/src/resources/sqlline b/distribution/src/resources/sqlline
index c526f03..ff83ab1 100755
--- a/distribution/src/resources/sqlline
+++ b/distribution/src/resources/sqlline
@@ -1,5 +1,5 @@
#!/bin/bash
-
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -16,8 +16,7 @@
# limitations under the License.
ARGS=();
-while [ $# -gt 0 ]
-do
+while [ $# -gt 0 ] ; do
case "$1" in
-q) shift;
QUERY=$1;;
@@ -35,47 +34,12 @@ bin=`cd "$bin">/dev/null; pwd`
. "$bin"/drill-config.sh
-if [ -z $JAVA_HOME ]
-then
- JAVA=`which java`
-else
- JAVA=`find -L $JAVA_HOME -name java | head -n 1`
-fi
-
-if [ -e $JAVA ]; then
- echo ""
-else
- echo "Java not found."
- exit 1
-fi
-
-$JAVA -version 2>&1 | grep "version" | egrep -e "1.7" > /dev/null
-if [ $? -ne 0 ]; then
- echo "Java 1.7 is required to run Apache Drill."
- exit 1
-fi
-
-# get log directory
-if [ "$DRILL_LOG_DIR" = "" ]; then
- export DRILL_LOG_DIR=/var/log/drill
-fi
-
-
-CP=$DRILL_HOME/jars/*:$CP
-CP=$DRILL_HOME/lib/*:$CP
-CP=$DRILL_HOME/contrib/*:$CP
-CP=$HADOOP_CLASSPATH:$CP
-
-CP=$DRILL_CONF_DIR:$CP
-
DRILL_SHELL_JAVA_OPTS="$DRILL_SHELL_JAVA_OPTS -Dlog.path=$DRILL_LOG_DIR/sqlline.log"
-if [ -n "$QUERY" ]
-then
- echo $QUERY | exec $JAVA $DRILL_SHELL_JAVA_OPTS $DRILL_JAVA_OPTS -cp $CP sqlline.SqlLine -d org.apache.drill.jdbc.Driver "${ARGS[@]}"
-elif [ -n "$FILE" ]
-then
- exec $JAVA $DRILL_SHELL_JAVA_OPTS $DRILL_JAVA_OPTS -cp $CP sqlline.SqlLine "${ARGS[@]}" --run=$FILE
+if [ -n "$QUERY" ] ; then
+ echo $QUERY | exec "$JAVA" $DRILL_SHELL_JAVA_OPTS $DRILL_JAVA_OPTS -cp $CP sqlline.SqlLine -d org.apache.drill.jdbc.Driver "${ARGS[@]}"
+elif [ -n "$FILE" ] ; then
+ exec "$JAVA" $DRILL_SHELL_JAVA_OPTS $DRILL_JAVA_OPTS -cp $CP sqlline.SqlLine "${ARGS[@]}" --run=$FILE
else
- exec $JAVA $DRILL_SHELL_JAVA_OPTS $DRILL_JAVA_OPTS -cp $CP sqlline.SqlLine "${ARGS[@]}"
+ exec "$JAVA" $DRILL_SHELL_JAVA_OPTS $DRILL_JAVA_OPTS -cp $CP sqlline.SqlLine "${ARGS[@]}"
fi
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/822cbb06/distribution/src/resources/submit_plan
----------------------------------------------------------------------
diff --git a/distribution/src/resources/submit_plan b/distribution/src/resources/submit_plan
index 3c9184a..36666cc 100755
--- a/distribution/src/resources/submit_plan
+++ b/distribution/src/resources/submit_plan
@@ -1,5 +1,5 @@
#!/bin/bash
-
+#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -20,37 +20,6 @@ bin=`cd "$bin">/dev/null; pwd`
. "$bin"/drill-config.sh
-if [ -z $JAVA_HOME ]
-then
- JAVA=`which java`
-else
- JAVA=`find -L $JAVA_HOME -name java | head -n 1`
-fi
-
-if [ -e $JAVA ]; then
- echo ""
-else
- echo "Java not found."
- exit 1
-fi
-
-$JAVA -version 2>&1 | grep "version" | egrep -e "1.7" > /dev/null
-if [ $? -ne 0 ]; then
- echo "Java 1.7 is required to run Apache Drill."
- exit 1
-fi
-
-# get log directory
-if [ "$DRILL_LOG_DIR" = "" ]; then
- export DRILL_LOG_DIR=/var/log/drill
-fi
-
-CP=$DRILL_HOME/jars/*:$CP
-CP=$DRILL_HOME/lib/*:$CP
-
-CP=$DRILL_CONF_DIR:$CP
-CP=$HADOOP_CLASSPATH:$CP
-
DRILL_SHELL_JAVA_OPTS="$DRILL_SHELL_JAVA_OPTS -Dlog.path=$DRILL_LOG_DIR/submitter.log"
exec $JAVA $DRILL_SHELL_JAVA_OPTS $DRILL_JAVA_OPTS -cp $CP org.apache.drill.exec.client.QuerySubmitter $@
[12/12] git commit: DRILL-429: Remove extraneous casts.
Posted by ja...@apache.org.
DRILL-429: Remove extraneous casts.
Change cast from a function call to a new, separate type of expression.
Add test to confirm functionality.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/30ce7d71
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/30ce7d71
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/30ce7d71
Branch: refs/heads/master
Commit: 30ce7d710a1bb876a272fd454ff286b5909ff717
Parents: 822cbb0
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sat Mar 22 11:50:31 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sat Mar 22 11:50:31 2014 -0700
----------------------------------------------------------------------
.../drill/common/expression/CastExpression.java | 65 +++++++++++++++
.../expression/ExpressionStringBuilder.java | 56 +++++++++++++
.../common/expression/FunctionCallFactory.java | 12 +--
.../visitors/AbstractExprVisitor.java | 6 ++
.../expression/visitors/AggregateChecker.java | 6 ++
.../expression/visitors/ConstantChecker.java | 6 ++
.../common/expression/visitors/ExprVisitor.java | 7 +-
.../visitors/ExpressionValidator.java | 6 ++
.../sig/ConstantExpressionIdentifier.java | 7 ++
.../drill/exec/expr/EvaluationVisitor.java | 24 +++++-
.../exec/expr/ExpressionTreeMaterializer.java | 85 +++++++++++++++++++-
.../drill/exec/planner/logical/DrillOptiq.java | 32 ++++----
.../drill/exec/rpc/RpcExceptionHandler.java | 2 +-
.../apache/drill/jdbc/test/TestJdbcQuery.java | 1 -
14 files changed, 280 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/30ce7d71/common/src/main/java/org/apache/drill/common/expression/CastExpression.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/CastExpression.java b/common/src/main/java/org/apache/drill/common/expression/CastExpression.java
new file mode 100644
index 0000000..7e5eea0
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/expression/CastExpression.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import java.util.Collections;
+import java.util.Iterator;
+
+import org.apache.drill.common.expression.visitors.ExprVisitor;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+
+public class CastExpression extends LogicalExpressionBase implements Iterable<LogicalExpression>{
+
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CastExpression.class);
+
+ private final LogicalExpression input;
+ private final MajorType type;
+
+ public CastExpression(LogicalExpression input, MajorType type, ExpressionPosition pos) {
+ super(pos);
+ this.input = input;
+ this.type = type;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitCastExpression(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Collections.singleton(input).iterator();
+ }
+
+ public LogicalExpression getInput() {
+ return input;
+ }
+
+ @Override
+ public MajorType getMajorType() {
+ return type;
+ }
+
+ @Override
+ public String toString() {
+ return "CastExpression [input=" + input + ", type=" + type + "]";
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/30ce7d71/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java b/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
index 80dfbac..80fd41b 100644
--- a/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
+++ b/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
@@ -20,9 +20,12 @@ package org.apache.drill.common.expression;
import org.apache.drill.common.expression.IfExpression.IfCondition;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
+import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
import org.apache.drill.common.expression.ValueExpressions.LongExpression;
import org.apache.drill.common.expression.ValueExpressions.QuotedString;
import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
+import org.apache.drill.common.types.TypeProtos.MajorType;
import com.google.common.collect.ImmutableList;
@@ -96,6 +99,59 @@ public class ExpressionStringBuilder extends AbstractExprVisitor<Void, StringBui
sb.append("\"");
return null;
}
+
+ @Override
+ public Void visitCastExpression(CastExpression e, StringBuilder sb) throws RuntimeException {
+ MajorType mt = e.getMajorType();
+
+ sb.append("cast( (");
+ e.getInput().accept(this, sb);
+ sb.append(" ) as ");
+ sb.append(mt.getMinorType().name());
+
+ switch(mt.getMinorType()){
+ case FLOAT4:
+ case FLOAT8:
+ case INT:
+ case SMALLINT:
+ case BIGINT:
+ case UINT1:
+ case UINT2:
+ case UINT4:
+ case UINT8:
+ // do nothing else.
+ break;
+ case VAR16CHAR:
+ case VARBINARY:
+ case VARCHAR:
+ case FIXED16CHAR:
+ case FIXEDBINARY:
+ case FIXEDCHAR:
+ // add size in parens
+ sb.append("(");
+ sb.append(mt.getWidth());
+ sb.append(")");
+ break;
+ default:
+ throw new UnsupportedOperationException(String.format("Unable to convert cast expression %s into string.", e));
+ }
+ sb.append(" )");
+ return null;
+ }
+
+ @Override
+ public Void visitFloatConstant(FloatExpression fExpr, StringBuilder sb) throws RuntimeException {
+ sb.append(fExpr.getFloat());
+ return null;
+ }
+
+ @Override
+ public Void visitIntConstant(IntExpression intExpr, StringBuilder sb) throws RuntimeException {
+ sb.append(intExpr.getInt());
+ return null;
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/30ce7d71/common/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java b/common/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java
index df4fcb7..c6c7074 100644
--- a/common/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java
+++ b/common/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java
@@ -72,17 +72,7 @@ public class FunctionCallFactory {
* expr -- input expression
*/
public static LogicalExpression createCast(MajorType type, ExpressionPosition ep, LogicalExpression expr){
- String castFuncWithType = "cast" + type.getMinorType().name();
-
- List<LogicalExpression> newArgs = Lists.newArrayList();
- newArgs.add(expr); //input_expr
-
- //VarLen type
- if (!Types.isFixedWidthType(type)) {
- newArgs.add(new ValueExpressions.LongExpression(type.getWidth(), null));
- }
-
- return new FunctionCall(castFuncWithType, newArgs, ep);
+ return new CastExpression(expr, type, ep);
}
public static LogicalExpression createExpression(String functionName, List<LogicalExpression> args){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/30ce7d71/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java b/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
index ef96cab..727a3dc 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.common.expression.visitors;
+import org.apache.drill.common.expression.CastExpression;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.IfExpression;
@@ -83,6 +84,11 @@ public abstract class AbstractExprVisitor<T, VAL, EXCEP extends Exception> imple
}
@Override
+ public T visitCastExpression(CastExpression e, VAL value) throws EXCEP {
+ return visitUnknown(e, value);
+ }
+
+ @Override
public T visitUnknown(LogicalExpression e, VAL value) throws EXCEP {
throw new UnsupportedOperationException(String.format("Expression of type %s not handled by visitor type %s.", e.getClass().getCanonicalName(), this.getClass().getCanonicalName()));
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/30ce7d71/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java b/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
index 71ac82e..630d00a 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.common.expression.visitors;
+import org.apache.drill.common.expression.CastExpression;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.IfExpression;
@@ -111,6 +112,11 @@ public final class AggregateChecker extends SimpleExprVisitor<Boolean>{
public Boolean visitUnknown(LogicalExpression e, Void value) throws RuntimeException {
return false;
}
+
+ @Override
+ public Boolean visitCastExpression(CastExpression e, Void value) throws RuntimeException {
+ return e.getInput().accept(this, value);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/30ce7d71/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java b/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
index 1391c84..60e997f 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.common.expression.visitors;
+import org.apache.drill.common.expression.CastExpression;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.FunctionHolderExpression;
@@ -115,4 +116,9 @@ final class ConstantChecker implements ExprVisitor<Boolean, ErrorCollector, Runt
return false;
}
+ @Override
+ public Boolean visitCastExpression(CastExpression e, ErrorCollector value) throws RuntimeException {
+ return e.getInput().accept(this, value);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/30ce7d71/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java b/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
index 10b0acf..e199f21 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.common.expression.visitors;
+import org.apache.drill.common.expression.CastExpression;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.IfExpression;
@@ -24,10 +25,10 @@ import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
-import org.apache.drill.common.expression.ValueExpressions.LongExpression;
-import org.apache.drill.common.expression.ValueExpressions.QuotedString;
import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
import org.apache.drill.common.expression.ValueExpressions.IntExpression;
+import org.apache.drill.common.expression.ValueExpressions.LongExpression;
+import org.apache.drill.common.expression.ValueExpressions.QuotedString;
public interface ExprVisitor<T, VAL, EXCEP extends Exception> {
@@ -42,4 +43,6 @@ public interface ExprVisitor<T, VAL, EXCEP extends Exception> {
public T visitBooleanConstant(BooleanExpression e, VAL value) throws EXCEP;
public T visitQuotedStringConstant(QuotedString e, VAL value) throws EXCEP;
public T visitUnknown(LogicalExpression e, VAL value) throws EXCEP;
+ public T visitCastExpression(CastExpression e, VAL value) throws EXCEP;
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/30ce7d71/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java b/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java
index 396517a..96666bd 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.common.expression.visitors;
+import org.apache.drill.common.expression.CastExpression;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.FunctionHolderExpression;
@@ -137,4 +138,9 @@ public class ExpressionValidator implements ExprVisitor<Void, ErrorCollector, Ru
return null;
}
+ @Override
+ public Void visitCastExpression(CastExpression e, ErrorCollector value) throws RuntimeException {
+ return e.accept(this, value);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/30ce7d71/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
index fbb7cc8..e476897 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
@@ -22,6 +22,7 @@ import java.util.IdentityHashMap;
import java.util.List;
import java.util.Set;
+import org.apache.drill.common.expression.CastExpression;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.IfExpression;
@@ -139,6 +140,12 @@ public class ConstantExpressionIdentifier implements ExprVisitor<Boolean, Identi
}
@Override
+ public Boolean visitCastExpression(CastExpression e, IdentityHashMap<LogicalExpression, Object> value)
+ throws RuntimeException {
+ return e.getInput().accept(this, value);
+ }
+
+ @Override
public Boolean visitUnknown(LogicalExpression e, IdentityHashMap<LogicalExpression, Object> value){
return checkChildren(e, value, false);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/30ce7d71/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
index c7f7336..8dfba76 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
@@ -17,9 +17,10 @@
*/
package org.apache.drill.exec.expr;
-import java.util.Collections;
+import java.util.List;
import java.util.Set;
+import org.apache.drill.common.expression.CastExpression;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.IfExpression;
@@ -27,6 +28,7 @@ import org.apache.drill.common.expression.IfExpression.IfCondition;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.expression.TypedNullConstant;
+import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
import org.apache.drill.common.expression.ValueExpressions.LongExpression;
@@ -35,10 +37,9 @@ import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
-
+import org.apache.drill.exec.compile.sig.ConstantExpressionIdentifier;
import org.apache.drill.exec.expr.ClassGenerator.BlockType;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
-import org.apache.drill.exec.compile.sig.ConstantExpressionIdentifier;
import org.apache.drill.exec.expr.fn.DrillFuncHolder;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.expr.fn.HiveFuncHolder;
@@ -46,6 +47,7 @@ import org.apache.drill.exec.physical.impl.filter.ReturnValueExpression;
import org.apache.drill.exec.record.NullExpression;
import org.apache.drill.exec.vector.ValueHolderHelper;
+import com.google.common.collect.Lists;
import com.sun.codemodel.JBlock;
import com.sun.codemodel.JClass;
import com.sun.codemodel.JConditional;
@@ -317,6 +319,22 @@ public class EvaluationVisitor {
setup.assign(var, ((JClass)generator.getModel().ref(ValueHolderHelper.class)).staticInvoke("getVarCharHolder").arg(stringLiteral));
return new HoldingContainer(majorType, var, null, null);
}
+
+ @Override
+ public HoldingContainer visitCastExpression(CastExpression e, ClassGenerator<?> value) throws RuntimeException {
+ // we create
+ MajorType type = e.getMajorType();
+ String castFuncWithType = "cast" + type.getMinorType().name();
+
+ List<LogicalExpression> newArgs = Lists.newArrayList();
+ newArgs.add(e.getInput()); //input_expr
+
+ //VarLen type
+ if (!Types.isFixedWidthType(type)) {
+ newArgs.add(new ValueExpressions.LongExpression(type.getWidth(), null));
+ }
+ FunctionCall fc = new FunctionCall(castFuncWithType, newArgs, e.getPosition());
+ return fc.accept(this, value); }
}
private class ConstantFilter extends EvalVisitor {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/30ce7d71/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
index def54d9..f764d32 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.expr;
import java.util.List;
+import org.apache.drill.common.expression.CastExpression;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FunctionCall;
@@ -27,6 +28,7 @@ import org.apache.drill.common.expression.IfExpression;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.expression.TypedNullConstant;
+import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
@@ -37,6 +39,7 @@ import org.apache.drill.common.expression.fn.CastFunctions;
import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
import org.apache.drill.common.expression.visitors.ExpressionValidator;
import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
@@ -90,8 +93,8 @@ public class ExpressionTreeMaterializer {
@Override
public LogicalExpression visitFunctionHolderExpression(FunctionHolderExpression holder, FunctionImplementationRegistry value) throws RuntimeException {
- throw new UnsupportedOperationException(
- String.format("Can't support materialize %s", holder.getClass().getCanonicalName()));
+ // a function holder is already materialized, no need to rematerialize. generally this won't be used unless we materialize a partial tree and rematerialize the whole tree.
+ return holder;
}
@Override
@@ -242,5 +245,83 @@ public class ExpressionTreeMaterializer {
public LogicalExpression visitQuotedStringConstant(QuotedString e, FunctionImplementationRegistry registry) {
return e;
}
+
+ @Override
+ public LogicalExpression visitCastExpression(CastExpression e, FunctionImplementationRegistry value){
+
+ // if the cast is pointless, remove it.
+ LogicalExpression input = e.getInput().accept(this, value);
+
+ MajorType newMajor = e.getMajorType();
+ MinorType newMinor = input.getMajorType().getMinorType();
+
+ if(castEqual(e.getPosition(), newMajor, input.getMajorType())) return input; // don't do pointless cast.
+
+
+ if(newMinor == MinorType.LATE || newMinor == MinorType.NULL){
+ // if the type still isn't fully bound, leave as cast expression.
+ return new CastExpression(input, e.getMajorType(), e.getPosition());
+ }else{
+ // if the type is fully bound, convert to functioncall and materialze the function.
+ MajorType type = e.getMajorType();
+ String castFuncWithType = "cast" + type.getMinorType().name();
+
+ List<LogicalExpression> newArgs = Lists.newArrayList();
+ newArgs.add(e.getInput()); //input_expr
+
+ //VarLen type
+ if (!Types.isFixedWidthType(type)) {
+ newArgs.add(new ValueExpressions.LongExpression(type.getWidth(), null));
+ }
+ FunctionCall fc = new FunctionCall(castFuncWithType, newArgs, e.getPosition());
+ return fc.accept(this, value);
+ }
+
+
+
+ }
+
+ private boolean castEqual(ExpressionPosition pos, MajorType from, MajorType to){
+ if(!from.getMinorType().equals(to.getMinorType())) return false;
+ switch(from.getMinorType()){
+ case FLOAT4:
+ case FLOAT8:
+ case INT:
+ case BIGINT:
+ case BIT:
+ case TINYINT:
+ case UINT1:
+ case UINT2:
+ case UINT4:
+ case UINT8:
+ // nothing else matters.
+ return true;
+
+ case FIXED16CHAR:
+ case FIXEDBINARY:
+ case FIXEDCHAR:
+ // width always matters
+ this.errorCollector.addGeneralError(pos, "Casting fixed width types are not yet supported..");
+ return false;
+
+ case VAR16CHAR:
+ case VARBINARY:
+ case VARCHAR:
+ if(to.getWidth() < from.getWidth() && to.getWidth() > 0){
+ this.errorCollector.addGeneralError(pos, "Casting from a longer variable length type to a shorter variable length type is not currently supported.");
+ return false;
+ }else{
+ return true;
+ }
+
+ default:
+ errorCollector.addGeneralError(pos, String.format("Casting rules are unknown for type %s.", from));
+ return false;
+
+ }
+
}
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/30ce7d71/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
index b39b230..7398d78 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
@@ -21,16 +21,17 @@ import java.math.BigDecimal;
import java.util.Collections;
import java.util.List;
+import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.FunctionCallFactory;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.expression.ValueExpressions;
-import org.apache.drill.common.expression.ValueExpressions.LongExpression;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
import org.apache.drill.exec.record.NullExpression;
import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.RelOptPlanner;
import org.eigenbase.reltype.RelDataTypeField;
import org.eigenbase.rex.RexCall;
import org.eigenbase.rex.RexCorrelVariable;
@@ -45,9 +46,9 @@ import org.eigenbase.rex.RexRangeRef;
import org.eigenbase.rex.RexVisitorImpl;
import org.eigenbase.sql.SqlSyntax;
import org.eigenbase.sql.fun.SqlStdOperatorTable;
+import org.eigenbase.util.NlsString;
import com.google.common.collect.Lists;
-import org.eigenbase.util.NlsString;
/**
* Utilities for Drill's planner.
@@ -163,20 +164,21 @@ public class DrillOptiq {
private LogicalExpression getDrillCastFunctionFromOptiq(RexCall call){
LogicalExpression arg = call.getOperands().get(0).accept(this);
- List<LogicalExpression> args = Collections.singletonList(arg);
- String fname = null;
+ MajorType castType = null;
+
switch(call.getType().getSqlTypeName().getName()){
- case "VARCHAR": {
- args = Lists.newArrayList(arg, new LongExpression(call.getType().getPrecision()));
- return FunctionCallFactory.createExpression("castVARCHAR", args);
- }
- case "INTEGER": fname = "castINT"; break;
- case "FLOAT": fname = "castFLOAT4"; break;
- case "DOUBLE": fname = "castFLOAT8"; break;
+ case "VARCHAR":
+ castType = Types.required(MinorType.VARCHAR).toBuilder().setWidth(call.getType().getPrecision()).build();
+ break;
+
+ case "INTEGER": castType = Types.required(MinorType.INT); break;
+ case "FLOAT": Types.required(MinorType.FLOAT4); break;
+ case "DOUBLE": Types.required(MinorType.FLOAT8); break;
case "DECIMAL": throw new UnsupportedOperationException("Need to add decimal.");
- default: fname = "cast" + call.getType().getSqlTypeName().getName();
+ default: castType = Types.required(MinorType.valueOf(call.getType().getSqlTypeName().getName()));
}
- return FunctionCallFactory.createExpression(fname, args);
+
+ return FunctionCallFactory.createCast(castType, ExpressionPosition.UNKNOWN, arg);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/30ce7d71/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcExceptionHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcExceptionHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcExceptionHandler.java
index 42f733b..ee42ee5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcExceptionHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcExceptionHandler.java
@@ -30,7 +30,7 @@ public class RpcExceptionHandler implements ChannelHandler{
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception {
- if(!ctx.channel().isOpen()){
+ if(!ctx.channel().isOpen() || cause.getMessage().equals("Connection reset by peer")){
logger.warn("Exception with closed channel", cause);
return;
}else{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/30ce7d71/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java b/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
index 8ea758d..21de9f4 100644
--- a/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
+++ b/sqlparser/src/test/java/org/apache/drill/jdbc/test/TestJdbcQuery.java
@@ -101,7 +101,6 @@ public class TestJdbcQuery {
}
@Test
- @Ignore // failing due to a bug in cast to varchar function
public void testVarCharLiteral() throws Exception {
testQuery(String.format("select cast('test literal' as VARCHAR) from INFORMATION_SCHEMA.TABLES LIMIT 1"));
}
[02/12] DRILL-387: Support using Hive simple UDFs in Drill * As part
of this change FunctionDefinition (and related code) * are deleted,
instead the same information available in * Function Holders are used whenever
required * Freemarker/CodeModel codege
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
new file mode 100644
index 0000000..61c3e54
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
@@ -0,0 +1,260 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.fn;
+
+import com.sun.codemodel.JBlock;
+import com.sun.codemodel.JCatchBlock;
+import com.sun.codemodel.JClass;
+import com.sun.codemodel.JCodeModel;
+import com.sun.codemodel.JConditional;
+import com.sun.codemodel.JExpr;
+import com.sun.codemodel.JInvocation;
+import com.sun.codemodel.JTryBlock;
+import com.sun.codemodel.JVar;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.expr.ClassGenerator;
+import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.expr.fn.impl.hive.DrillDeferredObject;
+import org.apache.drill.exec.expr.fn.impl.hive.ObjectInspectorHelper;
+import org.apache.drill.exec.expr.holders.ValueHolder;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+
+public class HiveFuncHolder {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionImplementationRegistry.class);
+
+ private MajorType[] argTypes;
+ private ObjectInspector returnOI;
+ private MajorType returnType;
+ private Class<? extends GenericUDF> genericUdfClazz;
+ private boolean isGenericUDF = true;
+ private Class<? extends UDF> udfClazz = null;
+ private String udfName = "";
+ private boolean isRandom;
+
+
+ /**
+ * Create holder for GenericUDF
+ * @param genericUdfClazz implementation class
+ * @param argTypes
+ * @param returnOI
+ * @param returnType
+ */
+ public HiveFuncHolder(Class<? extends GenericUDF> genericUdfClazz, MajorType[] argTypes,
+ ObjectInspector returnOI, MajorType returnType, boolean isRandom) {
+ this.genericUdfClazz = genericUdfClazz;
+ this.argTypes = argTypes;
+ this.returnOI = returnOI;
+ this.returnType = returnType;
+ this.isRandom = isRandom;
+ }
+
+ /**
+ * Create holder for UDF
+ * @param udfName name of the UDF class
+ * @param udfClazz UDF implementation class
+ * @param argTypes
+ * @param returnOI
+ * @param returnType
+ */
+ public HiveFuncHolder(String udfName, Class< ? extends UDF> udfClazz, MajorType[] argTypes,
+ ObjectInspector returnOI, MajorType returnType, boolean isRandom) {
+ this(GenericUDFBridge.class, argTypes, returnOI, returnType, isRandom);
+ this.isGenericUDF = false;
+ this.udfClazz = udfClazz;
+ this.udfName = udfName;
+ }
+
+ /**
+ * UDF return type
+ */
+ public MajorType getReturnType() {
+ return returnType;
+ }
+
+ /**
+ * Aggregate function
+ */
+ public boolean isAggregating() {
+ // currently only simple UDFS are supported
+ return false;
+ }
+
+ /**
+ * is the function non-deterministic?
+ */
+ public boolean isRandom() {
+ return isRandom;
+ }
+
+ /**
+ * Start generating code
+ * @return workspace variables
+ */
+ public JVar[] renderStart(ClassGenerator<?> g, HoldingContainer[] inputVariables){
+ JVar[] workspaceJVars = new JVar[5];
+
+ workspaceJVars[0] = g.declareClassField("returnOI", g.getModel()._ref(ObjectInspector.class));
+ workspaceJVars[1] = g.declareClassField("udfInstance", g.getModel()._ref(GenericUDF.class));
+ workspaceJVars[2] = g.declareClassField("deferredObjects", g.getModel()._ref(DrillDeferredObject[].class));
+ workspaceJVars[3] = g.declareClassField("arguments", g.getModel()._ref(DrillDeferredObject[].class));
+ workspaceJVars[4] = g.declareClassField("returnValueHolder",
+ TypeHelper.getHolderType(g.getModel(), returnType.getMinorType(), TypeProtos.DataMode.OPTIONAL));
+
+ return workspaceJVars;
+ }
+
+ /**
+ * Complete code generation
+ * @param g
+ * @param inputVariables
+ * @param workspaceJVars
+ * @return HoldingContainer for return value
+ */
+ public HoldingContainer renderEnd(ClassGenerator<?> g, HoldingContainer[] inputVariables, JVar[] workspaceJVars) {
+ generateSetup(g, workspaceJVars);
+ return generateEval(g, inputVariables, workspaceJVars);
+ }
+
+ private JInvocation getUDFInstance(JCodeModel m) {
+ if (isGenericUDF) {
+ return JExpr._new(m.directClass(genericUdfClazz.getCanonicalName()));
+ } else {
+ return JExpr._new(m.directClass(GenericUDFBridge.class.getCanonicalName()))
+ .arg(JExpr.lit(udfName))
+ .arg(JExpr.lit(false))
+ .arg(JExpr.dotclass(m.directClass(udfClazz.getCanonicalName())));
+ }
+ }
+
+ private void generateSetup(ClassGenerator<?> g, JVar[] workspaceJVars) {
+ JCodeModel m = g.getModel();
+ JBlock sub = new JBlock(true, true);
+
+ // declare and instantiate argument ObjectInspector's
+ JVar oiArray = sub.decl(
+ m._ref(ObjectInspector[].class),
+ "argOIs",
+ JExpr.newArray(m._ref(ObjectInspector.class), argTypes.length));
+
+ JClass oih = m.directClass(ObjectInspectorHelper.class.getCanonicalName());
+ JClass mt = m.directClass(TypeProtos.MinorType.class.getCanonicalName());
+ for(int i=0; i<argTypes.length; i++) {
+ sub.assign(
+ oiArray.component(JExpr.lit(i)),
+ oih.staticInvoke("getDrillObjectInspector")
+ .arg(mt.staticInvoke("valueOf")
+ .arg(JExpr.lit(argTypes[i].getMinorType().getNumber()))));
+ }
+
+ // declare and instantiate DeferredObject array
+ sub.assign(workspaceJVars[2], JExpr.newArray(m._ref(DrillDeferredObject.class), argTypes.length));
+
+ for(int i=0; i<argTypes.length; i++) {
+ sub.assign(
+ workspaceJVars[2].component(JExpr.lit(i)),
+ JExpr._new(m.directClass(DrillDeferredObject.class.getCanonicalName())));
+ }
+
+ // declare empty array for argument deferred objects
+ sub.assign(workspaceJVars[3], JExpr.newArray(m._ref(DrillDeferredObject.class), argTypes.length));
+
+ // create new instance of the UDF class
+ sub.assign(workspaceJVars[1], getUDFInstance(m));
+
+ // create try..catch block to initialize the UDF instance with argument OIs
+ JTryBlock udfInitTry = sub._try();
+ udfInitTry.body().assign(
+ workspaceJVars[0],
+ workspaceJVars[1].invoke("initialize")
+ .arg(oiArray));
+
+ JCatchBlock udfInitCatch = udfInitTry._catch(m.directClass(Exception.class.getCanonicalName()));
+ JVar exVar = udfInitCatch.param("ex");
+ udfInitCatch.body()
+ ._throw(JExpr._new(m.directClass(RuntimeException.class.getCanonicalName()))
+ .arg(JExpr.lit(String.format("Failed to initialize GenericUDF"))).arg(exVar));
+
+ sub.add(ObjectInspectorHelper.initReturnValueHolder(m, workspaceJVars[4], returnOI, returnType.getMinorType()));
+
+ // now add it to the doSetup block in Generated class
+ JBlock setup = g.getBlock(ClassGenerator.BlockType.SETUP);
+ setup.directStatement(String.format("/** start %s for function %s **/ ",
+ ClassGenerator.BlockType.SETUP.name(), genericUdfClazz.getName() + (!isGenericUDF ? "("+udfName+")" : "")));
+
+ setup.add(sub);
+
+ setup.directStatement(String.format("/** end %s for function %s **/ ",
+ ClassGenerator.BlockType.SETUP.name(), genericUdfClazz.getName() + (!isGenericUDF ? "("+udfName+")" : "")));
+ }
+
+ private HoldingContainer generateEval(ClassGenerator<?> g, HoldingContainer[] inputVariables, JVar[] workspaceJVars) {
+
+ HoldingContainer out = g.declare(returnType);
+
+ JCodeModel m = g.getModel();
+ JBlock sub = new JBlock(true, true);
+
+ // initialize DeferredObject's. For an optional type, assign the value holder only if it is not null
+ for(int i=0; i<argTypes.length; i++) {
+ if (inputVariables[i].isOptional()) {
+ JBlock conditionalBlock = new JBlock(false, false);
+ JConditional jc = conditionalBlock._if(inputVariables[i].getIsSet().ne(JExpr.lit(0)));
+ jc._then().assign(workspaceJVars[3].component(JExpr.lit(i)), workspaceJVars[2].component(JExpr.lit(i)));
+ jc._then().assign(JExpr.ref(workspaceJVars[3].component(JExpr.lit(i)), "valueHolder"), inputVariables[i].getHolder());
+ jc._else().assign(workspaceJVars[3].component(JExpr.lit(i)), JExpr._null());
+ sub.add(conditionalBlock);
+ } else {
+ sub.assign(workspaceJVars[3].component(JExpr.lit(i)), workspaceJVars[2].component(JExpr.lit(i)));
+ sub.assign(JExpr.ref(workspaceJVars[3].component(JExpr.lit(i)), "valueHolder"), inputVariables[i].getHolder());
+ }
+ }
+
+ // declare generic object for storing return value from GenericUDF.evaluate
+ JVar retVal = sub.decl(m._ref(Object.class), "ret");
+
+ // create try..catch block to call the GenericUDF instance with given input
+ JTryBlock udfEvalTry = sub._try();
+ udfEvalTry.body().assign(retVal,
+ workspaceJVars[1].invoke("evaluate").arg(workspaceJVars[3]));
+
+ JCatchBlock udfEvalCatch = udfEvalTry._catch(m.directClass(Exception.class.getCanonicalName()));
+ JVar exVar = udfEvalCatch.param("ex");
+ udfEvalCatch.body()
+ ._throw(JExpr._new(m.directClass(RuntimeException.class.getCanonicalName()))
+ .arg(JExpr.lit(String.format("GenericUDF.evaluate method failed"))).arg(exVar));
+
+ // get the ValueHolder from retVal and return ObjectInspector
+ sub.add(ObjectInspectorHelper.getDrillObject(m, returnOI, workspaceJVars[0], workspaceJVars[4], retVal));
+ sub.assign(out.getHolder(), workspaceJVars[4]);
+
+ // now add it to the doEval block in Generated class
+ JBlock setup = g.getBlock(ClassGenerator.BlockType.EVAL);
+ setup.directStatement(String.format("/** start %s for function %s **/ ",
+ ClassGenerator.BlockType.EVAL.name(), genericUdfClazz.getName() + (!isGenericUDF ? "("+udfName+")" : "")));
+ setup.add(sub);
+ setup.directStatement(String.format("/** end %s for function %s **/ ",
+ ClassGenerator.BlockType.EVAL.name(), genericUdfClazz.getName() + (!isGenericUDF ? "("+udfName+")" : "")));
+
+ return out;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionImplementationRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionImplementationRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionImplementationRegistry.java
new file mode 100644
index 0000000..0c19a08
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionImplementationRegistry.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.fn;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.common.util.PathScanner;
+import org.apache.drill.exec.expr.fn.impl.hive.ObjectInspectorHelper;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.udf.UDFType;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+
+import com.google.common.collect.ArrayListMultimap;
+
+public class HiveFunctionImplementationRegistry {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillFunctionImplementationRegistry.class);
+
+ private ArrayListMultimap<String, Class<? extends GenericUDF>> methodsGenericUDF = ArrayListMultimap.create();
+ private ArrayListMultimap<String, Class<? extends UDF>> methodsUDF = ArrayListMultimap.create();
+ private HashSet<Class> nonDeterministicUDFs = new HashSet<>();
+
+ /**
+ * Scan the classpath for implementation of GenericUDF/UDF interfaces,
+ * extracts function annotation and store the
+ * (function name) --> (implementation class) mappings.
+ * @param config
+ */
+ public HiveFunctionImplementationRegistry(DrillConfig config){
+ Set<Class<? extends GenericUDF>> genericUDFClasses = PathScanner.scanForImplementations(GenericUDF.class, null);
+ for (Class<? extends GenericUDF> clazz : genericUDFClasses)
+ register(clazz, methodsGenericUDF);
+
+ Set<Class<? extends UDF>> udfClasses = PathScanner.scanForImplementations(UDF.class, null);
+ for (Class<? extends UDF> clazz : udfClasses)
+ register(clazz, methodsUDF);
+ }
+
+ private <C,I> void register(Class<? extends I> clazz, ArrayListMultimap<String,Class<? extends I>> methods) {
+ Description desc = clazz.getAnnotation(Description.class);
+ if (desc != null) {
+ // if the function is non-deterministic add it to the list
+ UDFType type = clazz.getAnnotation(UDFType.class);
+ if (type != null && type.deterministic())
+ nonDeterministicUDFs.add(clazz);
+
+ String[] names = desc.name().split(",");
+ for(int i=0; i<names.length; i++)
+ names[i] = names[i].trim();
+
+ for(int i=0; i<names.length;i++)
+ methods.put(names[i], clazz);
+ } else {
+ logger.warn("Unable to initialize function for class {}", clazz.getName());
+ }
+ }
+
+ /**
+ * Find the UDF class for given function name and check if it accepts the given input argument
+ * types. If a match is found, create a holder and return
+ * @param call
+ * @return
+ */
+ public HiveFuncHolder getFunction(FunctionCall call){
+ HiveFuncHolder holder;
+ MajorType[] argTypes = new MajorType[call.args.size()];
+ ObjectInspector[] argOIs = new ObjectInspector[call.args.size()];
+ for(int i=0; i<call.args.size(); i++) {
+ argTypes[i] = call.args.get(i).getMajorType();
+ argOIs[i] = ObjectInspectorHelper.getDrillObjectInspector(argTypes[i].getMinorType());
+ }
+
+ // search in GenericUDF list
+ for(Class<? extends GenericUDF> clazz: methodsGenericUDF.get(call.getName())) {
+ holder = matchAndCreateGenericUDFHolder(clazz, argTypes, argOIs);
+ if(holder != null)
+ return holder;
+ }
+
+ // search in UDF list
+ for (Class<? extends UDF> clazz : methodsUDF.get(call.getName())) {
+ holder = matchAndCreateUDFHolder(call.getName(), clazz, argTypes, argOIs);
+ if (holder != null)
+ return holder;
+ }
+
+ return null;
+ }
+
+ private HiveFuncHolder matchAndCreateGenericUDFHolder(Class<? extends GenericUDF> udfClazz,
+ MajorType[] argTypes,
+ ObjectInspector[] argOIs) {
+ // probe UDF to find if the arg types and acceptable
+ // if acceptable create a holder object
+ try {
+ GenericUDF udfInstance = udfClazz.newInstance();
+ ObjectInspector returnOI = udfInstance.initialize(argOIs);
+ return new HiveFuncHolder(
+ udfClazz,
+ argTypes,
+ returnOI,
+ Types.optional(ObjectInspectorHelper.getDrillType(returnOI)),
+ nonDeterministicUDFs.contains(udfClazz));
+ } catch(IllegalAccessException | InstantiationException e) {
+ logger.debug("Failed to instantiate class", e);
+ } catch(UDFArgumentException e) { /*ignore this*/ }
+
+ return null;
+ }
+
+ private HiveFuncHolder matchAndCreateUDFHolder(String udfName,
+ Class<? extends UDF> udfClazz,
+ MajorType[] argTypes,
+ ObjectInspector[] argOIs) {
+ try {
+ GenericUDF udfInstance = new GenericUDFBridge(udfName, false/* is operator */, udfClazz);
+ ObjectInspector returnOI = udfInstance.initialize(argOIs);
+
+ return new HiveFuncHolder(
+ udfName,
+ udfClazz,
+ argTypes,
+ returnOI,
+ Types.optional(ObjectInspectorHelper.getDrillType(returnOI)),
+ nonDeterministicUDFs.contains(udfClazz));
+ } catch(UDFArgumentException e) { /*ignore this*/ }
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Alternator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Alternator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Alternator.java
index f65af3b..bdd227c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Alternator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Alternator.java
@@ -17,12 +17,6 @@
*/
package org.apache.drill.exec.expr.fn.impl;
-import org.apache.drill.common.expression.CallProvider;
-import org.apache.drill.common.expression.FunctionDefinition;
-import org.apache.drill.common.expression.NoArgValidator;
-import org.apache.drill.common.expression.OutputTypeDeterminer;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
@@ -31,11 +25,9 @@ import org.apache.drill.exec.expr.annotations.Workspace;
import org.apache.drill.exec.expr.holders.BigIntHolder;
import org.apache.drill.exec.record.RecordBatch;
-
-
public class Alternator {
- @FunctionTemplate(name = "alternate", scope = FunctionScope.SIMPLE)
+ @FunctionTemplate(name = "alternate", isRandom = true, scope = FunctionScope.SIMPLE)
public static class Alternate2 implements DrillSimpleFunc{
@Workspace int val;
@Output BigIntHolder out;
@@ -55,7 +47,7 @@ public class Alternator {
}
}
- @FunctionTemplate(name = "alternate3", scope = FunctionScope.SIMPLE)
+ @FunctionTemplate(name = "alternate3", isRandom = true, scope = FunctionScope.SIMPLE)
public static class Alternate3 implements DrillSimpleFunc{
@Workspace int val;
@Output BigIntHolder out;
@@ -76,16 +68,4 @@ public class Alternator {
}
}
}
-
- public static class Provider implements CallProvider{
-
- @Override
- public FunctionDefinition[] getFunctionDefintions() {
- return new FunctionDefinition[]{
- FunctionDefinition.simpleRandom("alternate", NoArgValidator.VALIDATOR, new OutputTypeDeterminer.FixedType(Types.required(MinorType.BIGINT)), "alternate"),
- FunctionDefinition.simpleRandom("alternate3", NoArgValidator.VALIDATOR, new OutputTypeDeterminer.FixedType(Types.required(MinorType.BIGINT)), "alternate3")
- };
- }
-
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/BitFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/BitFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/BitFunctions.java
index 41bb7c6..9b75574 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/BitFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/BitFunctions.java
@@ -29,7 +29,7 @@ import org.apache.drill.exec.record.RecordBatch;
public class BitFunctions {
- @FunctionTemplate(name = "or", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
+ @FunctionTemplate(names = {"or", "||"}, scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
public static class BitOr implements DrillSimpleFunc {
@Param BitHolder left;
@@ -43,7 +43,7 @@ public class BitFunctions {
}
}
- @FunctionTemplate(name = "and", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
+ @FunctionTemplate(names = {"and", "&&"}, scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
public static class BitAnd implements DrillSimpleFunc {
@Param BitHolder left;
@@ -58,7 +58,7 @@ public class BitFunctions {
}
- @FunctionTemplate(name = "xor", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
+ @FunctionTemplate(names = {"xor", "^"}, scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
public static class IntXor implements DrillSimpleFunc {
@Param IntHolder left;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteSubstring.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteSubstring.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteSubstring.java
index 964e4af..d55de78 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteSubstring.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ByteSubstring.java
@@ -17,13 +17,6 @@
*/
package org.apache.drill.exec.expr.fn.impl;
-import org.apache.drill.common.expression.Arg;
-import org.apache.drill.common.expression.BasicArgumentValidator;
-import org.apache.drill.common.expression.CallProvider;
-import org.apache.drill.common.expression.FunctionDefinition;
-import org.apache.drill.common.expression.OutputTypeDeterminer;
-import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.types.Types;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.Output;
@@ -48,7 +41,7 @@ import org.apache.drill.exec.record.RecordBatch;
*
* - If the substring is invalid, return an empty string.
*/
-@FunctionTemplate(name = "bytesubstring",
+@FunctionTemplate(names = {"bytesubstring", "byte_substr"},
scope = FunctionTemplate.FunctionScope.SIMPLE,
nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
public class ByteSubstring implements DrillSimpleFunc {
@@ -87,20 +80,4 @@ public class ByteSubstring implements DrillSimpleFunc {
}
}
-
- public static class Provider implements CallProvider {
-
- @Override
- public FunctionDefinition[] getFunctionDefintions() {
- return new FunctionDefinition[] {
- FunctionDefinition.simple("bytesubstring",
- new BasicArgumentValidator(new Arg(Types.required(TypeProtos.MinorType.VARBINARY),
- Types.optional(TypeProtos.MinorType.VARBINARY)),
- new Arg(false, false, "offset", TypeProtos.MinorType.BIGINT),
- new Arg(false, false, "length", TypeProtos.MinorType.BIGINT)),
- new OutputTypeDeterminer.SameAsFirstInput(),
- "byte_substr")
- };
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CharSubstring.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CharSubstring.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CharSubstring.java
index 0b43028..9b18f58 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CharSubstring.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/CharSubstring.java
@@ -17,13 +17,6 @@
*/
package org.apache.drill.exec.expr.fn.impl;
-import org.apache.drill.common.expression.Arg;
-import org.apache.drill.common.expression.BasicArgumentValidator;
-import org.apache.drill.common.expression.CallProvider;
-import org.apache.drill.common.expression.FunctionDefinition;
-import org.apache.drill.common.expression.OutputTypeDeterminer;
-import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.types.Types;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.Output;
@@ -52,7 +45,7 @@ import org.apache.drill.exec.record.RecordBatch;
*
* - TODO: implement optional length parameter
*/
-@FunctionTemplate(name = "charsubstring",
+@FunctionTemplate(names = {"charsubstring", "substring2", "substr2"},
scope = FunctionTemplate.FunctionScope.SIMPLE,
nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
public class CharSubstring implements DrillSimpleFunc {
@@ -141,21 +134,4 @@ public class CharSubstring implements DrillSimpleFunc {
}
}
}
-
- public static class Provider implements CallProvider {
-
- @Override
- public FunctionDefinition[] getFunctionDefintions() {
- return new FunctionDefinition[] {
- FunctionDefinition.simple("charsubstring",
- new BasicArgumentValidator(new Arg(Types.required(TypeProtos.MinorType.VARCHAR),
- Types.optional(TypeProtos.MinorType.VARCHAR)),
- new Arg(false, false, "offset", TypeProtos.MinorType.BIGINT),
- new Arg(false, false, "length", TypeProtos.MinorType.BIGINT)),
- new OutputTypeDeterminer.SameAsFirstInput(),
- "substring2",
- "substr2")
- };
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java
deleted file mode 100644
index cd0eb21..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparatorFunctions.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.expr.fn.impl;
-
-import org.apache.drill.common.expression.ArgumentValidators;
-import org.apache.drill.common.expression.CallProvider;
-import org.apache.drill.common.expression.FunctionDefinition;
-import org.apache.drill.common.expression.OutputTypeDeterminer;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.expr.DrillSimpleFunc;
-import org.apache.drill.exec.expr.annotations.FunctionTemplate;
-import org.apache.drill.exec.expr.annotations.Output;
-import org.apache.drill.exec.expr.annotations.Param;
-import org.apache.drill.exec.expr.holders.BigIntHolder;
-import org.apache.drill.exec.expr.holders.IntHolder;
-import org.apache.drill.exec.expr.holders.NullableBigIntHolder;
-import org.apache.drill.exec.expr.holders.NullableIntHolder;
-import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
-import org.apache.drill.exec.expr.holders.VarBinaryHolder;
-import org.apache.drill.exec.expr.holders.VarCharHolder;
-import org.apache.drill.exec.record.RecordBatch;
-
-public class ComparatorFunctions {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComparatorFunctions.class);
-
-
- public static final FunctionDefinition COMPARE_TO = FunctionDefinition.simple("compare_to", new ArgumentValidators.AllowedTypeList(2, Types.required(MinorType.INT)), new OutputTypeDeterminer.FixedType(Types.required(MinorType.INT)));
- public static class Provider implements CallProvider{
-
- @Override
- public FunctionDefinition[] getFunctionDefintions() {
- return new FunctionDefinition[]{
- COMPARE_TO
- };
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctionsNullable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctionsNullable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctionsNullable.java
index 9be620a..570aaeb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctionsNullable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctionsNullable.java
@@ -31,7 +31,7 @@ public class ComparisonFunctionsNullable {
private ComparisonFunctionsNullable() {}
- @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ @FunctionTemplate(names = {"equal", "=", "=="}, scope = FunctionTemplate.FunctionScope.SIMPLE)
public static class NullableIntEqual implements DrillSimpleFunc {
@Param NullableIntHolder left;
@@ -49,7 +49,7 @@ public class ComparisonFunctionsNullable {
}
}
- @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ @FunctionTemplate(names = {"equal", "=", "=="}, scope = FunctionTemplate.FunctionScope.SIMPLE)
public static class NullableBigIntEqual implements DrillSimpleFunc {
@Param NullableBigIntHolder left;
@@ -67,7 +67,7 @@ public class ComparisonFunctionsNullable {
}
}
- @FunctionTemplate(name = "not equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ @FunctionTemplate(names = {"not equal", "<>", "!="}, scope = FunctionTemplate.FunctionScope.SIMPLE)
public static class NullableIntNotEqual implements DrillSimpleFunc {
@Param NullableIntHolder left;
@@ -85,7 +85,7 @@ public class ComparisonFunctionsNullable {
}
}
- @FunctionTemplate(name = "not equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ @FunctionTemplate(names = {"not equal", "<>", "!="}, scope = FunctionTemplate.FunctionScope.SIMPLE)
public static class NullableBigIntNotEqual implements DrillSimpleFunc {
@Param NullableBigIntHolder left;
@@ -103,7 +103,7 @@ public class ComparisonFunctionsNullable {
}
}
- @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ @FunctionTemplate(names = {"less than","<"}, scope = FunctionTemplate.FunctionScope.SIMPLE)
public static class NullableIntLessThan implements DrillSimpleFunc {
@Param NullableIntHolder left;
@@ -121,7 +121,7 @@ public class ComparisonFunctionsNullable {
}
}
- @FunctionTemplate(name = "less than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ @FunctionTemplate(names = {"less than","<"}, scope = FunctionTemplate.FunctionScope.SIMPLE)
public static class NullableBigIntLessThan implements DrillSimpleFunc {
@Param NullableBigIntHolder left;
@@ -139,7 +139,7 @@ public class ComparisonFunctionsNullable {
}
}
- @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ @FunctionTemplate(names = {"less than or equal to","<="}, scope = FunctionTemplate.FunctionScope.SIMPLE)
public static class NullableIntLessThanEqual implements DrillSimpleFunc {
@Param NullableIntHolder left;
@@ -157,7 +157,7 @@ public class ComparisonFunctionsNullable {
}
}
- @FunctionTemplate(name = "less than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ @FunctionTemplate(names = {"less than or equal to","<="}, scope = FunctionTemplate.FunctionScope.SIMPLE)
public static class NullableBigIntLessThanEqual implements DrillSimpleFunc {
@Param NullableBigIntHolder left;
@@ -175,7 +175,7 @@ public class ComparisonFunctionsNullable {
}
}
- @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ @FunctionTemplate(names = {"greater than",">"}, scope = FunctionTemplate.FunctionScope.SIMPLE)
public static class NullableIntGreaterThan implements DrillSimpleFunc {
@Param NullableIntHolder left;
@@ -193,7 +193,7 @@ public class ComparisonFunctionsNullable {
}
}
- @FunctionTemplate(name = "greater than", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ @FunctionTemplate(names = {"greater than",">"}, scope = FunctionTemplate.FunctionScope.SIMPLE)
public static class NullableBigIntGreaterThan implements DrillSimpleFunc {
@Param NullableBigIntHolder left;
@@ -211,7 +211,7 @@ public class ComparisonFunctionsNullable {
}
}
- @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ @FunctionTemplate(names = {"greater than or equal to",">="}, scope = FunctionTemplate.FunctionScope.SIMPLE)
public static class NullableIntGreaterThanEqual implements DrillSimpleFunc {
@Param NullableIntHolder left;
@@ -229,7 +229,8 @@ public class ComparisonFunctionsNullable {
}
}
- @FunctionTemplate(name = "greater than or equal to", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ @FunctionTemplate(names = {"greater than or equal to",">="},
+ scope = FunctionTemplate.FunctionScope.SIMPLE)
public static class NullableBigIntGreaterThanEqual implements DrillSimpleFunc {
@Param NullableBigIntHolder left;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
index 3cc82b4..5a6943e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
@@ -17,10 +17,6 @@
*/
package org.apache.drill.exec.expr.fn.impl;
-import org.apache.drill.common.expression.ArgumentValidators;
-import org.apache.drill.common.expression.CallProvider;
-import org.apache.drill.common.expression.FunctionDefinition;
-import org.apache.drill.common.expression.OutputTypeDeterminer;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
@@ -31,18 +27,6 @@ import org.apache.drill.exec.record.RecordBatch;
public class HashFunctions {
- public static final FunctionDefinition HASH = FunctionDefinition.simple("hash", new ArgumentValidators.AnyTypeAllowed(1),
- OutputTypeDeterminer.FIXED_INT, "hash");
-
- public static class Provider implements CallProvider {
-
- @Override
- public FunctionDefinition[] getFunctionDefintions() {
- return new FunctionDefinition[] { HASH };
- }
-
- }
-
@FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
public static class NullableFloatHash implements DrillSimpleFunc {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNotNull.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNotNull.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNotNull.java
index 5ee5ccb..ad7a93e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNotNull.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNotNull.java
@@ -17,10 +17,6 @@
*/
package org.apache.drill.exec.expr.fn.impl;
-import org.apache.drill.common.expression.ArgumentValidators;
-import org.apache.drill.common.expression.CallProvider;
-import org.apache.drill.common.expression.FunctionDefinition;
-import org.apache.drill.common.expression.OutputTypeDeterminer;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.Output;
@@ -29,7 +25,7 @@ import org.apache.drill.exec.expr.holders.BitHolder;
import org.apache.drill.exec.expr.holders.NullableFloat8Holder;
import org.apache.drill.exec.record.RecordBatch;
-@FunctionTemplate(name = "isNotNull", scope = FunctionTemplate.FunctionScope.SIMPLE)
+@FunctionTemplate(names = {"isNotNull", "isnotnull"}, scope = FunctionTemplate.FunctionScope.SIMPLE)
public class IsNotNull implements DrillSimpleFunc {
@Param NullableFloat8Holder input;
@@ -40,20 +36,5 @@ public class IsNotNull implements DrillSimpleFunc {
public void eval() {
out.value = (input.isSet == 0 ? 0 : 1);
}
-
- public static class Provider implements CallProvider {
-
- @Override
- public FunctionDefinition[] getFunctionDefintions() {
- return new FunctionDefinition[] {
- FunctionDefinition.simple("isNotNull",
- new ArgumentValidators.AnyTypeAllowed(1),
- OutputTypeDeterminer.FIXED_BIT,
- "isnotnull")
- };
- }
-
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNull.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNull.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNull.java
index a1d3368..02905d4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNull.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNull.java
@@ -17,10 +17,6 @@
*/
package org.apache.drill.exec.expr.fn.impl;
-import org.apache.drill.common.expression.ArgumentValidators;
-import org.apache.drill.common.expression.CallProvider;
-import org.apache.drill.common.expression.FunctionDefinition;
-import org.apache.drill.common.expression.OutputTypeDeterminer;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.Output;
@@ -29,7 +25,7 @@ import org.apache.drill.exec.expr.holders.BitHolder;
import org.apache.drill.exec.expr.holders.NullableFloat8Holder;
import org.apache.drill.exec.record.RecordBatch;
-@FunctionTemplate(name = "isNull", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+@FunctionTemplate(names = {"isNull", "isnull"}, scope = FunctionTemplate.FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
public class IsNull implements DrillSimpleFunc {
@Param NullableFloat8Holder input;
@@ -40,20 +36,5 @@ public class IsNull implements DrillSimpleFunc {
public void eval() {
out.value = (input.isSet == 0 ? 1 : 0);
}
-
- public static class Provider implements CallProvider {
-
- @Override
- public FunctionDefinition[] getFunctionDefintions() {
- return new FunctionDefinition[] {
- FunctionDefinition.simple("isNull",
- new ArgumentValidators.AnyTypeAllowed(1),
- OutputTypeDeterminer.FIXED_BIT,
- "isnull")
- };
- }
-
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
index 3fcf5c9..59be472 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/MathFunctions.java
@@ -34,7 +34,7 @@ public class MathFunctions{
private MathFunctions(){}
- @FunctionTemplate(name = "negative", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
+ @FunctionTemplate(names = {"negative", "u-", "-"}, scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
public static class Negative implements DrillSimpleFunc{
@Param BigIntHolder input;
@@ -63,7 +63,8 @@ public class MathFunctions{
}
- @FunctionTemplate(name = "random", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
+ @FunctionTemplate(name = "random", isRandom = true,
+ scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
public static class Random implements DrillSimpleFunc{
@Output Float8Holder out;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SimpleRepeatedFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SimpleRepeatedFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SimpleRepeatedFunctions.java
index 2f838b1..cc4343e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SimpleRepeatedFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/SimpleRepeatedFunctions.java
@@ -17,13 +17,6 @@
*/
package org.apache.drill.exec.expr.fn.impl;
-import org.apache.drill.common.expression.Arg;
-import org.apache.drill.common.expression.BasicArgumentValidator;
-import org.apache.drill.common.expression.CallProvider;
-import org.apache.drill.common.expression.FunctionDefinition;
-import org.apache.drill.common.expression.OutputTypeDeterminer;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.common.types.Types;
import org.apache.drill.exec.expr.DrillSimpleFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.annotations.Output;
@@ -36,7 +29,6 @@ import org.apache.drill.exec.expr.holders.RepeatedIntHolder;
import org.apache.drill.exec.record.RecordBatch;
public class SimpleRepeatedFunctions {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MathFunctions.class);
private SimpleRepeatedFunctions() {
}
@@ -91,26 +83,4 @@ public class SimpleRepeatedFunctions {
}
}
-
- public static class Provider implements CallProvider {
-
- @Override
- public FunctionDefinition[] getFunctionDefintions() {
- return new FunctionDefinition[] {
- FunctionDefinition.simple("repeated_contains", new BasicArgumentValidator( //
- new Arg("repeatedToSearch", //
- Types.repeated(MinorType.BIGINT), //
- Types.repeated(MinorType.INT)), //
- new Arg("targetValue", Types.required(MinorType.BIGINT))), //
- OutputTypeDeterminer.FixedType.FIXED_BIT),
-
- FunctionDefinition.simple(
- "repeated_count",
- new BasicArgumentValidator(new Arg("repeatedToSearch", Types.repeated(MinorType.BIGINT), Types
- .repeated(MinorType.INT))), new OutputTypeDeterminer.FixedType(Types.required(MinorType.INT)))
-
- };
- }
-
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
index 9212867..0780ca8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
@@ -20,11 +20,6 @@ package org.apache.drill.exec.expr.fn.impl;
import io.netty.buffer.ByteBuf;
-import org.apache.drill.common.expression.Arg;
-import org.apache.drill.common.expression.ArgumentValidators.*;
-import org.apache.drill.common.expression.BasicArgumentValidator;
-import org.apache.drill.common.expression.CallProvider;
-import org.apache.drill.common.expression.FunctionDefinition;
import org.apache.drill.common.expression.OutputTypeDeterminer;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MinorType;
@@ -115,7 +110,7 @@ public class StringFunctions{
}
- @FunctionTemplate(name = "char_length", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
+ @FunctionTemplate(names = {"char_length", "character_length", "length"}, scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
public static class CharLength implements DrillSimpleFunc{
@Param VarCharHolder input;
@@ -279,7 +274,7 @@ public class StringFunctions{
// Follow Postgre.
// -- Valid "offset": [1, string_length],
// -- Valid "length": [1, up to string_length - offset + 1], if length > string_length - offset +1, get the substr up to the string_lengt.
- @FunctionTemplate(name = "substring", scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
+ @FunctionTemplate(names = {"substring", "substr"}, scope = FunctionScope.SIMPLE, nulls = NullHandling.NULL_IF_NULL)
public static class Substring implements DrillSimpleFunc{
@Param VarCharHolder string;
@@ -791,75 +786,4 @@ public class StringFunctions{
}
}
-
-
- /*
- * Function Definitions
- */
-
- public static class Provider implements CallProvider {
-
- @Override
- public FunctionDefinition[] getFunctionDefintions() {
- return new FunctionDefinition[] {
- FunctionDefinition.simple("like", new AllowedTypeList(2, Types.required(MinorType.VARCHAR), Types.optional(MinorType.VARCHAR)), new OutputTypeDeterminer.NullIfNullType(MinorType.BIT), "like"),
- FunctionDefinition.simple("similar", new AllowedTypeList(2, Types.required(MinorType.VARCHAR), Types.optional(MinorType.VARCHAR)), new OutputTypeDeterminer.NullIfNullType(MinorType.BIT), "similar"),
- FunctionDefinition.simple("regexp_replace", new AllowedTypeList(3, Types.required(MinorType.VARCHAR), Types.optional(MinorType.VARCHAR)), new OutputTypeDeterminer.NullIfNullType(MinorType.VARCHAR), "regexp_replace"),
- FunctionDefinition.simple("char_length", new AllowedTypeList(1, Types.required(MinorType.VARCHAR), Types.optional(MinorType.VARCHAR)), new OutputTypeDeterminer.NullIfNullType(MinorType.BIGINT), "char_length","character_length","length"),
- FunctionDefinition.simple("octet_length", new AllowedTypeList(1, Types.required(MinorType.VARCHAR), Types.optional(MinorType.VARCHAR)), new OutputTypeDeterminer.NullIfNullType(MinorType.BIGINT), "octet_length"),
- FunctionDefinition.simple("bit_length", new AllowedTypeList(1, Types.required(MinorType.VARCHAR), Types.optional(MinorType.VARCHAR)), new OutputTypeDeterminer.NullIfNullType(MinorType.BIGINT), "bit_length"),
- FunctionDefinition.simple("position", new AllowedTypeList(2, Types.required(MinorType.VARCHAR), Types.optional(MinorType.VARCHAR)), new OutputTypeDeterminer.NullIfNullType(MinorType.BIGINT), "position"),
- FunctionDefinition.simple("strpos", new AllowedTypeList(2, Types.required(MinorType.VARCHAR), Types.optional(MinorType.VARCHAR)), new OutputTypeDeterminer.NullIfNullType(MinorType.BIGINT), "strpos"),
- FunctionDefinition.simple("lower", new AllowedTypeList(1, Types.required(MinorType.VARCHAR), Types.optional(MinorType.VARCHAR)), new OutputTypeDeterminer.NullIfNullType(MinorType.VARCHAR), "lower"),
- FunctionDefinition.simple("upper", new AllowedTypeList(1, Types.required(MinorType.VARCHAR), Types.optional(MinorType.VARCHAR)), new OutputTypeDeterminer.NullIfNullType(MinorType.VARCHAR), "upper"),
- FunctionDefinition.simple("initcap", new AllowedTypeList(1, Types.required(MinorType.VARCHAR), Types.optional(MinorType.VARCHAR)), new OutputTypeDeterminer.NullIfNullType(MinorType.VARCHAR), "initcap"),
- FunctionDefinition.simple("substring", new BasicArgumentValidator(new Arg(Types.required(TypeProtos.MinorType.VARCHAR), Types.optional(TypeProtos.MinorType.VARCHAR)),
- new Arg(Types.required(TypeProtos.MinorType.BIGINT), Types.optional(TypeProtos.MinorType.BIGINT)),
- new Arg(Types.required(TypeProtos.MinorType.BIGINT), Types.optional(TypeProtos.MinorType.BIGINT))
- ),
- new OutputTypeDeterminer.NullIfNullType(MinorType.VARCHAR),
- "substring",
- "substr"),
- FunctionDefinition.simple("left", new BasicArgumentValidator(new Arg(Types.required(TypeProtos.MinorType.VARCHAR), Types.optional(TypeProtos.MinorType.VARCHAR)),
- new Arg(Types.required(TypeProtos.MinorType.BIGINT), Types.optional(TypeProtos.MinorType.BIGINT))
- ),
- new OutputTypeDeterminer.NullIfNullType(MinorType.VARCHAR),
- "left"),
- FunctionDefinition.simple("right",new BasicArgumentValidator(new Arg(Types.required(TypeProtos.MinorType.VARCHAR), Types.optional(TypeProtos.MinorType.VARCHAR)),
- new Arg(Types.required(TypeProtos.MinorType.BIGINT), Types.optional(TypeProtos.MinorType.BIGINT))
- ),
- new OutputTypeDeterminer.NullIfNullType(MinorType.VARCHAR),
- "right"),
- FunctionDefinition.simple("replace",new BasicArgumentValidator(new Arg(Types.required(TypeProtos.MinorType.VARCHAR), Types.optional(TypeProtos.MinorType.VARCHAR)),
- new Arg(Types.required(TypeProtos.MinorType.VARCHAR), Types.optional(TypeProtos.MinorType.VARCHAR)),
- new Arg(Types.required(TypeProtos.MinorType.VARCHAR), Types.optional(TypeProtos.MinorType.VARCHAR))
- ),
- new OutputTypeDeterminer.NullIfNullType(MinorType.VARCHAR),
- "replace"),
- FunctionDefinition.simple("lpad",new BasicArgumentValidator(new Arg(Types.required(TypeProtos.MinorType.VARCHAR), Types.optional(TypeProtos.MinorType.VARCHAR)),
- new Arg(Types.required(TypeProtos.MinorType.BIGINT), Types.optional(TypeProtos.MinorType.BIGINT)),
- new Arg(Types.required(TypeProtos.MinorType.VARCHAR), Types.optional(TypeProtos.MinorType.VARCHAR))
- ),
- new OutputTypeDeterminer.NullIfNullType(MinorType.VARCHAR),
- "lpad") ,
- FunctionDefinition.simple("rpad",new BasicArgumentValidator(new Arg(Types.required(TypeProtos.MinorType.VARCHAR), Types.optional(TypeProtos.MinorType.VARCHAR)),
- new Arg(Types.required(TypeProtos.MinorType.BIGINT), Types.optional(TypeProtos.MinorType.BIGINT)),
- new Arg(Types.required(TypeProtos.MinorType.VARCHAR), Types.optional(TypeProtos.MinorType.VARCHAR))
- ),
- new OutputTypeDeterminer.NullIfNullType(MinorType.VARCHAR),
- "rpad"),
- FunctionDefinition.simple("ltrim",new BasicArgumentValidator(new Arg(Types.required(TypeProtos.MinorType.VARCHAR), Types.optional(TypeProtos.MinorType.VARCHAR)),
- new Arg(Types.required(TypeProtos.MinorType.VARCHAR), Types.optional(TypeProtos.MinorType.VARCHAR))
- ),
- new OutputTypeDeterminer.NullIfNullType(MinorType.VARCHAR),
- "ltrim"),
- FunctionDefinition.simple("rtrim",new BasicArgumentValidator(new Arg(Types.required(TypeProtos.MinorType.VARCHAR), Types.optional(TypeProtos.MinorType.VARCHAR)),
- new Arg(Types.required(TypeProtos.MinorType.VARCHAR), Types.optional(TypeProtos.MinorType.VARCHAR))
- ),
- new OutputTypeDeterminer.NullIfNullType(MinorType.VARCHAR), "rtrim") ,
- FunctionDefinition.simple("concat", new AllowedTypeList(2, Types.required(MinorType.VARCHAR), Types.optional(MinorType.VARCHAR)), new OutputTypeDeterminer.NullIfNullType(MinorType.VARCHAR), "concat"),
-
- };
- }
- }
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractPrimitiveObjectInspector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractPrimitiveObjectInspector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractPrimitiveObjectInspector.java
new file mode 100644
index 0000000..04b552e
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractPrimitiveObjectInspector.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.fn.impl.hive;
+
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.BaseTypeParams;
+
+
+public abstract class AbstractPrimitiveObjectInspector implements PrimitiveObjectInspector {
+
+ @Override
+ public Class<?> getPrimitiveWritableClass() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object getPrimitiveWritableObject(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Class<?> getJavaPrimitiveClass() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object getPrimitiveJavaObject(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object copyObject(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean preferWritable() {
+ return false;
+ }
+
+ @Override
+ public BaseTypeParams getTypeParams() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setTypeParams(BaseTypeParams baseTypeParams) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Category getCategory() {
+ return Category.PRIMITIVE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/DrillDeferredObject.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/DrillDeferredObject.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/DrillDeferredObject.java
new file mode 100644
index 0000000..fbc5f05
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/DrillDeferredObject.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.fn.impl.hive;
+
+import org.apache.drill.exec.expr.holders.ValueHolder;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+
+public class DrillDeferredObject implements GenericUDF.DeferredObject {
+ public ValueHolder valueHolder;
+
+ @Override
+ public void prepare(int version) throws HiveException {}
+
+ @Override
+ public Object get() throws HiveException {
+ return valueHolder;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
index 36cfb53..fa66926 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
@@ -20,7 +20,6 @@ package org.apache.drill.exec.ops;
import java.util.Collection;
import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.FunctionRegistry;
import org.apache.drill.exec.cache.DistributedCache;
import org.apache.drill.exec.planner.PhysicalPlanReader;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
@@ -82,10 +81,6 @@ public class QueryContext {
return workBus;
}
- public FunctionRegistry getFunctionRegistry(){
- return drillbitContext.getFunctionRegistry();
- }
-
public DrillSchemaFactory getFactory(){
return drillbitContext.getSchemaFactory();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggBatch.java
index ad60e8f..3286cbd 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggBatch.java
@@ -38,7 +38,7 @@ import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.HoldingContainerExpression;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.ValueVectorWriteExpression;
-import org.apache.drill.exec.expr.fn.impl.ComparatorFunctions;
+import org.apache.drill.exec.expr.fn.ComparatorFunctionHelper;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.StreamingAggregate;
import org.apache.drill.exec.record.AbstractRecordBatch;
@@ -215,9 +215,9 @@ public class AggBatch extends AbstractRecordBatch<StreamingAggregate> {
HoldingContainer first = cg.addExpr(expr, false);
cg.setMappingSet(isSameI2Mapping);
HoldingContainer second = cg.addExpr(expr, false);
-
- FunctionCall f = new FunctionCall(ComparatorFunctions.COMPARE_TO, ImmutableList.of((LogicalExpression) new HoldingContainerExpression(first), new HoldingContainerExpression(second)), ExpressionPosition.UNKNOWN);
- HoldingContainer out = cg.addExpr(f, false);
+
+ LogicalExpression fh = ComparatorFunctionHelper.get(first, second, context.getFunctionRegistry());
+ HoldingContainer out = cg.addExpr(fh, false);
cg.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)))._then()._return(JExpr.FALSE);
}
cg.getEvalBlock()._return(JExpr.TRUE);
@@ -237,8 +237,8 @@ public class AggBatch extends AbstractRecordBatch<StreamingAggregate> {
cg.setMappingSet(isaB2Mapping);
HoldingContainer second = cg.addExpr(expr, false);
- FunctionCall f = new FunctionCall(ComparatorFunctions.COMPARE_TO, ImmutableList.of((LogicalExpression) new HoldingContainerExpression(first), new HoldingContainerExpression(second)), ExpressionPosition.UNKNOWN);
- HoldingContainer out = cg.addExpr(f, false);
+ LogicalExpression fh = ComparatorFunctionHelper.get(first, second, context.getFunctionRegistry());
+ HoldingContainer out = cg.addExpr(fh, false);
cg.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)))._then()._return(JExpr.FALSE);
}
cg.getEvalBlock()._return(JExpr.TRUE);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
index 6e98543..1fa34b6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
@@ -36,7 +36,7 @@ import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
import org.apache.drill.exec.expr.HoldingContainerExpression;
import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.expr.fn.impl.ComparatorFunctions;
+import org.apache.drill.exec.expr.fn.ComparatorFunctionHelper;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.MergeJoinPOP;
import org.apache.drill.exec.physical.impl.filter.ReturnValueExpression;
@@ -296,8 +296,10 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
._return(JExpr.lit(1));
}
- FunctionCall f = new FunctionCall(ComparatorFunctions.COMPARE_TO, ImmutableList.of((LogicalExpression) new HoldingContainerExpression(compareLeftExprHolder), (LogicalExpression) new HoldingContainerExpression(compareRightExprHolder)), ExpressionPosition.UNKNOWN);
- cg.addExpr(new ReturnValueExpression(f, false), false);
+ LogicalExpression fh = ComparatorFunctionHelper.get(compareLeftExprHolder,
+ compareRightExprHolder,
+ context.getFunctionRegistry());
+ cg.addExpr(new ReturnValueExpression(fh, false), false);
//
// // equality
// cg.getEvalBlock()._if(compareLeftExprHolder.getValue().eq(compareRightExprHolder.getValue()))
@@ -344,8 +346,11 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP> {
}
// check value equality
- FunctionCall g = new FunctionCall(ComparatorFunctions.COMPARE_TO, ImmutableList.of((LogicalExpression) new HoldingContainerExpression(compareThisLeftExprHolder), (LogicalExpression) new HoldingContainerExpression(compareNextLeftExprHolder)), ExpressionPosition.UNKNOWN);
- cg.addExpr(new ReturnValueExpression(g, false), false);
+
+ LogicalExpression gh = ComparatorFunctionHelper.get(compareThisLeftExprHolder,
+ compareNextLeftExprHolder,
+ context.getFunctionRegistry());
+ cg.addExpr(new ReturnValueExpression(gh, false), false);
// generate copyLeft()
//////////////////////
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
index 8572085..84bc897 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/mergereceiver/MergingRecordBatch.java
@@ -38,10 +38,9 @@ import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
-import org.apache.drill.exec.expr.HoldingContainerExpression;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.ValueVectorReadExpression;
-import org.apache.drill.exec.expr.fn.impl.ComparatorFunctions;
+import org.apache.drill.exec.expr.fn.ComparatorFunctionHelper;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.MergingReceiverPOP;
import org.apache.drill.exec.proto.UserBitShared;
@@ -541,11 +540,8 @@ public class MergingRecordBatch implements RecordBatch {
rightVar.ref("isSet"));
// generate the comparison function
- FunctionCall f = new FunctionCall(ComparatorFunctions.COMPARE_TO,
- ImmutableList.of((LogicalExpression) new HoldingContainerExpression(left),
- new HoldingContainerExpression(right)),
- ExpressionPosition.UNKNOWN);
- ClassGenerator.HoldingContainer out = cg.addExpr(f, false);
+ LogicalExpression fh = ComparatorFunctionHelper.get(left, right, context.getFunctionRegistry());
+ ClassGenerator.HoldingContainer out = cg.addExpr(fh, false);
// generate less than/greater than checks (fixing results for ASCending vs. DESCending)
cg.getEvalBlock()._if(out.getValue().eq(JExpr.lit(1)))
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
index 3413046..867200e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/orderedpartitioner/OrderedPartitionRecordBatch.java
@@ -27,10 +27,8 @@ import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FieldReference;
-import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.logical.data.Order;
import org.apache.drill.common.logical.data.Order.Ordering;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.Types;
@@ -45,11 +43,10 @@ import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
-import org.apache.drill.exec.expr.HoldingContainerExpression;
import org.apache.drill.exec.expr.TypeHelper;
import org.apache.drill.exec.expr.ValueVectorReadExpression;
import org.apache.drill.exec.expr.ValueVectorWriteExpression;
-import org.apache.drill.exec.expr.fn.impl.ComparatorFunctions;
+import org.apache.drill.exec.expr.fn.ComparatorFunctionHelper;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.OrderedPartitionSender;
import org.apache.drill.exec.physical.impl.sort.SortBatch;
@@ -72,7 +69,6 @@ import org.apache.drill.exec.vector.ValueVector;
import org.eigenbase.rel.RelFieldCollation.Direction;
import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.sun.codemodel.JConditional;
import com.sun.codemodel.JExpr;
@@ -533,10 +529,8 @@ public class OrderedPartitionRecordBatch extends AbstractRecordBatch<OrderedPart
new ValueVectorReadExpression(new TypedFieldId(expr.getMajorType(), count++)), false);
cg.setMappingSet(mainMapping);
- FunctionCall f = new FunctionCall(ComparatorFunctions.COMPARE_TO, ImmutableList.of(
- (LogicalExpression) new HoldingContainerExpression(left), new HoldingContainerExpression(right)),
- ExpressionPosition.UNKNOWN);
- ClassGenerator.HoldingContainer out = cg.addExpr(f, false);
+ LogicalExpression fh = ComparatorFunctionHelper.get(left, right, context.getFunctionRegistry());
+ ClassGenerator.HoldingContainer out = cg.addExpr(fh, false);
JConditional jc = cg.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)));
if (od.getDirection() == Direction.Ascending) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
index 3364a45..9a1b5ab 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/sort/SortBatch.java
@@ -22,8 +22,6 @@ import java.util.List;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.logical.data.Order.Ordering;
import org.apache.drill.exec.compile.sig.MappingSet;
@@ -33,8 +31,7 @@ import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
import org.apache.drill.exec.expr.CodeGenerator;
import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
-import org.apache.drill.exec.expr.HoldingContainerExpression;
-import org.apache.drill.exec.expr.fn.impl.ComparatorFunctions;
+import org.apache.drill.exec.expr.fn.ComparatorFunctionHelper;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.Sort;
import org.apache.drill.exec.record.AbstractRecordBatch;
@@ -46,7 +43,6 @@ import org.apache.drill.exec.record.selection.SelectionVector2;
import org.apache.drill.exec.record.selection.SelectionVector4;
import org.eigenbase.rel.RelFieldCollation.Direction;
-import com.google.common.collect.ImmutableList;
import com.sun.codemodel.JConditional;
import com.sun.codemodel.JExpr;
@@ -190,8 +186,8 @@ public class SortBatch extends AbstractRecordBatch<Sort> {
g.setMappingSet(mainMapping);
// next we wrap the two comparison sides and add the expression block for the comparison.
- FunctionCall f = new FunctionCall(ComparatorFunctions.COMPARE_TO, ImmutableList.of((LogicalExpression) new HoldingContainerExpression(left), new HoldingContainerExpression(right)), ExpressionPosition.UNKNOWN);
- HoldingContainer out = g.addExpr(f, false);
+ LogicalExpression fh = ComparatorFunctionHelper.get(left, right, context.getFunctionRegistry());
+ HoldingContainer out = g.addExpr(fh, false);
JConditional jc = g.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)));
if(od.getDirection() == Direction.Ascending){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRel.java
index b4c1bf9..e1337ef 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillAggregateRel.java
@@ -25,6 +25,7 @@ import net.hydromatic.optiq.util.BitSets;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.FunctionCallFactory;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.common.logical.data.GroupingAggregate;
@@ -96,7 +97,7 @@ public class DrillAggregateRel extends AggregateRelBase implements DrillRel {
// for count(1).
if(args.isEmpty()) args.add(new ValueExpressions.LongExpression(1l));
- LogicalExpression expr = implementor.getContext().getRegistry().createExpression(call.getAggregation().getName().toLowerCase(), ExpressionPosition.UNKNOWN, args);
+ LogicalExpression expr = FunctionCallFactory.createExpression(call.getAggregation().getName().toLowerCase(), ExpressionPosition.UNKNOWN, args);
return expr;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
index 13f5494..4fb9a01 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillOptiq.java
@@ -22,6 +22,8 @@ import java.util.Collections;
import java.util.List;
import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.FunctionCallFactory;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.expression.ValueExpressions;
@@ -92,7 +94,7 @@ public class DrillOptiq {
args = Lists.reverse(args);
LogicalExpression lastArg = args.get(0);
for(int i = 1; i < args.size(); i++){
- lastArg = context.getRegistry().createExpression(funcName, Lists.newArrayList(args.get(i), lastArg));
+ lastArg = FunctionCallFactory.createExpression(funcName, Lists.newArrayList(args.get(i), lastArg));
}
return lastArg;
@@ -102,7 +104,7 @@ public class DrillOptiq {
for(RexNode n : call.getOperands()){
exprs.add(n.accept(this));
}
- return context.getRegistry().createExpression(call.getOperator().getName().toLowerCase(), Lists.newArrayList(exprs));
+ return FunctionCallFactory.createExpression(call.getOperator().getName().toLowerCase(), Lists.newArrayList(exprs));
case SPECIAL:
logger.debug("Special");
switch(call.getKind()){
@@ -165,7 +167,7 @@ public class DrillOptiq {
switch(call.getType().getSqlTypeName().getName()){
case "VARCHAR": {
args = Lists.newArrayList(arg, new LongExpression(call.getType().getPrecision()));
- return context.getRegistry().createExpression("castVARCHAR", args);
+ return FunctionCallFactory.createExpression("castVARCHAR", args);
}
case "INTEGER": fname = "castINT"; break;
case "FLOAT": fname = "castFLOAT4"; break;
@@ -173,7 +175,7 @@ public class DrillOptiq {
case "DECIMAL": throw new UnsupportedOperationException("Need to add decimal.");
default: fname = "cast" + call.getType().getSqlTypeName().getName();
}
- return context.getRegistry().createExpression(fname, args);
+ return FunctionCallFactory.createExpression(fname, args);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillParseContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillParseContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillParseContext.java
index b82fef5..be4474f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillParseContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillParseContext.java
@@ -17,20 +17,12 @@
*/
package org.apache.drill.exec.planner.logical;
-import org.apache.drill.common.expression.FunctionRegistry;
-
+// TODO: Can we remove this completely? Or is there any plan to add any props in future for parsing?
public class DrillParseContext {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillParseContext.class);
-
- private final FunctionRegistry registry;
-
- public DrillParseContext(FunctionRegistry registry) {
+
+ public DrillParseContext() {
super();
- this.registry = registry;
}
- public FunctionRegistry getRegistry(){
- return registry;
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
index 0fdff3c..bdbb642 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
@@ -24,7 +24,6 @@ import net.hydromatic.optiq.tools.RelConversionException;
import net.hydromatic.optiq.tools.RuleSet;
import net.hydromatic.optiq.tools.ValidationException;
-import org.apache.drill.common.expression.FunctionRegistry;
import org.apache.drill.common.logical.LogicalPlan;
import org.apache.drill.common.logical.PlanProperties.Generator.ResultMode;
import org.apache.drill.exec.planner.logical.DrillImplementor;
@@ -45,11 +44,9 @@ import org.eigenbase.sql.parser.SqlParseException;
public class DrillSqlWorker {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillSqlWorker.class);
- private final FunctionRegistry registry;
private final Planner planner;
- public DrillSqlWorker(DrillSchemaFactory schemaFactory, FunctionRegistry functionRegistry) throws Exception {
- this.registry = functionRegistry;
+ public DrillSqlWorker(DrillSchemaFactory schemaFactory) throws Exception {
this.planner = Frameworks.getPlanner(ConnectionConfig.Lex.MYSQL, schemaFactory, SqlStdOperatorTable.instance(), new RuleSet[]{DrillRuleSets.DRILL_BASIC_RULES});
}
@@ -81,7 +78,7 @@ public class DrillSqlWorker {
}else{
convertedRelNode = new DrillScreenRel(convertedRelNode.getCluster(), convertedRelNode.getTraitSet(), convertedRelNode);
}
- DrillImplementor implementor = new DrillImplementor(new DrillParseContext(registry), resultMode);
+ DrillImplementor implementor = new DrillImplementor(new DrillParseContext(), resultMode);
implementor.go( (DrillRel) convertedRelNode);
planner.close();
planner.reset();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
index 0e6b7bf..8edbcc5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
@@ -23,7 +23,6 @@ import java.util.Collection;
import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.common.expression.FunctionRegistry;
import org.apache.drill.exec.cache.DistributedCache;
import org.apache.drill.exec.coord.ClusterCoordinator;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
@@ -56,7 +55,6 @@ public class DrillbitContext {
private final Controller controller;
private final WorkEventBus workBus;
private final FunctionImplementationRegistry functionRegistry;
- private final FunctionRegistry functionRegistryX;
public DrillbitContext(DrillbitEndpoint endpoint, BootStrapContext context, ClusterCoordinator coord, Controller controller, DataConnectionCreator connectionsPool, DistributedCache cache, WorkEventBus workBus) {
super();
@@ -75,11 +73,6 @@ public class DrillbitContext {
this.reader = new PhysicalPlanReader(context.getConfig(), context.getConfig().getMapper(), endpoint, storagePlugins);
this.operatorCreatorRegistry = new OperatorCreatorRegistry(context.getConfig());
this.functionRegistry = new FunctionImplementationRegistry(context.getConfig());
- this.functionRegistryX = new FunctionRegistry(context.getConfig());
- }
-
- public FunctionRegistry getFunctionRegistry(){
- return functionRegistryX;
}
public FunctionImplementationRegistry getFunctionImplementationRegistry() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
index baecc3f..08cb599 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
@@ -346,7 +346,7 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
private void runSQL(String sql) {
try{
- DrillSqlWorker sqlWorker = new DrillSqlWorker(context.getFactory(), context.getFunctionRegistry());
+ DrillSqlWorker sqlWorker = new DrillSqlWorker(context.getFactory());
LogicalPlan plan = sqlWorker.getPlan(sql);
[08/12] git commit: DRILL-335: Implement Hash Aggregation
Posted by ja...@apache.org.
DRILL-335: Implement Hash Aggregation
1. Implementation of the hash aggregation execution operator - this has two main parts: the HashAggTemplate and the HashAggBatch.
2. Implementation of a hash table which is used by the hash aggregation. The hash table hash two main parts: the HashTableTemplate and the ChainedHashTable. The hash table internally uses the notion of 'BatchHolder' to keep track of all keys that can fit within one batch of 64K values. New BatchHolder objects are created as needed. Each BatchHolder has its own vector container. The HashAggregate also has a similar structure and it keeps track of the workspace variables.
(NOTE: An initial design document for the hash aggregation and hash table was already attached with Drill-335. The document has not yet been updated with the latest implementation ... will try to do that in the near future).
3. Jinfeng's changes to use workspace vectors in the generated code for aggregate functions (previously, for streaming aggregate we only needed to maintain workspace variable for 1 running group; however for hash aggregate we need to maintain it for all groups).
4. Fix for Drill-318: because of #3 above, the previous fix for Drill-318 is not valid anymore. I modified the template generation code for the aggregate functions such that they conform to the new infrastructure.
5. The original AggTemplate, AggBatch and Aggregator classes have been moved to corresponding StreamingAggTemplate, StreamingAggBatch and StreamingAggregator in order to differentiate it from hash aggregation. These appear as new files but the code there has not changed.
I have run several tests manually as part of TestHashAggr...these tests use TPC-H data and in particular a relatively large 'Orders' table. However, I have not yet packaged the tests to run as part of JUnit since the location and size of the parquet files needs to be figured out. I will continue to work on that.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/8d725f69
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/8d725f69
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/8d725f69
Branch: refs/heads/master
Commit: 8d725f695703a10c5ee045d1639e90dd3e4f2bfb
Parents: 693bd35
Author: Aman Sinha <as...@maprtech.com>
Authored: Sat Mar 22 09:57:13 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sat Mar 22 09:57:13 2014 -0700
----------------------------------------------------------------------
exec/java-exec/src/main/codegen/config.fmpp | 2 +
.../src/main/codegen/data/AggrTypes1.tdd | 64 ++
.../src/main/codegen/data/AggrTypes2.tdd | 31 +
.../codegen/templates/AggrTypeFunctions1.java | 152 +++++
.../codegen/templates/AggrTypeFunctions2.java | 106 ++++
.../src/main/codegen/templates/TypeHelper.java | 62 ++
.../exec/compile/sig/GeneratorMapping.java | 2 +-
.../drill/exec/compile/sig/MappingSet.java | 85 ++-
.../apache/drill/exec/expr/ClassGenerator.java | 31 +-
.../exec/expr/fn/ComparatorFunctionHelper.java | 59 --
.../drill/exec/expr/fn/DrillAggFuncHolder.java | 138 ++++-
.../drill/exec/expr/fn/DrillFuncHolder.java | 110 ++--
.../drill/exec/expr/fn/FunctionConverter.java | 21 +-
.../exec/expr/fn/FunctionGenerationHelper.java | 86 +++
.../exec/expr/fn/agg/impl/CountFunctions.java | 86 ---
.../exec/expr/fn/agg/impl/SumFunctions.java | 86 ---
.../apache/drill/exec/ops/FragmentContext.java | 2 +-
.../apache/drill/exec/opt/BasicOptimizer.java | 4 +-
.../drill/exec/physical/OperatorCost.java | 8 +
.../physical/base/AbstractPhysicalVisitor.java | 5 +
.../exec/physical/base/PhysicalVisitor.java | 1 +
.../exec/physical/config/HashAggregate.java | 119 ++++
.../exec/physical/impl/aggregate/AggBatch.java | 327 ----------
.../impl/aggregate/AggBatchCreator.java | 40 --
.../physical/impl/aggregate/AggTemplate.java | 305 ---------
.../physical/impl/aggregate/Aggregator.java | 43 --
.../physical/impl/aggregate/HashAggBatch.java | 278 +++++++++
.../impl/aggregate/HashAggBatchCreator.java | 41 ++
.../impl/aggregate/HashAggTemplate.java | 417 +++++++++++++
.../physical/impl/aggregate/HashAggregator.java | 59 ++
.../impl/aggregate/StreamingAggBatch.java | 324 ++++++++++
.../aggregate/StreamingAggBatchCreator.java | 40 ++
.../impl/aggregate/StreamingAggTemplate.java | 304 +++++++++
.../impl/aggregate/StreamingAggregator.java | 46 ++
.../physical/impl/common/ChainedHashTable.java | 280 +++++++++
.../exec/physical/impl/common/HashTable.java | 63 ++
.../physical/impl/common/HashTableConfig.java | 62 ++
.../physical/impl/common/HashTableTemplate.java | 616 +++++++++++++++++++
.../exec/physical/impl/join/MergeJoinBatch.java | 6 +-
.../impl/mergereceiver/MergingRecordBatch.java | 4 +-
.../OrderedPartitionRecordBatch.java | 4 +-
.../impl/project/ProjectRecordBatch.java | 2 +-
.../exec/physical/impl/sort/SortBatch.java | 4 +-
.../apache/drill/exec/expr/ExpressionTest.java | 2 +-
.../exec/physical/impl/SimpleRootExec.java | 4 +
.../exec/physical/impl/TestSimpleFunctions.java | 11 +-
.../exec/physical/impl/agg/TestHashAggr.java | 61 ++
.../physical/impl/common/TestHashTable.java | 97 +++
.../test/resources/common/test_hashtable1.json | 40 ++
49 files changed, 3676 insertions(+), 1064 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/codegen/config.fmpp
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/config.fmpp b/exec/java-exec/src/main/codegen/config.fmpp
index f2e2c69..c1742e4 100644
--- a/exec/java-exec/src/main/codegen/config.fmpp
+++ b/exec/java-exec/src/main/codegen/config.fmpp
@@ -21,6 +21,8 @@ data: {
MathFunctionTypes: tdd(../data/MathFunctionTypes.tdd)
mathFunc:tdd(../data/MathFunc.tdd)
drillOI:tdd(../data/HiveTypes.tdd)
+ aggrtypes1: tdd(../data/AggrTypes1.tdd),
+ aggrtypes2: tdd(../data/AggrTypes2.tdd)
}
freemarkerLinks: {
includes: includes/
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/codegen/data/AggrTypes1.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/AggrTypes1.tdd b/exec/java-exec/src/main/codegen/data/AggrTypes1.tdd
new file mode 100644
index 0000000..ac5d6eb
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/data/AggrTypes1.tdd
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+ aggrtypes: [
+ {className: "Min", funcName: "min", types: [
+ {inputType: "Int", outputType: "Int", runningType: "Int"},
+ {inputType: "BigInt", outputType: "BigInt", runningType: "BigInt"},
+ {inputType: "NullableInt", outputType: "Int", runningType: "Int"},
+ {inputType: "NullableBigInt", outputType: "BigInt", runningType: "BigInt"},
+ {inputType: "Float4", outputType: "Float4", runningType: "Float4"},
+ {inputType: "Float8", outputType: "Float8", runningType: "Float8"},
+ {inputType: "NullableFloat4", outputType: "Float4", runningType: "Float4"},
+ {inputType: "NullableFloat8", outputType: "Float8", runningType: "Float8"}
+ ]
+ },
+ {className: "Max", funcName: "max", types: [
+ {inputType: "Int", outputType: "Int", runningType: "Int"},
+ {inputType: "BigInt", outputType: "BigInt", runningType: "BigInt"},
+ {inputType: "NullableInt", outputType: "Int", runningType: "Int"},
+ {inputType: "NullableBigInt", outputType: "BigInt", runningType: "BigInt"},
+ {inputType: "Float4", outputType: "Float4", runningType: "Float4"},
+ {inputType: "Float8", outputType: "Float8", runningType: "Float8"},
+ {inputType: "NullableFloat4", outputType: "Float4", runningType: "Float4"},
+ {inputType: "NullableFloat8", outputType: "Float8", runningType: "Float8"}
+ ]
+ },
+ {className: "Sum", funcName: "sum", types: [
+ {inputType: "Int", outputType: "BigInt", runningType: "BigInt"},
+ {inputType: "BigInt", outputType: "BigInt", runningType: "BigInt"},
+ {inputType: "NullableInt", outputType: "BigInt", runningType: "BigInt"},
+ {inputType: "NullableBigInt", outputType: "BigInt", runningType: "BigInt"},
+ {inputType: "Float4", outputType: "Float8", runningType: "Float8"},
+ {inputType: "Float8", outputType: "Float8", runningType: "Float8"},
+ {inputType: "NullableFloat4", outputType: "Float8", runningType: "Float8"},
+ {inputType: "NullableFloat8", outputType: "Float8", runningType: "Float8"}
+ ]
+ },
+ {className: "Count", funcName: "count", types: [
+ {inputType: "Int", outputType: "BigInt", runningType: "BigInt"},
+ {inputType: "BigInt", outputType: "BigInt", runningType: "BigInt"},
+ {inputType: "NullableInt", outputType: "BigInt", runningType: "BigInt"},
+ {inputType: "NullableBigInt", outputType: "BigInt", runningType: "BigInt"},
+ {inputType: "Float4", outputType: "BigInt", runningType: "BigInt"},
+ {inputType: "Float8", outputType: "BigInt", runningType: "BigInt"},
+ {inputType: "NullableFloat4", outputType: "BigInt", runningType: "BigInt"},
+ {inputType: "NullableFloat8", outputType: "BigInt", runningType: "BigInt"}
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/codegen/data/AggrTypes2.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/AggrTypes2.tdd b/exec/java-exec/src/main/codegen/data/AggrTypes2.tdd
new file mode 100644
index 0000000..a05d074
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/data/AggrTypes2.tdd
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http:# www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+ aggrtypes: [
+ {className: "Avg", funcName: "avg", types: [
+ {inputType: "Int", outputType: "Float8", sumRunningType: "BigInt", countRunningType: "BigInt"},
+ {inputType: "BigInt", outputType: "Float8", sumRunningType: "BigInt", countRunningType: "BigInt"},
+ {inputType: "NullableInt", outputType: "Float8", sumRunningType: "BigInt", countRunningType: "BigInt"},
+ {inputType: "NullableBigInt", outputType: "Float8", sumRunningType: "BigInt", countRunningType: "BigInt"},
+ {inputType: "Float4", outputType: "Float8", sumRunningType: "BigInt", countRunningType: "BigInt"},
+ {inputType: "Float8", outputType: "Float8", sumRunningType: "BigInt", countRunningType: "BigInt"},
+ {inputType: "NullableFloat4", outputType: "Float8", sumRunningType: "BigInt", countRunningType: "BigInt"},
+ {inputType: "NullableFloat8", outputType: "Float8", sumRunningType: "BigInt", countRunningType: "BigInt"}
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/codegen/templates/AggrTypeFunctions1.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/AggrTypeFunctions1.java b/exec/java-exec/src/main/codegen/templates/AggrTypeFunctions1.java
new file mode 100644
index 0000000..9abc768
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/templates/AggrTypeFunctions1.java
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+<@pp.dropOutputFile />
+
+
+
+<#list aggrtypes1.aggrtypes as aggrtype>
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gaggr/${aggrtype.className}Functions.java" />
+
+<#include "/@includes/license.ftl" />
+
+<#-- A utility class that is used to generate java code for aggr functions that maintain a single -->
+<#-- running counter to hold the result. This includes: MIN, MAX, SUM, COUNT. -->
+
+/*
+ * This class is automatically generated from AggrTypeFunctions1.tdd using FreeMarker.
+ */
+
+package org.apache.drill.exec.expr.fn.impl.gaggr;
+
+import org.apache.drill.exec.expr.DrillAggFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.annotations.Workspace;
+import org.apache.drill.exec.expr.holders.*;
+import org.apache.drill.exec.record.RecordBatch;
+
+@SuppressWarnings("unused")
+
+public class ${aggrtype.className}Functions {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${aggrtype.className}Functions.class);
+
+<#list aggrtype.types as type>
+
+@FunctionTemplate(name = "${aggrtype.funcName}", scope = FunctionTemplate.FunctionScope.POINT_AGGREGATE)
+public static class ${type.inputType}${aggrtype.className} implements DrillAggFunc{
+
+ @Param ${type.inputType}Holder in;
+ @Workspace ${type.runningType}Holder value;
+ @Output ${type.outputType}Holder out;
+
+ public void setup(RecordBatch b) {
+ value = new ${type.runningType}Holder();
+ <#if aggrtype.funcName == "sum" || aggrtype.funcName == "count">
+ value.value = 0;
+ <#elseif aggrtype.funcName == "min">
+ <#if type.runningType?starts_with("Int")>
+ value.value = Integer.MAX_VALUE;
+ <#elseif type.runningType?starts_with("BigInt")>
+ value.value = Long.MAX_VALUE;
+ <#elseif type.runningType?starts_with("Float4")>
+ value.value = Float.MAX_VALUE;
+ <#elseif type.runningType?starts_with("Float8")>
+ value.value = Double.MAX_VALUE;
+ </#if>
+ <#elseif aggrtype.funcName == "max">
+ <#if type.runningType?starts_with("Int")>
+ value.value = Integer.MIN_VALUE;
+ <#elseif type.runningType?starts_with("BigInt")>
+ value.value = Long.MIN_VALUE;
+ <#elseif type.runningType?starts_with("Float4")>
+ value.value = Float.MIN_VALUE;
+ <#elseif type.runningType?starts_with("Float8")>
+ value.value = Double.MIN_VALUE;
+ </#if>
+ </#if>
+
+ }
+
+ @Override
+ public void add() {
+ <#if type.inputType?starts_with("Nullable")>
+ sout: {
+ if (in.isSet == 0) {
+ // processing nullable input and the value is null, so don't do anything...
+ break sout;
+ }
+ </#if>
+ <#if aggrtype.funcName == "min">
+ value.value = Math.min(value.value, in.value);
+ <#elseif aggrtype.funcName == "max">
+ value.value = Math.max(value.value, in.value);
+ <#elseif aggrtype.funcName == "sum">
+ value.value += in.value;
+ <#elseif aggrtype.funcName == "count">
+ value.value++;
+ <#else>
+ // TODO: throw an error ?
+ </#if>
+ <#if type.inputType?starts_with("Nullable")>
+ } // end of sout block
+ </#if>
+ }
+
+ @Override
+ public void output() {
+ out.value = value.value;
+ }
+
+ @Override
+ public void reset() {
+
+ <#if aggrtype.funcName == "sum" || aggrtype.funcName == "count">
+ value.value = 0;
+ <#elseif aggrtype.funcName == "min">
+ <#if type.runningType?starts_with("Int")>
+ value.value = Integer.MAX_VALUE;
+ <#elseif type.runningType?starts_with("BigInt")>
+ value.value = Long.MAX_VALUE;
+ <#elseif type.runningType?starts_with("Float4")>
+ value.value = Float.MAX_VALUE;
+ <#elseif type.runningType?starts_with("Float8")>
+ value.value = Double.MAX_VALUE;
+ </#if>
+ <#elseif aggrtype.funcName == "max">
+ <#if type.runningType?starts_with("Int")>
+ value.value = Integer.MIN_VALUE;
+ <#elseif type.runningType?starts_with("BigInt")>
+ value.value = Long.MIN_VALUE;
+ <#elseif type.runningType?starts_with("Float4")>
+ value.value = Float.MIN_VALUE;
+ <#elseif type.runningType?starts_with("Float8")>
+ value.value = Double.MIN_VALUE;
+ </#if>
+ </#if>
+
+ }
+
+ }
+
+
+</#list>
+}
+</#list>
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/codegen/templates/AggrTypeFunctions2.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/AggrTypeFunctions2.java b/exec/java-exec/src/main/codegen/templates/AggrTypeFunctions2.java
new file mode 100644
index 0000000..ca9adb5
--- /dev/null
+++ b/exec/java-exec/src/main/codegen/templates/AggrTypeFunctions2.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+<@pp.dropOutputFile />
+
+
+
+<#list aggrtypes2.aggrtypes as aggrtype>
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/gaggr/${aggrtype.className}Functions.java" />
+
+<#include "/@includes/license.ftl" />
+
+<#-- A utility class that is used to generate java code for aggr functions that maintain a sum -->
+<#-- and a running count. For now, this includes: AVG. -->
+
+/*
+ * This class is automatically generated from AggrTypeFunctions2.tdd using FreeMarker.
+ */
+
+
+package org.apache.drill.exec.expr.fn.impl.gaggr;
+
+import org.apache.drill.exec.expr.DrillAggFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.annotations.Workspace;
+import org.apache.drill.exec.expr.holders.*;
+import org.apache.drill.exec.record.RecordBatch;
+
+@SuppressWarnings("unused")
+
+public class ${aggrtype.className}Functions {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${aggrtype.className}Functions.class);
+
+<#list aggrtype.types as type>
+
+@FunctionTemplate(name = "${aggrtype.funcName}", scope = FunctionTemplate.FunctionScope.POINT_AGGREGATE)
+
+public static class ${type.inputType}${aggrtype.className} implements DrillAggFunc{
+
+ @Param ${type.inputType}Holder in;
+ @Workspace ${type.sumRunningType}Holder sum;
+ @Workspace ${type.countRunningType}Holder count;
+ @Output ${type.outputType}Holder out;
+
+ public void setup(RecordBatch b) {
+ sum = new ${type.sumRunningType}Holder();
+ count = new ${type.countRunningType}Holder();
+ sum.value = 0;
+ count.value = 0;
+ }
+
+ @Override
+ public void add() {
+ <#if type.inputType?starts_with("Nullable")>
+ sout: {
+ if (in.isSet == 0) {
+ // processing nullable input and the value is null, so don't do anything...
+ break sout;
+ }
+ </#if>
+ <#if aggrtype.funcName == "avg">
+ sum.value += in.value;
+ count.value++;
+ <#else>
+ // TODO: throw an error ?
+ </#if>
+ <#if type.inputType?starts_with("Nullable")>
+ } // end of sout block
+ </#if>
+ }
+
+ @Override
+ public void output() {
+ out.value = sum.value/count.value;
+ }
+
+ @Override
+ public void reset() {
+ sum.value = 0;
+ count.value = 0;
+ }
+
+ }
+
+
+</#list>
+}
+</#list>
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/codegen/templates/TypeHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/TypeHelper.java b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
index 8c72cf7..84b5f70 100644
--- a/exec/java-exec/src/main/codegen/templates/TypeHelper.java
+++ b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
@@ -132,4 +132,66 @@ public class TypeHelper {
throw new UnsupportedOperationException(type.getMinorType() + " type is not supported. Mode: " + type.getMode());
}
+ public static ValueHolder getValue(ValueVector vector, int index) {
+ MajorType type = vector.getField().getType();
+ ValueHolder holder;
+ switch(type.getMinorType()) {
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+ <#if minor.class?starts_with("Var")>
+ throw new UnsupportedOperationException(type.getMinorType() + " type is not supported.");
+ <#else>
+ holder = new ${minor.class}Holder();
+ ((${minor.class}Holder)holder).value = ((${minor.class}Vector) vector).getAccessor().get(index);
+ break;
+ </#if>
+ </#list>
+</#list>
+ default:
+ throw new UnsupportedOperationException(type.getMinorType() + " type is not supported.");
+ }
+ return holder;
+ }
+
+ public static void setValue(ValueVector vector, int index, ValueHolder holder) {
+ MajorType type = vector.getField().getType();
+
+ switch(type.getMinorType()) {
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+ ((${minor.class}Vector) vector).getMutator().set(index, (${minor.class}Holder) holder);
+ break;
+ </#list>
+</#list>
+ default:
+ throw new UnsupportedOperationException(type.getMinorType() + " type is not supported.");
+ }
+ }
+
+ public static boolean compareValues(ValueVector v1, int v1index, ValueVector v2, int v2index) {
+ MajorType type1 = v1.getField().getType();
+ MajorType type2 = v2.getField().getType();
+
+ if (type1.getMinorType() != type2.getMinorType()) {
+ return false;
+ }
+
+ switch(type1.getMinorType()) {
+<#list vv.types as type>
+ <#list type.minor as minor>
+ case ${minor.class?upper_case} :
+ if ( ((${minor.class}Vector) v1).getAccessor().get(v1index) ==
+ ((${minor.class}Vector) v2).getAccessor().get(v2index) )
+ return true;
+ break;
+ </#list>
+</#list>
+ default:
+ break;
+ }
+ return false;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/GeneratorMapping.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/GeneratorMapping.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/GeneratorMapping.java
index 8604956..70a3cda 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/GeneratorMapping.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/GeneratorMapping.java
@@ -67,7 +67,7 @@ public class GeneratorMapping {
Preconditions.checkNotNull(eval, "The current mapping does not have an eval method defined.");
return eval;
case RESET:
- Preconditions.checkNotNull(reset, "The current mapping does not have a cleanup method defined.");
+ Preconditions.checkNotNull(reset, "The current mapping does not have a reset method defined.");
return reset;
case SETUP:
Preconditions.checkNotNull(setup, "The current mapping does not have a setup method defined.");
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/MappingSet.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/MappingSet.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/MappingSet.java
index c116077..ac0e413 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/MappingSet.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/MappingSet.java
@@ -22,13 +22,11 @@ import java.util.Arrays;
import org.apache.drill.exec.expr.DirectExpression;
import com.google.common.base.Preconditions;
-import com.sun.codemodel.JExpr;
-import com.sun.codemodel.JExpression;
public class MappingSet {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MappingSet.class);
-
+
private GeneratorMapping constant;
private GeneratorMapping[] mappings;
private int mappingIndex;
@@ -37,12 +35,20 @@ public class MappingSet {
private DirectExpression writeIndex;
private DirectExpression incoming;
private DirectExpression outgoing;
-
-
+ private DirectExpression workspace;
+ private DirectExpression workspaceIndex;
+
public MappingSet(GeneratorMapping mapping) {
- this("inIndex", "outIndex", new GeneratorMapping[]{mapping, mapping});
+ this("inIndex", "outIndex", new GeneratorMapping[] { mapping, mapping });
+ }
+
+ public MappingSet(String readIndex, String writeIndex, String workspaceIndex, String incoming, String outgoing,
+ String workspace, GeneratorMapping... mappings) {
+ this(readIndex, writeIndex, incoming, outgoing, mappings);
+ this.workspaceIndex = DirectExpression.direct(workspaceIndex);
+ this.workspace = DirectExpression.direct(workspace);
}
-
+
public MappingSet(GeneratorMapping... mappings) {
this("inIndex", "outIndex", mappings);
}
@@ -50,7 +56,7 @@ public class MappingSet {
public MappingSet(String readIndex, String writeIndex, GeneratorMapping... mappings) {
this(readIndex, writeIndex, "incoming", "outgoing", mappings);
}
-
+
public MappingSet(String readIndex, String writeIndex, String incoming, String outgoing, GeneratorMapping... mappings) {
this.readIndex = DirectExpression.direct(readIndex);
this.writeIndex = DirectExpression.direct(writeIndex);
@@ -58,64 +64,79 @@ public class MappingSet {
this.outgoing = DirectExpression.direct(outgoing);
Preconditions.checkArgument(mappings.length >= 2);
this.constant = mappings[0];
-
- //Make sure the constant GM is different from other GM. If it is identical, clone another copy.
- for (int i = 1; i< mappings.length; i++) {
+
+ // Make sure the constant GM is different from other GM. If it is identical, clone another copy.
+ for (int i = 1; i < mappings.length; i++) {
if (mappings[0] == mappings[i]) {
this.constant = new GeneratorMapping(mappings[0]);
break;
}
}
-
+
this.mappings = Arrays.copyOfRange(mappings, 1, mappings.length);
this.current = this.mappings[0];
}
- public void enterConstant(){
+ public void enterConstant() {
assert constant != current;
current = constant;
}
-
- public void exitConstant(){
+
+ public void exitConstant() {
assert constant == current;
current = mappings[mappingIndex];
}
-
+
public boolean isWithinConstant() {
return constant == current;
}
-
- public void enterChild(){
+
+ public void enterChild() {
assert current == mappings[mappingIndex];
mappingIndex++;
- if(mappingIndex >= mappings.length) throw new IllegalStateException("This generator does not support mappings beyond");
+ if (mappingIndex >= mappings.length)
+ throw new IllegalStateException("This generator does not support mappings beyond");
current = mappings[mappingIndex];
}
-
- public void exitChild(){
+
+ public void exitChild() {
assert current == mappings[mappingIndex];
mappingIndex--;
- if(mappingIndex < 0) throw new IllegalStateException("You tried to traverse higher than the provided mapping provides.");
+ if (mappingIndex < 0)
+ throw new IllegalStateException("You tried to traverse higher than the provided mapping provides.");
current = mappings[mappingIndex];
}
-
- public GeneratorMapping getCurrentMapping(){
+
+ public GeneratorMapping getCurrentMapping() {
return current;
}
-
- public DirectExpression getValueWriteIndex(){
+
+ public DirectExpression getValueWriteIndex() {
return writeIndex;
}
-
- public DirectExpression getValueReadIndex(){
+
+ public DirectExpression getValueReadIndex() {
return readIndex;
}
-
- public DirectExpression getOutgoing(){
+
+ public DirectExpression getOutgoing() {
return outgoing;
}
-
- public DirectExpression getIncoming(){
+
+ public DirectExpression getIncoming() {
return incoming;
}
+
+ public DirectExpression getWorkspaceIndex() {
+ return workspaceIndex;
+ }
+
+ public DirectExpression getWorkspace() {
+ return workspace;
+ }
+
+ public boolean isHashAggMapping() {
+ return workspace != null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java
index 6d00435..2280aec 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ClassGenerator.java
@@ -20,7 +20,9 @@ package org.apache.drill.exec.expr;
import static org.apache.drill.exec.compile.sig.GeneratorMapping.GM;
import java.lang.reflect.Modifier;
+import java.util.ArrayList;
import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import org.apache.drill.common.expression.LogicalExpression;
@@ -33,6 +35,7 @@ import org.apache.drill.exec.compile.sig.GeneratorMapping;
import org.apache.drill.exec.compile.sig.MappingSet;
import org.apache.drill.exec.compile.sig.SignatureHolder;
import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.expr.fn.DrillFuncHolder.WorkspaceReference;
import org.apache.drill.exec.record.TypedFieldId;
import com.beust.jcommander.internal.Lists;
@@ -44,6 +47,7 @@ import com.sun.codemodel.JClassAlreadyExistsException;
import com.sun.codemodel.JCodeModel;
import com.sun.codemodel.JDefinedClass;
import com.sun.codemodel.JExpr;
+import com.sun.codemodel.JExpression;
import com.sun.codemodel.JFieldRef;
import com.sun.codemodel.JMethod;
import com.sun.codemodel.JMod;
@@ -63,7 +67,9 @@ public class ClassGenerator<T>{
private final EvaluationVisitor evaluationVisitor;
private final Map<ValueVectorSetup, JVar> vvDeclaration = Maps.newHashMap();
private final Map<String, ClassGenerator<T>> innerClasses = Maps.newHashMap();
- private final CodeGenerator<T> classGenerator;
+ private final List<TypedFieldId> workspaceTypes = Lists.newArrayList();
+ private final Map<WorkspaceReference, JVar> workspaceVectors = Maps.newHashMap();
+ private final CodeGenerator<T> codeGenerator;
public final JDefinedClass clazz;
private final LinkedList<JBlock>[] blocks;
@@ -78,8 +84,8 @@ public class ClassGenerator<T>{
@SuppressWarnings("unchecked")
- ClassGenerator(CodeGenerator<T> classGenerator, MappingSet mappingSet, SignatureHolder signature, EvaluationVisitor eval, JDefinedClass clazz, JCodeModel model) throws JClassAlreadyExistsException {
- this.classGenerator = classGenerator;
+ ClassGenerator(CodeGenerator<T> codeGenerator, MappingSet mappingSet, SignatureHolder signature, EvaluationVisitor eval, JDefinedClass clazz, JCodeModel model) throws JClassAlreadyExistsException {
+ this.codeGenerator = codeGenerator;
this.clazz = clazz;
this.mappings = mappingSet;
this.sig = signature;
@@ -94,7 +100,7 @@ public class ClassGenerator<T>{
for(SignatureHolder child : signature.getChildHolders()){
String innerClassName = child.getSignatureClass().getSimpleName();
JDefinedClass innerClazz = clazz._class(Modifier.FINAL + Modifier.PRIVATE, innerClassName);
- innerClasses.put(innerClassName, new ClassGenerator<>(classGenerator, mappingSet, child, eval, innerClazz, model));
+ innerClasses.put(innerClassName, new ClassGenerator<>(codeGenerator, mappingSet, child, eval, innerClazz, model));
}
}
@@ -112,8 +118,8 @@ public class ClassGenerator<T>{
this.mappings = mappings;
}
- public CodeGenerator<T> getClassGenerator() {
- return classGenerator;
+ public CodeGenerator<T> getCodeGenerator() {
+ return codeGenerator;
}
private GeneratorMapping getCurrentMapping(){
@@ -242,6 +248,9 @@ public class ClassGenerator<T>{
return clazz.field(JMod.NONE, t, prefix + index++);
}
+ public JVar declareClassField(String prefix, JType t, JExpression init){
+ return clazz.field(JMod.NONE, t, prefix + index++, init);
+ }
public HoldingContainer declare(MajorType t){
return declare(t, true);
@@ -263,6 +272,14 @@ public class ClassGenerator<T>{
return new HoldingContainer(t, var, var.ref("value"), outputSet);
}
+ public List<TypedFieldId> getWorkspaceTypes() {
+ return this.workspaceTypes;
+ }
+
+ public Map<WorkspaceReference, JVar> getWorkspaceVectors() {
+ return this.workspaceVectors;
+ }
+
private static class ValueVectorSetup{
final DirectExpression batch;
final TypedFieldId fieldId;
@@ -366,4 +383,4 @@ public class ClassGenerator<T>{
public JType getHolderType(MajorType t){
return TypeHelper.getHolderType(model, t.getMinorType(), t.getMode());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ComparatorFunctionHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ComparatorFunctionHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ComparatorFunctionHelper.java
deleted file mode 100644
index 8bceefd..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/ComparatorFunctionHelper.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.expr.fn;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.FunctionHolderExpression;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.types.TypeProtos;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
-import org.apache.drill.exec.expr.DrillFuncHolderExpr;
-import org.apache.drill.exec.expr.HoldingContainerExpression;
-
-public class ComparatorFunctionHelper {
- public static final String COMPARE_TO = "compare_to";
-
- /**
- * Given materialized arguments find the "compare_to" FunctionHolderExpression
- * @param left
- * @param right
- * @param registry
- * @return FunctionHolderExpression containing the function implementation
- */
- public static FunctionHolderExpression get(HoldingContainer left,
- HoldingContainer right,
- FunctionImplementationRegistry registry) {
-
- ImmutableList<TypeProtos.MajorType> args = ImmutableList.of(left.getMajorType(), right.getMajorType()) ;
- TypeProtos.MajorType returnType = Types.required(TypeProtos.MinorType.INT);
-
- for (DrillFuncHolder h : registry.getDrillRegistry().getMethods().get(COMPARE_TO)) {
- if (h.matches(returnType, args)) {
- return new DrillFuncHolderExpr(COMPARE_TO, h,
- ImmutableList.of((LogicalExpression)
- new HoldingContainerExpression(left),
- new HoldingContainerExpression(right)),
- ExpressionPosition.UNKNOWN);
- }
- }
-
- return null;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
index 90d4521..48c35f2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillAggFuncHolder.java
@@ -20,15 +20,23 @@ package org.apache.drill.exec.expr.fn;
import java.util.List;
import java.util.Map;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.Types;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.ClassGenerator.BlockType;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
import org.apache.drill.exec.expr.annotations.FunctionTemplate.NullHandling;
+import org.apache.drill.exec.record.TypedFieldId;
import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
import com.sun.codemodel.JBlock;
+import com.sun.codemodel.JClass;
import com.sun.codemodel.JExpr;
+import com.sun.codemodel.JExpression;
+import com.sun.codemodel.JForLoop;
+import com.sun.codemodel.JInvocation;
import com.sun.codemodel.JMod;
import com.sun.codemodel.JVar;
@@ -67,9 +75,39 @@ class DrillAggFuncHolder extends DrillFuncHolder{
@Override
public JVar[] renderStart(ClassGenerator<?> g, HoldingContainer[] inputVariables) {
- JVar[] workspaceJVars = declareWorkspaceVariables(g);
- generateBody(g, BlockType.SETUP, setup, null, workspaceJVars, false);
- return workspaceJVars;
+ if (!g.getMappingSet().isHashAggMapping()) { //Declare workspace vars for non-hash-aggregation.
+ JVar[] workspaceJVars = declareWorkspaceVariables(g);
+ generateBody(g, BlockType.SETUP, setup, null, workspaceJVars, false);
+ return workspaceJVars;
+ } else { //Declare workspace vars and workspace vectors for hash aggregation.
+
+ JVar[] workspaceJVars = declareWorkspaceVectors(g);
+
+ JBlock setupBlock = g.getSetupBlock();
+
+ //Loop through all workspace vectors, to get the minimum of size of all workspace vectors.
+ JVar sizeVar = setupBlock.decl(g.getModel().INT, "vectorSize", JExpr.lit(Integer.MAX_VALUE));
+ JClass mathClass = g.getModel().ref(Math.class);
+ for (int id = 0; id<workspaceVars.length; id ++) {
+ setupBlock.assign(sizeVar,mathClass.staticInvoke("min").arg(sizeVar).arg(g.getWorkspaceVectors().get(workspaceVars[id]).invoke("getValueCapacity")));
+ }
+
+ for(int i =0 ; i < workspaceVars.length; i++) {
+ setupBlock.assign(workspaceJVars[i], JExpr._new(g.getHolderType(workspaceVars[i].majorType)));
+ }
+
+ //Use for loop to initialize entries in the workspace vectors.
+ JForLoop forLoop = setupBlock._for();
+ JVar ivar = forLoop.init(g.getModel().INT, "i", JExpr.lit(0));
+ forLoop.test(ivar.lt(sizeVar));
+ forLoop.update(ivar.assignPlus(JExpr.lit(1)));
+
+ JBlock subBlock = generateInitWorkspaceBlockHA(g, BlockType.SETUP, setup, workspaceJVars, ivar);
+ forLoop.body().add(subBlock);
+ return workspaceJVars;
+ }
+
+
}
@@ -87,11 +125,101 @@ class DrillAggFuncHolder extends DrillFuncHolder{
JVar internalOutput = sub.decl(JMod.FINAL, g.getHolderType(returnValue.type), returnValue.name, JExpr._new(g.getHolderType(returnValue.type)));
addProtectedBlock(g, sub, output, null, workspaceJVars, false);
sub.assign(out.getHolder(), internalOutput);
+ //hash aggregate uses workspace vectors. Initialization is done in "setup" and does not require "reset" block.
+ if (!g.getMappingSet().isHashAggMapping()) {
+ generateBody(g, BlockType.RESET, reset, null, workspaceJVars, false);
+ }
+ generateBody(g, BlockType.CLEANUP, cleanup, null, workspaceJVars, false);
- generateBody(g, BlockType.RESET, reset, null, workspaceJVars, false);
- generateBody(g, BlockType.CLEANUP, cleanup, null, workspaceJVars, false);
return out;
}
+ private JVar[] declareWorkspaceVectors(ClassGenerator<?> g) {
+ JVar[] workspaceJVars = new JVar[workspaceVars.length];
+
+ for(int i =0 ; i < workspaceVars.length; i++){
+ Preconditions.checkState(Types.isFixedWidthType(workspaceVars[i].majorType), String.format("Workspace variable '%s' in aggregation function '%s' is not allowed to have variable length type.", workspaceVars[i].name, registeredNames[0]));
+ Preconditions.checkState(workspaceVars[i].majorType.getMode()==DataMode.REQUIRED, String.format("Workspace variable '%s' in aggregation function '%s' is not allowed to have null or repeated type.", workspaceVars[i].name, registeredNames[0]));
+
+ //workspaceJVars[i] = g.declareClassField("work", g.getHolderType(workspaceVars[i].majorType), JExpr._new(g.getHolderType(workspaceVars[i].majorType)));
+ workspaceJVars[i] = g.declareClassField("work", g.getHolderType(workspaceVars[i].majorType));
+
+ //Declare a workspace vector for the workspace var.
+ TypedFieldId typedFieldId = new TypedFieldId(workspaceVars[i].majorType, g.getWorkspaceTypes().size());
+ JVar vv = g.declareVectorValueSetupAndMember(g.getMappingSet().getWorkspace(), typedFieldId);
+
+ g.getWorkspaceTypes().add(typedFieldId);
+ g.getWorkspaceVectors().put(workspaceVars[i], vv);
+ }
+ return workspaceJVars;
+ }
+
+ private JBlock generateInitWorkspaceBlockHA(ClassGenerator<?> g, BlockType bt, String body, JVar[] workspaceJVars, JExpression wsIndexVariable){
+ JBlock initBlock = new JBlock(true, true);
+ if(!Strings.isNullOrEmpty(body) && !body.trim().isEmpty()){
+ JBlock sub = new JBlock(true, true);
+ addProtectedBlockHA(g, sub, body, null, workspaceJVars, wsIndexVariable);
+ initBlock.directStatement(String.format("/** start %s for function %s **/ ", bt.name(), registeredNames[0]));
+ initBlock.add(sub);
+ initBlock.directStatement(String.format("/** end %s for function %s **/ ", bt.name(), registeredNames[0]));
+ }
+ return initBlock;
+ }
+
+ @Override
+ protected void addProtectedBlock(ClassGenerator<?> g, JBlock sub, String body, HoldingContainer[] inputVariables, JVar[] workspaceJVars, boolean decConstantInputOnly){
+ if (!g.getMappingSet().isHashAggMapping()) {
+ super.addProtectedBlock(g, sub, body, inputVariables, workspaceJVars, decConstantInputOnly);
+ } else {
+ JExpression indexVariable = g.getMappingSet().getWorkspaceIndex();
+ addProtectedBlockHA(g, sub, body, inputVariables, workspaceJVars, indexVariable);
+ }
+ }
+
+ /*
+ * This is customized version of "addProtectedBlock" for hash aggregation. It take one additional parameter "wsIndexVariable".
+ */
+ private void addProtectedBlockHA(ClassGenerator<?> g, JBlock sub, String body, HoldingContainer[] inputVariables, JVar[] workspaceJVars, JExpression wsIndexVariable){
+ if (inputVariables != null){
+ for(int i =0; i < inputVariables.length; i++){
+ ValueReference parameter = parameters[i];
+ HoldingContainer inputVariable = inputVariables[i];
+ sub.decl(inputVariable.getHolder().type(), parameter.name, inputVariable.getHolder());
+ }
+ }
+
+ JVar[] internalVars = new JVar[workspaceJVars.length];
+ for(int i =0; i < workspaceJVars.length; i++){
+ //sub.assign(workspaceJVars[i], JExpr._new(g.getHolderType(workspaceVars[i].majorType)));
+ //Access workspaceVar through workspace vector.
+ JInvocation getValueAccessor = g.getWorkspaceVectors().get(workspaceVars[i]).invoke("getAccessor").invoke("get");
+ if (Types.usesHolderForGet(workspaceVars[i].majorType)) {
+ sub.add(getValueAccessor.arg(wsIndexVariable).arg(workspaceJVars[i]));
+ } else {
+ sub.assign(workspaceJVars[i].ref("value"), getValueAccessor.arg(wsIndexVariable));
+ }
+ internalVars[i] = sub.decl(g.getHolderType(workspaceVars[i].majorType), workspaceVars[i].name, workspaceJVars[i]);
+ }
+
+ Preconditions.checkNotNull(body);
+ sub.directStatement(body);
+
+ // reassign workspace variables back.
+ for(int i =0; i < workspaceJVars.length; i++){
+ sub.assign(workspaceJVars[i], internalVars[i]);
+
+ //Change workspaceVar through workspace vector.
+ JInvocation setMeth;
+ if (Types.usesHolderForGet(workspaceVars[i].majorType)) {
+ setMeth = g.getWorkspaceVectors().get(workspaceVars[i]).invoke("getMutator").invoke("setSafe").arg(wsIndexVariable).arg(workspaceJVars[i]);
+ }else{
+ setMeth = g.getWorkspaceVectors().get(workspaceVars[i]).invoke("getMutator").invoke("setSafe").arg(wsIndexVariable).arg(workspaceJVars[i].ref("value"));
+ }
+
+ sub.add(setMeth);
+ }
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
index 5868421..47b8408 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFuncHolder.java
@@ -49,7 +49,7 @@ import com.sun.codemodel.JVar;
public abstract class DrillFuncHolder {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionImplementationRegistry.class);
-
+
protected final FunctionTemplate.FunctionScope scope;
protected final FunctionTemplate.NullHandling nullHandling;
protected final boolean isBinaryCommutative;
@@ -60,10 +60,10 @@ public abstract class DrillFuncHolder {
protected final ValueReference[] parameters;
protected final ValueReference returnValue;
protected final ImmutableMap<String, String> methodMap;
-
+
public DrillFuncHolder(FunctionScope scope, NullHandling nullHandling, boolean isBinaryCommutative, boolean isRandom,
- String[] registeredNames, ValueReference[] parameters, ValueReference returnValue,
- WorkspaceReference[] workspaceVars, Map<String, String> methods, List<String> imports) {
+ String[] registeredNames, ValueReference[] parameters, ValueReference returnValue,
+ WorkspaceReference[] workspaceVars, Map<String, String> methods, List<String> imports) {
super();
this.scope = scope;
this.nullHandling = nullHandling;
@@ -81,12 +81,16 @@ public abstract class DrillFuncHolder {
return imports;
}
- public JVar[] renderStart(ClassGenerator<?> g, HoldingContainer[] inputVariables){
+ public JVar[] renderStart(ClassGenerator<?> g, HoldingContainer[] inputVariables) {
return declareWorkspaceVariables(g);
};
-
- public void renderMiddle(ClassGenerator<?> g, HoldingContainer[] inputVariables, JVar[] workspaceJVars){};
- public abstract HoldingContainer renderEnd(ClassGenerator<?> g, HoldingContainer[] inputVariables, JVar[] workspaceJVars);
+
+ public void renderMiddle(ClassGenerator<?> g, HoldingContainer[] inputVariables, JVar[] workspaceJVars) {
+ };
+
+ public abstract HoldingContainer renderEnd(ClassGenerator<?> g, HoldingContainer[] inputVariables,
+ JVar[] workspaceJVars);
+
public abstract boolean isNested();
public boolean isAggregating() {
@@ -97,19 +101,20 @@ public abstract class DrillFuncHolder {
return isRandom;
}
- protected JVar[] declareWorkspaceVariables(ClassGenerator<?> g){
+ protected JVar[] declareWorkspaceVariables(ClassGenerator<?> g) {
JVar[] workspaceJVars = new JVar[workspaceVars.length];
- for(int i =0 ; i < workspaceVars.length; i++){
+ for (int i = 0; i < workspaceVars.length; i++) {
workspaceJVars[i] = g.declareClassField("work", g.getModel()._ref(workspaceVars[i].type));
}
return workspaceJVars;
}
- protected void generateBody(ClassGenerator<?> g, BlockType bt, String body, HoldingContainer[] inputVariables, JVar[] workspaceJVars, boolean decConstantInputOnly){
- if(!Strings.isNullOrEmpty(body) && !body.trim().isEmpty()){
+ protected void generateBody(ClassGenerator<?> g, BlockType bt, String body, HoldingContainer[] inputVariables,
+ JVar[] workspaceJVars, boolean decConstantInputOnly) {
+ if (!Strings.isNullOrEmpty(body) && !body.trim().isEmpty()) {
JBlock sub = new JBlock(true, true);
if (decConstantInputOnly) {
- addProtectedBlock(g, sub, body, inputVariables, workspaceJVars, true);
+ addProtectedBlock(g, sub, body, inputVariables, workspaceJVars, true);
} else {
addProtectedBlock(g, sub, body, null, workspaceJVars, false);
}
@@ -119,59 +124,63 @@ public abstract class DrillFuncHolder {
}
}
- protected void addProtectedBlock(ClassGenerator<?> g, JBlock sub, String body, HoldingContainer[] inputVariables, JVar[] workspaceJVars, boolean decConstInputOnly){
- if(inputVariables != null){
- for(int i =0; i < inputVariables.length; i++){
+ protected void addProtectedBlock(ClassGenerator<?> g, JBlock sub, String body, HoldingContainer[] inputVariables,
+ JVar[] workspaceJVars, boolean decConstInputOnly) {
+ if (inputVariables != null) {
+ for (int i = 0; i < inputVariables.length; i++) {
if (decConstInputOnly && !inputVariables[i].isConstant())
continue;
ValueReference parameter = parameters[i];
HoldingContainer inputVariable = inputVariables[i];
- sub.decl(inputVariable.getHolder().type(), parameter.name, inputVariable.getHolder());
+ sub.decl(inputVariable.getHolder().type(), parameter.name, inputVariable.getHolder());
}
}
JVar[] internalVars = new JVar[workspaceJVars.length];
- for(int i =0; i < workspaceJVars.length; i++){
- internalVars[i] = sub.decl(g.getModel()._ref(workspaceVars[i].type), workspaceVars[i].name, workspaceJVars[i]);
+ for (int i = 0; i < workspaceJVars.length; i++) {
+ internalVars[i] = sub.decl(g.getModel()._ref(workspaceVars[i].type), workspaceVars[i].name, workspaceJVars[i]);
}
-
+
Preconditions.checkNotNull(body);
sub.directStatement(body);
-
+
// reassign workspace variables back to global space.
- for(int i =0; i < workspaceJVars.length; i++){
+ for (int i = 0; i < workspaceJVars.length; i++) {
sub.assign(workspaceJVars[i], internalVars[i]);
}
}
- public boolean matches(MajorType returnType, List<MajorType> argTypes){
-
- if(!softCompare(returnType, returnValue.type)){
-// logger.debug(String.format("Call [%s] didn't match as return type [%s] was different than expected [%s]. ", call.getDefinition().getName(), returnValue.type, call.getMajorType()));
+ public boolean matches(MajorType returnType, List<MajorType> argTypes) {
+
+ if (!softCompare(returnType, returnValue.type)) {
+ // logger.debug(String.format("Call [%s] didn't match as return type [%s] was different than expected [%s]. ",
+ // call.getDefinition().getName(), returnValue.type, call.getMajorType()));
return false;
}
-
- if(argTypes.size() != parameters.length){
-// logger.debug(String.format("Call [%s] didn't match as the number of arguments provided [%d] were different than expected [%d]. ", call.getDefinition().getName(), parameters.length, call.args.size()));
+
+ if (argTypes.size() != parameters.length) {
+ // logger.debug(String.format("Call [%s] didn't match as the number of arguments provided [%d] were different than expected [%d]. ",
+ // call.getDefinition().getName(), parameters.length, call.args.size()));
return false;
}
- for(int i =0; i < parameters.length; i++){
- if(!softCompare(parameters[i].type, argTypes.get(i))){
-// logger.debug(String.format("Call [%s] didn't match as the argument [%s] didn't match the expected type [%s]. ", call.getDefinition().getName(), arg.getMajorType(), param.type));
+ for (int i = 0; i < parameters.length; i++) {
+ if (!softCompare(parameters[i].type, argTypes.get(i))) {
+ // logger.debug(String.format("Call [%s] didn't match as the argument [%s] didn't match the expected type [%s]. ",
+ // call.getDefinition().getName(), arg.getMajorType(), param.type));
return false;
}
}
-
+
return true;
}
-
+
public MajorType getParmMajorType(int i) {
return this.parameters[i].type;
}
-
- public int getParmSize(){
+
+ public int getParmSize() {
return this.parameters.length;
}
@@ -191,23 +200,24 @@ public abstract class DrillFuncHolder {
return returnValue.type;
}
-
+
public NullHandling getNullHandling() {
- return this.nullHandling ;
+ return this.nullHandling;
}
-
- private boolean softCompare(MajorType a, MajorType b){
+
+ private boolean softCompare(MajorType a, MajorType b) {
return Types.softEquals(a, b, nullHandling == NullHandling.NULL_IF_NULL);
}
-
+
public String[] getRegisteredNames() {
return registeredNames;
}
- public static class ValueReference{
+ public static class ValueReference {
MajorType type;
String name;
boolean isConstant;
+
public ValueReference(MajorType type, String name) {
super();
Preconditions.checkNotNull(type);
@@ -216,23 +226,22 @@ public abstract class DrillFuncHolder {
this.name = name;
isConstant = false;
}
-
+
public void setConstant(boolean isConstant) {
this.isConstant = isConstant;
}
-
+
@Override
public String toString() {
return "ValueReference [type=" + type + ", name=" + name + "]";
}
-
+
}
-
- public static class WorkspaceReference{
+ public static class WorkspaceReference {
Class<?> type;
String name;
-
+ MajorType majorType;
public WorkspaceReference(Class<?> type, String name) {
super();
@@ -241,5 +250,10 @@ public abstract class DrillFuncHolder {
this.type = type;
this.name = name;
}
+
+ void setMajorType(MajorType majorType) {
+ this.majorType = majorType;
+ }
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
index 4dff94d..0f80d11 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionConverter.java
@@ -29,6 +29,7 @@ import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.expr.DrillFunc;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
import org.apache.drill.exec.expr.annotations.Output;
import org.apache.drill.exec.expr.annotations.Param;
import org.apache.drill.exec.expr.annotations.Workspace;
@@ -154,7 +155,25 @@ public class FunctionConverter {
}else{
// workspace work.
// logger.debug("Found workspace field {}:{}", field.getType(), field.getName());
- workspaceFields.add(new WorkspaceReference(field.getType(), field.getName()));
+ //workspaceFields.add(new WorkspaceReference(field.getType(), field.getName()));
+ WorkspaceReference wsReference = new WorkspaceReference(field.getType(), field.getName());
+
+ if (template.scope() == FunctionScope.POINT_AGGREGATE && !ValueHolder.class.isAssignableFrom(field.getType()) ) {
+ return failure(String.format("Aggregate function '%s' workspace variable '%s' is of type '%s'. Please change it to Holder type.", template.name(), field.getName(), field.getType()), clazz, field);
+ }
+
+ //If the workspace var is of Holder type, get its MajorType and assign to WorkspaceReference.
+ if(ValueHolder.class.isAssignableFrom(field.getType())){
+ MajorType majorType = null;
+ try{
+ majorType = getStaticFieldValue("TYPE", field.getType(), MajorType.class);
+ }catch(Exception e){
+ return failure("Failure while trying to access the ValueHolder's TYPE static variable. All ValueHolders must contain a static TYPE variable that defines their MajorType.", e, clazz, field.getName());
+ }
+ wsReference.setMajorType(majorType);
+ }
+
+ workspaceFields.add(wsReference);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java
new file mode 100644
index 0000000..21281be
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionGenerationHelper.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.fn;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FunctionHolderExpression;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
+import org.apache.drill.exec.expr.DrillFuncHolderExpr;
+import org.apache.drill.exec.expr.HoldingContainerExpression;
+
+import com.google.common.collect.ImmutableList;
+
+public class FunctionGenerationHelper {
+ public static final String COMPARE_TO = "compare_to";
+
+ /**
+ * Given materialized arguments find the "compare_to" FunctionHolderExpression
+ * @param left
+ * @param right
+ * @param registry
+ * @return FunctionHolderExpression containing the function implementation
+ */
+ public static FunctionHolderExpression getComparator(HoldingContainer left,
+ HoldingContainer right,
+ FunctionImplementationRegistry registry) {
+ return getFunctionExpression(COMPARE_TO, Types.required(MinorType.INT), registry, left, right);
+ }
+
+ public static FunctionHolderExpression getFunctionExpression(String name, MajorType returnType, FunctionImplementationRegistry registry, HoldingContainer... args){
+
+ List<MajorType> argTypes = new ArrayList<MajorType>(args.length);
+ List<LogicalExpression> argExpressions = new ArrayList<LogicalExpression>(args.length);
+ for(HoldingContainer c : args){
+ argTypes.add(c.getMajorType());
+ argExpressions.add(new HoldingContainerExpression(c));
+ }
+
+ for (DrillFuncHolder h : registry.getDrillRegistry().getMethods().get(name)) {
+ if (h.matches(returnType, argTypes)) {
+ return new DrillFuncHolderExpr(name, h, argExpressions, ExpressionPosition.UNKNOWN);
+ }
+ }
+
+ StringBuilder sb = new StringBuilder();
+ sb.append("Failure finding function that runtime code generation expected. Signature: ");
+ sb.append(name);
+ sb.append("( ");
+ for(int i =0; i < args.length; i++){
+ MajorType mt = args[i].getMajorType();
+ appendType(mt, sb);
+ if(i != 0) sb.append(", ");
+ }
+ sb.append(" ) returns ");
+ appendType(returnType, sb);
+ throw new UnsupportedOperationException(sb.toString());
+ }
+
+ private static final void appendType(MajorType mt, StringBuilder sb){
+ sb.append(mt.getMinorType().name());
+ sb.append(":");
+ sb.append(mt.getMode().name());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/agg/impl/CountFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/agg/impl/CountFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/agg/impl/CountFunctions.java
deleted file mode 100644
index b0939f1..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/agg/impl/CountFunctions.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.expr.fn.agg.impl;
-
-import org.apache.drill.exec.expr.DrillAggFunc;
-import org.apache.drill.exec.expr.annotations.FunctionTemplate;
-import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
-import org.apache.drill.exec.expr.annotations.Output;
-import org.apache.drill.exec.expr.annotations.Param;
-import org.apache.drill.exec.expr.annotations.Workspace;
-import org.apache.drill.exec.expr.holders.BigIntHolder;
-import org.apache.drill.exec.expr.holders.IntHolder;
-import org.apache.drill.exec.record.RecordBatch;
-
-public class CountFunctions {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CountFunctions.class);
-
- @FunctionTemplate(name = "count", scope = FunctionScope.POINT_AGGREGATE)
- public static class BigIntCount implements DrillAggFunc{
-
- @Param BigIntHolder in;
- @Workspace long count;
- @Output BigIntHolder out;
-
- public void setup(RecordBatch incoming) {
- }
-
- @Override
- public void add() {
- count++;
- }
-
- @Override
- public void output() {
- out.value = count;
- }
-
- @Override
- public void reset() {
- count = 0;
- }
-
- }
-
- @FunctionTemplate(name = "count", scope = FunctionScope.POINT_AGGREGATE)
- public static class IntCount implements DrillAggFunc{
-
- @Param IntHolder in;
- @Workspace long count;
- @Output BigIntHolder out;
-
- public void setup(RecordBatch incoming) {
- }
-
- @Override
- public void add() {
- count++;
- }
-
- @Override
- public void output() {
- out.value = count;
- }
-
- @Override
- public void reset() {
- count = 0;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/agg/impl/SumFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/agg/impl/SumFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/agg/impl/SumFunctions.java
deleted file mode 100644
index 57905fe..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/agg/impl/SumFunctions.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.expr.fn.agg.impl;
-
-import org.apache.drill.exec.expr.DrillAggFunc;
-import org.apache.drill.exec.expr.annotations.FunctionTemplate;
-import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
-import org.apache.drill.exec.expr.annotations.Output;
-import org.apache.drill.exec.expr.annotations.Param;
-import org.apache.drill.exec.expr.annotations.Workspace;
-import org.apache.drill.exec.expr.holders.BigIntHolder;
-import org.apache.drill.exec.expr.holders.IntHolder;
-import org.apache.drill.exec.record.RecordBatch;
-
-public class SumFunctions {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SumFunctions.class);
-
- @FunctionTemplate(name = "sum", scope = FunctionScope.POINT_AGGREGATE)
- public static class BigIntSum implements DrillAggFunc{
-
- @Param BigIntHolder in;
- @Workspace long sum;
- @Output BigIntHolder out;
-
- public void setup(RecordBatch incoming) {
- }
-
- @Override
- public void add() {
- sum += in.value;
- }
-
- @Override
- public void output() {
- out.value = sum;
- }
-
- @Override
- public void reset() {
- sum = 0;
- }
-
- }
-
- @FunctionTemplate(name = "sum", scope = FunctionScope.POINT_AGGREGATE)
- public static class IntSum implements DrillAggFunc{
-
- @Param IntHolder in;
- @Workspace int sum;
- @Output IntHolder out;
-
- public void setup(RecordBatch incoming) {
- }
-
- @Override
- public void add() {
- sum += in.value;
- }
-
- @Override
- public void output() {
- out.value = sum;
- }
-
- @Override
- public void reset() {
- sum = 0;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
index 8462622..a17fe45 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java
@@ -128,7 +128,7 @@ public class FragmentContext implements Closeable {
}
public <T> T getImplementationClass(ClassGenerator<T> cg) throws ClassTransformationException, IOException {
- return getImplementationClass(cg.getClassGenerator());
+ return getImplementationClass(cg.getCodeGenerator());
}
public <T> T getImplementationClass(CodeGenerator<T> cg) throws ClassTransformationException, IOException {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
index 4ea87e0..42de28b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
@@ -53,6 +53,8 @@ import org.apache.drill.exec.physical.config.Screen;
import org.apache.drill.exec.physical.config.SelectionVectorRemover;
import org.apache.drill.exec.physical.config.Sort;
import org.apache.drill.exec.physical.config.StreamingAggregate;
+import org.apache.drill.exec.physical.config.HashAggregate;
+import org.apache.drill.exec.physical.OperatorCost;
import org.apache.drill.exec.store.StoragePlugin;
import org.eigenbase.rel.RelFieldCollation.Direction;
import org.eigenbase.rel.RelFieldCollation.NullDirection;
@@ -154,8 +156,6 @@ public class BasicOptimizer extends Optimizer{
return new SelectionVectorRemover(new Limit(input, limit.getFirst(), limit.getLast()));
}
-
-
@Override
public PhysicalOperator visitJoin(Join join, Object value) throws OptimizerException {
PhysicalOperator leftOp = join.getLeft().accept(this, value);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/OperatorCost.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/OperatorCost.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/OperatorCost.java
index 0bade41..cc474b5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/OperatorCost.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/OperatorCost.java
@@ -63,4 +63,12 @@ public class OperatorCost {
return combine(this, c2);
}
+ public int compare(OperatorCost c2) {
+ float thisTotal = this.network + this.disk + this.memory + this.cpu;
+ float c2Total = c2.network + c2.disk + c2.memory + c2.cpu;
+ if (thisTotal < c2Total) return -1;
+ else if (thisTotal == c2Total) return 0;
+ else return 1;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
index ec7244a..5f37487 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractPhysicalVisitor.java
@@ -63,6 +63,11 @@ public abstract class AbstractPhysicalVisitor<T, X, E extends Throwable> impleme
}
@Override
+ public T visitHashAggregate(HashAggregate agg, X value) throws E {
+ return visitOp(agg, value);
+ }
+
+ @Override
public T visitSender(Sender sender, X value) throws E {
return visitOp(sender, value);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java
index 2e2d7fd..712fafe 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/PhysicalVisitor.java
@@ -44,6 +44,7 @@ public interface PhysicalVisitor<RETURN, EXTRA, EXCEP extends Throwable> {
public RETURN visitSender(Sender sender, EXTRA value) throws EXCEP;
public RETURN visitReceiver(Receiver receiver, EXTRA value) throws EXCEP;
public RETURN visitStreamingAggregate(StreamingAggregate agg, EXTRA value) throws EXCEP;
+ public RETURN visitHashAggregate(HashAggregate agg, EXTRA value) throws EXCEP;
public RETURN visitOp(PhysicalOperator op, EXTRA value) throws EXCEP;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashAggregate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashAggregate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashAggregate.java
new file mode 100644
index 0000000..eb77d78
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/HashAggregate.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.config;
+
+import org.apache.drill.common.logical.data.NamedExpression;
+import org.apache.drill.exec.physical.OperatorCost;
+import org.apache.drill.exec.physical.base.AbstractSingle;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.base.PhysicalVisitor;
+import org.apache.drill.exec.physical.base.Size;
+import org.apache.drill.exec.physical.impl.common.HashTable;
+import org.apache.drill.exec.physical.impl.common.HashTableConfig;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+
+@JsonTypeName("hash-aggregate")
+public class HashAggregate extends AbstractSingle {
+
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HashAggregate.class);
+
+ private final NamedExpression[] groupByExprs;
+ private final NamedExpression[] aggrExprs;
+
+ private final float cardinality;
+
+ // configuration parameters for the hash table
+ private final HashTableConfig htConfig;
+
+ @JsonCreator
+ public HashAggregate(@JsonProperty("child") PhysicalOperator child, @JsonProperty("keys") NamedExpression[] groupByExprs, @JsonProperty("exprs") NamedExpression[] aggrExprs, @JsonProperty("cardinality") float cardinality) {
+ super(child);
+ this.groupByExprs = groupByExprs;
+ this.aggrExprs = aggrExprs;
+ this.cardinality = cardinality;
+
+ int initial_capacity = cardinality > HashTable.DEFAULT_INITIAL_CAPACITY ?
+ (int) cardinality : HashTable.DEFAULT_INITIAL_CAPACITY;
+
+ this.htConfig = new HashTableConfig(initial_capacity,
+ HashTable.DEFAULT_LOAD_FACTOR,
+ groupByExprs,
+ null /* no probe exprs */) ;
+ }
+
+ public NamedExpression[] getGroupByExprs() {
+ return groupByExprs;
+ }
+
+ public NamedExpression[] getAggrExprs() {
+ return aggrExprs;
+ }
+
+ public double getCardinality() {
+ return cardinality;
+ }
+
+ public HashTableConfig getHtConfig() {
+ return htConfig;
+ }
+
+ @Override
+ public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E{
+ return physicalVisitor.visitHashAggregate(this, value);
+ }
+
+ @Override
+ public OperatorCost getCost() {
+
+ final float hashCpuCost = (float) 10000.0; // temporarily set this high until we calibrate hashaggr vs. streaming-aggr costs.
+ Size childSize = child.getSize();
+ long n = childSize.getRecordCount();
+ long width = childSize.getRecordSize();
+ int numExprs = getGroupByExprs().length;
+
+ double cpuCost = n * numExprs * hashCpuCost;
+ double diskCost = 0; // for now assume hash table fits in memory
+
+ return new OperatorCost(0, (float) diskCost, (float) n*width, (float) cpuCost);
+ }
+
+ public void logCostInfo(OperatorCost HACost, OperatorCost SACost) {
+ logger.debug("HashAggregate cost: cpu = {}, disk = {}, memory = {}, network = {}.", HACost.getCpu(), HACost.getDisk(), HACost.getMemory(), HACost.getNetwork());
+ logger.debug("Streaming aggregate cost: cpu = {}, disk = {}, memory = {}, network = {}.", SACost.getCpu(), SACost.getDisk(), SACost.getMemory(), SACost.getNetwork());
+ }
+
+ @Override
+ protected PhysicalOperator getNewWithChild(PhysicalOperator child) {
+ return new HashAggregate(child, groupByExprs, aggrExprs, cardinality);
+ }
+
+ @Override
+ public Size getSize() {
+ // not a great hack...
+ return new Size( (long) (child.getSize().getRecordCount()*cardinality), child.getSize().getRecordSize());
+ }
+
+
+
+
+
+
+}
[04/12] git commit: DRILL-387: Support using Hive simple UDFs in
Drill * As part of this change FunctionDefinition (and related code) * are
deleted,
instead the same information available in * Function Holders are used whenever
required * Freemarker/Code
Posted by ja...@apache.org.
DRILL-387: Support using Hive simple UDFs in Drill
* As part of this change FunctionDefinition (and related code)
* are deleted, instead the same information available in
* Function Holders are used whenever required
* Freemarker/CodeModel codegen for Drill ObjectInspectors
* Comparator function cleanup
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/f8fbb5d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/f8fbb5d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/f8fbb5d4
Branch: refs/heads/master
Commit: f8fbb5d42a783dcb8e86a7cda1e5bc5b539c7941
Parents: 2ce6756
Author: vkorukanti <ve...@gmail.com>
Authored: Tue Mar 18 16:26:16 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sat Mar 22 08:09:03 2014 -0700
----------------------------------------------------------------------
.../drill/common/expression/parser/ExprParser.g | 30 +--
.../org/apache/drill/common/expression/Arg.java | 93 -------
.../common/expression/ArgumentValidator.java | 28 --
.../common/expression/ArgumentValidators.java | 181 -------------
.../expression/BasicArgumentValidator.java | 62 -----
.../drill/common/expression/CallProvider.java | 22 --
.../expression/ExpressionStringBuilder.java | 38 +--
.../common/expression/ExpressionValidator.java | 16 +-
.../drill/common/expression/FunctionCall.java | 19 +-
.../common/expression/FunctionCallFactory.java | 117 +++++++++
.../common/expression/FunctionDefinition.java | 120 ---------
.../expression/FunctionHolderExpression.java | 70 +++++
.../common/expression/FunctionInstance.java | 34 ---
.../common/expression/FunctionRegistry.java | 122 ---------
.../common/expression/LogicalExpression.java | 2 -
.../drill/common/expression/NoArgValidator.java | 38 ---
.../expression/fn/AggregationFunctions.java | 35 ---
.../common/expression/fn/BooleanFunctions.java | 48 ----
.../common/expression/fn/CastFunctionDefs.java | 72 -----
.../common/expression/fn/CastFunctions.java | 48 ++++
.../common/expression/fn/MathFunctions.java | 46 ----
.../common/expression/fn/StringFunctions.java | 40 ---
.../common/expression/fn/TypeFunctions.java | 35 ---
.../common/expression/fn/UnaryFunctions.java | 39 ---
.../visitors/AbstractExprVisitor.java | 8 +-
.../expression/visitors/AggregateChecker.java | 18 +-
.../expression/visitors/ConstantChecker.java | 16 +-
.../common/expression/visitors/ExprVisitor.java | 18 +-
.../expression/visitors/SimpleExprVisitor.java | 7 +
.../common/expression/parser/TreeTest.java | 3 -
.../src/main/resources/drill-module.conf | 5 +
exec/java-exec/src/main/codegen/config.fmpp | 1 +
.../src/main/codegen/data/HiveTypes.tdd | 92 +++++++
.../codegen/templates/ComparisonFunctions.java | 12 +-
.../main/codegen/templates/MathFunctions.java | 39 ---
.../templates/ObjectInspectorHelper.java | 182 +++++++++++++
.../codegen/templates/ObjectInspectors.java | 93 +++++++
.../sig/ConstantExpressionIdentifier.java | 8 +-
.../drill/exec/expr/DrillFuncHolderExpr.java | 69 +++++
.../drill/exec/expr/EvaluationVisitor.java | 67 +++--
.../exec/expr/ExpressionTreeMaterializer.java | 155 +++++++++--
.../drill/exec/expr/HiveFuncHolderExpr.java | 69 +++++
.../drill/exec/expr/ImplicitCastBuilder.java | 168 ------------
.../exec/expr/annotations/FunctionTemplate.java | 18 +-
.../exec/expr/fn/ComparatorFunctionHelper.java | 59 +++++
.../drill/exec/expr/fn/DrillAggFuncHolder.java | 13 +-
.../drill/exec/expr/fn/DrillFuncHolder.java | 70 +++--
.../fn/DrillFunctionImplementationRegistry.java | 54 ++++
.../exec/expr/fn/DrillSimpleFuncHolder.java | 10 +-
.../drill/exec/expr/fn/FunctionConverter.java | 21 +-
.../expr/fn/FunctionImplementationRegistry.java | 77 +-----
.../drill/exec/expr/fn/HiveFuncHolder.java | 260 +++++++++++++++++++
.../fn/HiveFunctionImplementationRegistry.java | 154 +++++++++++
.../drill/exec/expr/fn/impl/Alternator.java | 24 +-
.../drill/exec/expr/fn/impl/BitFunctions.java | 6 +-
.../drill/exec/expr/fn/impl/ByteSubstring.java | 25 +-
.../drill/exec/expr/fn/impl/CharSubstring.java | 26 +-
.../exec/expr/fn/impl/ComparatorFunctions.java | 54 ----
.../fn/impl/ComparisonFunctionsNullable.java | 25 +-
.../drill/exec/expr/fn/impl/HashFunctions.java | 16 --
.../drill/exec/expr/fn/impl/IsNotNull.java | 21 +-
.../apache/drill/exec/expr/fn/impl/IsNull.java | 21 +-
.../drill/exec/expr/fn/impl/MathFunctions.java | 5 +-
.../expr/fn/impl/SimpleRepeatedFunctions.java | 30 ---
.../exec/expr/fn/impl/StringFunctions.java | 82 +-----
.../hive/AbstractPrimitiveObjectInspector.java | 70 +++++
.../expr/fn/impl/hive/DrillDeferredObject.java | 34 +++
.../org/apache/drill/exec/ops/QueryContext.java | 5 -
.../exec/physical/impl/aggregate/AggBatch.java | 12 +-
.../exec/physical/impl/join/MergeJoinBatch.java | 15 +-
.../impl/mergereceiver/MergingRecordBatch.java | 10 +-
.../OrderedPartitionRecordBatch.java | 12 +-
.../exec/physical/impl/sort/SortBatch.java | 10 +-
.../exec/planner/logical/DrillAggregateRel.java | 3 +-
.../drill/exec/planner/logical/DrillOptiq.java | 10 +-
.../exec/planner/logical/DrillParseContext.java | 14 +-
.../drill/exec/planner/sql/DrillSqlWorker.java | 7 +-
.../drill/exec/server/DrillbitContext.java | 7 -
.../apache/drill/exec/work/foreman/Foreman.java | 2 +-
.../apache/drill/exec/expr/ExpressionTest.java | 2 -
.../drill/exec/fn/impl/GeneratorFunctions.java | 36 +--
.../drill/exec/physical/impl/TestHiveUDFs.java | 213 +++++++++++++++
.../exec/physical/impl/TestSimpleFunctions.java | 7 +-
.../record/ExpressionTreeMaterializerTest.java | 27 +-
.../resources/functions/hive/GenericUDF.json | 45 ++++
.../src/test/resources/functions/hive/UDF.json | 39 +++
86 files changed, 2152 insertions(+), 1884 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
----------------------------------------------------------------------
diff --git a/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g b/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
index b5cf292..edb8507 100644
--- a/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
+++ b/common/src/main/antlr3/org/apache/drill/common/expression/parser/ExprParser.g
@@ -41,12 +41,8 @@ import org.apache.drill.common.types.TypeProtos.*;
}
@members{
- private FunctionRegistry registry;
private String fullExpression;
private int tokenPos;
- public void setRegistry(FunctionRegistry registry){
- this.registry = registry;
- }
public static void p(String s){
System.out.println(s);
@@ -66,7 +62,7 @@ parse returns [LogicalExpression e]
;
functionCall returns [LogicalExpression e]
- : Identifier OParen exprList? CParen {$e = registry.createExpression($Identifier.text, pos($Identifier), $exprList.listE); }
+ : Identifier OParen exprList? CParen {$e = FunctionCallFactory.createExpression($Identifier.text, pos($Identifier), $exprList.listE); }
;
castCall returns [LogicalExpression e]
@@ -76,9 +72,9 @@ castCall returns [LogicalExpression e]
}
: Cast OParen expression As dataType repeat? CParen
{ if ($repeat.isRep!=null && $repeat.isRep.compareTo(Boolean.TRUE)==0)
- $e = registry.createCast(TypeProtos.MajorType.newBuilder().mergeFrom($dataType.type).setMode(DataMode.REPEATED).build(), pos($Cast), $expression.e);
- else
- $e = registry.createCast($dataType.type, pos($Cast), $expression.e);}
+ $e = FunctionCallFactory.createCast(TypeProtos.MajorType.newBuilder().mergeFrom($dataType.type).setMode(DataMode.REPEATED).build(), pos($Cast), $expression.e);
+ else
+ $e = FunctionCallFactory.createCast($dataType.type, pos($Cast), $expression.e);}
;
repeat returns [Boolean isRep]
@@ -167,7 +163,7 @@ orExpr returns [LogicalExpression e]
if(exprs.size() == 1){
$e = exprs.get(0);
}else{
- $e = registry.createExpression("||", p, exprs);
+ $e = FunctionCallFactory.createExpression("||", p, exprs);
}
}
: a1=andExpr { exprs.add($a1.e); p = pos( $a1.start );} (Or a2=andExpr { exprs.add($a2.e); })*
@@ -182,7 +178,7 @@ andExpr returns [LogicalExpression e]
if(exprs.size() == 1){
$e = exprs.get(0);
}else{
- $e = registry.createExpression("&&", p, exprs);
+ $e = FunctionCallFactory.createExpression("&&", p, exprs);
}
}
: e1=equExpr { exprs.add($e1.e); p = pos( $e1.start ); } ( And e2=equExpr { exprs.add($e2.e); })*
@@ -195,14 +191,14 @@ equExpr returns [LogicalExpression e]
ExpressionPosition p = null;
}
@after{
- $e = registry.createByOp(exprs, p, cmps);
+ $e = FunctionCallFactory.createByOp(exprs, p, cmps);
}
: r1=relExpr { exprs.add($r1.e); p = pos( $r1.start );
} ( cmpr= ( Equals | NEquals ) r2=relExpr {exprs.add($r2.e); cmps.add($cmpr.text); })*
;
relExpr returns [LogicalExpression e]
- : left=addExpr {$e = $left.e; } (cmpr = (GTEquals | LTEquals | GT | LT) right=addExpr {$e = registry.createExpression($cmpr.text, pos($left.start), $left.e, $right.e); } )?
+ : left=addExpr {$e = $left.e; } (cmpr = (GTEquals | LTEquals | GT | LT) right=addExpr {$e = FunctionCallFactory.createExpression($cmpr.text, pos($left.start), $left.e, $right.e); } )?
;
addExpr returns [LogicalExpression e]
@@ -212,7 +208,7 @@ addExpr returns [LogicalExpression e]
ExpressionPosition p = null;
}
@after{
- $e = registry.createByOp(exprs, p, ops);
+ $e = FunctionCallFactory.createByOp(exprs, p, ops);
}
: m1=mulExpr {exprs.add($m1.e); p = pos($m1.start); } ( op=(Plus|Minus) m2=mulExpr {exprs.add($m2.e); ops.add($op.text); })*
;
@@ -224,7 +220,7 @@ mulExpr returns [LogicalExpression e]
ExpressionPosition p = null;
}
@after{
- $e = registry.createByOp(exprs, p, ops);
+ $e = FunctionCallFactory.createByOp(exprs, p, ops);
}
: p1=xorExpr {exprs.add($p1.e); p = pos($p1.start);} (op=(Asterisk|ForwardSlash|Percent) p2=xorExpr {exprs.add($p2.e); ops.add($op.text); } )*
;
@@ -236,14 +232,14 @@ xorExpr returns [LogicalExpression e]
ExpressionPosition p = null;
}
@after{
- $e = registry.createByOp(exprs, p, ops);
+ $e = FunctionCallFactory.createByOp(exprs, p, ops);
}
: u1=unaryExpr {exprs.add($u1.e); p = pos($u1.start);} (Caret u2=unaryExpr {exprs.add($u2.e); ops.add($Caret.text);} )*
;
unaryExpr returns [LogicalExpression e]
- : Minus atom {$e = registry.createExpression("u-", pos($atom.start), $atom.e); }
- | Excl atom {$e= registry.createExpression("!", pos($atom.start), $atom.e); }
+ : Minus atom {$e = FunctionCallFactory.createExpression("u-", pos($atom.start), $atom.e); }
+ | Excl atom {$e= FunctionCallFactory.createExpression("!", pos($atom.start), $atom.e); }
| atom {$e = $atom.e; }
;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/Arg.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/Arg.java b/common/src/main/java/org/apache/drill/common/expression/Arg.java
deleted file mode 100644
index 0dd47ff..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/Arg.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.util.List;
-
-import org.apache.drill.common.expression.visitors.ConstantChecker;
-import org.apache.drill.common.types.Types;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-
-import com.google.common.collect.Lists;
-
-
-public class Arg {
- private final String name;
- private final MajorType[] allowedTypes;
- private final boolean constantsOnly;
-
-
- /**
- * Create an arg that supports any of the listed minor types using opt or req. Does not allow repeated types.
- * @param name
- * @param constantsOnly
- * @param types
- */
- public Arg(boolean constantsOnly, boolean allowNulls, String name, MinorType... types){
- this(constantsOnly, name, getMajorTypes(allowNulls, types));
- }
-
- public Arg(MajorType... allowedTypes){
- this(false, null, allowedTypes);
- }
-
- public Arg(String name, MajorType... allowedTypes) {
- this(false, name, allowedTypes);
- }
-
- public Arg(boolean constantsOnly, String name, MajorType... allowedTypes) {
- this.name = name;
- this.allowedTypes = allowedTypes;
- this.constantsOnly = constantsOnly;
- }
-
- public String getName(){
- return name;
- }
-
- public void confirmDataType(ExpressionPosition expr, int argIndex, LogicalExpression e, ErrorCollector errors){
- if(constantsOnly){
- if(ConstantChecker.onlyIncludesConstants(e)) errors.addExpectedConstantValue(expr, argIndex, name);
- }
- MajorType dt = e.getMajorType();
- if(dt.getMinorType() != MinorType.LATE){
-
- // change among allowed types.
- for(MajorType a : allowedTypes){
- if(dt.equals(a)) return;
- }
-
- // didn't find an allowed type.
- errors.addUnexpectedArgumentType(expr, name, dt, allowedTypes, argIndex);
-
- }
-
-
- }
-
- private static MajorType[] getMajorTypes(boolean allowNulls, MinorType... types){
- List<MajorType> mts = Lists.newArrayList();
- for(MinorType t : types){
- if(allowNulls) mts.add(Types.optional(t));
- mts.add(Types.required(t));
- }
- return mts.toArray(new MajorType[mts.size()]);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/ArgumentValidator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ArgumentValidator.java b/common/src/main/java/org/apache/drill/common/expression/ArgumentValidator.java
deleted file mode 100644
index 18456b5..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/ArgumentValidator.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.util.List;
-
-/**
- * Validates whether the set of arguments are acceptable
- */
-public interface ArgumentValidator {
- public void validateArguments(ExpressionPosition expr, List<LogicalExpression> expressions, ErrorCollector errors);
- public String[] getArgumentNamesByPosition();
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/ArgumentValidators.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ArgumentValidators.java b/common/src/main/java/org/apache/drill/common/expression/ArgumentValidators.java
deleted file mode 100644
index da46645..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/ArgumentValidators.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.util.List;
-
-import org.apache.commons.lang3.ArrayUtils;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.Types;
-
-import com.google.common.base.Predicate;
-import com.google.common.collect.Range;
-import com.google.common.collect.Ranges;
-
-public class ArgumentValidators {
-
- private static final String[] EMPTY_NAMES = new String[0];
-
- public static class AnyTypeAllowed implements ArgumentValidator {
-
- private final Range<Integer> argumentCount;
-
- public AnyTypeAllowed(int argCount) {
- super();
- argumentCount = Range.singleton(argCount);
- }
-
- public AnyTypeAllowed(int minArguments, int maxArguments) {
- super();
- argumentCount = Range.closedOpen(minArguments, maxArguments);
- }
-
- @Override
- public void validateArguments(ExpressionPosition expr, List<LogicalExpression> expressions, ErrorCollector errors) {
- // only need to check argument count since any type is allowed.
- if (!argumentCount.contains(expressions.size()))
- errors.addUnexpectedArgumentCount(expr, expressions.size(), argumentCount);
- }
-
- @Override
- public String[] getArgumentNamesByPosition() {
- return EMPTY_NAMES;
- }
-
- }
-
- private static class PredicateValidator implements ArgumentValidator {
- private final Range<Integer> argumentCount;
- private Predicate<MajorType> predicate;
- private boolean allSame;
-
- public PredicateValidator(int argCount, Predicate<MajorType> predicate, boolean allSame) {
- super();
- this.argumentCount = Range.singleton(argCount);
- this.predicate = predicate;
- this.allSame = allSame;
- }
-
- public PredicateValidator(int minArguments, int maxArguments, Predicate<MajorType> predicate, boolean allSame) {
- super();
- this.argumentCount = Range.closedOpen(minArguments, maxArguments);
- this.predicate = predicate;
- this.allSame = allSame;
- }
-
- @Override
- public void validateArguments(ExpressionPosition expr, List<LogicalExpression> expressions, ErrorCollector errors) {
- int i = -1;
- MajorType t = null;
- for (LogicalExpression le : expressions) {
- i++;
- MajorType majorType = le.getMajorType();
- if (t == null) t = majorType;
-
- if (!predicate.apply(majorType)) {
- errors.addUnexpectedType(expr, i, majorType);
- continue;
- }
-
- if (allSame && !Types.isLateBind(t) && !Types.isLateBind(majorType) && !Types.softEquals(t, majorType, true)) {
- errors.addUnexpectedType(expr, i, majorType);
- }
-
- }
- if (!argumentCount.contains(expressions.size()))
- errors.addUnexpectedArgumentCount(expr, expressions.size(), argumentCount);
- }
-
- @Override
- public String[] getArgumentNamesByPosition() {
- return EMPTY_NAMES;
- }
- }
-
- public static class ComparableArguments extends PredicateValidator {
-
- public ComparableArguments(int argCount, MajorType... allowedTypes) {
- super(argCount, new ComparableChecker(), false);
- }
-
- public ComparableArguments(int minArguments, int maxArguments, MajorType... allowedTypes) {
- super(minArguments, maxArguments, new ComparableChecker(), false);
- }
-
- public static class ComparableChecker implements Predicate<MajorType> {
-
- public boolean apply(MajorType dt) {
- Types.Comparability comparability = Types.getComparability(dt);
- return comparability.equals(Types.Comparability.ORDERED) || comparability.equals(Types.Comparability.UNKNOWN);
- }
- }
- }
-
- public static class AllowedTypeList extends PredicateValidator {
-
- public AllowedTypeList(int argCount, MajorType... allowedTypes) {
- super(argCount, new AllowedTypeChecker(allowedTypes), false);
- }
-
- public AllowedTypeList(int minArguments, int maxArguments, MajorType... allowedTypes) {
- super(minArguments, maxArguments, new AllowedTypeChecker(allowedTypes), false);
- }
-
- public AllowedTypeList(int argCount, boolean allSame, MajorType... allowedTypes) {
- super(argCount, new AllowedTypeChecker(allowedTypes), allSame);
- }
-
- public AllowedTypeList(int minArguments, int maxArguments, boolean allSame, MajorType... allowedTypes) {
- super(minArguments, maxArguments, new AllowedTypeChecker(allowedTypes), allSame);
- }
-
- public static class AllowedTypeChecker implements Predicate<MajorType> {
-
- private MajorType[] allowedTypes;
-
- public AllowedTypeChecker(MajorType... allowedTypes) {
- this.allowedTypes = allowedTypes;
- }
-
- public boolean apply(MajorType dt) {
- return ArrayUtils.contains(allowedTypes, dt);
- }
- }
-
- }
-
- public static class NumericTypeAllowed extends PredicateValidator {
-
- public NumericTypeAllowed(int argCount, boolean allSame) {
- super(argCount, new NumericTypeChecker(), allSame);
-
- }
-
- public NumericTypeAllowed(int minArguments, int maxArguments, boolean allSame) {
- super(minArguments, maxArguments, new NumericTypeChecker(), allSame);
- }
-
- public static class NumericTypeChecker implements Predicate<MajorType> {
-
- public boolean apply(MajorType dt) {
- return Types.isNumericType(dt);
- }
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/BasicArgumentValidator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/BasicArgumentValidator.java b/common/src/main/java/org/apache/drill/common/expression/BasicArgumentValidator.java
deleted file mode 100644
index f6c94d9..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/BasicArgumentValidator.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.util.List;
-
-import org.apache.drill.common.types.TypeProtos.MajorType;
-
-public class BasicArgumentValidator implements ArgumentValidator {
-
- private final Arg[] args;
-
- public BasicArgumentValidator(MajorType... types) {
- this.args = new Arg[] { new Arg("single", types) };
- }
-
- public BasicArgumentValidator(Arg... args) {
- this.args = args;
- }
-
- @Override
- public void validateArguments(ExpressionPosition expr, List<LogicalExpression> expressions, ErrorCollector errors) {
- if (expressions.size() != args.length) errors.addUnexpectedArgumentCount(expr, expressions.size(), args.length);
-
- int i = 0;
- for (LogicalExpression e : expressions) {
- args[i].confirmDataType(expr, i, e, errors);
-
- i++;
- }
- }
-
- public Arg arg(String name, MajorType... allowedTypes) {
- return new Arg(name, allowedTypes);
- }
-
- @Override
- public String[] getArgumentNamesByPosition() {
- String[] names = new String[args.length];
- for(int i =0; i < names.length; i++){
- names[i] = args[i].getName();
- }
- return names;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/CallProvider.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/CallProvider.java b/common/src/main/java/org/apache/drill/common/expression/CallProvider.java
deleted file mode 100644
index 2339261..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/CallProvider.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-public interface CallProvider {
- public FunctionDefinition[] getFunctionDefintions();
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java b/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
index 69dd9b3..80dfbac 100644
--- a/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
+++ b/common/src/main/java/org/apache/drill/common/expression/ExpressionStringBuilder.java
@@ -31,39 +31,23 @@ public class ExpressionStringBuilder extends AbstractExprVisitor<Void, StringBui
@Override
public Void visitFunctionCall(FunctionCall call, StringBuilder sb) throws RuntimeException {
- FunctionDefinition func = call.getDefinition();
ImmutableList<LogicalExpression> args = call.args;
- if (func.isOperator()) {
- if (args.size() == 1) { // unary
- func.addRegisteredName(sb);
- sb.append("(");
- args.get(0).accept(this, sb);
- sb.append(")");
- } else {
- for (int i = 0; i < args.size(); i++) {
- if (i != 0) {
- sb.append(" ");
- func.addRegisteredName(sb);
- }
- sb.append(" (");
- args.get(i).accept(this, sb);
- sb.append(") ");
- }
- }
- } else { // normal function
-
- func.addRegisteredName(sb);
- sb.append("(");
- for (int i = 0; i < args.size(); i++) {
- if (i != 0) sb.append(", ");
- args.get(i).accept(this, sb);
- }
- sb.append(") ");
+ sb.append(call.getName());
+ sb.append("(");
+ for (int i = 0; i < args.size(); i++) {
+ if (i != 0) sb.append(", ");
+ args.get(i).accept(this, sb);
}
+ sb.append(") ");
return null;
}
@Override
+ public Void visitFunctionHolderExpression(FunctionHolderExpression holder, StringBuilder sb) throws RuntimeException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public Void visitIfExpression(IfExpression ifExpr, StringBuilder sb) throws RuntimeException {
ImmutableList<IfCondition> conditions = ifExpr.conditions;
sb.append(" ( ");
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/ExpressionValidator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ExpressionValidator.java b/common/src/main/java/org/apache/drill/common/expression/ExpressionValidator.java
index 076c150..3c7e132 100644
--- a/common/src/main/java/org/apache/drill/common/expression/ExpressionValidator.java
+++ b/common/src/main/java/org/apache/drill/common/expression/ExpressionValidator.java
@@ -22,6 +22,8 @@ import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
import org.apache.drill.common.expression.ValueExpressions.LongExpression;
import org.apache.drill.common.expression.ValueExpressions.QuotedString;
+import org.apache.drill.common.expression.visitors.AggregateChecker;
+import org.apache.drill.common.expression.visitors.ConstantChecker;
import org.apache.drill.common.expression.visitors.ExprVisitor;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
@@ -32,8 +34,18 @@ public class ExpressionValidator implements ExprVisitor<Void, ErrorCollector, Ru
@Override
public Void visitFunctionCall(FunctionCall call, ErrorCollector errors) throws RuntimeException {
- call.getDefinition().getArgumentValidator()
- .validateArguments(call.getPosition(), call.args, errors);
+ throw new UnsupportedOperationException("FunctionCall is not expected here. "+
+ "It should have been converted to FunctionHolderExpression in materialization");
+ }
+
+ @Override
+ public Void visitFunctionHolderExpression(FunctionHolderExpression holder, ErrorCollector errors) throws RuntimeException {
+ // make sure aggregate functions are not nested inside aggregate functions
+ AggregateChecker.isAggregating(holder);
+
+ // make sure arguments are constant if the function implementation expects constants for any arguments
+ ConstantChecker.onlyIncludesConstants(holder);
+
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/FunctionCall.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/FunctionCall.java b/common/src/main/java/org/apache/drill/common/expression/FunctionCall.java
index e3ed4c5..53a5b48 100644
--- a/common/src/main/java/org/apache/drill/common/expression/FunctionCall.java
+++ b/common/src/main/java/org/apache/drill/common/expression/FunctionCall.java
@@ -29,13 +29,13 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
public class FunctionCall extends LogicalExpressionBase implements Iterable<LogicalExpression> {
- private final FunctionDefinition func;
+ private final String name;
public final ImmutableList<LogicalExpression> args;
private final ExpressionPosition pos;
- public FunctionCall(FunctionDefinition func, List<LogicalExpression> args, ExpressionPosition pos) {
+ public FunctionCall(String name, List<LogicalExpression> args, ExpressionPosition pos) {
super(pos);
- this.func = func;
+ this.name = name;
if(args == null) args = Lists.newArrayList();
@@ -46,6 +46,10 @@ public class FunctionCall extends LogicalExpressionBase implements Iterable<Logi
this.pos = pos;
}
+ public String getName() {
+ return name;
+ }
+
@Override
public ExpressionPosition getPosition() {
return pos;
@@ -61,19 +65,16 @@ public class FunctionCall extends LogicalExpressionBase implements Iterable<Logi
return args.iterator();
}
- public FunctionDefinition getDefinition(){
- return func;
- }
-
@Override
public MajorType getMajorType() {
- return func.getDataType(this.args);
+ throw new UnsupportedOperationException(
+ "No return type as FunctionCall is not a materialized expression");
}
@Override
public String toString() {
final int maxLen = 10;
- return "FunctionCall [func=" + func + ", args="
+ return "FunctionCall [func=" + name + ", args="
+ (args != null ? args.subList(0, Math.min(args.size(), maxLen)) : null) + ", pos=" + pos + "]";
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java b/common/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java
new file mode 100644
index 0000000..df4fcb7
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/expression/FunctionCallFactory.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.drill.common.config.CommonConstants;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.common.exceptions.ExpressionParsingException;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.util.PathScanner;
+import org.apache.drill.common.types.Types;
+
+import com.google.common.collect.Lists;
+
+public class FunctionCallFactory {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionCallFactory.class);
+
+ private static Map<String, String> opToFuncTable = new HashMap<>();
+
+ static {
+ opToFuncTable.put("+", "add");
+ opToFuncTable.put("-", "subtract");
+ opToFuncTable.put("/", "divide");
+ opToFuncTable.put("*", "multiply");
+ opToFuncTable.put("%", "modulo");
+ opToFuncTable.put("^", "xor");
+
+ opToFuncTable.put("||", "or");
+ opToFuncTable.put("&&", "and");
+ opToFuncTable.put(">", "greater than");
+ opToFuncTable.put("<", "less than");
+ opToFuncTable.put("==", "equal");
+ opToFuncTable.put("=", "equal");
+ opToFuncTable.put("!=", "not equal");
+ opToFuncTable.put("<>", "not equal");
+ opToFuncTable.put(">=", "greater than or equal to");
+ opToFuncTable.put("<=", "less than or equal to");
+
+ opToFuncTable.put("!", "not");
+ opToFuncTable.put("u-", "negative");
+ }
+
+ private static String replaceOpWithFuncName(String op) {
+ return (opToFuncTable.containsKey(op)) ? (opToFuncTable.get(op)) : op;
+ }
+
+ /*
+ * create a cast function.
+ * arguments : type -- targetType
+ * ep -- input expression position
+ * expr -- input expression
+ */
+ public static LogicalExpression createCast(MajorType type, ExpressionPosition ep, LogicalExpression expr){
+ String castFuncWithType = "cast" + type.getMinorType().name();
+
+ List<LogicalExpression> newArgs = Lists.newArrayList();
+ newArgs.add(expr); //input_expr
+
+ //VarLen type
+ if (!Types.isFixedWidthType(type)) {
+ newArgs.add(new ValueExpressions.LongExpression(type.getWidth(), null));
+ }
+
+ return new FunctionCall(castFuncWithType, newArgs, ep);
+ }
+
+ public static LogicalExpression createExpression(String functionName, List<LogicalExpression> args){
+ return createExpression(functionName, ExpressionPosition.UNKNOWN, args);
+ }
+
+ public static LogicalExpression createExpression(String functionName, ExpressionPosition ep, List<LogicalExpression> args){
+ return new FunctionCall(replaceOpWithFuncName(functionName), args, ep);
+ }
+
+ public static LogicalExpression createExpression(String unaryName, ExpressionPosition ep, LogicalExpression... e){
+ return new FunctionCall(replaceOpWithFuncName(unaryName), Lists.newArrayList(e), ep);
+ }
+
+ public static LogicalExpression createByOp(List<LogicalExpression> args, ExpressionPosition ep, List<String> opTypes) {
+ if (args.size() == 1) {
+ return args.get(0);
+ }
+
+ if (args.size() - 1 != opTypes.size())
+ throw new DrillRuntimeException("Must receive one more expression then the provided number of operators.");
+
+ LogicalExpression first = args.get(0);
+ for (int i = 0; i < opTypes.size(); i++) {
+ List<LogicalExpression> l2 = new ArrayList<LogicalExpression>();
+ l2.add(first);
+ l2.add(args.get(i + 1));
+ first = createExpression(opTypes.get(i), ep, args);
+ }
+ return first;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/FunctionDefinition.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/FunctionDefinition.java b/common/src/main/java/org/apache/drill/common/expression/FunctionDefinition.java
deleted file mode 100644
index f9e408b..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/FunctionDefinition.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.commons.lang3.ArrayUtils;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-
-public class FunctionDefinition {
-
- private final String name;
- private final String[] registeredNames;
- private final ArgumentValidator argumentValidator;
- private final OutputTypeDeterminer outputType;
- private final boolean aggregating;
- private final boolean isOperator;
- private final boolean isRandom;
-
- private FunctionDefinition(String name, ArgumentValidator argumentValidator, OutputTypeDeterminer outputType,
- boolean aggregating, boolean isOperator, String[] registeredNames) {
- this(name, argumentValidator, outputType, aggregating, isOperator, registeredNames, false);
- }
-
- private FunctionDefinition(String name, ArgumentValidator argumentValidator, OutputTypeDeterminer outputType,
- boolean aggregating, boolean isOperator, String[] registeredNames, boolean isRandom) {
- this.name = name;
- this.argumentValidator = argumentValidator;
- this.outputType = outputType;
- this.aggregating = aggregating;
- this.registeredNames = ArrayUtils.isEmpty(registeredNames) ? new String[]{name} : registeredNames;
- this.isOperator = isOperator;
- this.isRandom = isRandom;
- }
-
- public MajorType getDataType(List<LogicalExpression> args){
- return outputType.getOutputType(args);
- }
-
- public String[] getArgumentNames(){
- return argumentValidator.getArgumentNamesByPosition();
- }
-
- public ArgumentValidator getArgumentValidator() {
- return argumentValidator;
- }
-
- public OutputTypeDeterminer getOutputTypeDeterminer() {
- return outputType;
- }
-
- public static FunctionDefinition simple(String name, ArgumentValidator argumentValidator, OutputTypeDeterminer outputType, String... registeredNames){
- return new FunctionDefinition(name, argumentValidator, outputType, false, false, registeredNames);
- }
-
- public static FunctionDefinition simpleRandom(String name, ArgumentValidator argumentValidator, OutputTypeDeterminer outputType, String... registeredNames){
- return new FunctionDefinition(name, argumentValidator, outputType, false, false, registeredNames, true);
- }
-
- public static FunctionDefinition aggregator(String name, ArgumentValidator argumentValidator, OutputTypeDeterminer outputType, String... registeredNames){
- return new FunctionDefinition(name, argumentValidator, outputType, true, false, registeredNames);
- }
-
- public static FunctionDefinition operator(String name, ArgumentValidator argumentValidator, OutputTypeDeterminer outputType, String... registeredNames){
- return new FunctionDefinition(name, argumentValidator, outputType, false, true, registeredNames);
- }
-
- public boolean isOperator(){
- return isOperator;
- }
-
- public boolean isAggregating(){
- return aggregating;
- }
-
- public String[] getRegisteredNames(){
- return this.registeredNames;
- }
-
- public String getName(){
- return this.name;
- }
-
- public boolean isRandom() {
- return this.isRandom;
- }
-
- public FunctionCall newCall(List<LogicalExpression> args, ExpressionPosition pos){
- return new FunctionCall(this, args, pos);
- }
-
- public void addRegisteredName(StringBuilder sb){
- sb.append(registeredNames[0]);
- }
-
- @Override
- public String toString() {
- return "FunctionDefinition [name=" + name + ", registeredNames=" + Arrays.toString(registeredNames)
- + ", aggregating=" + aggregating + ", isOperator=" + isOperator + "]";
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/FunctionHolderExpression.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/FunctionHolderExpression.java b/common/src/main/java/org/apache/drill/common/expression/FunctionHolderExpression.java
new file mode 100644
index 0000000..b17e00d
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/expression/FunctionHolderExpression.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import org.apache.drill.common.expression.visitors.ExprVisitor;
+
+import java.util.List;
+
+public abstract class FunctionHolderExpression extends LogicalExpressionBase {
+ public final ImmutableList<LogicalExpression> args;
+ public final String nameUsed;
+
+ public FunctionHolderExpression(String nameUsed, ExpressionPosition pos, List<LogicalExpression> args) {
+ super(pos);
+ if(args == null) args = Lists.newArrayList();
+
+ if (!(args instanceof ImmutableList)) {
+ args = ImmutableList.copyOf(args);
+ }
+ this.args = (ImmutableList<LogicalExpression>) args;
+ this.nameUsed = nameUsed;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitFunctionHolderExpression(this, value);
+ }
+
+ /**
+ * A function can have multiple names, it returns the function name used in the query
+ * @return
+ */
+ public String getName() {
+ return nameUsed;
+ }
+
+ /**
+ * constant input expected for i'th argument?
+ * @param i
+ * @return
+ */
+ public abstract boolean argConstantOnly(int i);
+
+ /**
+ * @return aggregating function or not
+ */
+ public abstract boolean isAggregating();
+
+ /**
+ * is the function output non-deterministic?
+ */
+ public abstract boolean isRandom();
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/FunctionInstance.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/FunctionInstance.java b/common/src/main/java/org/apache/drill/common/expression/FunctionInstance.java
deleted file mode 100644
index cda49e9..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/FunctionInstance.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.util.List;
-
-public class FunctionInstance {
-
- private FunctionDefinition definition;
- private List<LogicalExpression> arguments;
-
- public FunctionInstance(FunctionDefinition definition, List<LogicalExpression> arguments) {
- super();
- this.definition = definition;
- this.arguments = arguments;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/FunctionRegistry.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/FunctionRegistry.java b/common/src/main/java/org/apache/drill/common/expression/FunctionRegistry.java
deleted file mode 100644
index 7cb386a..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/FunctionRegistry.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.drill.common.config.CommonConstants;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.common.exceptions.ExpressionParsingException;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.util.PathScanner;
-import org.apache.drill.common.types.Types;
-
-import com.google.common.collect.Lists;
-
-public class FunctionRegistry {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FunctionRegistry.class);
-
- private final Map<String, FunctionDefinition> funcMap;
-
- public FunctionRegistry(DrillConfig config){
- try{
- Set<Class<? extends CallProvider>> providerClasses = PathScanner.scanForImplementations(CallProvider.class, config.getStringList(CommonConstants.LOGICAL_FUNCTION_SCAN_PACKAGES));
- Map<String, FunctionDefinition> funcs = new HashMap<String, FunctionDefinition>();
- for (Class<? extends CallProvider> c : providerClasses) {
- CallProvider p = c.newInstance();
- FunctionDefinition[] defs = p.getFunctionDefintions();
- for(FunctionDefinition d : defs){
- for(String rn : d.getRegisteredNames()){
-
- FunctionDefinition d2 = funcs.put(rn, d);
-// logger.debug("Registering function {}", d);
- if(d2 != null){
- throw new ExceptionInInitializerError(String.format("Failure while registering functions. The function %s tried to register with the name %s but the function %s already registered with that name.", d.getName(), rn, d2.getName()) );
- }
- }
- }
- }
- funcMap = funcs;
- }catch(Exception e){
- throw new RuntimeException("Failure while setting up FunctionRegistry.", e);
- }
- }
-
- /*
- * create a cast function.
- * arguments : type -- targetType
- * ep -- input expression position
- * expr -- input expression
- */
- public LogicalExpression createCast(MajorType type, ExpressionPosition ep, LogicalExpression expr){
- String castFuncWithType = "cast" + type.getMinorType().name();
-
- FunctionDefinition d = funcMap.get(castFuncWithType);
- if(d == null) throw new ExpressionParsingException(String.format("Unable to find function definition for function named '%s'", castFuncWithType));
-
- List<LogicalExpression> newArgs = Lists.newArrayList();
- newArgs.add(expr); //input_expr
-
- //VarLen type
- if (!Types.isFixedWidthType(type)) {
- newArgs.add(new ValueExpressions.LongExpression(type.getWidth(), null));
- }
-
- FunctionDefinition castFuncDef = FunctionDefinition.simple(castFuncWithType, d.getArgumentValidator(), d.getOutputTypeDeterminer()) ;
-
- return new FunctionCall(castFuncDef, newArgs, ep);
- }
-
- public LogicalExpression createExpression(String functionName, List<LogicalExpression> args){
- return createExpression(functionName, ExpressionPosition.UNKNOWN, args);
- }
-
- public LogicalExpression createExpression(String functionName, ExpressionPosition ep, List<LogicalExpression> args){
- FunctionDefinition d = funcMap.get(functionName);
- if(d == null) throw new ExpressionParsingException(String.format("Unable to find function definition for function named '%s'", functionName));
- return d.newCall(args, ep);
- }
-
- public LogicalExpression createExpression(String unaryName, ExpressionPosition ep, LogicalExpression... e){
- return funcMap.get(unaryName).newCall(Lists.newArrayList(e), ep);
- }
-
- public LogicalExpression createByOp(List<LogicalExpression> args, ExpressionPosition ep, List<String> opTypes) {
- // logger.debug("Generating new comparison expressions.");
- if (args.size() == 1) {
- return args.get(0);
- }
-
- if (args.size() - 1 != opTypes.size())
- throw new DrillRuntimeException("Must receive one more expression then the provided number of operators.");
-
- LogicalExpression first = args.get(0);
- for (int i = 0; i < opTypes.size(); i++) {
- List<LogicalExpression> l2 = new ArrayList<LogicalExpression>();
- l2.add(first);
- l2.add(args.get(i + 1));
- first = createExpression(opTypes.get(i), ep, args);
- }
- return first;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java b/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java
index 70c2617..4df2c8e 100644
--- a/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java
+++ b/common/src/main/java/org/apache/drill/common/expression/LogicalExpression.java
@@ -75,8 +75,6 @@ public interface LogicalExpression extends Iterable<LogicalExpression>{
//TODO: move functionregistry and error collector to injectables.
//ctxt.findInjectableValue(valueId, forProperty, beanInstance)
-
- parser.setRegistry(new FunctionRegistry(config));
parse_return ret = parser.parse();
// ret.e.resolveAndValidate(expr, errorCollector);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/NoArgValidator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/NoArgValidator.java b/common/src/main/java/org/apache/drill/common/expression/NoArgValidator.java
deleted file mode 100644
index eaabfeb..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/NoArgValidator.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import java.util.List;
-
-public class NoArgValidator implements ArgumentValidator{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(NoArgValidator.class);
-
- @Override
- public void validateArguments(ExpressionPosition expr, List<LogicalExpression> expressions, ErrorCollector errors) {
- if(!expressions.isEmpty()){
- errors.addGeneralError(expr, "Expected zero arguments, received more than that.");
- }
- }
-
- @Override
- public String[] getArgumentNamesByPosition() {
- return new String[0];
- }
-
- public static final NoArgValidator VALIDATOR = new NoArgValidator();
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/fn/AggregationFunctions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/fn/AggregationFunctions.java b/common/src/main/java/org/apache/drill/common/expression/fn/AggregationFunctions.java
deleted file mode 100644
index 5b46b78..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/fn/AggregationFunctions.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.fn;
-
-import org.apache.drill.common.expression.ArgumentValidators.AnyTypeAllowed;
-import org.apache.drill.common.expression.CallProvider;
-import org.apache.drill.common.expression.FunctionDefinition;
-import org.apache.drill.common.expression.OutputTypeDeterminer;
-
-public class AggregationFunctions implements CallProvider {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AggregationFunctions.class);
-
- @Override
- public FunctionDefinition[] getFunctionDefintions() {
- return new FunctionDefinition[] {
- FunctionDefinition.aggregator("count", new AnyTypeAllowed(1), OutputTypeDeterminer.FIXED_BIGINT),
- FunctionDefinition.aggregator("sum", new AnyTypeAllowed(1), new OutputTypeDeterminer.SameAsFirstInput())
- };
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/fn/BooleanFunctions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/fn/BooleanFunctions.java b/common/src/main/java/org/apache/drill/common/expression/fn/BooleanFunctions.java
deleted file mode 100644
index f21b750..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/fn/BooleanFunctions.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.fn;
-
-import org.apache.drill.common.expression.ArgumentValidators.AllowedTypeList;
-import org.apache.drill.common.expression.ArgumentValidators.ComparableArguments;
-import org.apache.drill.common.expression.CallProvider;
-import org.apache.drill.common.expression.FunctionDefinition;
-import org.apache.drill.common.expression.OutputTypeDeterminer;
-import org.apache.drill.common.types.Types;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class BooleanFunctions implements CallProvider {
- static final Logger logger = LoggerFactory.getLogger(BooleanFunctions.class);
-
- @Override
- public FunctionDefinition[] getFunctionDefintions() {
- return new FunctionDefinition[] {
- FunctionDefinition.operator("or", new AllowedTypeList(2, Integer.MAX_VALUE, Types.REQUIRED_BIT), OutputTypeDeterminer.FIXED_BIT, "or", "||"),
- FunctionDefinition.operator("and", new AllowedTypeList(2, Integer.MAX_VALUE, Types.REQUIRED_BIT), OutputTypeDeterminer.FIXED_BIT, "and", "&&"),
- FunctionDefinition.operator("greater than", new ComparableArguments(2), OutputTypeDeterminer.FIXED_BIT, ">"),
- FunctionDefinition.operator("less than", new ComparableArguments(2), OutputTypeDeterminer.FIXED_BIT, "<"),
- FunctionDefinition.operator("equal", new ComparableArguments(2), OutputTypeDeterminer.FIXED_BIT, "==", "="),
- FunctionDefinition.operator("not equal", new ComparableArguments(2), OutputTypeDeterminer.FIXED_BIT, "<>", "!="),
- FunctionDefinition.operator("greater than or equal to", new ComparableArguments(2), OutputTypeDeterminer.FIXED_BIT, ">="),
- FunctionDefinition.operator("less than or equal to", new ComparableArguments(2), OutputTypeDeterminer.FIXED_BIT, "<="), };
-
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java b/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java
deleted file mode 100644
index 4be3820..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.fn;
-
-import org.apache.drill.common.expression.ArgumentValidators;
-import org.apache.drill.common.expression.CallProvider;
-import org.apache.drill.common.expression.FunctionDefinition;
-import org.apache.drill.common.expression.OutputTypeDeterminer;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-
-public class CastFunctionDefs implements CallProvider{
-
- private static final FunctionDefinition CAST_BIG_INT = FunctionDefinition.simple("castBIGINT", new ArgumentValidators.AnyTypeAllowed(1,3), new OutputTypeDeterminer.NullIfNullType(MinorType.BIGINT));
- private static final FunctionDefinition CAST_INT = FunctionDefinition.simple("castINT", new ArgumentValidators.AnyTypeAllowed(1,3), new OutputTypeDeterminer.NullIfNullType(MinorType.INT));
- private static final FunctionDefinition CAST_FLOAT4 = FunctionDefinition.simple("castFLOAT4", new ArgumentValidators.AnyTypeAllowed(1,3), new OutputTypeDeterminer.NullIfNullType(MinorType.FLOAT4));
- private static final FunctionDefinition CAST_FLOAT8 = FunctionDefinition.simple("castFLOAT8", new ArgumentValidators.AnyTypeAllowed(1,3), new OutputTypeDeterminer.NullIfNullType(MinorType.FLOAT8));
- private static final FunctionDefinition CAST_VARCHAR = FunctionDefinition.simple("castVARCHAR", new ArgumentValidators.AnyTypeAllowed(1,4), new OutputTypeDeterminer.NullIfNullType(MinorType.VARCHAR));
- private static final FunctionDefinition CAST_VARBINARY = FunctionDefinition.simple("castVARBINARY", new ArgumentValidators.AnyTypeAllowed(1,4), new OutputTypeDeterminer.NullIfNullType(MinorType.VARBINARY));
- private static final FunctionDefinition CAST_VAR16CHAR = FunctionDefinition.simple("castVAR16CHAR", new ArgumentValidators.AnyTypeAllowed(1,4), new OutputTypeDeterminer.NullIfNullType(MinorType.VAR16CHAR));
-
-
- @Override
- public FunctionDefinition[] getFunctionDefintions() {
- return new FunctionDefinition[]{
- CAST_BIG_INT,
- CAST_INT,
- CAST_FLOAT4,
- CAST_FLOAT8,
- CAST_VARCHAR,
- CAST_VARBINARY,
- CAST_VAR16CHAR
- };
- }
-
- public static FunctionDefinition getCastFuncDef(MinorType targetMinorType) {
- switch (targetMinorType) {
- case BIGINT :
- return CAST_BIG_INT;
- case INT :
- return CAST_INT;
- case FLOAT4:
- return CAST_FLOAT4;
- case FLOAT8:
- return CAST_FLOAT8;
- case VARCHAR:
- return CAST_VARCHAR;
- case VARBINARY:
- return CAST_VARBINARY;
- case VAR16CHAR:
- return CAST_VAR16CHAR;
-
- default:
- throw new RuntimeException(String.format("cast function for type %s is not defined", targetMinorType.name()));
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java b/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java
new file mode 100644
index 0000000..c752857
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctions.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.fn;
+
+import org.apache.drill.common.types.TypeProtos.MinorType;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class CastFunctions {
+
+ private static Map<MinorType, String> TYPE2FUNC = new HashMap<>();
+
+ static {
+ TYPE2FUNC.put(MinorType.BIGINT, "castBIGINT");
+ TYPE2FUNC.put(MinorType.INT, "castINT");
+ TYPE2FUNC.put(MinorType.FLOAT4, "castFLOAT4");
+ TYPE2FUNC.put(MinorType.FLOAT8, "castFLOAT8");
+ TYPE2FUNC.put(MinorType.VARCHAR, "castVARCHAR");
+ TYPE2FUNC.put(MinorType.VAR16CHAR, "castVAR16CHAR");
+ TYPE2FUNC.put(MinorType.VARBINARY, "castVARBINARY");
+ }
+
+ public static String getCastFunc(MinorType targetMinorType) {
+ String func = TYPE2FUNC.get(targetMinorType);
+ if (func != null)
+ return func;
+
+ throw new RuntimeException(
+ String.format("cast function for type %s is not defined", targetMinorType.name()));
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/fn/MathFunctions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/fn/MathFunctions.java b/common/src/main/java/org/apache/drill/common/expression/fn/MathFunctions.java
deleted file mode 100644
index e1d3390..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/fn/MathFunctions.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.fn;
-
-import org.apache.drill.common.expression.ArgumentValidators;
-import org.apache.drill.common.expression.FunctionDefinition;
-import org.apache.drill.common.expression.CallProvider;
-import org.apache.drill.common.expression.OutputTypeDeterminer;
-
-
-
-public class MathFunctions implements CallProvider{
-
-
- @Override
- public FunctionDefinition[] getFunctionDefintions() {
- return new FunctionDefinition[]{
- FunctionDefinition.operator("add", new ArgumentValidators.NumericTypeAllowed(1, Integer.MAX_VALUE, true), new OutputTypeDeterminer.SameAsAnySoft(), "+"),
- FunctionDefinition.operator("subtract", new ArgumentValidators.NumericTypeAllowed(1, Integer.MAX_VALUE, true), new OutputTypeDeterminer.SameAsAnySoft(), "-"),
- FunctionDefinition.operator("divide", new ArgumentValidators.NumericTypeAllowed(1, Integer.MAX_VALUE, true), new OutputTypeDeterminer.SameAsAnySoft(), "/"),
- FunctionDefinition.operator("multiply", new ArgumentValidators.NumericTypeAllowed(1, Integer.MAX_VALUE, true), new OutputTypeDeterminer.SameAsAnySoft(), "*"),
- FunctionDefinition.operator("modulo", new ArgumentValidators.NumericTypeAllowed(1, Integer.MAX_VALUE, true), new OutputTypeDeterminer.SameAsAnySoft(), "%"),
- XOR,
- FunctionDefinition.simple("power", new ArgumentValidators.NumericTypeAllowed(1, 3, true), new OutputTypeDeterminer.SameAsAnySoft()),
- FunctionDefinition.simpleRandom("random", new ArgumentValidators.NumericTypeAllowed(1, 1, true), new OutputTypeDeterminer.SameAsAnySoft())
- };
-
- }
-
- public static final FunctionDefinition XOR = FunctionDefinition.operator("xor", new ArgumentValidators.NumericTypeAllowed(1, Integer.MAX_VALUE, true), new OutputTypeDeterminer.SameAsAnySoft(), "^");
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/fn/StringFunctions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/fn/StringFunctions.java b/common/src/main/java/org/apache/drill/common/expression/fn/StringFunctions.java
deleted file mode 100644
index 1158115..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/fn/StringFunctions.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.fn;
-
-import org.apache.drill.common.expression.Arg;
-import org.apache.drill.common.expression.BasicArgumentValidator;
-import org.apache.drill.common.expression.CallProvider;
-import org.apache.drill.common.expression.FunctionDefinition;
-import org.apache.drill.common.expression.OutputTypeDeterminer.FixedType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-
-public class StringFunctions implements CallProvider{
-
-
- @Override
- public FunctionDefinition[] getFunctionDefintions() {
- return new FunctionDefinition[]{
- FunctionDefinition.simple("regex_like", new BasicArgumentValidator( //
- new Arg(true, false, "pattern", MinorType.VARCHAR, MinorType.VARCHAR), //
- new Arg(false, true, "value", MinorType.FIXEDCHAR, MinorType.VARCHAR, MinorType.VARCHAR) ), FixedType.FIXED_BIT),
- };
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/fn/TypeFunctions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/fn/TypeFunctions.java b/common/src/main/java/org/apache/drill/common/expression/fn/TypeFunctions.java
deleted file mode 100644
index 23ca87c..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/fn/TypeFunctions.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.fn;
-
-import org.apache.drill.common.expression.ArgumentValidators.AnyTypeAllowed;
-import org.apache.drill.common.expression.FunctionDefinition;
-import org.apache.drill.common.expression.CallProvider;
-import org.apache.drill.common.expression.OutputTypeDeterminer;
-
-public class TypeFunctions implements CallProvider{
-
- @Override
- public FunctionDefinition[] getFunctionDefintions() {
- return new FunctionDefinition[]{
- FunctionDefinition.simple("isNumber", new AnyTypeAllowed(1), OutputTypeDeterminer.FIXED_BIT),
- };
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/fn/UnaryFunctions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/fn/UnaryFunctions.java b/common/src/main/java/org/apache/drill/common/expression/fn/UnaryFunctions.java
deleted file mode 100644
index 0a678fd..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/fn/UnaryFunctions.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression.fn;
-
-import org.apache.drill.common.expression.ArgumentValidators;
-import org.apache.drill.common.expression.BasicArgumentValidator;
-import org.apache.drill.common.expression.CallProvider;
-import org.apache.drill.common.expression.FunctionDefinition;
-import org.apache.drill.common.expression.OutputTypeDeterminer;
-import org.apache.drill.common.types.Types;
-
-public class UnaryFunctions implements CallProvider{
-
- @Override
- public FunctionDefinition[] getFunctionDefintions() {
- return new FunctionDefinition[]{
- FunctionDefinition.operator("isNull", new ArgumentValidators.AnyTypeAllowed(1), new OutputTypeDeterminer.FixedType(Types.REQUIRED_BIT)),
- FunctionDefinition.operator("not", new BasicArgumentValidator(Types.REQUIRED_BIT), new OutputTypeDeterminer.FixedType(Types.REQUIRED_BIT), "!"),
- FunctionDefinition.operator("negative", new ArgumentValidators.NumericTypeAllowed(1, true), new OutputTypeDeterminer.SameAsFirstInput(), "u-"),
- };
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java b/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
index 469cedd..ef96cab 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
@@ -18,6 +18,7 @@
package org.apache.drill.common.expression.visitors;
import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.IfExpression;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
@@ -33,7 +34,12 @@ public abstract class AbstractExprVisitor<T, VAL, EXCEP extends Exception> imple
@Override
public T visitFunctionCall(FunctionCall call, VAL value) throws EXCEP {
- return null;
+ return visitUnknown(call, value);
+ }
+
+ @Override
+ public T visitFunctionHolderExpression(FunctionHolderExpression holder, VAL value) throws EXCEP {
+ return visitUnknown(holder, value);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java b/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
index 36e6a52..71ac82e 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
@@ -18,6 +18,7 @@
package org.apache.drill.common.expression.visitors;
import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.IfExpression;
import org.apache.drill.common.expression.IfExpression.IfCondition;
import org.apache.drill.common.expression.LogicalExpression;
@@ -41,13 +42,22 @@ public final class AggregateChecker extends SimpleExprVisitor<Boolean>{
@Override
public Boolean visitFunctionCall(FunctionCall call) {
- if(call.getDefinition().isAggregating()){
- for(LogicalExpression e : call){
- if(e.accept(this, null)) throw new IllegalArgumentException(String.format("Your aggregating function call %s also includes arguments that contain aggregations. This isn't allowed.", call.getDefinition().toString()));
+ throw new UnsupportedOperationException("FunctionCall is not expected here. "+
+ "It should have been converted to FunctionHolderExpression in materialization");
+ }
+
+ @Override
+ public Boolean visitFunctionHolderExpression(FunctionHolderExpression holder) throws RuntimeException {
+ if(holder.isAggregating()){
+ for(LogicalExpression e : holder.args){
+ if(e.accept(this, null))
+ throw new IllegalArgumentException(String.format(
+ "Your aggregating function call %s also includes arguments that contain aggregations."+
+ " This isn't allowed.", holder.getName()));
}
return true;
}else{
- for(LogicalExpression e : call){
+ for(LogicalExpression e : holder.args){
if(e.accept(this, null)) return true;
}
return false;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java b/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
index a951a60..fcb2489 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
@@ -18,6 +18,7 @@
package org.apache.drill.common.expression.visitors;
import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.IfExpression;
import org.apache.drill.common.expression.IfExpression.IfCondition;
import org.apache.drill.common.expression.LogicalExpression;
@@ -41,8 +42,19 @@ public final class ConstantChecker extends SimpleExprVisitor<Boolean>{
@Override
public Boolean visitFunctionCall(FunctionCall call) {
- for(LogicalExpression e : call){
- if(!e.accept(this, null)) return false;
+ throw new UnsupportedOperationException("FunctionCall is not expected here. "+
+ "It should have been converted to FunctionHolderExpression in materialization");
+ }
+
+ @Override
+ public Boolean visitFunctionHolderExpression(FunctionHolderExpression holder) {
+ for(int i=0; i<holder.args.size(); i++) {
+ if (holder.argConstantOnly(i) && !holder.args.get(i).accept(this, null)) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Function '").append(holder.getName())
+ .append("' expects constant input for argument number ").append(i);
+ throw new UnsupportedOperationException(sb.toString());
+ }
}
return true;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/f8fbb5d4/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java b/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
index 87d146a..10b0acf 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
@@ -18,6 +18,7 @@
package org.apache.drill.common.expression.visitors;
import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.IfExpression;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
@@ -30,14 +31,15 @@ import org.apache.drill.common.expression.ValueExpressions.IntExpression;
public interface ExprVisitor<T, VAL, EXCEP extends Exception> {
- public T visitFunctionCall(FunctionCall call, VAL value) throws EXCEP;
- public T visitIfExpression(IfExpression ifExpr, VAL value) throws EXCEP;
- public T visitSchemaPath(SchemaPath path, VAL value) throws EXCEP;
+ public T visitFunctionCall(FunctionCall call, VAL value) throws EXCEP;
+ public T visitFunctionHolderExpression(FunctionHolderExpression holder, VAL value) throws EXCEP;
+ public T visitIfExpression(IfExpression ifExpr, VAL value) throws EXCEP;
+ public T visitSchemaPath(SchemaPath path, VAL value) throws EXCEP;
public T visitIntConstant(IntExpression intExpr, VAL value) throws EXCEP;
public T visitFloatConstant(FloatExpression fExpr, VAL value) throws EXCEP;
- public T visitLongConstant(LongExpression intExpr, VAL value) throws EXCEP;
- public T visitDoubleConstant(DoubleExpression dExpr, VAL value) throws EXCEP;
- public T visitBooleanConstant(BooleanExpression e, VAL value) throws EXCEP;
- public T visitQuotedStringConstant(QuotedString e, VAL value) throws EXCEP;
- public T visitUnknown(LogicalExpression e, VAL value) throws EXCEP;
+ public T visitLongConstant(LongExpression intExpr, VAL value) throws EXCEP;
+ public T visitDoubleConstant(DoubleExpression dExpr, VAL value) throws EXCEP;
+ public T visitBooleanConstant(BooleanExpression e, VAL value) throws EXCEP;
+ public T visitQuotedStringConstant(QuotedString e, VAL value) throws EXCEP;
+ public T visitUnknown(LogicalExpression e, VAL value) throws EXCEP;
}
[07/12] DRILL-335: Implement Hash Aggregation
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggBatch.java
deleted file mode 100644
index 3286cbd..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggBatch.java
+++ /dev/null
@@ -1,327 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical.impl.aggregate;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.drill.common.expression.ErrorCollector;
-import org.apache.drill.common.expression.ErrorCollectorImpl;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.FunctionCall;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.logical.data.NamedExpression;
-import org.apache.drill.exec.compile.sig.GeneratorMapping;
-import org.apache.drill.exec.compile.sig.MappingSet;
-import org.apache.drill.exec.exception.ClassTransformationException;
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.expr.ClassGenerator;
-import org.apache.drill.exec.expr.ClassGenerator.BlockType;
-import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
-import org.apache.drill.exec.expr.CodeGenerator;
-import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
-import org.apache.drill.exec.expr.HoldingContainerExpression;
-import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.expr.ValueVectorWriteExpression;
-import org.apache.drill.exec.expr.fn.ComparatorFunctionHelper;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.physical.config.StreamingAggregate;
-import org.apache.drill.exec.record.AbstractRecordBatch;
-import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.record.TypedFieldId;
-import org.apache.drill.exec.record.selection.SelectionVector2;
-import org.apache.drill.exec.record.selection.SelectionVector4;
-import org.apache.drill.exec.vector.ValueVector;
-import org.apache.drill.exec.vector.allocator.VectorAllocator;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.sun.codemodel.JExpr;
-import com.sun.codemodel.JVar;
-
-public class AggBatch extends AbstractRecordBatch<StreamingAggregate> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AggBatch.class);
-
- private Aggregator aggregator;
- private final RecordBatch incoming;
- private boolean done = false;
-
- public static enum AggOutcome {
- RETURN_OUTCOME, CLEANUP_AND_RETURN, UPDATE_AGGREGATOR;
- }
-
- public AggBatch(StreamingAggregate popConfig, RecordBatch incoming, FragmentContext context) {
- super(popConfig, context);
- this.incoming = incoming;
- }
-
- @Override
- public int getRecordCount() {
- if(done) return 0;
- return aggregator.getOutputCount();
- }
-
- @Override
- public IterOutcome next() {
-
- // this is only called on the first batch. Beyond this, the aggregator manages batches.
- if (aggregator == null) {
- IterOutcome outcome = incoming.next();
- logger.debug("Next outcome of {}", outcome);
- switch (outcome) {
- case NONE:
- case NOT_YET:
- case STOP:
- return outcome;
- case OK_NEW_SCHEMA:
- if (!createAggregator()){
- done = true;
- return IterOutcome.STOP;
- }
- break;
- case OK:
- throw new IllegalStateException("You should never get a first batch without a new schema");
- default:
- throw new IllegalStateException(String.format("unknown outcome %s", outcome));
- }
- }
-
- while(true){
- AggOutcome out = aggregator.doWork();
- logger.debug("Aggregator response {}, records {}", out, aggregator.getOutputCount());
- switch(out){
- case CLEANUP_AND_RETURN:
- container.zeroVectors();
- done = true;
- return aggregator.getOutcome();
- case RETURN_OUTCOME:
- return aggregator.getOutcome();
- case UPDATE_AGGREGATOR:
- aggregator = null;
- if(!createAggregator()){
- return IterOutcome.STOP;
- }
- continue;
- default:
- throw new IllegalStateException(String.format("Unknown state %s.", out));
- }
- }
-
- }
-
-
-
- /**
- * Creates a new Aggregator based on the current schema. If setup fails, this method is responsible for cleaning up
- * and informing the context of the failure state, as well is informing the upstream operators.
- *
- * @return true if the aggregator was setup successfully. false if there was a failure.
- */
- private boolean createAggregator() {
- logger.debug("Creating new aggregator.");
- try{
- this.aggregator = createAggregatorInternal();
- return true;
- }catch(SchemaChangeException | ClassTransformationException | IOException ex){
- context.fail(ex);
- container.clear();
- incoming.kill();
- return false;
- }
- }
-
-
-
-
- private Aggregator createAggregatorInternal() throws SchemaChangeException, ClassTransformationException, IOException{
- ClassGenerator<Aggregator> cg = CodeGenerator.getRoot(AggTemplate.TEMPLATE_DEFINITION, context.getFunctionRegistry());
- container.clear();
- List<VectorAllocator> allocators = Lists.newArrayList();
-
- LogicalExpression[] keyExprs = new LogicalExpression[popConfig.getKeys().length];
- LogicalExpression[] valueExprs = new LogicalExpression[popConfig.getExprs().length];
- TypedFieldId[] keyOutputIds = new TypedFieldId[popConfig.getKeys().length];
-
- ErrorCollector collector = new ErrorCollectorImpl();
-
- for(int i =0; i < keyExprs.length; i++){
- NamedExpression ne = popConfig.getKeys()[i];
- final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector,context.getFunctionRegistry() );
- if(expr == null) continue;
- keyExprs[i] = expr;
- final MaterializedField outputField = MaterializedField.create(ne.getRef(), expr.getMajorType());
- ValueVector vector = TypeHelper.getNewVector(outputField, context.getAllocator());
- allocators.add(VectorAllocator.getAllocator(vector, 50));
- keyOutputIds[i] = container.add(vector);
- }
-
- for(int i =0; i < valueExprs.length; i++){
- NamedExpression ne = popConfig.getExprs()[i];
- final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry());
- if(expr == null) continue;
-
- final MaterializedField outputField = MaterializedField.create(ne.getRef(), expr.getMajorType());
- ValueVector vector = TypeHelper.getNewVector(outputField, context.getAllocator());
- allocators.add(VectorAllocator.getAllocator(vector, 50));
- TypedFieldId id = container.add(vector);
- valueExprs[i] = new ValueVectorWriteExpression(id, expr, true);
- }
-
- if(collector.hasErrors()) throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
-
- setupIsSame(cg, keyExprs);
- setupIsSameApart(cg, keyExprs);
- addRecordValues(cg, valueExprs);
- outputRecordKeys(cg, keyOutputIds, keyExprs);
- outputRecordKeysPrev(cg, keyOutputIds, keyExprs);
-
- cg.getBlock("resetValues")._return(JExpr.TRUE);
- getIndex(cg);
-
- container.buildSchema(SelectionVectorMode.NONE);
- Aggregator agg = context.getImplementationClass(cg);
- agg.setup(context, incoming, this, allocators.toArray(new VectorAllocator[allocators.size()]));
- return agg;
- }
-
-
-
- private static final GeneratorMapping IS_SAME = GeneratorMapping.create("setupInterior", "isSame", null, null);
- private final MappingSet isSameI1Mapping = new MappingSet("index1", null, IS_SAME, IS_SAME);
- private final MappingSet isSameI2Mapping = new MappingSet("index2", null, IS_SAME, IS_SAME);
-
- private void setupIsSame(ClassGenerator<Aggregator> cg, LogicalExpression[] keyExprs){
- cg.setMappingSet(isSameI1Mapping);
- for(LogicalExpression expr : keyExprs){
- // first, we rewrite the evaluation stack for each side of the comparison.
- cg.setMappingSet(isSameI1Mapping);
- HoldingContainer first = cg.addExpr(expr, false);
- cg.setMappingSet(isSameI2Mapping);
- HoldingContainer second = cg.addExpr(expr, false);
-
- LogicalExpression fh = ComparatorFunctionHelper.get(first, second, context.getFunctionRegistry());
- HoldingContainer out = cg.addExpr(fh, false);
- cg.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)))._then()._return(JExpr.FALSE);
- }
- cg.getEvalBlock()._return(JExpr.TRUE);
- }
-
- private static final GeneratorMapping IS_SAME_PREV_INTERNAL_BATCH_READ = GeneratorMapping.create("isSamePrev", "isSamePrev", null, null); // the internal batch changes each time so we need to redo setup.
- private static final GeneratorMapping IS_SAME_PREV = GeneratorMapping.create("setupInterior", "isSamePrev", null, null);
- private final MappingSet isaB1Mapping = new MappingSet("b1Index", null, "b1", null, IS_SAME_PREV_INTERNAL_BATCH_READ, IS_SAME_PREV_INTERNAL_BATCH_READ);
- private final MappingSet isaB2Mapping = new MappingSet("b2Index", null, "incoming", null, IS_SAME_PREV, IS_SAME_PREV);
-
- private void setupIsSameApart(ClassGenerator<Aggregator> cg, LogicalExpression[] keyExprs){
- cg.setMappingSet(isaB1Mapping);
- for(LogicalExpression expr : keyExprs){
- // first, we rewrite the evaluation stack for each side of the comparison.
- cg.setMappingSet(isaB1Mapping);
- HoldingContainer first = cg.addExpr(expr, false);
- cg.setMappingSet(isaB2Mapping);
- HoldingContainer second = cg.addExpr(expr, false);
-
- LogicalExpression fh = ComparatorFunctionHelper.get(first, second, context.getFunctionRegistry());
- HoldingContainer out = cg.addExpr(fh, false);
- cg.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)))._then()._return(JExpr.FALSE);
- }
- cg.getEvalBlock()._return(JExpr.TRUE);
- }
-
- private static final GeneratorMapping EVAL_INSIDE = GeneratorMapping.create("setupInterior", "addRecord", null, null);
- private static final GeneratorMapping EVAL_OUTSIDE = GeneratorMapping.create("setupInterior", "outputRecordValues", "resetValues", "cleanup");
- private final MappingSet evalMapping = new MappingSet("index", "outIndex", EVAL_INSIDE, EVAL_OUTSIDE, EVAL_INSIDE);
-
- private void addRecordValues(ClassGenerator<Aggregator> cg, LogicalExpression[] valueExprs){
- cg.setMappingSet(evalMapping);
- for(LogicalExpression ex : valueExprs){
- HoldingContainer hc = cg.addExpr(ex);
- cg.getBlock(BlockType.EVAL)._if(hc.getValue().eq(JExpr.lit(0)))._then()._return(JExpr.FALSE);
- }
- cg.getBlock(BlockType.EVAL)._return(JExpr.TRUE);
- }
-
- private final MappingSet recordKeysMapping = new MappingSet(GeneratorMapping.create("setupInterior", "outputRecordKeys", null, null));
-
- private void outputRecordKeys(ClassGenerator<Aggregator> cg, TypedFieldId[] keyOutputIds, LogicalExpression[] keyExprs){
- cg.setMappingSet(recordKeysMapping);
- for(int i =0; i < keyExprs.length; i++){
- HoldingContainer hc = cg.addExpr(new ValueVectorWriteExpression(keyOutputIds[i], keyExprs[i], true));
- cg.getBlock(BlockType.EVAL)._if(hc.getValue().eq(JExpr.lit(0)))._then()._return(JExpr.FALSE);
- }
- cg.getBlock(BlockType.EVAL)._return(JExpr.TRUE);
- }
-
- private static final GeneratorMapping PREVIOUS_KEYS_OUT = GeneratorMapping.create("setupInterior", "outputRecordKeysPrev", null, null);
- private final MappingSet recordKeysPrevOutMapping = new MappingSet("previousIndex", "outIndex", "previous", "outgoing", PREVIOUS_KEYS_OUT, PREVIOUS_KEYS_OUT);
-
- private static final GeneratorMapping PREVIOUS_KEYS = GeneratorMapping.create("outputRecordKeysPrev", "outputRecordKeysPrev", null, null);
- private final MappingSet recordKeysPrevMapping = new MappingSet("previousIndex", "outIndex", "previous", null, PREVIOUS_KEYS, PREVIOUS_KEYS);
-
- private void outputRecordKeysPrev(ClassGenerator<Aggregator> cg, TypedFieldId[] keyOutputIds, LogicalExpression[] keyExprs){
- cg.setMappingSet(recordKeysPrevMapping);
-
- for(int i =0; i < keyExprs.length; i++){
- // IMPORTANT: there is an implicit assertion here that the TypedFieldIds for the previous batch and the current batch are the same. This is possible because InternalBatch guarantees this.
- logger.debug("Writing out expr {}", keyExprs[i]);
- cg.rotateBlock();
- cg.setMappingSet(recordKeysPrevMapping);
- HoldingContainer innerExpression = cg.addExpr(keyExprs[i], false);
- cg.setMappingSet(recordKeysPrevOutMapping);
- HoldingContainer outerExpression = cg.addExpr(new ValueVectorWriteExpression(keyOutputIds[i], new HoldingContainerExpression(innerExpression), true), false);
- cg.getBlock(BlockType.EVAL)._if(outerExpression.getValue().eq(JExpr.lit(0)))._then()._return(JExpr.FALSE);
-
- }
- cg.getBlock(BlockType.EVAL)._return(JExpr.TRUE);
- }
-
- private void getIndex(ClassGenerator<Aggregator> g){
- switch(incoming.getSchema().getSelectionVectorMode()){
- case FOUR_BYTE: {
- JVar var = g.declareClassField("sv4_", g.getModel()._ref(SelectionVector4.class));
- g.getBlock("setupInterior").assign(var, JExpr.direct("incoming").invoke("getSelectionVector4"));
- g.getBlock("getVectorIndex")._return(var.invoke("get").arg(JExpr.direct("recordIndex")));;
- return;
- }
- case NONE: {
- g.getBlock("getVectorIndex")._return(JExpr.direct("recordIndex"));;
- return;
- }
- case TWO_BYTE: {
- JVar var = g.declareClassField("sv2_", g.getModel()._ref(SelectionVector2.class));
- g.getBlock("setupInterior").assign(var, JExpr.direct("incoming").invoke("getSelectionVector2"));
- g.getBlock("getVectorIndex")._return(var.invoke("get").arg(JExpr.direct("recordIndex")));;
- return;
- }
-
- default:
- throw new IllegalStateException();
-
- }
-
- }
-
-
-
- @Override
- protected void killIncoming() {
- incoming.kill();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggBatchCreator.java
deleted file mode 100644
index 36344f9..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggBatchCreator.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical.impl.aggregate;
-
-import java.util.List;
-
-import org.apache.drill.common.exceptions.ExecutionSetupException;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.physical.config.StreamingAggregate;
-import org.apache.drill.exec.physical.impl.BatchCreator;
-import org.apache.drill.exec.record.RecordBatch;
-
-import com.google.common.base.Preconditions;
-
-public class AggBatchCreator implements BatchCreator<StreamingAggregate>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AggBatchCreator.class);
-
- @Override
- public RecordBatch getBatch(FragmentContext context, StreamingAggregate config, List<RecordBatch> children) throws ExecutionSetupException {
- Preconditions.checkArgument(children.size() == 1);
- return new AggBatch(config, children.iterator().next(), context);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggTemplate.java
deleted file mode 100644
index a8f39e3..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/AggTemplate.java
+++ /dev/null
@@ -1,305 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical.impl.aggregate;
-
-import javax.inject.Named;
-
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.physical.impl.aggregate.AggBatch.AggOutcome;
-import org.apache.drill.exec.record.BatchSchema;
-import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.record.RecordBatch.IterOutcome;
-import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.vector.allocator.VectorAllocator;
-
-public abstract class AggTemplate implements Aggregator {
- private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Aggregator.class);
- private static final boolean EXTRA_DEBUG = false;
- private static final String TOO_BIG_ERROR = "Couldn't add value to an empty batch. This likely means that a single value is too long for a varlen field.";
- private IterOutcome lastOutcome = null;
- private boolean first = true;
- private boolean newSchema = false;
- private int previousIndex = 0;
- private int underlyingIndex = 0;
- private int currentIndex;
- private int addedRecordCount = 0;
- private boolean pendingOutput = false;
- private IterOutcome outcome;
- private int outputCount = 0;
- private RecordBatch incoming;
- private BatchSchema schema;
- private RecordBatch outgoing;
- private VectorAllocator[] allocators;
- private FragmentContext context;
- private InternalBatch remainderBatch;
-
-
- @Override
- public void setup(FragmentContext context, RecordBatch incoming, RecordBatch outgoing, VectorAllocator[] allocators) throws SchemaChangeException {
- this.allocators = allocators;
- this.context = context;
- this.incoming = incoming;
- this.schema = incoming.getSchema();
- this.allocators = allocators;
- this.outgoing = outgoing;
- setupInterior(incoming, outgoing);
- this.currentIndex = this.getVectorIndex(underlyingIndex);
- }
-
-
- private void allocateOutgoing() {
- for (VectorAllocator a : allocators) {
- if(EXTRA_DEBUG) logger.debug("Allocating {} with {} records.", a, 20000);
- a.alloc(20000);
- }
- }
-
- @Override
- public IterOutcome getOutcome() {
- return outcome;
- }
-
- @Override
- public int getOutputCount() {
- return outputCount;
- }
-
- private AggOutcome tooBigFailure(){
- context.fail(new Exception(TOO_BIG_ERROR));
- this.outcome = IterOutcome.STOP;
- return AggOutcome.CLEANUP_AND_RETURN;
- }
-
- @Override
- public AggOutcome doWork() {
- try{ // outside loop to ensure that first is set to false after the first run.
-
- // if we're in the first state, allocate outgoing.
- if(first){
- allocateOutgoing();
- }
-
- // pick up a remainder batch if we have one.
- if(remainderBatch != null){
- if (!outputToBatch( previousIndex )) return tooBigFailure();
- remainderBatch.clear();
- remainderBatch = null;
- return setOkAndReturn();
- }
-
-
- // setup for new output and pick any remainder.
- if (pendingOutput) {
- allocateOutgoing();
- pendingOutput = false;
- if (!outputToBatch( previousIndex)) return tooBigFailure();
- }
-
- if(newSchema){
- return AggOutcome.UPDATE_AGGREGATOR;
- }
-
- if(lastOutcome != null){
- outcome = lastOutcome;
- return AggOutcome.CLEANUP_AND_RETURN;
- }
-
- outside: while(true){
- // loop through existing records, adding as necessary.
- for (; underlyingIndex < incoming.getRecordCount(); incIndex()) {
- if(EXTRA_DEBUG) logger.debug("Doing loop with values underlying {}, current {}", underlyingIndex, currentIndex);
- if (isSame( previousIndex, currentIndex )) {
- if(EXTRA_DEBUG) logger.debug("Values were found the same, adding.");
- addRecordInc(currentIndex);
- } else {
- if(EXTRA_DEBUG) logger.debug("Values were different, outputting previous batch.");
- if (outputToBatch(previousIndex)) {
- if(EXTRA_DEBUG) logger.debug("Output successful.");
- addRecordInc(currentIndex);
- } else {
- if(EXTRA_DEBUG) logger.debug("Output failed.");
- if(outputCount == 0) return tooBigFailure();
-
- // mark the pending output but move forward for the next cycle.
- pendingOutput = true;
- previousIndex = currentIndex;
- incIndex();
- return setOkAndReturn();
-
- }
- }
- previousIndex = currentIndex;
- }
-
-
- InternalBatch previous = null;
-
- try{
- while(true){
- previous = new InternalBatch(incoming);
- IterOutcome out = incoming.next();
- if(EXTRA_DEBUG) logger.debug("Received IterOutcome of {}", out);
- switch(out){
- case NONE:
- lastOutcome = out;
- if(addedRecordCount > 0){
- if( !outputToBatchPrev( previous, previousIndex, outputCount) ) remainderBatch = previous;
- if(EXTRA_DEBUG) logger.debug("Received no more batches, returning.");
- return setOkAndReturn();
- }else{
- outcome = out;
- return AggOutcome.CLEANUP_AND_RETURN;
- }
-
-
-
- case NOT_YET:
- this.outcome = out;
- return AggOutcome.RETURN_OUTCOME;
-
- case OK_NEW_SCHEMA:
- if(EXTRA_DEBUG) logger.debug("Received new schema. Batch has {} records.", incoming.getRecordCount());
- if(addedRecordCount > 0){
- if( !outputToBatchPrev( previous, previousIndex, outputCount) ) remainderBatch = previous;
- if(EXTRA_DEBUG) logger.debug("Wrote out end of previous batch, returning.");
- newSchema = true;
- return setOkAndReturn();
- }
- cleanup();
- return AggOutcome.UPDATE_AGGREGATOR;
- case OK:
- resetIndex();
- if(incoming.getRecordCount() == 0){
- continue;
- }else{
- if(isSamePrev(previousIndex , previous, currentIndex)){
- if(EXTRA_DEBUG) logger.debug("New value was same as last value of previous batch, adding.");
- addRecordInc(currentIndex);
- previousIndex = currentIndex;
- incIndex();
- if(EXTRA_DEBUG) logger.debug("Continuing outside");
- continue outside;
- }else{ // not the same
- if(EXTRA_DEBUG) logger.debug("This is not the same as the previous, add record and continue outside.");
- previousIndex = currentIndex;
- if(addedRecordCount > 0){
- if( !outputToBatchPrev( previous, previousIndex, outputCount) ){
- remainderBatch = previous;
- return setOkAndReturn();
- }
- continue outside;
- }
- }
- }
- case STOP:
- default:
- lastOutcome = out;
- outcome = out;
- return AggOutcome.CLEANUP_AND_RETURN;
- }
-
-
- }
- }finally{
- // make sure to clear previous if we haven't saved it.
- if(remainderBatch == null && previous != null){
- previous.clear();
- }
- }
- }
- }finally{
- if(first) first = !first;
- }
-
- }
-
-
- private final void incIndex(){
- underlyingIndex++;
- if(underlyingIndex >= incoming.getRecordCount()){
- currentIndex = Integer.MAX_VALUE;
- return;
- }
- currentIndex = getVectorIndex(underlyingIndex);
- }
-
- private final void resetIndex(){
- underlyingIndex = -1;
- incIndex();
- }
-
- private final AggOutcome setOkAndReturn(){
- if(first){
- this.outcome = IterOutcome.OK_NEW_SCHEMA;
- }else{
- this.outcome = IterOutcome.OK;
- }
- for(VectorWrapper<?> v : outgoing){
- v.getValueVector().getMutator().setValueCount(outputCount);
- }
- return AggOutcome.RETURN_OUTCOME;
- }
-
- private final boolean outputToBatch(int inIndex){
- boolean success = outputRecordKeys(inIndex, outputCount) //
- && outputRecordValues(outputCount) //
- && resetValues();
- if(success){
- if(EXTRA_DEBUG) logger.debug("Outputting values to {}", outputCount);
- outputCount++;
- addedRecordCount = 0;
- }
-
- return success;
- }
-
- private final boolean outputToBatchPrev(InternalBatch b1, int inIndex, int outIndex){
- boolean success = outputRecordKeysPrev(b1, inIndex, outIndex) //
- && outputRecordValues(outIndex) //
- && resetValues();
- if(success){
- outputCount++;
- addedRecordCount = 0;
- }
-
- return success;
- }
-
- private void addRecordInc(int index){
- addRecord(index);
- this.addedRecordCount++;
- }
-
- @Override
- public void cleanup(){
- if(remainderBatch != null) remainderBatch.clear();
- }
-
-
- public abstract void setupInterior(@Named("incoming") RecordBatch incoming, @Named("outgoing") RecordBatch outgoing) throws SchemaChangeException;
- public abstract boolean isSame(@Named("index1") int index1, @Named("index2") int index2);
- public abstract boolean isSamePrev(@Named("b1Index") int b1Index, @Named("b1") InternalBatch b1, @Named("b2Index") int b2Index);
- public abstract void addRecord(@Named("index") int index);
- public abstract boolean outputRecordKeys(@Named("inIndex") int inIndex, @Named("outIndex") int outIndex);
- public abstract boolean outputRecordKeysPrev(@Named("previous") InternalBatch previous, @Named("previousIndex") int previousIndex, @Named("outIndex") int outIndex);
- public abstract boolean outputRecordValues(@Named("outIndex") int outIndex);
- public abstract int getVectorIndex(@Named("recordIndex") int recordIndex);
- public abstract boolean resetValues();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/Aggregator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/Aggregator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/Aggregator.java
deleted file mode 100644
index 2683045..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/Aggregator.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical.impl.aggregate;
-
-import org.apache.drill.exec.compile.TemplateClassDefinition;
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.physical.impl.aggregate.AggBatch.AggOutcome;
-import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.record.RecordBatch.IterOutcome;
-import org.apache.drill.exec.vector.allocator.VectorAllocator;
-
-public interface Aggregator {
-
- public static TemplateClassDefinition<Aggregator> TEMPLATE_DEFINITION = new TemplateClassDefinition<Aggregator>(Aggregator.class, AggTemplate.class);
-
- public abstract void setup(FragmentContext context, RecordBatch incoming, RecordBatch outgoing,
- VectorAllocator[] allocators) throws SchemaChangeException;
-
- public abstract IterOutcome getOutcome();
-
- public abstract int getOutputCount();
-
- public abstract AggOutcome doWork();
-
- public abstract void cleanup();
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
new file mode 100644
index 0000000..d720390
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatch.java
@@ -0,0 +1,278 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.aggregate;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.drill.common.expression.ErrorCollector;
+import org.apache.drill.common.expression.ErrorCollectorImpl;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.NamedExpression;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.exec.compile.sig.GeneratorMapping;
+import org.apache.drill.exec.compile.sig.MappingSet;
+import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.expr.CodeGenerator;
+import org.apache.drill.exec.expr.ClassGenerator;
+import org.apache.drill.exec.expr.ClassGenerator.BlockType;
+import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
+import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
+import org.apache.drill.exec.expr.HoldingContainerExpression;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.expr.ValueVectorWriteExpression;
+import org.apache.drill.exec.expr.holders.IntHolder;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.config.HashAggregate;
+import org.apache.drill.exec.record.AbstractRecordBatch;
+import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.selection.SelectionVector2;
+import org.apache.drill.exec.record.selection.SelectionVector4;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.allocator.VectorAllocator;
+import org.apache.drill.exec.physical.impl.aggregate.HashAggregator.AggOutcome;
+import org.apache.drill.exec.physical.impl.common.ChainedHashTable;
+import org.apache.drill.exec.physical.impl.common.HashTable;
+import org.apache.drill.exec.record.VectorWrapper;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.sun.codemodel.JExpr;
+import com.sun.codemodel.JVar;
+
+public class HashAggBatch extends AbstractRecordBatch<HashAggregate> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HashAggBatch.class);
+
+ private HashAggregator aggregator;
+ private final RecordBatch incoming;
+ private boolean done = false;
+ private LogicalExpression[] aggrExprs;
+ private TypedFieldId[] groupByOutFieldIds ;
+ private TypedFieldId[] aggrOutFieldIds ; // field ids for the outgoing batch
+
+ private final GeneratorMapping UPDATE_AGGR_INSIDE =
+ GeneratorMapping.create("setupInterior" /* setup method */, "updateAggrValuesInternal" /* eval method */,
+ "resetValues" /* reset */, "cleanup" /* cleanup */) ;
+
+ private final GeneratorMapping UPDATE_AGGR_OUTSIDE =
+ GeneratorMapping.create("setupInterior" /* setup method */, "outputRecordValues" /* eval method */,
+ "resetValues" /* reset */, "cleanup" /* cleanup */) ;
+
+ private final MappingSet UpdateAggrValuesMapping = new MappingSet("incomingRowIdx" /* read index */, "outRowIdx" /* write index */, "htRowIdx" /* workspace index */, "incoming" /* read container */, "outgoing" /* write container */, "aggrValuesContainer" /* workspace container */, UPDATE_AGGR_INSIDE, UPDATE_AGGR_OUTSIDE, UPDATE_AGGR_INSIDE);
+
+
+ public HashAggBatch(HashAggregate popConfig, RecordBatch incoming, FragmentContext context) {
+ super(popConfig, context);
+ this.incoming = incoming;
+ }
+
+ @Override
+ public int getRecordCount() {
+ if(done) return 0;
+ return aggregator.getOutputCount();
+ }
+
+ @Override
+ public IterOutcome next() {
+
+ // this is only called on the first batch. Beyond this, the aggregator manages batches.
+ if (aggregator == null) {
+ IterOutcome outcome = incoming.next();
+ logger.debug("Next outcome of {}", outcome);
+ switch (outcome) {
+ case NONE:
+ case NOT_YET:
+ case STOP:
+ return outcome;
+ case OK_NEW_SCHEMA:
+ if (!createAggregator()){
+ done = true;
+ return IterOutcome.STOP;
+ }
+ break;
+ case OK:
+ throw new IllegalStateException("You should never get a first batch without a new schema");
+ default:
+ throw new IllegalStateException(String.format("unknown outcome %s", outcome));
+ }
+ }
+
+ if (aggregator.allFlushed()) {
+ return IterOutcome.NONE;
+ }
+
+ while(true){
+ AggOutcome out = aggregator.doWork();
+ logger.debug("Aggregator response {}, records {}", out, aggregator.getOutputCount());
+ switch(out){
+ case CLEANUP_AND_RETURN:
+ container.zeroVectors();
+ aggregator.cleanup();
+ done = true;
+ return aggregator.getOutcome();
+ case RETURN_OUTCOME:
+ return aggregator.getOutcome();
+ case UPDATE_AGGREGATOR:
+ aggregator = null;
+ if(!createAggregator()){
+ return IterOutcome.STOP;
+ }
+ continue;
+ default:
+ throw new IllegalStateException(String.format("Unknown state %s.", out));
+ }
+ }
+
+ }
+
+ /**
+ * Creates a new Aggregator based on the current schema. If setup fails, this method is responsible for cleaning up
+ * and informing the context of the failure state, as well is informing the upstream operators.
+ *
+ * @return true if the aggregator was setup successfully. false if there was a failure.
+ */
+ private boolean createAggregator() {
+ logger.debug("Creating new aggregator.");
+ try{
+ this.aggregator = createAggregatorInternal();
+ return true;
+ }catch(SchemaChangeException | ClassTransformationException | IOException ex){
+ context.fail(ex);
+ container.clear();
+ incoming.kill();
+ return false;
+ }
+ }
+
+ private HashAggregator createAggregatorInternal() throws SchemaChangeException, ClassTransformationException, IOException{
+ CodeGenerator<HashAggregator> top = CodeGenerator.get(HashAggregator.TEMPLATE_DEFINITION, context.getFunctionRegistry());
+ ClassGenerator<HashAggregator> cg = top.getRoot();
+ ClassGenerator<HashAggregator> cgInner = cg.getInnerGenerator("BatchHolder");
+
+ container.clear();
+ List<VectorAllocator> keyAllocators = Lists.newArrayList();
+ List<VectorAllocator> valueAllocators = Lists.newArrayList();
+
+ aggrExprs = new LogicalExpression[popConfig.getAggrExprs().length];
+ groupByOutFieldIds = new TypedFieldId[popConfig.getGroupByExprs().length];
+ aggrOutFieldIds = new TypedFieldId[popConfig.getAggrExprs().length];
+
+ ErrorCollector collector = new ErrorCollectorImpl();
+
+ int i;
+
+ for(i = 0; i < popConfig.getGroupByExprs().length; i++) {
+ NamedExpression ne = popConfig.getGroupByExprs()[i];
+ final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry() );
+ if(expr == null) continue;
+
+ final MaterializedField outputField = MaterializedField.create(ne.getRef(), expr.getMajorType());
+ ValueVector vv = TypeHelper.getNewVector(outputField, context.getAllocator());
+ keyAllocators.add(VectorAllocator.getAllocator(vv, 50));
+
+ // add this group-by vector to the output container
+ groupByOutFieldIds[i] = container.add(vv);
+ }
+
+ for(i = 0; i < aggrExprs.length; i++){
+ NamedExpression ne = popConfig.getAggrExprs()[i];
+ final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry() );
+
+ if(collector.hasErrors()) throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
+
+ if(expr == null) continue;
+
+ final MaterializedField outputField = MaterializedField.create(ne.getRef(), expr.getMajorType());
+ ValueVector vv = TypeHelper.getNewVector(outputField, context.getAllocator());
+ valueAllocators.add(VectorAllocator.getAllocator(vv, 50));
+ aggrOutFieldIds[i] = container.add(vv);
+
+ aggrExprs[i] = new ValueVectorWriteExpression(aggrOutFieldIds[i], expr, true);
+ }
+
+ setupUpdateAggrValues(cgInner);
+ setupGetIndex(cg);
+ cg.getBlock("resetValues")._return(JExpr.TRUE);
+
+ container.buildSchema(SelectionVectorMode.NONE);
+ HashAggregator agg = context.getImplementationClass(top);
+
+ agg.setup(popConfig, context, incoming, this,
+ aggrExprs,
+ cgInner.getWorkspaceTypes(),
+ groupByOutFieldIds,
+ keyAllocators.toArray(new VectorAllocator[keyAllocators.size()]),
+ valueAllocators.toArray(new VectorAllocator[valueAllocators.size()]));
+
+ return agg;
+ }
+
+
+ private void setupUpdateAggrValues(ClassGenerator<HashAggregator> cg) {
+ cg.setMappingSet(UpdateAggrValuesMapping);
+
+ for (LogicalExpression aggr : aggrExprs) {
+ HoldingContainer hc = cg.addExpr(aggr);
+ cg.getBlock(BlockType.EVAL)._if(hc.getValue().eq(JExpr.lit(0)))._then()._return(JExpr.FALSE);
+ }
+
+ cg.getBlock(BlockType.EVAL)._return(JExpr.TRUE);
+ }
+
+ private void setupGetIndex(ClassGenerator<HashAggregator> cg){
+ switch(incoming.getSchema().getSelectionVectorMode()){
+ case FOUR_BYTE: {
+ JVar var = cg.declareClassField("sv4_", cg.getModel()._ref(SelectionVector4.class));
+ cg.getBlock("doSetup").assign(var, JExpr.direct("incoming").invoke("getSelectionVector4"));
+ cg.getBlock("getVectorIndex")._return(var.invoke("get").arg(JExpr.direct("recordIndex")));;
+ return;
+ }
+ case NONE: {
+ cg.getBlock("getVectorIndex")._return(JExpr.direct("recordIndex"));;
+ return;
+ }
+ case TWO_BYTE: {
+ JVar var = cg.declareClassField("sv2_", cg.getModel()._ref(SelectionVector2.class));
+ cg.getBlock("doSetup").assign(var, JExpr.direct("incoming").invoke("getSelectionVector2"));
+ cg.getBlock("getVectorIndex")._return(var.invoke("getIndex").arg(JExpr.direct("recordIndex")));;
+ return;
+ }
+
+ default:
+ throw new IllegalStateException();
+
+ }
+
+ }
+
+
+ @Override
+ protected void killIncoming() {
+ incoming.kill();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatchCreator.java
new file mode 100644
index 0000000..5e0167e
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggBatchCreator.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.aggregate;
+
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.config.HashAggregate;
+import org.apache.drill.exec.physical.impl.BatchCreator;
+import org.apache.drill.exec.physical.impl.aggregate.HashAggBatch;
+import org.apache.drill.exec.record.RecordBatch;
+
+import com.google.common.base.Preconditions;
+
+public class HashAggBatchCreator implements BatchCreator<HashAggregate>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HashAggBatchCreator.class);
+
+ @Override
+ public RecordBatch getBatch(FragmentContext context, HashAggregate config, List<RecordBatch> children) throws ExecutionSetupException {
+ Preconditions.checkArgument(children.size() == 1);
+ return new HashAggBatch(config, children.iterator().next(), context);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
new file mode 100644
index 0000000..3109124
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.java
@@ -0,0 +1,417 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.aggregate;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import javax.inject.Named;
+
+import org.apache.drill.common.expression.ErrorCollector;
+import org.apache.drill.common.expression.ErrorCollectorImpl;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.NamedExpression;
+import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.expr.holders.IntHolder;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.config.HashAggregate;
+import org.apache.drill.exec.physical.impl.common.ChainedHashTable;
+import org.apache.drill.exec.physical.impl.common.HashTable;
+import org.apache.drill.exec.physical.impl.common.HashTableConfig;
+import org.apache.drill.exec.record.BatchSchema;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.RecordBatch.IterOutcome;
+import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.record.VectorContainer;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.allocator.VectorAllocator;
+import org.apache.drill.exec.compile.sig.RuntimeOverridden;
+import org.apache.drill.exec.vector.BigIntVector;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+
+import com.google.common.collect.Lists;
+
+public abstract class HashAggTemplate implements HashAggregator {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HashAggregator.class);
+
+ private static final boolean EXTRA_DEBUG = false;
+ private static final String TOO_BIG_ERROR = "Couldn't add value to an empty batch. This likely means that a single value is too long for a varlen field.";
+ private boolean first = true;
+ private boolean newSchema = false;
+ private int underlyingIndex = 0;
+ private int currentIndex = 0;
+ private IterOutcome outcome;
+ private int outputCount = 0;
+ private int numGroupedRecords = 0;
+ private RecordBatch incoming;
+ private BatchSchema schema;
+ private RecordBatch outgoing;
+ private VectorAllocator[] keyAllocators;
+ private VectorAllocator[] valueAllocators;
+ private FragmentContext context;
+
+ private HashAggregate hashAggrConfig;
+ private HashTable htable;
+ private ArrayList<BatchHolder> batchHolders;
+ private IntHolder htIdxHolder; // holder for the Hashtable's internal index returned by put()
+
+ List<VectorAllocator> wsAllocators = Lists.newArrayList(); // allocators for the workspace vectors
+ ErrorCollector collector = new ErrorCollectorImpl();
+
+ private MaterializedField[] materializedValueFields;
+ private boolean allFlushed = false;
+
+ public class BatchHolder {
+
+ private VectorContainer aggrValuesContainer; // container for aggr values (workspace variables)
+ int maxOccupiedIdx = 0;
+
+ private BatchHolder() {
+
+ aggrValuesContainer = new VectorContainer();
+
+ ValueVector vector ;
+
+ for(int i = 0; i < materializedValueFields.length; i++) {
+ MaterializedField outputField = materializedValueFields[i];
+ // Create a type-specific ValueVector for this value
+ vector = TypeHelper.getNewVector(outputField, context.getAllocator()) ;
+ VectorAllocator.getAllocator(vector, 50 /* avg. width */).alloc(HashTable.BATCH_SIZE) ;
+
+ aggrValuesContainer.add(vector) ;
+ }
+
+ }
+
+ private boolean updateAggrValues(int incomingRowIdx, int idxWithinBatch) {
+ updateAggrValuesInternal(incomingRowIdx, idxWithinBatch);
+ maxOccupiedIdx = Math.max(maxOccupiedIdx, idxWithinBatch);
+ return true;
+ }
+
+ private void setup(int idx) {
+ setupInterior(incoming, outgoing, aggrValuesContainer);
+ }
+
+ private boolean outputValues() {
+ for (int i = 0; i <= maxOccupiedIdx; i++) {
+ if (outputRecordValues(i, outputCount) ) {
+ if (EXTRA_DEBUG) logger.debug("Outputting values to {}", outputCount) ;
+ outputCount++;
+ } else {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ // Code-generated methods (implemented in HashAggBatch)
+
+ @RuntimeOverridden
+ public void setupInterior(@Named("incoming") RecordBatch incoming, @Named("outgoing") RecordBatch outgoing, @Named("aggrValuesContainer") VectorContainer aggrValuesContainer) {}
+
+ @RuntimeOverridden
+ public void updateAggrValuesInternal(@Named("incomingRowIdx") int incomingRowIdx, @Named("htRowIdx") int htRowIdx) {}
+
+ @RuntimeOverridden
+ public boolean outputRecordValues(@Named("htRowIdx") int htRowIdx, @Named("outRowIdx") int outRowIdx) {return true;}
+ }
+
+
+ @Override
+ public void setup(HashAggregate hashAggrConfig, FragmentContext context, RecordBatch incoming, RecordBatch outgoing,
+ LogicalExpression[] valueExprs,
+ List<TypedFieldId> valueFieldIds,
+ TypedFieldId[] groupByOutFieldIds,
+ VectorAllocator[] keyAllocators, VectorAllocator[] valueAllocators)
+ throws SchemaChangeException, ClassTransformationException, IOException {
+
+ if (valueFieldIds.size() < valueExprs.length) throw new IllegalArgumentException("Wrong number of workspace variables.");
+
+ this.context = context;
+ this.incoming = incoming;
+ this.schema = incoming.getSchema();
+ this.keyAllocators = keyAllocators;
+ this.valueAllocators = valueAllocators;
+ this.outgoing = outgoing;
+
+ this.hashAggrConfig = hashAggrConfig;
+
+ // currently, hash aggregation is only applicable if there are group-by expressions.
+ // For non-grouped (a.k.a Plain) aggregations that don't involve DISTINCT, there is no
+ // need to create hash table. However, for plain aggregations with DISTINCT ..
+ // e.g SELECT COUNT(DISTINCT a1) FROM t1 ;
+ // we need to build a hash table on the aggregation column a1.
+ // TODO: This functionality will be added later.
+ if (hashAggrConfig.getGroupByExprs().length == 0) {
+ throw new IllegalArgumentException("Currently, hash aggregation is only applicable if there are group-by expressions.");
+ }
+
+ this.htIdxHolder = new IntHolder();
+ materializedValueFields = new MaterializedField[valueFieldIds.size()];
+
+ int i = 0;
+ FieldReference ref = new FieldReference("dummy", ExpressionPosition.UNKNOWN, valueFieldIds.get(0).getType());
+ for (TypedFieldId id : valueFieldIds) {
+ materializedValueFields[i++] = MaterializedField.create(ref, id.getType());
+ }
+
+ ChainedHashTable ht = new ChainedHashTable(hashAggrConfig.getHtConfig(), context, incoming, null /* no incoming probe */, outgoing) ;
+ this.htable = ht.createAndSetupHashTable(groupByOutFieldIds) ;
+
+ batchHolders = new ArrayList<BatchHolder>();
+ addBatchHolder();
+
+ doSetup(incoming);
+ }
+
+ @Override
+ public AggOutcome doWork() {
+ try{
+ // Note: Keeping the outer and inner try blocks here to maintain some similarity with
+ // StreamingAggregate which does somethings conditionally in the outer try block.
+ // In the future HashAggregate may also need to perform some actions conditionally
+ // in the outer try block.
+
+ outside: while(true) {
+ // loop through existing records, aggregating the values as necessary.
+ for (; underlyingIndex < incoming.getRecordCount(); incIndex()) {
+ if(EXTRA_DEBUG) logger.debug("Doing loop with values underlying {}, current {}", underlyingIndex, currentIndex);
+ checkGroupAndAggrValues(currentIndex);
+ }
+
+ try{
+
+ while(true){
+ IterOutcome out = incoming.next();
+ if(EXTRA_DEBUG) logger.debug("Received IterOutcome of {}", out);
+ switch(out){
+ case NOT_YET:
+ this.outcome = out;
+ return AggOutcome.RETURN_OUTCOME;
+
+ case OK_NEW_SCHEMA:
+ if(EXTRA_DEBUG) logger.debug("Received new schema. Batch has {} records.", incoming.getRecordCount());
+ newSchema = true;
+
+ // TODO: new schema case needs to be handled appropriately
+ return AggOutcome.UPDATE_AGGREGATOR;
+
+ case OK:
+ resetIndex();
+ if(incoming.getRecordCount() == 0){
+ continue;
+ } else {
+ checkGroupAndAggrValues(currentIndex);
+ incIndex();
+
+ if(EXTRA_DEBUG) logger.debug("Continuing outside loop");
+ continue outside;
+ }
+
+ case NONE:
+ outcome = out;
+ outputKeysAndValues() ;
+
+ return setOkAndReturn();
+
+ case STOP:
+ default:
+ outcome = out;
+ return AggOutcome.CLEANUP_AND_RETURN;
+ }
+ }
+
+ } finally {
+ // placeholder...
+ }
+ }
+ } finally{
+ if(first) first = !first;
+ }
+ }
+
+ private void allocateOutgoing() {
+ for (VectorAllocator a : keyAllocators) {
+ if(EXTRA_DEBUG) logger.debug("Outgoing batch: Allocating {} with {} records.", a, numGroupedRecords);
+ a.alloc(numGroupedRecords);
+ }
+
+ for (VectorAllocator a : valueAllocators) {
+ if(EXTRA_DEBUG) logger.debug("Outgoing batch: Allocating {} with {} records.", a, numGroupedRecords);
+ a.alloc(numGroupedRecords);
+ }
+ }
+
+ @Override
+ public IterOutcome getOutcome() {
+ return outcome;
+ }
+
+ @Override
+ public int getOutputCount() {
+ return outputCount;
+ }
+
+ @Override
+ public void cleanup(){
+ htable.clear();
+ htable = null;
+ htIdxHolder = null;
+ materializedValueFields = null;
+ batchHolders.clear();
+ batchHolders = null;
+ }
+
+ private AggOutcome tooBigFailure(){
+ context.fail(new Exception(TOO_BIG_ERROR));
+ this.outcome = IterOutcome.STOP;
+ return AggOutcome.CLEANUP_AND_RETURN;
+ }
+
+ private final AggOutcome setOkAndReturn(){
+ if(first){
+ this.outcome = IterOutcome.OK_NEW_SCHEMA;
+ }else{
+ this.outcome = IterOutcome.OK;
+ }
+ for(VectorWrapper<?> v : outgoing){
+ v.getValueVector().getMutator().setValueCount(outputCount);
+ }
+ return AggOutcome.RETURN_OUTCOME;
+ }
+
+ private final void incIndex(){
+ underlyingIndex++;
+ if(underlyingIndex >= incoming.getRecordCount()){
+ currentIndex = Integer.MAX_VALUE;
+ return;
+ }
+ currentIndex = getVectorIndex(underlyingIndex);
+ }
+
+ private final void resetIndex(){
+ underlyingIndex = -1;
+ incIndex();
+ }
+
+ private void addBatchHolder() {
+ BatchHolder bh = new BatchHolder();
+ batchHolders.add(bh);
+
+ if (EXTRA_DEBUG) logger.debug("HashAggregate: Added new batch; num batches = {}.", batchHolders.size());
+
+ int batchIdx = batchHolders.size() - 1;
+ bh.setup(batchIdx);
+ }
+
+ private boolean outputKeysAndValues() {
+
+ allocateOutgoing();
+
+ this.htable.outputKeys();
+
+ for (BatchHolder bh : batchHolders) {
+ if (! bh.outputValues() ) {
+ return false;
+ }
+ }
+
+ allFlushed = true ;
+ return true;
+ }
+
+ public boolean allFlushed() {
+ return allFlushed;
+ }
+
+ // Check if a group is present in the hash table; if not, insert it in the hash table.
+ // The htIdxHolder contains the index of the group in the hash table container; this same
+ // index is also used for the aggregation values maintained by the hash aggregate.
+ private boolean checkGroupAndAggrValues(int incomingRowIdx) {
+ if (incomingRowIdx < 0) {
+ throw new IllegalArgumentException("Invalid incoming row index.");
+ }
+
+ /** for debugging
+ Object tmp = (incoming).getValueAccessorById(0, BigIntVector.class).getValueVector();
+ BigIntVector vv0 = null;
+ BigIntHolder holder = null;
+
+ if (tmp != null) {
+ vv0 = ((BigIntVector) tmp);
+ holder = new BigIntHolder();
+ holder.value = vv0.getAccessor().get(incomingRowIdx) ;
+ }
+ */
+
+ HashTable.PutStatus putStatus = htable.put(incomingRowIdx, htIdxHolder) ;
+
+ if (putStatus != HashTable.PutStatus.PUT_FAILED) {
+ int currentIdx = htIdxHolder.value;
+
+ // get the batch index and index within the batch
+ if (currentIdx >= batchHolders.size() * HashTable.BATCH_SIZE) {
+ addBatchHolder();
+ }
+ BatchHolder bh = batchHolders.get( (currentIdx >>> 16) & HashTable.BATCH_MASK);
+ int idxWithinBatch = currentIdx & HashTable.BATCH_MASK;
+
+ if (putStatus == HashTable.PutStatus.KEY_PRESENT) {
+ if (EXTRA_DEBUG) logger.debug("Group-by key already present in hash table, updating the aggregate values");
+
+ // debugging
+ //if (holder.value == 100018 || holder.value == 100021) {
+ // logger.debug("group-by key = {} already present at hash table index = {}", holder.value, currentIdx) ;
+ //}
+
+ }
+ else if (putStatus == HashTable.PutStatus.KEY_ADDED) {
+ if (EXTRA_DEBUG) logger.debug("Group-by key was added to hash table, inserting new aggregate values") ;
+
+ // debugging
+ // if (holder.value == 100018 || holder.value == 100021) {
+ // logger.debug("group-by key = {} added at hash table index = {}", holder.value, currentIdx) ;
+ //}
+ }
+
+ if (bh.updateAggrValues(incomingRowIdx, idxWithinBatch)) {
+ numGroupedRecords++;
+ return true;
+ }
+
+ }
+
+ return false;
+ }
+
+ // Code-generated methods (implemented in HashAggBatch)
+ public abstract void doSetup(@Named("incoming") RecordBatch incoming);
+ public abstract int getVectorIndex(@Named("recordIndex") int recordIndex);
+ public abstract boolean resetValues();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java
new file mode 100644
index 0000000..b23dbee
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/HashAggregator.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.aggregate;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Collection;
+
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.exec.compile.TemplateClassDefinition;
+import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.config.HashAggregate;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.RecordBatch.IterOutcome;
+import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.vector.allocator.VectorAllocator;
+
+public interface HashAggregator {
+
+ public static TemplateClassDefinition<HashAggregator> TEMPLATE_DEFINITION = new TemplateClassDefinition<HashAggregator>(HashAggregator.class, HashAggTemplate.class);
+
+ public static enum AggOutcome {
+ RETURN_OUTCOME, CLEANUP_AND_RETURN, UPDATE_AGGREGATOR
+ }
+
+ public abstract void setup(HashAggregate hashAggrConfig, FragmentContext context, RecordBatch incoming,
+ RecordBatch outgoing, LogicalExpression[] valueExprs,
+ List<TypedFieldId> valueFieldIds,
+ TypedFieldId[] keyFieldIds,
+ VectorAllocator[] keyAllocators, VectorAllocator[] valueAllocators)
+ throws SchemaChangeException, IOException, ClassTransformationException;
+
+ public abstract IterOutcome getOutcome();
+
+ public abstract int getOutputCount();
+
+ public abstract AggOutcome doWork();
+
+ public abstract void cleanup();
+
+ public abstract boolean allFlushed();
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
new file mode 100644
index 0000000..fccdbd6
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatch.java
@@ -0,0 +1,324 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.aggregate;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.drill.common.expression.ErrorCollector;
+import org.apache.drill.common.expression.ErrorCollectorImpl;
+import org.apache.drill.common.expression.ExpressionPosition;
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.logical.data.NamedExpression;
+import org.apache.drill.exec.compile.sig.GeneratorMapping;
+import org.apache.drill.exec.compile.sig.MappingSet;
+import org.apache.drill.exec.exception.ClassTransformationException;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.expr.ClassGenerator;
+import org.apache.drill.exec.expr.ClassGenerator.BlockType;
+import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
+import org.apache.drill.exec.expr.CodeGenerator;
+import org.apache.drill.exec.expr.ExpressionTreeMaterializer;
+import org.apache.drill.exec.expr.HoldingContainerExpression;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.expr.ValueVectorWriteExpression;
+import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.config.StreamingAggregate;
+import org.apache.drill.exec.physical.impl.aggregate.StreamingAggregator.AggOutcome;
+import org.apache.drill.exec.record.AbstractRecordBatch;
+import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.record.selection.SelectionVector2;
+import org.apache.drill.exec.record.selection.SelectionVector4;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.allocator.VectorAllocator;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.sun.codemodel.JExpr;
+import com.sun.codemodel.JVar;
+
+public class StreamingAggBatch extends AbstractRecordBatch<StreamingAggregate> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StreamingAggBatch.class);
+
+ private StreamingAggregator aggregator;
+ private final RecordBatch incoming;
+ private boolean done = false;
+
+ public StreamingAggBatch(StreamingAggregate popConfig, RecordBatch incoming, FragmentContext context) {
+ super(popConfig, context);
+ this.incoming = incoming;
+ }
+
+ @Override
+ public int getRecordCount() {
+ if(done) return 0;
+ return aggregator.getOutputCount();
+ }
+
+ @Override
+ public IterOutcome next() {
+
+ // this is only called on the first batch. Beyond this, the aggregator manages batches.
+ if (aggregator == null) {
+ IterOutcome outcome = incoming.next();
+ logger.debug("Next outcome of {}", outcome);
+ switch (outcome) {
+ case NONE:
+ case NOT_YET:
+ case STOP:
+ return outcome;
+ case OK_NEW_SCHEMA:
+ if (!createAggregator()){
+ done = true;
+ return IterOutcome.STOP;
+ }
+ break;
+ case OK:
+ throw new IllegalStateException("You should never get a first batch without a new schema");
+ default:
+ throw new IllegalStateException(String.format("unknown outcome %s", outcome));
+ }
+ }
+
+ while(true){
+ AggOutcome out = aggregator.doWork();
+ logger.debug("Aggregator response {}, records {}", out, aggregator.getOutputCount());
+ switch(out){
+ case CLEANUP_AND_RETURN:
+ container.zeroVectors();
+ done = true;
+ return aggregator.getOutcome();
+ case RETURN_OUTCOME:
+ return aggregator.getOutcome();
+ case UPDATE_AGGREGATOR:
+ aggregator = null;
+ if(!createAggregator()){
+ return IterOutcome.STOP;
+ }
+ continue;
+ default:
+ throw new IllegalStateException(String.format("Unknown state %s.", out));
+ }
+ }
+
+ }
+
+
+
+ /**
+ * Creates a new Aggregator based on the current schema. If setup fails, this method is responsible for cleaning up
+ * and informing the context of the failure state, as well is informing the upstream operators.
+ *
+ * @return true if the aggregator was setup successfully. false if there was a failure.
+ */
+ private boolean createAggregator() {
+ logger.debug("Creating new aggregator.");
+ try{
+ this.aggregator = createAggregatorInternal();
+ return true;
+ }catch(SchemaChangeException | ClassTransformationException | IOException ex){
+ context.fail(ex);
+ container.clear();
+ incoming.kill();
+ return false;
+ }
+ }
+
+
+
+
+ private StreamingAggregator createAggregatorInternal() throws SchemaChangeException, ClassTransformationException, IOException{
+ ClassGenerator<StreamingAggregator> cg = CodeGenerator.getRoot(StreamingAggTemplate.TEMPLATE_DEFINITION, context.getFunctionRegistry());
+ container.clear();
+ List<VectorAllocator> allocators = Lists.newArrayList();
+
+ LogicalExpression[] keyExprs = new LogicalExpression[popConfig.getKeys().length];
+ LogicalExpression[] valueExprs = new LogicalExpression[popConfig.getExprs().length];
+ TypedFieldId[] keyOutputIds = new TypedFieldId[popConfig.getKeys().length];
+
+ ErrorCollector collector = new ErrorCollectorImpl();
+
+ for(int i =0; i < keyExprs.length; i++){
+ NamedExpression ne = popConfig.getKeys()[i];
+ final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector,context.getFunctionRegistry() );
+ if(expr == null) continue;
+ keyExprs[i] = expr;
+ final MaterializedField outputField = MaterializedField.create(ne.getRef(), expr.getMajorType());
+ ValueVector vector = TypeHelper.getNewVector(outputField, context.getAllocator());
+ allocators.add(VectorAllocator.getAllocator(vector, 50));
+ keyOutputIds[i] = container.add(vector);
+ }
+
+ for(int i =0; i < valueExprs.length; i++){
+ NamedExpression ne = popConfig.getExprs()[i];
+ final LogicalExpression expr = ExpressionTreeMaterializer.materialize(ne.getExpr(), incoming, collector, context.getFunctionRegistry());
+ if(expr == null) continue;
+
+ final MaterializedField outputField = MaterializedField.create(ne.getRef(), expr.getMajorType());
+ ValueVector vector = TypeHelper.getNewVector(outputField, context.getAllocator());
+ allocators.add(VectorAllocator.getAllocator(vector, 50));
+ TypedFieldId id = container.add(vector);
+ valueExprs[i] = new ValueVectorWriteExpression(id, expr, true);
+ }
+
+ if(collector.hasErrors()) throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
+
+ setupIsSame(cg, keyExprs);
+ setupIsSameApart(cg, keyExprs);
+ addRecordValues(cg, valueExprs);
+ outputRecordKeys(cg, keyOutputIds, keyExprs);
+ outputRecordKeysPrev(cg, keyOutputIds, keyExprs);
+
+ cg.getBlock("resetValues")._return(JExpr.TRUE);
+ getIndex(cg);
+
+ container.buildSchema(SelectionVectorMode.NONE);
+ StreamingAggregator agg = context.getImplementationClass(cg);
+ agg.setup(context, incoming, this, allocators.toArray(new VectorAllocator[allocators.size()]));
+ return agg;
+ }
+
+
+
+ private final GeneratorMapping IS_SAME = GeneratorMapping.create("setupInterior", "isSame", null, null);
+ private final MappingSet IS_SAME_I1 = new MappingSet("index1", null, IS_SAME, IS_SAME);
+ private final MappingSet IS_SAME_I2 = new MappingSet("index2", null, IS_SAME, IS_SAME);
+
+ private void setupIsSame(ClassGenerator<StreamingAggregator> cg, LogicalExpression[] keyExprs){
+ cg.setMappingSet(IS_SAME_I1);
+ for(LogicalExpression expr : keyExprs){
+ // first, we rewrite the evaluation stack for each side of the comparison.
+ cg.setMappingSet(IS_SAME_I1);
+ HoldingContainer first = cg.addExpr(expr, false);
+ cg.setMappingSet(IS_SAME_I2);
+ HoldingContainer second = cg.addExpr(expr, false);
+
+ LogicalExpression fh = FunctionGenerationHelper.getComparator(first, second, context.getFunctionRegistry());
+ HoldingContainer out = cg.addExpr(fh, false);
+ cg.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)))._then()._return(JExpr.FALSE);
+ }
+ cg.getEvalBlock()._return(JExpr.TRUE);
+ }
+
+ private final GeneratorMapping IS_SAME_PREV_INTERNAL_BATCH_READ = GeneratorMapping.create("isSamePrev", "isSamePrev", null, null); // the internal batch changes each time so we need to redo setup.
+ private final GeneratorMapping IS_SAME_PREV = GeneratorMapping.create("setupInterior", "isSamePrev", null, null);
+ private final MappingSet ISA_B1 = new MappingSet("b1Index", null, "b1", null, IS_SAME_PREV_INTERNAL_BATCH_READ, IS_SAME_PREV_INTERNAL_BATCH_READ);
+ private final MappingSet ISA_B2 = new MappingSet("b2Index", null, "incoming", null, IS_SAME_PREV, IS_SAME_PREV);
+
+ private void setupIsSameApart(ClassGenerator<StreamingAggregator> cg, LogicalExpression[] keyExprs){
+ cg.setMappingSet(ISA_B1);
+ for(LogicalExpression expr : keyExprs){
+ // first, we rewrite the evaluation stack for each side of the comparison.
+ cg.setMappingSet(ISA_B1);
+ HoldingContainer first = cg.addExpr(expr, false);
+ cg.setMappingSet(ISA_B2);
+ HoldingContainer second = cg.addExpr(expr, false);
+
+ LogicalExpression fh = FunctionGenerationHelper.getComparator(first, second, context.getFunctionRegistry());
+ HoldingContainer out = cg.addExpr(fh, false);
+ cg.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)))._then()._return(JExpr.FALSE);
+ }
+ cg.getEvalBlock()._return(JExpr.TRUE);
+ }
+
+ private final GeneratorMapping EVAL_INSIDE = GeneratorMapping.create("setupInterior", "addRecord", null, null);
+ private final GeneratorMapping EVAL_OUTSIDE = GeneratorMapping.create("setupInterior", "outputRecordValues", "resetValues", "cleanup");
+ private final MappingSet EVAL = new MappingSet("index", "outIndex", "incoming", "outgoing", EVAL_INSIDE, EVAL_OUTSIDE, EVAL_INSIDE);
+
+ private void addRecordValues(ClassGenerator<StreamingAggregator> cg, LogicalExpression[] valueExprs){
+ cg.setMappingSet(EVAL);
+ for(LogicalExpression ex : valueExprs){
+ HoldingContainer hc = cg.addExpr(ex);
+ cg.getBlock(BlockType.EVAL)._if(hc.getValue().eq(JExpr.lit(0)))._then()._return(JExpr.FALSE);
+ }
+ cg.getBlock(BlockType.EVAL)._return(JExpr.TRUE);
+ }
+
+ private final MappingSet RECORD_KEYS = new MappingSet(GeneratorMapping.create("setupInterior", "outputRecordKeys", null, null));
+
+ private void outputRecordKeys(ClassGenerator<StreamingAggregator> cg, TypedFieldId[] keyOutputIds, LogicalExpression[] keyExprs){
+ cg.setMappingSet(RECORD_KEYS);
+ for(int i =0; i < keyExprs.length; i++){
+ HoldingContainer hc = cg.addExpr(new ValueVectorWriteExpression(keyOutputIds[i], keyExprs[i], true));
+ cg.getBlock(BlockType.EVAL)._if(hc.getValue().eq(JExpr.lit(0)))._then()._return(JExpr.FALSE);
+ }
+ cg.getBlock(BlockType.EVAL)._return(JExpr.TRUE);
+ }
+
+ private final GeneratorMapping PREVIOUS_KEYS_OUT = GeneratorMapping.create("setupInterior", "outputRecordKeysPrev", null, null);
+ private final MappingSet RECORD_KEYS_PREV_OUT = new MappingSet("previousIndex", "outIndex", "previous", "outgoing", PREVIOUS_KEYS_OUT, PREVIOUS_KEYS_OUT);
+
+ private final GeneratorMapping PREVIOUS_KEYS = GeneratorMapping.create("outputRecordKeysPrev", "outputRecordKeysPrev", null, null);
+ private final MappingSet RECORD_KEYS_PREV = new MappingSet("previousIndex", "outIndex", "previous", null, PREVIOUS_KEYS, PREVIOUS_KEYS);
+
+ private void outputRecordKeysPrev(ClassGenerator<StreamingAggregator> cg, TypedFieldId[] keyOutputIds, LogicalExpression[] keyExprs){
+ cg.setMappingSet(RECORD_KEYS_PREV);
+
+ for(int i =0; i < keyExprs.length; i++){
+ // IMPORTANT: there is an implicit assertion here that the TypedFieldIds for the previous batch and the current batch are the same. This is possible because InternalBatch guarantees this.
+ logger.debug("Writing out expr {}", keyExprs[i]);
+ cg.rotateBlock();
+ cg.setMappingSet(RECORD_KEYS_PREV);
+ HoldingContainer innerExpression = cg.addExpr(keyExprs[i], false);
+ cg.setMappingSet(RECORD_KEYS_PREV_OUT);
+ HoldingContainer outerExpression = cg.addExpr(new ValueVectorWriteExpression(keyOutputIds[i], new HoldingContainerExpression(innerExpression), true), false);
+ cg.getBlock(BlockType.EVAL)._if(outerExpression.getValue().eq(JExpr.lit(0)))._then()._return(JExpr.FALSE);
+
+ }
+ cg.getBlock(BlockType.EVAL)._return(JExpr.TRUE);
+ }
+
+ private void getIndex(ClassGenerator<StreamingAggregator> g){
+ switch(incoming.getSchema().getSelectionVectorMode()){
+ case FOUR_BYTE: {
+ JVar var = g.declareClassField("sv4_", g.getModel()._ref(SelectionVector4.class));
+ g.getBlock("setupInterior").assign(var, JExpr.direct("incoming").invoke("getSelectionVector4"));
+ g.getBlock("getVectorIndex")._return(var.invoke("get").arg(JExpr.direct("recordIndex")));;
+ return;
+ }
+ case NONE: {
+ g.getBlock("getVectorIndex")._return(JExpr.direct("recordIndex"));;
+ return;
+ }
+ case TWO_BYTE: {
+ JVar var = g.declareClassField("sv2_", g.getModel()._ref(SelectionVector2.class));
+ g.getBlock("setupInterior").assign(var, JExpr.direct("incoming").invoke("getSelectionVector2"));
+ g.getBlock("getVectorIndex")._return(var.invoke("get").arg(JExpr.direct("recordIndex")));;
+ return;
+ }
+
+ default:
+ throw new IllegalStateException();
+
+ }
+
+ }
+
+
+
+ @Override
+ protected void killIncoming() {
+ incoming.kill();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8d725f69/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatchCreator.java
new file mode 100644
index 0000000..96b71fe
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/aggregate/StreamingAggBatchCreator.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.impl.aggregate;
+
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.config.StreamingAggregate;
+import org.apache.drill.exec.physical.impl.BatchCreator;
+import org.apache.drill.exec.record.RecordBatch;
+
+import com.google.common.base.Preconditions;
+
+public class StreamingAggBatchCreator implements BatchCreator<StreamingAggregate>{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StreamingAggBatchCreator.class);
+
+ @Override
+ public RecordBatch getBatch(FragmentContext context, StreamingAggregate config, List<RecordBatch> children) throws ExecutionSetupException {
+ Preconditions.checkArgument(children.size() == 1);
+ return new StreamingAggBatch(config, children.iterator().next(), context);
+ }
+
+
+}
[05/12] git commit: Small updates to Hive UDF: fixes for aggregate
and constant checker to add errors to error collector. Register hive udfs
without descriptions using fully qualified name with underscores instead of
periods.
Posted by ja...@apache.org.
Small updates to Hive UDF: fixes for aggregate and constant checker to add errors to error collector. Register hive udfs without descriptions using fully qualified name with underscores instead of periods.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/693bd353
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/693bd353
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/693bd353
Branch: refs/heads/master
Commit: 693bd35391b9106df4f2795bdf0ee5a73adb27af
Parents: f8fbb5d
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sat Mar 22 09:02:04 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Sat Mar 22 09:02:04 2014 -0700
----------------------------------------------------------------------
.../common/expression/ErrorCollectorImpl.java | 1 +
.../common/expression/ExpressionFunction.java | 1 +
.../expression/ExpressionValidationError.java | 31 ----
.../ExpressionValidationException.java | 38 -----
.../common/expression/ExpressionValidator.java | 125 -----------------
.../expression/visitors/ConstantChecker.java | 68 +++++----
.../visitors/ExpressionValidationError.java | 31 ++++
.../visitors/ExpressionValidationException.java | 38 +++++
.../visitors/ExpressionValidator.java | 140 +++++++++++++++++++
.../apache/drill/common/util/PathScanner.java | 1 +
.../exec/expr/ExpressionTreeMaterializer.java | 2 +-
.../fn/DrillFunctionImplementationRegistry.java | 3 +-
.../fn/HiveFunctionImplementationRegistry.java | 32 ++---
13 files changed, 267 insertions(+), 244 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/693bd353/common/src/main/java/org/apache/drill/common/expression/ErrorCollectorImpl.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ErrorCollectorImpl.java b/common/src/main/java/org/apache/drill/common/expression/ErrorCollectorImpl.java
index 9e5747b..bd67be3 100644
--- a/common/src/main/java/org/apache/drill/common/expression/ErrorCollectorImpl.java
+++ b/common/src/main/java/org/apache/drill/common/expression/ErrorCollectorImpl.java
@@ -20,6 +20,7 @@ package org.apache.drill.common.expression;
import java.util.Arrays;
import java.util.List;
+import org.apache.drill.common.expression.visitors.ExpressionValidationError;
import org.apache.drill.common.types.TypeProtos.MajorType;
import com.google.common.base.Joiner;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/693bd353/common/src/main/java/org/apache/drill/common/expression/ExpressionFunction.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ExpressionFunction.java b/common/src/main/java/org/apache/drill/common/expression/ExpressionFunction.java
index 0a514fd..3fe0d4c 100644
--- a/common/src/main/java/org/apache/drill/common/expression/ExpressionFunction.java
+++ b/common/src/main/java/org/apache/drill/common/expression/ExpressionFunction.java
@@ -21,6 +21,7 @@ import java.lang.reflect.Constructor;
import java.lang.reflect.Type;
import java.util.List;
+import org.apache.drill.common.expression.visitors.ExpressionValidationException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/693bd353/common/src/main/java/org/apache/drill/common/expression/ExpressionValidationError.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ExpressionValidationError.java b/common/src/main/java/org/apache/drill/common/expression/ExpressionValidationError.java
deleted file mode 100644
index 01bc6be..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/ExpressionValidationError.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-public class ExpressionValidationError {
- String message;
-
- public ExpressionValidationError(String message) {
- this.message = message;
- }
-
- @Override
- public String toString() {
- return message;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/693bd353/common/src/main/java/org/apache/drill/common/expression/ExpressionValidationException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ExpressionValidationException.java b/common/src/main/java/org/apache/drill/common/expression/ExpressionValidationException.java
deleted file mode 100644
index 32e70df..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/ExpressionValidationException.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-public class ExpressionValidationException extends RuntimeException {
-
- public ExpressionValidationException() {
- super();
- }
-
- public ExpressionValidationException(String arg0, Throwable arg1) {
- super(arg0, arg1);
- }
-
- public ExpressionValidationException(String arg0) {
- super(arg0);
- }
-
- public ExpressionValidationException(Throwable arg0) {
- super(arg0);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/693bd353/common/src/main/java/org/apache/drill/common/expression/ExpressionValidator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ExpressionValidator.java b/common/src/main/java/org/apache/drill/common/expression/ExpressionValidator.java
deleted file mode 100644
index 3c7e132..0000000
--- a/common/src/main/java/org/apache/drill/common/expression/ExpressionValidator.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.expression;
-
-import org.apache.drill.common.expression.IfExpression.IfCondition;
-import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
-import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
-import org.apache.drill.common.expression.ValueExpressions.LongExpression;
-import org.apache.drill.common.expression.ValueExpressions.QuotedString;
-import org.apache.drill.common.expression.visitors.AggregateChecker;
-import org.apache.drill.common.expression.visitors.ConstantChecker;
-import org.apache.drill.common.expression.visitors.ExprVisitor;
-import org.apache.drill.common.types.TypeProtos.DataMode;
-import org.apache.drill.common.types.TypeProtos.MajorType;
-import org.apache.drill.common.types.TypeProtos.MinorType;
-
-public class ExpressionValidator implements ExprVisitor<Void, ErrorCollector, RuntimeException> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpressionValidator.class);
-
- @Override
- public Void visitFunctionCall(FunctionCall call, ErrorCollector errors) throws RuntimeException {
- throw new UnsupportedOperationException("FunctionCall is not expected here. "+
- "It should have been converted to FunctionHolderExpression in materialization");
- }
-
- @Override
- public Void visitFunctionHolderExpression(FunctionHolderExpression holder, ErrorCollector errors) throws RuntimeException {
- // make sure aggregate functions are not nested inside aggregate functions
- AggregateChecker.isAggregating(holder);
-
- // make sure arguments are constant if the function implementation expects constants for any arguments
- ConstantChecker.onlyIncludesConstants(holder);
-
- return null;
- }
-
- @Override
- public Void visitIfExpression(IfExpression ifExpr, ErrorCollector errors) throws RuntimeException {
- // confirm that all conditions are required boolean values.
- int i = 0;
- for (IfCondition c : ifExpr.conditions) {
- MajorType mt = c.condition.getMajorType();
- if (mt.getMode() != DataMode.REQUIRED || mt.getMinorType() != MinorType.BIT){
- errors.addGeneralError(c.condition.getPosition(),String.format(
- "Failure composing If Expression. All conditions must return a required value and be of type boolean. Condition %d was DatMode %s and Type %s.",
- i, mt.getMode(), mt.getMinorType()));
- }
- i++;
- }
-
- // confirm that all outcomes are the same type.
- final MajorType mt = ifExpr.elseExpression.getMajorType();
- i = 0;
- for (IfCondition c : ifExpr.conditions) {
- MajorType innerT = c.expression.getMajorType();
- if (
- (innerT.getMode() == DataMode.REPEATED && mt.getMode() != DataMode.REPEATED) || //
- (innerT.getMinorType() != mt.getMinorType())
- ) {
- errors.addGeneralError(c.condition.getPosition(),String.format(
- "Failure composing If Expression. All expressions must return the same MajorType as the else expression. The %d if condition returned type type %s but the else expression was of type %s",
- i, innerT, mt));
- }
- i++;
- }
- return null;
- }
-
- @Override
- public Void visitSchemaPath(SchemaPath path, ErrorCollector errors) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitIntConstant(ValueExpressions.IntExpression intExpr, ErrorCollector value) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitFloatConstant(ValueExpressions.FloatExpression fExpr, ErrorCollector value) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitLongConstant(LongExpression intExpr, ErrorCollector errors) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitDoubleConstant(DoubleExpression dExpr, ErrorCollector errors) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitBooleanConstant(BooleanExpression e, ErrorCollector errors) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitQuotedStringConstant(QuotedString e, ErrorCollector errors) throws RuntimeException {
- return null;
- }
-
- @Override
- public Void visitUnknown(LogicalExpression e, ErrorCollector value) throws RuntimeException {
- return null;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/693bd353/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java b/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
index fcb2489..1391c84 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
@@ -17,6 +17,7 @@
*/
package org.apache.drill.common.expression.visitors;
+import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.IfExpression;
@@ -25,88 +26,93 @@ import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
-import org.apache.drill.common.expression.ValueExpressions.LongExpression;
-import org.apache.drill.common.expression.ValueExpressions.IntExpression;
import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
+import org.apache.drill.common.expression.ValueExpressions.LongExpression;
import org.apache.drill.common.expression.ValueExpressions.QuotedString;
-public final class ConstantChecker extends SimpleExprVisitor<Boolean>{
-
+final class ConstantChecker implements ExprVisitor<Boolean, ErrorCollector, RuntimeException> {
+
+
private final static ConstantChecker INSTANCE = new ConstantChecker();
private ConstantChecker(){}
- public static boolean onlyIncludesConstants(LogicalExpression e){
- return e.accept(INSTANCE, null);
+ public static void checkConstants(LogicalExpression e, ErrorCollector errors){
+ e.accept(INSTANCE, errors);
}
@Override
- public Boolean visitFunctionCall(FunctionCall call) {
- throw new UnsupportedOperationException("FunctionCall is not expected here. "+
- "It should have been converted to FunctionHolderExpression in materialization");
+ public Boolean visitFunctionCall(FunctionCall call, ErrorCollector errors) {
+ throw new UnsupportedOperationException("FunctionCall is not expected here. "
+ + "It should have been converted to FunctionHolderExpression in materialization");
}
@Override
- public Boolean visitFunctionHolderExpression(FunctionHolderExpression holder) {
- for(int i=0; i<holder.args.size(); i++) {
- if (holder.argConstantOnly(i) && !holder.args.get(i).accept(this, null)) {
- StringBuilder sb = new StringBuilder();
- sb.append("Function '").append(holder.getName())
- .append("' expects constant input for argument number ").append(i);
- throw new UnsupportedOperationException(sb.toString());
+ public Boolean visitFunctionHolderExpression(FunctionHolderExpression holder, ErrorCollector errors) {
+ boolean allArgsAreConstant = true;
+ for (int i = 0; i < holder.args.size(); i++) {
+ boolean thisArgIsConstant = holder.args.get(i).accept(this, errors);
+ if(!thisArgIsConstant){
+ allArgsAreConstant = false;
+ if (holder.argConstantOnly(i)) {
+ errors.addGeneralError( //
+ holder.args.get(i).getPosition(), //
+ String.format("Function %s expects constant input for argument number %d", holder.getName(), i));
+ }
}
}
- return true;
+ return allArgsAreConstant;
}
@Override
- public Boolean visitIfExpression(IfExpression ifExpr) {
- for(IfCondition c : ifExpr.conditions){
- if(!c.condition.accept(this, null) || !c.expression.accept(this, null)) return false;
+ public Boolean visitIfExpression(IfExpression ifExpr, ErrorCollector errors) {
+ for (IfCondition c : ifExpr.conditions) {
+ if (!c.condition.accept(this, errors) || !c.expression.accept(this, errors))
+ return false;
}
- if(!ifExpr.elseExpression.accept(this, null)) return false;
+ if (!ifExpr.elseExpression.accept(this, errors)) return false;
return true;
}
@Override
- public Boolean visitSchemaPath(SchemaPath path) {
+ public Boolean visitSchemaPath(SchemaPath path, ErrorCollector errors) {
return false;
}
@Override
- public Boolean visitIntConstant(IntExpression intExpr) {
+ public Boolean visitIntConstant(IntExpression intExpr, ErrorCollector errors) {
return true;
}
@Override
- public Boolean visitFloatConstant(FloatExpression fExpr) {
+ public Boolean visitFloatConstant(FloatExpression fExpr, ErrorCollector errors) {
return true;
}
@Override
- public Boolean visitLongConstant(LongExpression intExpr) {
+ public Boolean visitLongConstant(LongExpression intExpr, ErrorCollector errors) {
return true;
}
@Override
- public Boolean visitDoubleConstant(DoubleExpression dExpr) {
+ public Boolean visitDoubleConstant(DoubleExpression dExpr, ErrorCollector errors) {
return true;
}
@Override
- public Boolean visitBooleanConstant(BooleanExpression e) {
+ public Boolean visitBooleanConstant(BooleanExpression e, ErrorCollector errors) {
return true;
}
@Override
- public Boolean visitQuotedStringConstant(QuotedString e) {
+ public Boolean visitQuotedStringConstant(QuotedString e, ErrorCollector errors) {
return true;
}
@Override
- public Boolean visitUnknown(LogicalExpression e, Void value) throws RuntimeException {
+ public Boolean visitUnknown(LogicalExpression e, ErrorCollector errors){
return false;
}
-
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/693bd353/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationError.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationError.java b/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationError.java
new file mode 100644
index 0000000..e905638
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationError.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.visitors;
+
+public class ExpressionValidationError {
+ String message;
+
+ public ExpressionValidationError(String message) {
+ this.message = message;
+ }
+
+ @Override
+ public String toString() {
+ return message;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/693bd353/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationException.java b/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationException.java
new file mode 100644
index 0000000..3d944da
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidationException.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.visitors;
+
+public class ExpressionValidationException extends RuntimeException {
+
+ public ExpressionValidationException() {
+ super();
+ }
+
+ public ExpressionValidationException(String arg0, Throwable arg1) {
+ super(arg0, arg1);
+ }
+
+ public ExpressionValidationException(String arg0) {
+ super(arg0);
+ }
+
+ public ExpressionValidationException(Throwable arg0) {
+ super(arg0);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/693bd353/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java b/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java
new file mode 100644
index 0000000..396517a
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression.visitors;
+
+import org.apache.drill.common.expression.ErrorCollector;
+import org.apache.drill.common.expression.FunctionCall;
+import org.apache.drill.common.expression.FunctionHolderExpression;
+import org.apache.drill.common.expression.IfExpression;
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.ValueExpressions;
+import org.apache.drill.common.expression.IfExpression.IfCondition;
+import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
+import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
+import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
+import org.apache.drill.common.expression.ValueExpressions.LongExpression;
+import org.apache.drill.common.expression.ValueExpressions.QuotedString;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+
+public class ExpressionValidator implements ExprVisitor<Void, ErrorCollector, RuntimeException> {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExpressionValidator.class);
+
+ @Override
+ public Void visitFunctionCall(FunctionCall call, ErrorCollector errors) throws RuntimeException {
+ // we throw an exception here because this is a fundamental operator programming problem as opposed to an expression
+ // problem. At this point in an expression's lifecycle, all function calls should have been converted into
+ // FunctionHolders.
+ throw new UnsupportedOperationException("FunctionCall is not expected here. "
+ + "It should have been converted to FunctionHolderExpression in materialization");
+ }
+
+ @Override
+ public Void visitFunctionHolderExpression(FunctionHolderExpression holder, ErrorCollector errors)
+ throws RuntimeException {
+ // make sure aggregate functions are not nested inside aggregate functions
+ AggregateChecker.isAggregating(holder);
+
+ // make sure arguments are constant if the function implementation expects constants for any arguments
+ ConstantChecker.checkConstants(holder, errors);
+
+ return null;
+ }
+
+ @Override
+ public Void visitIfExpression(IfExpression ifExpr, ErrorCollector errors) throws RuntimeException {
+ // confirm that all conditions are required boolean values.
+ int i = 0;
+ for (IfCondition c : ifExpr.conditions) {
+ MajorType mt = c.condition.getMajorType();
+ if (mt.getMode() != DataMode.REQUIRED || mt.getMinorType() != MinorType.BIT) {
+ errors
+ .addGeneralError(
+ c.condition.getPosition(),
+ String
+ .format(
+ "Failure composing If Expression. All conditions must return a required value and be of type boolean. Condition %d was DatMode %s and Type %s.",
+ i, mt.getMode(), mt.getMinorType()));
+ }
+ i++;
+ }
+
+ // confirm that all outcomes are the same type.
+ final MajorType mt = ifExpr.elseExpression.getMajorType();
+ i = 0;
+ for (IfCondition c : ifExpr.conditions) {
+ MajorType innerT = c.expression.getMajorType();
+ if ((innerT.getMode() == DataMode.REPEATED && mt.getMode() != DataMode.REPEATED) || //
+ (innerT.getMinorType() != mt.getMinorType())) {
+ errors
+ .addGeneralError(
+ c.condition.getPosition(),
+ String
+ .format(
+ "Failure composing If Expression. All expressions must return the same MajorType as the else expression. The %d if condition returned type type %s but the else expression was of type %s",
+ i, innerT, mt));
+ }
+ i++;
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitSchemaPath(SchemaPath path, ErrorCollector errors) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitIntConstant(ValueExpressions.IntExpression intExpr, ErrorCollector value) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitFloatConstant(ValueExpressions.FloatExpression fExpr, ErrorCollector value) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitLongConstant(LongExpression intExpr, ErrorCollector errors) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitDoubleConstant(DoubleExpression dExpr, ErrorCollector errors) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitBooleanConstant(BooleanExpression e, ErrorCollector errors) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitQuotedStringConstant(QuotedString e, ErrorCollector errors) throws RuntimeException {
+ return null;
+ }
+
+ @Override
+ public Void visitUnknown(LogicalExpression e, ErrorCollector value) throws RuntimeException {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/693bd353/common/src/main/java/org/apache/drill/common/util/PathScanner.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/PathScanner.java b/common/src/main/java/org/apache/drill/common/util/PathScanner.java
index 21aaf48..65dcf23 100644
--- a/common/src/main/java/org/apache/drill/common/util/PathScanner.java
+++ b/common/src/main/java/org/apache/drill/common/util/PathScanner.java
@@ -79,6 +79,7 @@ public class PathScanner {
Set<Class<? extends T>> classes = getReflections().getSubTypesOf(baseClass);
for(Iterator<Class<? extends T>> i = classes.iterator(); i.hasNext();){
Class<? extends T> c = i.next();
+ assert baseClass.isAssignableFrom(c);
if(Modifier.isAbstract(c.getModifiers())) i.remove();
}
return classes;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/693bd353/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
index 82f45c0..def54d9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
@@ -21,7 +21,6 @@ import java.util.List;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.ExpressionValidator;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.IfExpression;
@@ -36,6 +35,7 @@ import org.apache.drill.common.expression.ValueExpressions.IntExpression;
import org.apache.drill.common.expression.ValueExpressions.QuotedString;
import org.apache.drill.common.expression.fn.CastFunctions;
import org.apache.drill.common.expression.visitors.AbstractExprVisitor;
+import org.apache.drill.common.expression.visitors.ExpressionValidator;
import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/693bd353/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionImplementationRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionImplementationRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionImplementationRegistry.java
index c78895e..64ceef5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionImplementationRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/DrillFunctionImplementationRegistry.java
@@ -40,8 +40,7 @@ public class DrillFunctionImplementationRegistry {
if(holder != null){
// register handle for each name the function can be referred to
String[] names = holder.getRegisteredNames();
- for(String name : names)
- methods.put(name, holder);
+ for(String name : names) methods.put(name, holder);
}else{
logger.warn("Unable to initialize function for class {}", clazz.getName());
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/693bd353/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionImplementationRegistry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionImplementationRegistry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionImplementationRegistry.java
index 0c19a08..634b24f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionImplementationRegistry.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionImplementationRegistry.java
@@ -38,11 +38,11 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import com.google.common.collect.ArrayListMultimap;
public class HiveFunctionImplementationRegistry {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillFunctionImplementationRegistry.class);
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveFunctionImplementationRegistry.class);
private ArrayListMultimap<String, Class<? extends GenericUDF>> methodsGenericUDF = ArrayListMultimap.create();
private ArrayListMultimap<String, Class<? extends UDF>> methodsUDF = ArrayListMultimap.create();
- private HashSet<Class> nonDeterministicUDFs = new HashSet<>();
+ private HashSet<Class<?>> nonDeterministicUDFs = new HashSet<>();
/**
* Scan the classpath for implementation of GenericUDF/UDF interfaces,
@@ -62,20 +62,20 @@ public class HiveFunctionImplementationRegistry {
private <C,I> void register(Class<? extends I> clazz, ArrayListMultimap<String,Class<? extends I>> methods) {
Description desc = clazz.getAnnotation(Description.class);
- if (desc != null) {
- // if the function is non-deterministic add it to the list
- UDFType type = clazz.getAnnotation(UDFType.class);
- if (type != null && type.deterministic())
- nonDeterministicUDFs.add(clazz);
-
- String[] names = desc.name().split(",");
- for(int i=0; i<names.length; i++)
- names[i] = names[i].trim();
-
- for(int i=0; i<names.length;i++)
- methods.put(names[i], clazz);
- } else {
- logger.warn("Unable to initialize function for class {}", clazz.getName());
+ String[] names;
+ if(desc != null){
+ names = desc.name().split(",");
+ for(int i=0; i<names.length; i++) names[i] = names[i].trim();
+ }else{
+ names = new String[]{clazz.getName().replace('.', '_')};
+ }
+
+ UDFType type = clazz.getAnnotation(UDFType.class);
+ if (type != null && type.deterministic()) nonDeterministicUDFs.add(clazz);
+
+
+ for(int i=0; i<names.length;i++){
+ methods.put(names[i], clazz);
}
}