You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/08/02 05:03:07 UTC
[07/13] git commit: nullable int and bigint equality operators. fixed
some bugs in code gen for NULL_IF_NULL
nullable int and bigint equality operators. fixed some bugs in code gen for NULL_IF_NULL
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/13bc4d1c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/13bc4d1c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/13bc4d1c
Branch: refs/heads/master
Commit: 13bc4d1c5b338ed444e5ee882f8ec304255756a9
Parents: 787041e
Author: Steven Phillips <sp...@maprtech.com>
Authored: Wed Jul 31 19:21:08 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Thu Aug 1 19:24:43 2013 -0700
----------------------------------------------------------------------
.../drill/exec/expr/fn/FunctionHolder.java | 7 +-
.../fn/impl/ComparisonFunctionsNullable.java | 88 ++++++++++++
.../impl/project/ProjectorTemplate.java | 6 +-
.../impl/TestComparisonFunctionsNullable.java | 143 +++++++++++++++++++
.../functions/nullableBigIntEqual.json | 35 +++++
.../functions/nullableBigIntNotEqual.json | 35 +++++
.../resources/functions/nullableIntEqual.json | 35 +++++
.../functions/nullableIntNotEqual.json | 35 +++++
8 files changed, 378 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/13bc4d1c/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java
index d249f4d..1848628 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/FunctionHolder.java
@@ -110,7 +110,7 @@ public class FunctionHolder {
//g.getBlock().directStatement(String.format("//---- start of eval portion of %s function. ----//", functionName));
JBlock sub = new JBlock(true, true);
-
+ JBlock topSub = sub;
HoldingContainer out = null;
// add outside null handling if it is defined.
@@ -129,7 +129,7 @@ public class FunctionHolder {
if(e != null){
// if at least one expression must be checked, set up the conditional.
returnValue.type = returnValue.type.toBuilder().setMode(DataMode.OPTIONAL).build();
- out = g.declare(returnValue.type, false);
+ out = g.declare(returnValue.type);
e = e.eq(JExpr.lit(0));
JConditional jc = sub._if(e);
jc._then().assign(out.getIsSet(), JExpr.lit(0));
@@ -140,11 +140,12 @@ public class FunctionHolder {
if(out == null) out = g.declare(returnValue.type);
// add the subblock after the out declaration.
- g.getBlock().add(sub);
+ g.getBlock().add(topSub);
JVar internalOutput = sub.decl(JMod.FINAL, g.getHolderType(returnValue.type), returnValue.name, JExpr._new(g.getHolderType(returnValue.type)));
addProtectedBlock(g, sub, evalBody, inputVariables);
+ if (sub != topSub) sub.assign(internalOutput.ref("isSet"),JExpr.lit(1));// Assign null if NULL_IF_NULL mode
sub.assign(out.getHolder(), internalOutput);
return out;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/13bc4d1c/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctionsNullable.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctionsNullable.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctionsNullable.java
new file mode 100644
index 0000000..c71174e
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/ComparisonFunctionsNullable.java
@@ -0,0 +1,88 @@
+package org.apache.drill.exec.expr.fn.impl;
+
+import org.apache.drill.exec.expr.DrillFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.record.RecordBatch;
+import org.apache.drill.exec.vector.NullableBigIntHolder;
+import org.apache.drill.exec.vector.BitHolder;
+import org.apache.drill.exec.vector.NullableIntHolder;
+
+public class ComparisonFunctionsNullable {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComparisonFunctionsNullable.class);
+
+ private ComparisonFunctionsNullable() {}
+
+ @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ public static class NullableIntEqual implements DrillFunc {
+
+ @Param NullableIntHolder left;
+ @Param NullableIntHolder right;
+ @Output BitHolder out;
+
+ public void setup(RecordBatch b) {}
+
+ public void eval() {
+ if (left.isSet == 0 || right.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = (left.value == right.value) ? 1 : 0;
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ public static class NullableBigIntEqual implements DrillFunc {
+
+ @Param NullableBigIntHolder left;
+ @Param NullableBigIntHolder right;
+ @Output BitHolder out;
+
+ public void setup(RecordBatch b) {}
+
+ public void eval() {
+ if (left.isSet == 0 || right.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = (left.value == right.value) ? 1 : 0;
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "not equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ public static class NullableIntNotEqual implements DrillFunc {
+
+ @Param NullableIntHolder left;
+ @Param NullableIntHolder right;
+ @Output BitHolder out;
+
+ public void setup(RecordBatch b) {}
+
+ public void eval() {
+ if (left.isSet == 0 || right.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = (left.value != right.value) ? 1 : 0;
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "not equal", scope = FunctionTemplate.FunctionScope.SIMPLE)
+ public static class NullableBigIntNotEqual implements DrillFunc {
+
+ @Param NullableBigIntHolder left;
+ @Param NullableBigIntHolder right;
+ @Output BitHolder out;
+
+ public void setup(RecordBatch b) {}
+
+ public void eval() {
+ if (left.isSet == 0 || right.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = (left.value != right.value) ? 1 : 0;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/13bc4d1c/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java
index 36c6109..8d47678 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/project/ProjectorTemplate.java
@@ -40,13 +40,13 @@ public abstract class ProjectorTemplate implements Projector {
case NONE:
- for(TransferPair t : transfers){
- t.transfer();
- }
final int countN = recordCount;
for (int i = 0; i < countN; i++, firstOutputIndex++) {
doEval(i, firstOutputIndex);
}
+ for(TransferPair t : transfers){
+ t.transfer();
+ }
return recordCount;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/13bc4d1c/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestComparisonFunctionsNullable.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestComparisonFunctionsNullable.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestComparisonFunctionsNullable.java
new file mode 100644
index 0000000..ceb3c83
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestComparisonFunctionsNullable.java
@@ -0,0 +1,143 @@
+package org.apache.drill.exec.physical.impl;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import com.yammer.metrics.MetricRegistry;
+import mockit.Injectable;
+import mockit.NonStrictExpectations;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.physical.PhysicalPlan;
+import org.apache.drill.exec.physical.base.FragmentRoot;
+import org.apache.drill.exec.planner.PhysicalPlanReader;
+import org.apache.drill.exec.proto.CoordinationProtos;
+import org.apache.drill.exec.proto.ExecProtos;
+import org.apache.drill.exec.rpc.user.UserServer;
+import org.apache.drill.exec.server.DrillbitContext;
+import org.junit.After;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestComparisonFunctionsNullable {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestComparisonFunctionsNullable.class);
+
+ DrillConfig c = DrillConfig.create();
+ @Test
+ public void testNullableIntEqual(@Injectable final DrillbitContext bitContext,
+ @Injectable UserServer.UserClientConnection connection) throws Throwable{
+
+ new NonStrictExpectations(){{
+ bitContext.getMetrics(); result = new MetricRegistry("test");
+ bitContext.getAllocator(); result = BufferAllocator.getAllocator(c);
+ }};
+
+ PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
+ PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/nullableIntEqual.json"), Charsets.UTF_8));
+ FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
+ FragmentContext context = new FragmentContext(bitContext, ExecProtos.FragmentHandle.getDefaultInstance(), connection, null, registry);
+ SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
+
+ while(exec.next()){
+ assertEquals(50, exec.getSelectionVector2().getCount());
+ }
+
+ if(context.getFailureCause() != null){
+ throw context.getFailureCause();
+ }
+
+ assertTrue(!context.isFailed());
+
+ }
+
+ @Test
+ public void testNullableBigIntEqual(@Injectable final DrillbitContext bitContext,
+ @Injectable UserServer.UserClientConnection connection) throws Throwable{
+
+ new NonStrictExpectations(){{
+ bitContext.getMetrics(); result = new MetricRegistry("test");
+ bitContext.getAllocator(); result = BufferAllocator.getAllocator(c);
+ }};
+
+ PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
+ PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/nullableBigIntEqual.json"), Charsets.UTF_8));
+ FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
+ FragmentContext context = new FragmentContext(bitContext, ExecProtos.FragmentHandle.getDefaultInstance(), connection, null, registry);
+ SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
+
+ while(exec.next()){
+ assertEquals(50, exec.getSelectionVector2().getCount());
+ }
+
+ if(context.getFailureCause() != null){
+ throw context.getFailureCause();
+ }
+
+ assertTrue(!context.isFailed());
+
+ }
+
+ @Test
+ public void testNullableIntNotEqual(@Injectable final DrillbitContext bitContext,
+ @Injectable UserServer.UserClientConnection connection) throws Throwable{
+
+ new NonStrictExpectations(){{
+ bitContext.getMetrics(); result = new MetricRegistry("test");
+ bitContext.getAllocator(); result = BufferAllocator.getAllocator(c);
+ }};
+
+ PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
+ PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/nullableIntNotEqual.json"), Charsets.UTF_8));
+ FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
+ FragmentContext context = new FragmentContext(bitContext, ExecProtos.FragmentHandle.getDefaultInstance(), connection, null, registry);
+ SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
+
+ while(exec.next()){
+ assertEquals(50, exec.getSelectionVector2().getCount());
+ }
+
+ if(context.getFailureCause() != null){
+ throw context.getFailureCause();
+ }
+
+ assertTrue(!context.isFailed());
+
+ }
+
+ @Test
+ public void testNullableBigIntNotEqual(@Injectable final DrillbitContext bitContext,
+ @Injectable UserServer.UserClientConnection connection) throws Throwable{
+
+ new NonStrictExpectations(){{
+ bitContext.getMetrics(); result = new MetricRegistry("test");
+ bitContext.getAllocator(); result = BufferAllocator.getAllocator(c);
+ }};
+
+ PhysicalPlanReader reader = new PhysicalPlanReader(c, c.getMapper(), CoordinationProtos.DrillbitEndpoint.getDefaultInstance());
+ PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(FileUtils.getResourceAsFile("/functions/nullableBigIntNotEqual.json"), Charsets.UTF_8));
+ FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
+ FragmentContext context = new FragmentContext(bitContext, ExecProtos.FragmentHandle.getDefaultInstance(), connection, null, registry);
+ SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
+
+ while(exec.next()){
+ assertEquals(50, exec.getSelectionVector2().getCount());
+ }
+
+ if(context.getFailureCause() != null){
+ throw context.getFailureCause();
+ }
+
+ assertTrue(!context.isFailed());
+
+ }
+
+ @After
+ public void tearDown() throws Exception{
+ // pause to get logger to catch up.
+ Thread.sleep(1000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/13bc4d1c/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntEqual.json
new file mode 100644
index 0000000..eca6d15
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntEqual.json
@@ -0,0 +1,35 @@
+{
+ head:{
+ type:"APACHE_DRILL_PHYSICAL",
+ version:"1",
+ generator:{
+ type:"manual"
+ }
+ },
+ graph:[
+ {
+ @id:1,
+ pop:"mock-scan",
+ url: "http://apache.org",
+ entries:[
+ {records: 100, types: [
+ {name: "blue", type: "INT", mode: "OPTIONAL"},
+ {name: "red", type: "BIGINT", mode: "OPTIONAL"},
+ {name: "yellow", type: "FLOAT8", mode: "REQUIRED"},
+ {name: "green", type: "INT", mode: "REQUIRED"}
+ ]}
+ ]
+ },
+ {
+ @id:2,
+ child: 1,
+ pop:"filter",
+ expr: "red == red"
+ },
+ {
+ @id: 3,
+ child: 2,
+ pop: "screen"
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/13bc4d1c/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntNotEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntNotEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntNotEqual.json
new file mode 100644
index 0000000..a501f50
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableBigIntNotEqual.json
@@ -0,0 +1,35 @@
+{
+ head:{
+ type:"APACHE_DRILL_PHYSICAL",
+ version:"1",
+ generator:{
+ type:"manual"
+ }
+ },
+ graph:[
+ {
+ @id:1,
+ pop:"mock-scan",
+ url: "http://apache.org",
+ entries:[
+ {records: 100, types: [
+ {name: "blue", type: "INT", mode: "OPTIONAL"},
+ {name: "red", type: "BIGINT", mode: "OPTIONAL"},
+ {name: "yellow", type: "FLOAT8", mode: "REQUIRED"},
+ {name: "green", type: "BIGINT", mode: "OPTIONAL"}
+ ]}
+ ]
+ },
+ {
+ @id:2,
+ child: 1,
+ pop:"filter",
+ expr: "red != (green + red)"
+ },
+ {
+ @id: 3,
+ child: 2,
+ pop: "screen"
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/13bc4d1c/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntEqual.json
new file mode 100644
index 0000000..60a00fb
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntEqual.json
@@ -0,0 +1,35 @@
+{
+ head:{
+ type:"APACHE_DRILL_PHYSICAL",
+ version:"1",
+ generator:{
+ type:"manual"
+ }
+ },
+ graph:[
+ {
+ @id:1,
+ pop:"mock-scan",
+ url: "http://apache.org",
+ entries:[
+ {records: 100, types: [
+ {name: "blue", type: "INT", mode: "OPTIONAL"},
+ {name: "red", type: "BIGINT", mode: "REQUIRED"},
+ {name: "yellow", type: "FLOAT8", mode: "REQUIRED"},
+ {name: "green", type: "INT", mode: "REQUIRED"}
+ ]}
+ ]
+ },
+ {
+ @id:2,
+ child: 1,
+ pop:"filter",
+ expr: "blue == blue"
+ },
+ {
+ @id: 3,
+ child: 2,
+ pop: "screen"
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/13bc4d1c/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntNotEqual.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntNotEqual.json b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntNotEqual.json
new file mode 100644
index 0000000..2cefc5c
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/functions/nullableIntNotEqual.json
@@ -0,0 +1,35 @@
+{
+ head:{
+ type:"APACHE_DRILL_PHYSICAL",
+ version:"1",
+ generator:{
+ type:"manual"
+ }
+ },
+ graph:[
+ {
+ @id:1,
+ pop:"mock-scan",
+ url: "http://apache.org",
+ entries:[
+ {records: 100, types: [
+ {name: "blue", type: "INT", mode: "OPTIONAL"},
+ {name: "red", type: "BIGINT", mode: "OPTIONAL"},
+ {name: "yellow", type: "FLOAT8", mode: "REQUIRED"},
+ {name: "green", type: "INT", mode: "OPTIONAL"}
+ ]}
+ ]
+ },
+ {
+ @id:2,
+ child: 1,
+ pop:"filter",
+ expr: "blue != (green + blue)"
+ },
+ {
+ @id: 3,
+ child: 2,
+ pop: "screen"
+ }
+ ]
+}
\ No newline at end of file