You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ve...@apache.org on 2015/03/20 19:31:50 UTC
[1/3] drill git commit: DRILL-2402: Update hash functions to use seed
strategy as opposed to xor strategy.
Repository: drill
Updated Branches:
refs/heads/master 9c9ee8c43 -> bb1d7615e
DRILL-2402: Update hash functions to use seed strategy as opposed to xor strategy.
Also: Simplify and consolidate expression materialization.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/bb1d7615
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/bb1d7615
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/bb1d7615
Branch: refs/heads/master
Commit: bb1d7615e7eb6c0c17c0c8a1cde0ca070393e257
Parents: cfd88db
Author: Jacques Nadeau <ja...@apache.org>
Authored: Wed Mar 4 19:47:05 2015 -0800
Committer: vkorukanti <ve...@gmail.com>
Committed: Thu Mar 19 22:16:46 2015 -0700
----------------------------------------------------------------------
.../exec/expr/ExpressionTreeMaterializer.java | 11 +
.../exec/expr/fn/impl/Hash64Functions.java | 585 +++++++++++++++++++
.../expr/fn/impl/Hash64FunctionsWithSeed.java | 581 ++++++++++++++++++
.../drill/exec/expr/fn/impl/HashFunctions.java | 12 +-
.../apache/drill/exec/expr/fn/impl/XXHash.java | 82 ++-
.../physical/impl/common/ChainedHashTable.java | 49 +-
.../drill/exec/planner/physical/PrelUtil.java | 37 +-
.../visitor/InsertLocalExchangeVisitor.java | 29 +-
.../exec/physical/impl/TestLocalExchange.java | 4 +-
9 files changed, 1290 insertions(+), 100 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/bb1d7615/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
index 0ac3e30..6d42136 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
@@ -25,6 +25,7 @@ import org.apache.drill.common.expression.BooleanOperator;
import org.apache.drill.common.expression.CastExpression;
import org.apache.drill.common.expression.ConvertExpression;
import org.apache.drill.common.expression.ErrorCollector;
+import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.FunctionHolderExpression;
@@ -59,6 +60,7 @@ import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.common.util.CoreDecimalUtility;
+import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.expr.annotations.FunctionTemplate;
import org.apache.drill.exec.expr.fn.AbstractFuncHolder;
import org.apache.drill.exec.expr.fn.DrillComplexWriterFuncHolder;
@@ -87,6 +89,15 @@ public class ExpressionTreeMaterializer {
return ExpressionTreeMaterializer.materialize(expr, batch, errorCollector, registry, false);
}
+ public static LogicalExpression materializeAndCheckErrors(LogicalExpression expr, VectorAccessible batch, FunctionImplementationRegistry registry) throws SchemaChangeException {
+ ErrorCollector collector = new ErrorCollectorImpl();
+ LogicalExpression e = ExpressionTreeMaterializer.materialize(expr, batch, collector, registry, false);
+ if (collector.hasErrors()) {
+ throw new SchemaChangeException(String.format("Failure while trying to materialize incoming schema. Errors:\n %s.", collector.toErrorString()));
+ }
+ return e;
+ }
+
public static LogicalExpression materialize(LogicalExpression expr, VectorAccessible batch, ErrorCollector errorCollector, FunctionImplementationRegistry registry,
boolean allowComplexWriterExpr) {
LogicalExpression out = expr.accept(new MaterializeVisitor(batch, errorCollector, allowComplexWriterExpr), registry);
http://git-wip-us.apache.org/repos/asf/drill/blob/bb1d7615/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Hash64Functions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Hash64Functions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Hash64Functions.java
new file mode 100644
index 0000000..57154ed
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Hash64Functions.java
@@ -0,0 +1,585 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.fn.impl;
+
+import org.apache.drill.exec.expr.DrillSimpleFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.BitHolder;
+import org.apache.drill.exec.expr.holders.DateHolder;
+import org.apache.drill.exec.expr.holders.Decimal18Holder;
+import org.apache.drill.exec.expr.holders.Decimal28SparseHolder;
+import org.apache.drill.exec.expr.holders.Decimal38SparseHolder;
+import org.apache.drill.exec.expr.holders.Decimal9Holder;
+import org.apache.drill.exec.expr.holders.Float4Holder;
+import org.apache.drill.exec.expr.holders.Float8Holder;
+import org.apache.drill.exec.expr.holders.IntHolder;
+import org.apache.drill.exec.expr.holders.NullableBigIntHolder;
+import org.apache.drill.exec.expr.holders.NullableBitHolder;
+import org.apache.drill.exec.expr.holders.NullableDateHolder;
+import org.apache.drill.exec.expr.holders.NullableDecimal18Holder;
+import org.apache.drill.exec.expr.holders.NullableDecimal28SparseHolder;
+import org.apache.drill.exec.expr.holders.NullableDecimal38SparseHolder;
+import org.apache.drill.exec.expr.holders.NullableDecimal9Holder;
+import org.apache.drill.exec.expr.holders.NullableFloat4Holder;
+import org.apache.drill.exec.expr.holders.NullableFloat8Holder;
+import org.apache.drill.exec.expr.holders.NullableIntHolder;
+import org.apache.drill.exec.expr.holders.NullableTimeHolder;
+import org.apache.drill.exec.expr.holders.NullableTimeStampHolder;
+import org.apache.drill.exec.expr.holders.NullableVar16CharHolder;
+import org.apache.drill.exec.expr.holders.NullableVarBinaryHolder;
+import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
+import org.apache.drill.exec.expr.holders.TimeHolder;
+import org.apache.drill.exec.expr.holders.TimeStampHolder;
+import org.apache.drill.exec.expr.holders.Var16CharHolder;
+import org.apache.drill.exec.expr.holders.VarBinaryHolder;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
+
+public class Hash64Functions {
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableFloatHash implements DrillSimpleFunc {
+
+ @Param
+ NullableFloat4Holder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, 0);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class FloatHash implements DrillSimpleFunc {
+
+ @Param
+ Float4Holder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, 0);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableDoubleHash implements DrillSimpleFunc {
+
+ @Param
+ NullableFloat8Holder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, 0);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class DoubleHash implements DrillSimpleFunc {
+
+ @Param
+ Float8Holder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, 0);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableVarBinaryHash implements DrillSimpleFunc {
+
+ @Param
+ NullableVarBinaryHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.start, in.end, in.buffer, 0);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableVarCharHash implements DrillSimpleFunc {
+
+ @Param
+ NullableVarCharHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.start, in.end, in.buffer, 0);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableVar16CharHash implements DrillSimpleFunc {
+
+ @Param
+ NullableVar16CharHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.start, in.end, in.buffer, 0);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableBigIntHash implements DrillSimpleFunc {
+
+ @Param
+ NullableBigIntHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, 0);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableIntHash implements DrillSimpleFunc {
+ @Param
+ NullableIntHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, 0);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class VarBinaryHash implements DrillSimpleFunc {
+
+ @Param
+ VarBinaryHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.start, in.end, in.buffer, 0);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class VarCharHash implements DrillSimpleFunc {
+
+ @Param
+ VarCharHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.start, in.end, in.buffer, 0);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class Var16CharHash implements DrillSimpleFunc {
+
+ @Param
+ Var16CharHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.start, in.end, in.buffer, 0);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class HashBigInt implements DrillSimpleFunc {
+
+ @Param
+ BigIntHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, 0);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class IntHash implements DrillSimpleFunc {
+ @Param
+ IntHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ // TODO: implement hash function for other types
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, 0);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class DateHash implements DrillSimpleFunc {
+ @Param
+ DateHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, 0);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableDateHash implements DrillSimpleFunc {
+ @Param
+ NullableDateHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, 0);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class TimeStampHash implements DrillSimpleFunc {
+ @Param
+ TimeStampHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, 0);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableTimeStampHash implements DrillSimpleFunc {
+ @Param
+ NullableTimeStampHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, 0);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class TimeHash implements DrillSimpleFunc {
+ @Param
+ TimeHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, 0);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableTimeHash implements DrillSimpleFunc {
+ @Param
+ NullableTimeHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, 0);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class Decimal9Hash implements DrillSimpleFunc {
+ @Param
+ Decimal9Holder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, 0);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableDecimal9Hash implements DrillSimpleFunc {
+ @Param
+ NullableDecimal9Holder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, 0);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class Decimal18Hash implements DrillSimpleFunc {
+ @Param
+ Decimal18Holder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, 0);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableDecimal18Hash implements DrillSimpleFunc {
+ @Param
+ NullableDecimal18Holder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, 0);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class Decimal28Hash implements DrillSimpleFunc {
+ @Param
+ Decimal28SparseHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.start, in.start + Decimal28SparseHolder.WIDTH,
+ in.buffer, 0);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableDecimal28Hash implements DrillSimpleFunc {
+ @Param
+ NullableDecimal28SparseHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.start, in.start
+ + NullableDecimal28SparseHolder.WIDTH, in.buffer, 0);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class Decimal38Hash implements DrillSimpleFunc {
+ @Param
+ Decimal38SparseHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.start, in.start + Decimal38SparseHolder.WIDTH,
+ in.buffer, 0);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableDecimal38Hash implements DrillSimpleFunc {
+ @Param
+ NullableDecimal38SparseHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.start, in.start
+ + NullableDecimal38SparseHolder.WIDTH, in.buffer, 0);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableBitHash implements DrillSimpleFunc {
+
+ @Param
+ NullableBitHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, 0);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class BitHash implements DrillSimpleFunc {
+
+ @Param
+ BitHolder in;
+ @Output
+ BigIntHolder out;
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, 0);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/bb1d7615/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Hash64FunctionsWithSeed.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Hash64FunctionsWithSeed.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Hash64FunctionsWithSeed.java
new file mode 100644
index 0000000..b9ec956
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/Hash64FunctionsWithSeed.java
@@ -0,0 +1,581 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.fn.impl;
+
+import org.apache.drill.exec.expr.DrillSimpleFunc;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate;
+import org.apache.drill.exec.expr.annotations.Output;
+import org.apache.drill.exec.expr.annotations.Param;
+import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
+import org.apache.drill.exec.expr.holders.BigIntHolder;
+import org.apache.drill.exec.expr.holders.BitHolder;
+import org.apache.drill.exec.expr.holders.DateHolder;
+import org.apache.drill.exec.expr.holders.Decimal18Holder;
+import org.apache.drill.exec.expr.holders.Decimal28SparseHolder;
+import org.apache.drill.exec.expr.holders.Decimal38SparseHolder;
+import org.apache.drill.exec.expr.holders.Decimal9Holder;
+import org.apache.drill.exec.expr.holders.Float4Holder;
+import org.apache.drill.exec.expr.holders.Float8Holder;
+import org.apache.drill.exec.expr.holders.IntHolder;
+import org.apache.drill.exec.expr.holders.NullableBigIntHolder;
+import org.apache.drill.exec.expr.holders.NullableBitHolder;
+import org.apache.drill.exec.expr.holders.NullableDateHolder;
+import org.apache.drill.exec.expr.holders.NullableDecimal18Holder;
+import org.apache.drill.exec.expr.holders.NullableDecimal28SparseHolder;
+import org.apache.drill.exec.expr.holders.NullableDecimal38SparseHolder;
+import org.apache.drill.exec.expr.holders.NullableDecimal9Holder;
+import org.apache.drill.exec.expr.holders.NullableFloat4Holder;
+import org.apache.drill.exec.expr.holders.NullableFloat8Holder;
+import org.apache.drill.exec.expr.holders.NullableIntHolder;
+import org.apache.drill.exec.expr.holders.NullableTimeHolder;
+import org.apache.drill.exec.expr.holders.NullableTimeStampHolder;
+import org.apache.drill.exec.expr.holders.NullableVar16CharHolder;
+import org.apache.drill.exec.expr.holders.NullableVarBinaryHolder;
+import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
+import org.apache.drill.exec.expr.holders.TimeHolder;
+import org.apache.drill.exec.expr.holders.TimeStampHolder;
+import org.apache.drill.exec.expr.holders.Var16CharHolder;
+import org.apache.drill.exec.expr.holders.VarBinaryHolder;
+import org.apache.drill.exec.expr.holders.VarCharHolder;
+
+public class Hash64FunctionsWithSeed {
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
+ public static class NullableFloatHash implements DrillSimpleFunc {
+
+ @Param NullableFloat4Holder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, seed.value);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
+ public static class FloatHash implements DrillSimpleFunc {
+
+ @Param Float4Holder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, seed.value);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
+ public static class NullableDoubleHash implements DrillSimpleFunc {
+
+ @Param NullableFloat8Holder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, seed.value);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
+ public static class DoubleHash implements DrillSimpleFunc {
+
+ @Param Float8Holder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, seed.value);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
+ public static class NullableVarBinaryHash implements DrillSimpleFunc {
+
+ @Param NullableVarBinaryHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.start, in.end, in.buffer, seed.value);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
+ public static class NullableVarCharHash implements DrillSimpleFunc {
+
+ @Param NullableVarCharHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.start, in.end, in.buffer, seed.value);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
+ public static class NullableVar16CharHash implements DrillSimpleFunc {
+
+ @Param NullableVar16CharHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.start, in.end, in.buffer, seed.value);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableBigIntHash implements DrillSimpleFunc {
+
+ @Param NullableBigIntHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ }
+ else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, seed.value);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableIntHash implements DrillSimpleFunc {
+ @Param NullableIntHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ }
+ else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, seed.value);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class VarBinaryHash implements DrillSimpleFunc {
+
+ @Param VarBinaryHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.start, in.end, in.buffer, seed.value);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class VarCharHash implements DrillSimpleFunc {
+
+ @Param VarCharHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.start, in.end, in.buffer, seed.value);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class Var16CharHash implements DrillSimpleFunc {
+
+ @Param Var16CharHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.start, in.end, in.buffer, seed.value);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class HashBigInt implements DrillSimpleFunc {
+
+ @Param BigIntHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, seed.value);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class IntHash implements DrillSimpleFunc {
+ @Param IntHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ // TODO: implement hash function for other types
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, seed.value);
+ }
+ }
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class DateHash implements DrillSimpleFunc {
+ @Param DateHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, seed.value);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableDateHash implements DrillSimpleFunc {
+ @Param NullableDateHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, seed.value);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class TimeStampHash implements DrillSimpleFunc {
+ @Param TimeStampHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, seed.value);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableTimeStampHash implements DrillSimpleFunc {
+ @Param NullableTimeStampHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, seed.value);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class TimeHash implements DrillSimpleFunc {
+ @Param TimeHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, seed.value);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableTimeHash implements DrillSimpleFunc {
+ @Param NullableTimeHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, seed.value);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class Decimal9Hash implements DrillSimpleFunc {
+ @Param Decimal9Holder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, seed.value);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableDecimal9Hash implements DrillSimpleFunc {
+ @Param NullableDecimal9Holder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, seed.value);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class Decimal18Hash implements DrillSimpleFunc {
+ @Param Decimal18Holder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, seed.value);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableDecimal18Hash implements DrillSimpleFunc {
+ @Param NullableDecimal18Holder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, seed.value);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class Decimal28Hash implements DrillSimpleFunc {
+ @Param Decimal28SparseHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.start, in.start + Decimal28SparseHolder.WIDTH, in.buffer, seed.value);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableDecimal28Hash implements DrillSimpleFunc {
+ @Param NullableDecimal28SparseHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.start, in.start + NullableDecimal28SparseHolder.WIDTH, in.buffer, seed.value);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class Decimal38Hash implements DrillSimpleFunc {
+ @Param Decimal38SparseHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.start, in.start + Decimal38SparseHolder.WIDTH, in.buffer, seed.value);
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+ public static class NullableDecimal38Hash implements DrillSimpleFunc {
+ @Param NullableDecimal38SparseHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.start, in.start + NullableDecimal38SparseHolder.WIDTH, in.buffer, seed.value);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
+ public static class NullableBitHash implements DrillSimpleFunc {
+
+ @Param NullableBitHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ if (in.isSet == 0) {
+ out.value = 0;
+ } else {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, seed.value);
+ }
+ }
+ }
+
+ @FunctionTemplate(name = "hash64", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL )
+ public static class BitHash implements DrillSimpleFunc {
+
+ @Param BitHolder in;
+ @Param BigIntHolder seed;
+ @Output BigIntHolder out;
+
+
+ public void setup() {
+ }
+
+ public void eval() {
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash64(in.value, seed.value);
+ }
+ }}
http://git-wip-us.apache.org/repos/asf/drill/blob/bb1d7615/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
index 28b0be7..4652cc0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
@@ -132,7 +132,7 @@ public class HashFunctions {
if (in.isSet == 0) {
out.value = 0;
} else {
- out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(in.start, in.end, in.buffer);
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(in.start, in.end, in.buffer, 0);
}
}
}
@@ -150,7 +150,7 @@ public class HashFunctions {
if (in.isSet == 0) {
out.value = 0;
} else {
- out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(in.start, in.end, in.buffer);
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(in.start, in.end, in.buffer, 0);
}
}
}
@@ -168,7 +168,7 @@ public class HashFunctions {
if (in.isSet == 0) {
out.value = 0;
} else {
- out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(in.start, in.end, in.buffer);
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(in.start, in.end, in.buffer, 0);
}
}
}
@@ -220,7 +220,7 @@ public class HashFunctions {
}
public void eval() {
- out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(in.start, in.end, in.buffer);
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(in.start, in.end, in.buffer, 0);
}
}
@@ -234,7 +234,7 @@ public class HashFunctions {
}
public void eval() {
- out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(in.start, in.end, in.buffer);
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(in.start, in.end, in.buffer, 0);
}
}
@@ -248,7 +248,7 @@ public class HashFunctions {
}
public void eval() {
- out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(in.start, in.end, in.buffer);
+ out.value = org.apache.drill.exec.expr.fn.impl.XXHash.hash32(in.start, in.end, in.buffer, 0);
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/bb1d7615/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/XXHash.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/XXHash.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/XXHash.java
index 8ad9611..81dbf5f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/XXHash.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/XXHash.java
@@ -35,7 +35,7 @@ public final class XXHash {
static final long PRIME64_4 = UnsignedLongs.decode("9650029242287828579");
static final long PRIME64_5 = UnsignedLongs.decode("2870177450012600261");
- private static long hash64(long start, long bEnd, long seed) {
+ private static long hash64bytes(long start, long bEnd, long seed) {
long len = bEnd - start;
long h64;
long p = start;
@@ -129,34 +129,26 @@ public final class XXHash {
return applyFinalHashComputation(h64);
}
- public static int hash32(int start, int end, DrillBuf buffer){
- if(BOUNDS_CHECKING_ENABLED){
- buffer.checkBytes(start, end);
- }
-
- long s = buffer.memoryAddress() + start;
- long e = buffer.memoryAddress() + end;
-
- return hash32(s, e, 0);
+ private static long applyFinalHashComputation(long h64) {
+ h64 ^= h64 >> 33;
+ h64 *= PRIME64_2;
+ h64 ^= h64 >> 29;
+ h64 *= PRIME64_3;
+ h64 ^= h64 >> 32;
+ return h64;
}
- public static int hash32(int val, int seed){
+
+ /* 64 bit variations */
+ public static long hash64(int val, long seed){
long h64 = seed + PRIME64_5;
h64 += 4; // add length (4 bytes) to hash value
h64 ^= val * PRIME64_1;
h64 = Long.rotateLeft(h64, 23) * PRIME64_2 + PRIME64_3;
- return (int) applyFinalHashComputation(h64);
- }
-
- public static int hash32(float val, int seed){
- return hash32(Float.floatToIntBits(val), seed);
- }
-
- public static int hash32(double val, int seed){
- return hash32(Double.doubleToLongBits(val), seed);
+ return applyFinalHashComputation(h64);
}
- public static int hash32(long val, int seed){
+ public static long hash64(long val, long seed){
long h64 = seed + PRIME64_5;
h64 += 8; // add length (8 bytes) to hash value
long k1 = val* PRIME64_2;
@@ -164,19 +156,47 @@ public final class XXHash {
k1 *= PRIME64_1;
h64 ^= k1;
h64 = Long.rotateLeft(h64, 27) * PRIME64_1 + PRIME64_4;
- return (int) applyFinalHashComputation(h64);
+ return applyFinalHashComputation(h64);
}
- private static int hash32(long start, long bEnd, long seed){
- return (int) hash64(start, bEnd, seed);
+ public static long hash64(float val, long seed){
+ return hash64(Float.floatToIntBits(val), seed);
}
- private static long applyFinalHashComputation(long h64) {
- h64 ^= h64 >> 33;
- h64 *= PRIME64_2;
- h64 ^= h64 >> 29;
- h64 *= PRIME64_3;
- h64 ^= h64 >> 32;
- return h64;
+ public static long hash64(double val, long seed){
+ return hash64(Double.doubleToLongBits(val), seed);
+ }
+
+ public static long hash64(int start, int end, DrillBuf buffer, long seed){
+ if(BOUNDS_CHECKING_ENABLED){
+ buffer.checkBytes(start, end);
+ }
+
+ long s = buffer.memoryAddress() + start;
+ long e = buffer.memoryAddress() + end;
+
+ return hash64bytes(s, e, seed);
+ }
+
+ /* 32 bit variations */
+ public static int hash32(int val, long seed){
+ return (int) hash64(val, seed);
+ }
+
+ public static int hash32(long val, long seed){
+ return (int) hash64(val, seed);
+ }
+
+ public static int hash32(float val, long seed){
+ return (int) hash64(val, seed);
+ }
+
+ public static int hash32(double val, long seed){
+ return (int) hash64(val, seed);
}
+
+ public static int hash32(int start, int end, DrillBuf buffer, long seed){
+ return (int) hash64(start, end, buffer, seed);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/drill/blob/bb1d7615/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
index 8f921f7..84a2956 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/common/ChainedHashTable.java
@@ -18,18 +18,15 @@
package org.apache.drill.exec.physical.impl.common;
import java.io.IOException;
+import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.expression.ErrorCollector;
import org.apache.drill.common.expression.ErrorCollectorImpl;
import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.FunctionCall;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.exceptions.DrillRuntimeException;
-import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.common.logical.data.NamedExpression;
-import org.apache.drill.common.types.TypeProtos;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.common.types.Types;
import org.apache.drill.exec.compile.sig.GeneratorMapping;
@@ -46,12 +43,13 @@ import org.apache.drill.exec.expr.ValueVectorWriteExpression;
import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.planner.physical.PrelUtil;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.TypedFieldId;
+import org.apache.drill.exec.record.VectorAccessible;
import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.resolver.TypeCastRules;
-import org.apache.drill.exec.vector.AllocationHelper;
import org.apache.drill.exec.vector.ValueVector;
import com.sun.codemodel.JConditional;
@@ -212,8 +210,8 @@ public class ChainedHashTable {
*/
addLeastRestrictiveCasts(keyExprsBuild, keyExprsProbe);
- setupGetHash(cg /* use top level code generator for getHash */, GetHashIncomingBuildMapping, keyExprsBuild, false);
- setupGetHash(cg /* use top level code generator for getHash */, GetHashIncomingProbeMapping, keyExprsProbe, true);
+ setupGetHash(cg /* use top level code generator for getHash */, GetHashIncomingBuildMapping, incomingBuild, keyExprsBuild, false);
+ setupGetHash(cg /* use top level code generator for getHash */, GetHashIncomingProbeMapping, incomingProbe, keyExprsProbe, true);
HashTable ht = context.getImplementationClass(top);
ht.setup(htConfig, context, allocator, incomingBuild, incomingProbe, outgoing, htContainerOrig);
@@ -341,7 +339,7 @@ public class ChainedHashTable {
}
}
- private void setupGetHash(ClassGenerator<HashTable> cg, MappingSet incomingMapping, LogicalExpression[] keyExprs,
+ private void setupGetHash(ClassGenerator<HashTable> cg, MappingSet incomingMapping, VectorAccessible batch, LogicalExpression[] keyExprs,
boolean isProbe) throws SchemaChangeException {
cg.setMappingSet(incomingMapping);
@@ -351,36 +349,11 @@ public class ChainedHashTable {
return;
}
- HoldingContainer combinedHashValue = null;
+ LogicalExpression hashExpression = PrelUtil.getHashExpression(Arrays.asList(keyExprs));
+ final LogicalExpression materializedExpr = ExpressionTreeMaterializer.materializeAndCheckErrors(hashExpression, batch, context.getFunctionRegistry());
+ HoldingContainer hash = cg.addExpr(materializedExpr);
+ cg.getEvalBlock()._return(hash.getValue());
- for (int i = 0; i < keyExprs.length; i++) {
- LogicalExpression expr = keyExprs[i];
-
- cg.setMappingSet(incomingMapping);
- HoldingContainer input = cg.addExpr(expr, false);
-
- // compute the hash(expr)
- LogicalExpression hashfunc =
- FunctionGenerationHelper.getFunctionExpression("hash", Types.required(MinorType.INT),
- context.getFunctionRegistry(), input);
- HoldingContainer hashValue = cg.addExpr(hashfunc, false);
-
- if (i == 0) {
- combinedHashValue = hashValue; // first expression..just use the hash value
- } else {
-
- // compute the combined hash value using XOR
- LogicalExpression xorfunc =
- FunctionGenerationHelper.getFunctionExpression("xor", Types.required(MinorType.INT),
- context.getFunctionRegistry(), hashValue, combinedHashValue);
- combinedHashValue = cg.addExpr(xorfunc, false);
- }
- }
- if (combinedHashValue != null) {
- cg.getEvalBlock()._return(combinedHashValue.getValue());
- } else {
- cg.getEvalBlock()._return(JExpr.lit(0));
- }
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/bb1d7615/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java
index d904c3a..f7c144f 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PrelUtil.java
@@ -17,11 +17,13 @@
*/
package org.apache.drill.exec.planner.physical;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
+import org.apache.drill.common.expression.CastExpression;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.FunctionCall;
@@ -31,6 +33,13 @@ import org.apache.drill.common.expression.PathSegment.ArraySegment;
import org.apache.drill.common.expression.PathSegment.NameSegment;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.logical.data.Order.Ordering;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.compile.sig.MappingSet;
+import org.apache.drill.exec.expr.ClassGenerator;
+import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
+import org.apache.drill.exec.expr.fn.FunctionGenerationHelper;
+import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.planner.physical.DrillDistributionTrait.DistributionField;
import org.apache.drill.exec.record.BatchSchema.SelectionVectorMode;
import org.eigenbase.rel.RelCollation;
@@ -74,8 +83,20 @@ public class PrelUtil {
}
/*
- * Return a hash expression : hash(field1) ^ hash(field2) ^ hash(field3) ... ^ hash(field_n)
+ * Return a hash expression : (int) hash(field1, hash(field2, hash(field3, 0)));
*/
+ public static LogicalExpression getHashExpression(List<LogicalExpression> fields){
+ assert fields.size() > 0;
+
+ FunctionCall func = new FunctionCall("hash64", ImmutableList.of(fields.get(0)), ExpressionPosition.UNKNOWN);
+ for (int i = 1; i<fields.size(); i++) {
+ func = new FunctionCall("hash64", ImmutableList.of(fields.get(i), func), ExpressionPosition.UNKNOWN);
+ }
+
+ return new CastExpression(func, Types.required(MinorType.INT), ExpressionPosition.UNKNOWN);
+
+ }
+
public static LogicalExpression getHashExpression(List<DistributionField> fields, RelDataType rowType) {
assert fields.size() > 0;
@@ -86,17 +107,11 @@ public class PrelUtil {
return new FieldReference(HASH_EXPR_NAME);
}
- FieldReference fr = new FieldReference(childFields.get(fields.get(0).getFieldId()), ExpressionPosition.UNKNOWN);
- FunctionCall func = new FunctionCall("hash", ImmutableList.of((LogicalExpression)fr), ExpressionPosition.UNKNOWN);
-
- for (int i = 1; i<fields.size(); i++) {
- fr = new FieldReference(childFields.get(fields.get(i).getFieldId()), ExpressionPosition.UNKNOWN);
- FunctionCall func2 = new FunctionCall("hash", ImmutableList.of((LogicalExpression)fr), ExpressionPosition.UNKNOWN);
-
- func = new FunctionCall("xor", ImmutableList.of((LogicalExpression)func, (LogicalExpression)func2), ExpressionPosition.UNKNOWN);
+ final List<LogicalExpression> expressions = new ArrayList<LogicalExpression>(childFields.size());
+ for(int i =0; i < fields.size(); i++){
+ expressions.add(new FieldReference(childFields.get(fields.get(i).getFieldId()), ExpressionPosition.UNKNOWN));
}
-
- return func;
+ return getHashExpression(expressions);
}
public static Iterator<Prel> iter(RelNode... nodes) {
http://git-wip-us.apache.org/repos/asf/drill/blob/bb1d7615/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
index b28198b..ccf890d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
@@ -42,6 +42,9 @@ import java.util.Collections;
import java.util.List;
public class InsertLocalExchangeVisitor extends BasePrelVisitor<Prel, Void, RuntimeException> {
+ private static final DrillSqlOperator SQL_OP_HASH64_WITH_NO_SEED = new DrillSqlOperator("hash64", 1, MajorType.getDefaultInstance(), true);
+ private static final DrillSqlOperator SQL_OP_HASH64_WITH_SEED = new DrillSqlOperator("hash64", 2, MajorType.getDefaultInstance(), true);
+ private static final DrillSqlOperator SQL_OP_CAST_INT = new DrillSqlOperator("castINT", 1, MajorType.getDefaultInstance(), true);
private final boolean isMuxEnabled;
private final boolean isDeMuxEnabled;
@@ -81,21 +84,23 @@ public class InsertLocalExchangeVisitor extends BasePrelVisitor<Prel, Void, Runt
if ( isMuxEnabled ) {
// Insert Project Operator with new column that will be a hash for HashToRandomExchange fields
List<DistributionField> fields = hashPrel.getFields();
- final DrillSqlOperator sqlOpH = new DrillSqlOperator("hash", 1, MajorType.getDefaultInstance(), true);
- final DrillSqlOperator sqlOpX = new DrillSqlOperator("xor", 2, MajorType.getDefaultInstance(), true);
- RexNode prevRex = null;
List<String> outputFieldNames = Lists.newArrayList(childFields);
final RexBuilder rexBuilder = prel.getCluster().getRexBuilder();
final List<RelDataTypeField> childRowTypeFields = child.getRowType().getFieldList();
- for ( DistributionField field : fields) {
- final int tmpField = field.getFieldId();
- RexNode rex = rexBuilder.makeInputRef(childRowTypeFields.get(tmpField).getType(), tmpField);
- RexNode rexFunc = rexBuilder.makeCall(sqlOpH, rex);
- if ( prevRex != null ) {
- rexFunc = rexBuilder.makeCall(sqlOpX, prevRex, rexFunc);
- }
- prevRex = rexFunc;
+
+ // First field has no seed argument for hash64 function.
+ final int firstFieldId = fields.get(0).getFieldId();
+ RexNode firstFieldInputRef = rexBuilder.makeInputRef(childRowTypeFields.get(firstFieldId).getType(), firstFieldId);
+ RexNode hashExpr = rexBuilder.makeCall(SQL_OP_HASH64_WITH_NO_SEED, firstFieldInputRef);
+
+ for (int i=1; i<fields.size(); i++) {
+ final int fieldId = fields.get(i).getFieldId();
+ RexNode inputRef = rexBuilder.makeInputRef(childRowTypeFields.get(fieldId).getType(), fieldId);
+ hashExpr = rexBuilder.makeCall(SQL_OP_HASH64_WITH_SEED, inputRef, hashExpr);
}
+
+ hashExpr = rexBuilder.makeCall(SQL_OP_CAST_INT, hashExpr);
+
List <RexNode> updatedExpr = Lists.newArrayList();
for ( RelDataTypeField field : childRowTypeFields) {
RexNode rex = rexBuilder.makeInputRef(field.getType(), field.getIndex());
@@ -104,7 +109,7 @@ public class InsertLocalExchangeVisitor extends BasePrelVisitor<Prel, Void, Runt
}
outputFieldNames.add(PrelUtil.HASH_EXPR_NAME);
- updatedExpr.add(prevRex);
+ updatedExpr.add(hashExpr);
RelDataType rowType = RexUtil.createStructType(prel.getCluster().getTypeFactory(), updatedExpr, outputFieldNames);
ProjectPrel addColumnprojectPrel = new ProjectPrel(child.getCluster(), child.getTraitSet(), child, updatedExpr, rowType);
http://git-wip-us.apache.org/repos/asf/drill/blob/bb1d7615/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestLocalExchange.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestLocalExchange.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestLocalExchange.java
index 7ccb882..08655e3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestLocalExchange.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestLocalExchange.java
@@ -208,7 +208,7 @@ public class TestLocalExchange extends PlanTestBase {
final String plan = getPlanInString("EXPLAIN PLAN FOR " + groupByMultipleQuery, JSON_FORMAT);
System.out.println("Plan: " + plan);
- jsonExchangeOrderChecker(plan, false, 1, "xor\\(xor\\(hash\\(.*\\) , hash\\(.*\\) \\) , hash\\(.*\\) \\) ");
+ jsonExchangeOrderChecker(plan, false, 1, "castint\\(hash64\\(.*, hash64\\(.*, hash64\\(.*\\) \\) \\) \\) ");
// Run the query and verify the output
final TestBuilder testBuilder = testBuilder()
@@ -286,7 +286,7 @@ public class TestLocalExchange extends PlanTestBase {
if ( isMuxOn ) {
// # of hash exchanges should be = # of mux exchanges + # of demux exchanges
assertEquals("HashExpr on the hash column should not happen", 2*expectedNumMuxes+expectedNumDeMuxes, StringUtils.countMatches(plan, PrelUtil.HASH_EXPR_NAME));
- jsonExchangeOrderChecker(plan, isDeMuxOn, expectedNumMuxes, "hash(.*) ");
+ jsonExchangeOrderChecker(plan, isDeMuxOn, expectedNumMuxes, "castint\\(hash64\\(.*\\) \\) ");
} else {
assertEquals("HashExpr on the hash column should not happen", 0, StringUtils.countMatches(plan, PrelUtil.HASH_EXPR_NAME));
}
[3/3] drill git commit: DRILL-2342: Store the nullability property of
column in view persistence store.
Posted by ve...@apache.org.
DRILL-2342: Store the nullability property of column in view persistence store.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/d7dc0b95
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/d7dc0b95
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/d7dc0b95
Branch: refs/heads/master
Commit: d7dc0b95b8086b63523ec2e6a1cc9236d1a5bc44
Parents: 9c9ee8c
Author: vkorukanti <ve...@gmail.com>
Authored: Tue Mar 17 17:38:55 2015 -0700
Committer: vkorukanti <ve...@gmail.com>
Committed: Thu Mar 19 22:16:46 2015 -0700
----------------------------------------------------------------------
.../org/apache/drill/exec/dotdrill/View.java | 26 +++++++++++---
.../org/apache/drill/TestMergeFilterPlan.java | 4 +--
.../apache/drill/exec/sql/TestViewSupport.java | 36 ++++++++++++++++++++
.../apache/drill/jdbc/test/TestMetadataDDL.java | 2 +-
.../org/apache/drill/jdbc/test/TestViews.java | 4 +--
5 files changed, 63 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/d7dc0b95/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java
index 8b6e1e3..a7b496b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/dotdrill/View.java
@@ -52,13 +52,23 @@ public class View {
public final SqlTypeName type;
public final Integer precision;
public final Integer scale;
+ public final Boolean isNullable;
@JsonCreator
- public FieldType(@JsonProperty("name") String name, @JsonProperty("type") SqlTypeName type, @JsonProperty("precision") Integer precision, @JsonProperty("scale") Integer scale){
+ public FieldType(
+ @JsonProperty("name") String name,
+ @JsonProperty("type") SqlTypeName type,
+ @JsonProperty("precision") Integer precision,
+ @JsonProperty("scale") Integer scale,
+ @JsonProperty("isNullable") Boolean isNullable){
this.name = name;
this.type = type;
this.precision = precision;
this.scale = scale;
+
+ // Property "isNullable" is not part of the initial view definition and added in DRILL-2342. If the
+ // default value is null, consider it as "true". It is safe to default to "nullable" than "required" type.
+ this.isNullable = (isNullable == null) ? true : isNullable;
}
public FieldType(String name, RelDataType dataType){
@@ -82,6 +92,7 @@ public class View {
this.precision = p;
this.scale = s;
+ this.isNullable = dataType.isNullable();
}
}
@@ -120,12 +131,19 @@ public class View {
for(FieldType field : fields){
names.add(field.name);
+ RelDataType type;
if(field.precision == null && field.scale == null){
- types.add(factory.createSqlType(field.type));
+ type = factory.createSqlType(field.type);
}else if(field.precision != null && field.scale == null){
- types.add(factory.createSqlType(field.type, field.precision));
+ type = factory.createSqlType(field.type, field.precision);
}else{
- types.add(factory.createSqlType(field.type, field.precision, field.scale));
+ type = factory.createSqlType(field.type, field.precision, field.scale);
+ }
+
+ if (field.isNullable) {
+ types.add(factory.createTypeWithNullability(type, true));
+ } else {
+ types.add(type);
}
}
return factory.createStructType(types, names);
http://git-wip-us.apache.org/repos/asf/drill/blob/d7dc0b95/exec/java-exec/src/test/java/org/apache/drill/TestMergeFilterPlan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestMergeFilterPlan.java b/exec/java-exec/src/test/java/org/apache/drill/TestMergeFilterPlan.java
index 65559b1..0b20b7a 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestMergeFilterPlan.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestMergeFilterPlan.java
@@ -55,8 +55,8 @@ public class TestMergeFilterPlan extends PlanTestBase {
"group by dat.store_id\n" +
"order by dat.store_id";
- String expectedPattern1 = "Filter(condition=[AND(OR(=($0, 1), =($0, 2), =($0, 3)), =(CAST($4):ANY NOT NULL, 'GRADUATE DEGREE'))])";
- String expectedPattern2 = "Filter(condition=[AND(OR(=($0, 1), =($0, 2), =($0, 3)), LIKE(CAST($1):ANY NOT NULL, '%VP%'))])";
+ String expectedPattern1 = "Filter(condition=[AND(OR(=($0, 1), =($0, 2), =($0, 3)), =($4, 'GRADUATE DEGREE'))])";
+ String expectedPattern2 = "Filter(condition=[AND(OR(=($0, 1), =($0, 2), =($0, 3)), LIKE($1, '%VP%'))])";
String excludedPattern = "Filter(condition=[OR(=($0, 1), =($0, 2), =($0, 3))])";
test("use dfs.tmp");
http://git-wip-us.apache.org/repos/asf/drill/blob/d7dc0b95/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
index bfd3f18..3b55b99 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/sql/TestViewSupport.java
@@ -40,5 +40,41 @@ public class TestViewSupport extends BaseTestQuery{
test(selectOutside);
}
+ /**
+ * DRILL-2342 This test is for case where output columns are nullable. Existing tests already cover the case
+ * where columns are required.
+ */
+ @Test
+ public void testNullabilityPropertyInViewPersistence() throws Exception {
+ final String viewName = "testNullabilityPropertyInViewPersistence";
+ try {
+
+ test("USE dfs_test.tmp");
+ test(String.format("CREATE OR REPLACE VIEW %s AS SELECT " +
+ "CAST(customer_id AS BIGINT) as cust_id, " +
+ "CAST(fname AS VARCHAR(25)) as fname, " +
+ "CAST(country AS VARCHAR(20)) as country " +
+ "FROM cp.`customer.json` " +
+ "ORDER BY customer_id " +
+ "LIMIT 1;", viewName));
+ testBuilder()
+ .sqlQuery(String.format("DESCRIBE %s", viewName))
+ .unOrdered()
+ .baselineColumns("COLUMN_NAME", "DATA_TYPE", "IS_NULLABLE")
+ .baselineValues("cust_id", "BIGINT", "YES")
+ .baselineValues("fname", "VARCHAR", "YES")
+ .baselineValues("country", "VARCHAR", "YES")
+ .go();
+
+ testBuilder()
+ .sqlQuery(String.format("SELECT * FROM %s", viewName))
+ .ordered()
+ .baselineColumns("cust_id", "fname", "country")
+ .baselineValues(1L, "Sheri", "Mexico")
+ .go();
+ } finally {
+ test("drop view " + viewName + ";");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/d7dc0b95/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
index 41a595d..3534f35 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestMetadataDDL.java
@@ -228,7 +228,7 @@ public class TestMetadataDDL extends JdbcTestQueryBase {
resultSet = statement.executeQuery("DESCRIBE `TABLES`");
Set<String> result = JdbcAssert.toStringSet(resultSet);
resultSet.close();
- ImmutableSet<String> expected = ImmutableSet.of("COLUMN_NAME=key; DATA_TYPE=INTEGER; IS_NULLABLE=NO");
+ ImmutableSet<String> expected = ImmutableSet.of("COLUMN_NAME=key; DATA_TYPE=INTEGER; IS_NULLABLE=YES");
assertTrue(String.format("Generated string:\n%s\ndoes not match:\n%s", result, expected), expected.equals(result));
// Test describe of `TABLES` with a schema qualifier which is not in default schema
http://git-wip-us.apache.org/repos/asf/drill/blob/d7dc0b95/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
index 8042a6d..3fe8e2f 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/TestViews.java
@@ -350,8 +350,8 @@ public class TestViews extends JdbcTestQueryBase {
result = JdbcAssert.toString(resultSet).trim();
resultSet.close();
expected =
- "COLUMN_NAME=key; DATA_TYPE=INTEGER; IS_NULLABLE=NO\n" +
- "COLUMN_NAME=value; DATA_TYPE=VARCHAR; IS_NULLABLE=NO";
+ "COLUMN_NAME=key; DATA_TYPE=INTEGER; IS_NULLABLE=YES\n" +
+ "COLUMN_NAME=value; DATA_TYPE=VARCHAR; IS_NULLABLE=YES";
assertTrue(String.format("Generated string:\n%s\ndoes not match:\n%s", result, expected),
expected.equals(result));
[2/3] drill git commit: DRILL-2010: MergeJoin: Store/restore the
right batch state when existing join loop due to output batch full.
Posted by ve...@apache.org.
DRILL-2010: MergeJoin: Store/restore the right batch state when existing join loop due to output batch full.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/cfd88dbf
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/cfd88dbf
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/cfd88dbf
Branch: refs/heads/master
Commit: cfd88dbf61101e86aea7de8ac0409e029ee30ffc
Parents: d7dc0b9
Author: vkorukanti <ve...@gmail.com>
Authored: Fri Mar 6 06:58:07 2015 -0800
Committer: vkorukanti <ve...@gmail.com>
Committed: Thu Mar 19 22:16:46 2015 -0700
----------------------------------------------------------------------
.../exec/physical/impl/join/JoinStatus.java | 26 ++++++++++++++++++++
.../exec/physical/impl/join/JoinTemplate.java | 24 +++++++++++++-----
2 files changed, 44 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/cfd88dbf/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinStatus.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinStatus.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinStatus.java
index 299fd3a..5fb2585 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinStatus.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinStatus.java
@@ -60,6 +60,10 @@ public final class JoinStatus {
public MergeJoinBatch outputBatch;
public SelectionVector4 sv4;
+ private boolean hasIntermediateData;
+ private int initialRightPosition = -1;
+ private boolean crossedBatchBoundaries;
+
private final JoinRelType joinType;
public boolean ok = true;
@@ -97,6 +101,28 @@ public final class JoinStatus {
+ "]";
}
+ public boolean hasIntermediateData() {
+ return hasIntermediateData;
+ }
+
+ public void resetIntermediateData() {
+ hasIntermediateData = false;
+ }
+
+ public void setIntermediateData(int initialRightPosition, boolean crossedBatchBoundaries) {
+ this.initialRightPosition = initialRightPosition;
+ this.crossedBatchBoundaries = crossedBatchBoundaries;
+ this.hasIntermediateData = true;
+ }
+
+ public int getInitialRightPosition() {
+ return initialRightPosition;
+ }
+
+ public boolean getCrossedBatchBoundaries() {
+ return crossedBatchBoundaries;
+ }
+
private final IterOutcome nextLeft() {
return outputBatch.next(LEFT_INPUT, left);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/cfd88dbf/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
index 7da9788..6da10d2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/JoinTemplate.java
@@ -95,7 +95,7 @@ public abstract class JoinTemplate implements JoinWorker {
// we've hit the end of the right record batch; copy any remaining values from the left batch
while (status.isLeftPositionAllowed()) {
if (status.isOutgoingBatchFull()) {
- return false;
+ return true;
}
doCopyLeft(status.getLeftPosition(), status.getOutPosition());
@@ -137,11 +137,21 @@ public abstract class JoinTemplate implements JoinWorker {
status.notifyLeftStoppedRepeating();
}
- boolean crossedBatchBoundaries = false;
- int initialRightPosition = status.getRightPosition();
+ boolean crossedBatchBoundaries;
+ int initialRightPosition;
+ if (status.hasIntermediateData()) {
+ crossedBatchBoundaries = status.getCrossedBatchBoundaries();
+ initialRightPosition = status.getInitialRightPosition();
+ status.resetIntermediateData();
+ } else {
+ crossedBatchBoundaries = false;
+ initialRightPosition = status.getRightPosition();
+ }
+
do {
if (status.isOutgoingBatchFull()) {
- return false;
+ status.setIntermediateData(initialRightPosition, crossedBatchBoundaries);
+ return true;
}
// copy all equal right keys to the output record batch
doCopyLeft(status.getLeftPosition(), status.getOutPosition());
@@ -170,7 +180,8 @@ public abstract class JoinTemplate implements JoinWorker {
}
status.advanceLeft();
- if (status.isLeftRepeating() && doCompareNextLeftKey(status.getLeftPosition()) != 0) {
+ if (status.isLeftRepeating() && status.isNextLeftPositionInCurrentBatch() &&
+ doCompareNextLeftKey(status.getLeftPosition()) != 0) {
// left no longer has duplicates. switch back to incoming batch mode
status.setDefaultAdvanceMode();
status.notifyLeftStoppedRepeating();
@@ -197,7 +208,8 @@ public abstract class JoinTemplate implements JoinWorker {
throw new IllegalStateException();
}
}
- return false;
+
+ return true;
}
// Generated Methods