You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by jn...@apache.org on 2017/03/03 22:10:23 UTC
[3/3] drill git commit: DRILL-5293: Change seed for distribution hash
function to differ from that of the hash table
DRILL-5293: Change seed for distribution hash function to differ from that of the hash table
close #765
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/3dfb4972
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/3dfb4972
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/3dfb4972
Branch: refs/heads/master
Commit: 3dfb497293a177164a158b246000ee8291ef258c
Parents: f0057e7
Author: Boaz Ben-Zvi <bo...@BBenZvi-E754-MBP13.local>
Authored: Fri Feb 24 16:48:42 2017 -0800
Committer: Jinfeng Ni <jn...@apache.org>
Committed: Fri Mar 3 10:20:37 2017 -0800
----------------------------------------------------------------------
.../drill/exec/planner/physical/HashPrelUtil.java | 14 ++++++++++----
.../physical/visitor/InsertLocalExchangeVisitor.java | 6 +++---
2 files changed, 13 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/3dfb4972/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashPrelUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashPrelUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashPrelUtil.java
index caf21bc..4300c82 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashPrelUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/HashPrelUtil.java
@@ -24,6 +24,7 @@ import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.FunctionCall;
import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.exec.planner.physical.DrillDistributionTrait.DistributionField;
import java.util.ArrayList;
@@ -36,6 +37,7 @@ public class HashPrelUtil {
public static final String HASH_EXPR_NAME = "E_X_P_R_H_A_S_H_F_I_E_L_D";
+ public static final int DIST_SEED = 1301011; // distribution seed
/**
* Interface for creating different forms of hash expression types.
* @param <T>
@@ -72,8 +74,9 @@ public class HashPrelUtil {
*/
public static <T> T createHashBasedPartitionExpression(
List<T> distFields,
+ T seed,
HashExpressionCreatorHelper<T> helper) {
- return createHashExpression(distFields, helper, true /*for distribution always hash as double*/);
+ return createHashExpression(distFields, seed, helper, true /*for distribution always hash as double*/);
}
/**
@@ -89,6 +92,7 @@ public class HashPrelUtil {
*/
public static <T> T createHashExpression(
List<T> inputExprs,
+ T seed,
HashExpressionCreatorHelper<T> helper,
boolean hashAsDouble) {
@@ -96,7 +100,7 @@ public class HashPrelUtil {
final String functionName = hashAsDouble ? HASH32_DOUBLE_FUNCTION_NAME : HASH32_FUNCTION_NAME;
- T func = helper.createCall(functionName, ImmutableList.of(inputExprs.get(0)));
+ T func = helper.createCall(functionName, ImmutableList.of(inputExprs.get(0), seed ));
for (int i = 1; i<inputExprs.size(); i++) {
func = helper.createCall(functionName, ImmutableList.of(inputExprs.get(i), func));
}
@@ -108,7 +112,8 @@ public class HashPrelUtil {
* Return a hash expression : hash32(field1, hash32(field2, hash32(field3, 0)));
*/
public static LogicalExpression getHashExpression(List<LogicalExpression> fields, boolean hashAsDouble){
- return createHashExpression(fields, HASH_HELPER_LOGICALEXPRESSION, hashAsDouble);
+ final LogicalExpression seed = ValueExpressions.getInt(0); // Hash Table seed
+ return createHashExpression(fields, seed, HASH_HELPER_LOGICALEXPRESSION, hashAsDouble);
}
@@ -134,6 +139,7 @@ public class HashPrelUtil {
expressions.add(new FieldReference(childFields.get(fields.get(i).getFieldId()), ExpressionPosition.UNKNOWN));
}
- return createHashBasedPartitionExpression(expressions, HASH_HELPER_LOGICALEXPRESSION);
+ final LogicalExpression distSeed = ValueExpressions.getInt(DIST_SEED);
+ return createHashBasedPartitionExpression(expressions, distSeed, HASH_HELPER_LOGICALEXPRESSION);
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/3dfb4972/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
index 6ebdba4..fba4cbe 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/InsertLocalExchangeVisitor.java
@@ -19,14 +19,12 @@ package org.apache.drill.exec.planner.physical.visitor;
import com.google.common.collect.Lists;
-import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.exec.planner.physical.ExchangePrel;
import org.apache.drill.exec.planner.physical.HashPrelUtil;
import org.apache.drill.exec.planner.physical.HashPrelUtil.HashExpressionCreatorHelper;
import org.apache.drill.exec.planner.physical.HashToRandomExchangePrel;
import org.apache.drill.exec.planner.physical.PlannerSettings;
import org.apache.drill.exec.planner.physical.Prel;
-import org.apache.drill.exec.planner.physical.PrelUtil;
import org.apache.drill.exec.planner.physical.ProjectPrel;
import org.apache.drill.exec.planner.physical.DrillDistributionTrait.DistributionField;
import org.apache.drill.exec.planner.physical.UnorderedDeMuxExchangePrel;
@@ -40,6 +38,7 @@ import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexUtil;
+import java.math.BigDecimal;
import java.util.Collections;
import java.util.List;
@@ -119,7 +118,8 @@ public class InsertLocalExchangeVisitor extends BasePrelVisitor<Prel, Void, Runt
}
outputFieldNames.add(HashPrelUtil.HASH_EXPR_NAME);
- updatedExpr.add(HashPrelUtil.createHashBasedPartitionExpression(distFieldRefs, hashHelper));
+ final RexNode distSeed = rexBuilder.makeBigintLiteral(BigDecimal.valueOf(HashPrelUtil.DIST_SEED)); // distribution seed
+ updatedExpr.add(HashPrelUtil.createHashBasedPartitionExpression(distFieldRefs, distSeed, hashHelper));
RelDataType rowType = RexUtil.createStructType(prel.getCluster().getTypeFactory(), updatedExpr, outputFieldNames);