You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kg...@apache.org on 2018/04/25 15:41:45 UTC
hive git commit: HIVE-19274: Add an OpTreeSignature persistence
checker hook (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/branch-3 86f0fcf10 -> 6535dce08
HIVE-19274: Add an OpTreeSignature persistence checker hook (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6535dce0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6535dce0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6535dce0
Branch: refs/heads/branch-3
Commit: 6535dce08e6e88961aca8fb48f20e8cdf25ec1c0
Parents: 86f0fcf
Author: Zoltan Haindrich <ki...@rxd.hu>
Authored: Wed Apr 25 17:39:01 2018 +0200
Committer: Zoltan Haindrich <ki...@rxd.hu>
Committed: Wed Apr 25 17:39:01 2018 +0200
----------------------------------------------------------------------
data/conf/llap/hive-site.xml | 2 +-
data/conf/perf-reg/tez/hive-site.xml | 2 +-
.../RuntimeStatsPersistenceCheckerHook.java | 71 ++++++++++++++++++++
.../ql/optimizer/signature/SignatureUtils.java | 2 +-
.../apache/hadoop/hive/ql/plan/JoinDesc.java | 6 +-
5 files changed, 77 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6535dce0/data/conf/llap/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/llap/hive-site.xml b/data/conf/llap/hive-site.xml
index 990b473..1507a56 100644
--- a/data/conf/llap/hive-site.xml
+++ b/data/conf/llap/hive-site.xml
@@ -163,7 +163,7 @@
<property>
<name>hive.exec.post.hooks</name>
- <value>org.apache.hadoop.hive.ql.hooks.PostExecutePrinter</value>
+ <value>org.apache.hadoop.hive.ql.hooks.PostExecutePrinter, org.apache.hadoop.hive.ql.hooks.RuntimeStatsPersistenceCheckerHook</value>
<description>Post Execute Hook for Tests</description>
</property>
http://git-wip-us.apache.org/repos/asf/hive/blob/6535dce0/data/conf/perf-reg/tez/hive-site.xml
----------------------------------------------------------------------
diff --git a/data/conf/perf-reg/tez/hive-site.xml b/data/conf/perf-reg/tez/hive-site.xml
index e11f8f8..78a5481 100644
--- a/data/conf/perf-reg/tez/hive-site.xml
+++ b/data/conf/perf-reg/tez/hive-site.xml
@@ -162,7 +162,7 @@
<property>
<name>hive.exec.post.hooks</name>
- <value>org.apache.hadoop.hive.ql.hooks.PostExecutePrinter</value>
+ <value>org.apache.hadoop.hive.ql.hooks.PostExecutePrinter, org.apache.hadoop.hive.ql.hooks.RuntimeStatsPersistenceCheckerHook</value>
<description>Post Execute Hook for Tests</description>
</property>
http://git-wip-us.apache.org/repos/asf/hive/blob/6535dce0/ql/src/java/org/apache/hadoop/hive/ql/hooks/RuntimeStatsPersistenceCheckerHook.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/RuntimeStatsPersistenceCheckerHook.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/RuntimeStatsPersistenceCheckerHook.java
new file mode 100644
index 0000000..b0bdad3
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/RuntimeStatsPersistenceCheckerHook.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.optimizer.signature.OpTreeSignature;
+import org.apache.hadoop.hive.ql.optimizer.signature.RuntimeStatsPersister;
+import org.apache.hadoop.hive.ql.plan.mapper.PlanMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This hook adds a persistence loop-back ensure that runtime statistics could be used.
+ */
+public class RuntimeStatsPersistenceCheckerHook implements ExecuteWithHookContext {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RuntimeStatsPersistenceCheckerHook.class);
+
+ @Override
+ public void run(HookContext hookContext) throws Exception {
+
+ PlanMapper pm = ((PrivateHookContext) hookContext).getContext().getPlanMapper();
+
+ List<OpTreeSignature> sigs = pm.getAll(OpTreeSignature.class);
+
+ for (OpTreeSignature sig : sigs) {
+ try {
+ OpTreeSignature sig2 = persistenceLoop(sig, OpTreeSignature.class);
+ sig.getSig().proveEquals(sig2.getSig());
+ } catch (Exception e) {
+ throw new RuntimeException("while checking the signature of: " + sig.getSig(), e);
+ }
+ }
+ for (OpTreeSignature sig : sigs) {
+ try {
+ OpTreeSignature sig2 = persistenceLoop(sig, OpTreeSignature.class);
+ if (!sig.equals(sig2)) {
+ throw new RuntimeException("signature mismatch");
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("while checking the signature of: " + sig.getSig(), e);
+ }
+ }
+ LOG.info("signature checked: " + sigs.size());
+ }
+
+ private <T> T persistenceLoop(T sig, Class<T> clazz) throws IOException {
+ RuntimeStatsPersister sp = RuntimeStatsPersister.INSTANCE;
+ String stored = sp.encode(sig);
+ return sp.decode(stored, clazz);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/6535dce0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/signature/SignatureUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/signature/SignatureUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/signature/SignatureUtils.java
index f599d33..6c86f9a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/signature/SignatureUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/signature/SignatureUtils.java
@@ -65,7 +65,7 @@ public final class SignatureUtils {
if (sigMethods.isEmpty()) {
// by supplying using "o" this enforces identity/equls matching
// which will most probably make the signature very unique
- ret.put(classLabel, o);
+ ret.put(classLabel, System.identityHashCode(o));
} else {
ret.put(classLabel, "1");
for (Method method : sigMethods) {
http://git-wip-us.apache.org/repos/asf/hive/blob/6535dce0/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
index 95990b8..b5ffcd9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
@@ -269,12 +269,12 @@ public class JoinDesc extends AbstractOperatorDesc {
*/
@Explain(displayName = "filter predicates")
@Signature
- public Map<Byte, String> getFiltersStringMap() {
+ public Map<String, String> getFiltersStringMap() {
if (getFilters() == null || getFilters().size() == 0) {
return null;
}
- LinkedHashMap<Byte, String> ret = new LinkedHashMap<Byte, String>();
+ LinkedHashMap<String, String> ret = new LinkedHashMap<>();
boolean filtersPresent = false;
for (Map.Entry<Byte, List<ExprNodeDesc>> ent : getFilters().entrySet()) {
@@ -295,7 +295,7 @@ public class JoinDesc extends AbstractOperatorDesc {
sb.append("}");
}
}
- ret.put(ent.getKey(), sb.toString());
+ ret.put(String.valueOf(ent.getKey()), sb.toString());
}
if (filtersPresent) {