You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by go...@apache.org on 2017/05/12 08:09:29 UTC
[1/2] hive git commit: HIVE-16635: Progressbar: Use different
timeouts for running queries (Gopal V, reviewed by Siddharth Seth)
Repository: hive
Updated Branches:
refs/heads/master 6bfa2491b -> 538c0088a
HIVE-16635: Progressbar: Use different timeouts for running queries (Gopal V, reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/40fe0d7e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/40fe0d7e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/40fe0d7e
Branch: refs/heads/master
Commit: 40fe0d7e03a03bf6082ff00f17322756e6f00ea9
Parents: 6bfa249
Author: Gopal V <go...@apache.org>
Authored: Fri May 12 00:07:22 2017 -0700
Committer: Gopal V <go...@apache.org>
Committed: Fri May 12 00:07:22 2017 -0700
----------------------------------------------------------------------
.../ql/exec/tez/monitoring/TezJobMonitor.java | 30 +++++++++++++++++---
1 file changed, 26 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/40fe0d7e/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java
index f2f97f3..049d7fd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.exec.tez.monitoring;
import com.google.common.base.Preconditions;
+
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Context;
@@ -39,6 +40,7 @@ import org.apache.tez.dag.api.client.DAGClient;
import org.apache.tez.dag.api.client.DAGStatus;
import org.apache.tez.dag.api.client.Progress;
import org.apache.tez.dag.api.client.StatusGetOpts;
+import org.apache.tez.util.StopWatch;
import java.io.IOException;
import java.io.InterruptedIOException;
@@ -47,6 +49,7 @@ import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
import static org.apache.tez.dag.api.client.DAGStatus.State.RUNNING;
@@ -58,8 +61,10 @@ import static org.apache.tez.dag.api.client.DAGStatus.State.RUNNING;
public class TezJobMonitor {
static final String CLASS_NAME = TezJobMonitor.class.getName();
- private static final int CHECK_INTERVAL = 200;
+ private static final int MIN_CHECK_INTERVAL = 200;
+ private static final int MAX_CHECK_INTERVAL = 1000;
private static final int MAX_RETRY_INTERVAL = 2500;
+ private static final int MAX_RETRY_FAILURES = (MAX_RETRY_INTERVAL / MAX_CHECK_INTERVAL) + 1;
private final PerfLogger perfLogger = SessionState.getPerfLogger();
private static final List<DAGClient> shutdownList;
@@ -124,6 +129,7 @@ public class TezJobMonitor {
boolean done = false;
boolean success = false;
int failedCounter = 0;
+ final StopWatch failureTimer = new StopWatch();
int rc = 0;
DAGStatus status = null;
Map<String, Progress> vertexProgressMap = null;
@@ -138,6 +144,7 @@ public class TezJobMonitor {
DAGStatus.State lastState = null;
boolean running = false;
+ int checkInterval = MIN_CHECK_INTERVAL;
while (true) {
try {
@@ -145,10 +152,13 @@ public class TezJobMonitor {
context.checkHeartbeaterLockException();
}
- status = dagClient.getDAGStatus(new HashSet<StatusGetOpts>(), CHECK_INTERVAL);
+ status = dagClient.getDAGStatus(new HashSet<StatusGetOpts>(), checkInterval);
vertexProgressMap = status.getVertexProgress();
DAGStatus.State state = status.getState();
+ failedCounter = 0; // AM is responsive again (recovery?)
+ failureTimer.reset();
+
if (state != lastState || state == RUNNING) {
lastState = state;
@@ -166,6 +176,8 @@ public class TezJobMonitor {
console.printInfo("Status: Running (" + dagClient.getExecutionContext() + ")\n");
this.executionStartTime = System.currentTimeMillis();
running = true;
+ // from running -> failed/succeeded, the AM breaks out of timeouts
+ checkInterval = MAX_CHECK_INTERVAL;
}
updateFunction.update(status, vertexProgressMap);
break;
@@ -204,9 +216,19 @@ public class TezJobMonitor {
} catch (Exception e) {
console.printInfo("Exception: " + e.getMessage());
boolean isInterrupted = hasInterruptedException(e);
- if (isInterrupted || (++failedCounter % MAX_RETRY_INTERVAL / CHECK_INTERVAL == 0)) {
+ if (failedCounter == 0) {
+ failureTimer.reset();
+ failureTimer.start();
+ }
+ if (isInterrupted
+ || (++failedCounter >= MAX_RETRY_FAILURES && failureTimer.now(TimeUnit.MILLISECONDS) > MAX_RETRY_INTERVAL)) {
try {
- console.printInfo("Killing DAG...");
+ if (isInterrupted) {
+ console.printInfo("Killing DAG...");
+ } else {
+ console.printInfo(String.format("Killing DAG... after %d seconds",
+ failureTimer.now(TimeUnit.SECONDS)));
+ }
dagClient.tryKillDAG();
} catch (IOException | TezException tezException) {
// best effort
[2/2] hive git commit: HIVE-16592: Vectorization: Long hashCodes
should bit-mix into lower bits (Gopal V, reviewed by Sergey Shelukhin)
Posted by go...@apache.org.
HIVE-16592: Vectorization: Long hashCodes should bit-mix into lower bits (Gopal V, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/538c0088
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/538c0088
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/538c0088
Branch: refs/heads/master
Commit: 538c0088ab89cdcc30f4d1ae2c7d2bd266d18235
Parents: 40fe0d7
Author: Gopal V <go...@apache.org>
Authored: Fri May 12 00:11:35 2017 -0700
Committer: Gopal V <go...@apache.org>
Committed: Fri May 12 00:11:44 2017 -0700
----------------------------------------------------------------------
.../apache/hive/common/util/HashCodeUtil.java | 30 ++++++--------------
.../hive/common/type/FastHiveDecimalImpl.java | 2 +-
2 files changed, 9 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/538c0088/common/src/java/org/apache/hive/common/util/HashCodeUtil.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hive/common/util/HashCodeUtil.java b/common/src/java/org/apache/hive/common/util/HashCodeUtil.java
index fa30273..b47479b 100644
--- a/common/src/java/org/apache/hive/common/util/HashCodeUtil.java
+++ b/common/src/java/org/apache/hive/common/util/HashCodeUtil.java
@@ -34,33 +34,19 @@ public class HashCodeUtil {
}
public static int calculateLongHashCode(long key) {
-
- key = (~key) + (key << 21); // key = (key << 21) - key - 1;
- key = key ^ (key >>> 24);
- key = (key + (key << 3)) + (key << 8); // key * 265
- key = key ^ (key >>> 14);
- key = (key + (key << 2)) + (key << 4); // key * 21
- key = key ^ (key >>> 28);
- key = key + (key << 31);
-
+ // Mixing down into the lower bits - this produces a worse hashcode in purely
+ // numeric terms, but leaving entropy in the higher bits is not useful for a
+ // 2^n bucketing scheme. See JSR166 ConcurrentHashMap r1.89 (released under Public Domain)
+ // Note: ConcurrentHashMap has since reverted this to retain entropy bits higher
+ // up, to support the 2-level hashing for segment which operates at a higher bitmask
+ key ^= (key >>> 7) ^ (key >>> 4);
+ key ^= (key >>> 20) ^ (key >>> 12);
return (int) key;
}
public static void calculateLongArrayHashCodes(long[] longs, int[] hashCodes, final int count) {
- long key;
for (int v = 0; v < count; v++) {
-
- key = longs[v];
-
- // Hash code logic from calculateLongHashCode.
- key = (~key) + (key << 21); // key = (key << 21) - key - 1;
- key = key ^ (key >>> 24);
- key = (key + (key << 3)) + (key << 8); // key * 265
- key = key ^ (key >>> 14);
- key = (key + (key << 2)) + (key << 4); // key * 21
- key = key ^ (key >>> 28);
- key = key + (key << 31);
- hashCodes[v] = (int) key;
+ hashCodes[v] = (int) calculateLongHashCode(longs[v]);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/538c0088/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java
index 88abf3c..ef9cbcf 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java
@@ -3913,7 +3913,7 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
long key = fast0;
- // Hash code logic from calculateLongHashCode.
+ // Hash code logic from original calculateLongHashCode
key = (~key) + (key << 21); // key = (key << 21) - key - 1;
key = key ^ (key >>> 24);