You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by we...@apache.org on 2017/05/30 21:17:13 UTC
[04/12] hive git commit: HIVE-16343: LLAP: Publish YARN's ProcFs
based memory usage to metrics for monitoring (Prasanth Jayachandran reviewed
by Siddharth Seth)
HIVE-16343: LLAP: Publish YARN's ProcFs based memory usage to metrics for monitoring (Prasanth Jayachandran reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c3c6175f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c3c6175f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c3c6175f
Branch: refs/heads/hive-14535
Commit: c3c6175fb4dbee40cd5f742a49414bf9ecb2f54a
Parents: 573a181
Author: Prasanth Jayachandran <pr...@apache.org>
Authored: Fri May 26 14:16:19 2017 -0700
Committer: Prasanth Jayachandran <pr...@apache.org>
Committed: Fri May 26 14:16:19 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/llap/LlapDaemonInfo.java | 14 +++++---
llap-server/bin/runLlapDaemon.sh | 2 +-
.../hive/llap/daemon/impl/LlapDaemon.java | 8 ++---
.../hive/llap/metrics/LlapDaemonJvmInfo.java | 2 ++
.../hive/llap/metrics/LlapDaemonJvmMetrics.java | 35 +++++++++++++++++---
.../hive/llap/daemon/MiniLlapCluster.java | 2 +-
6 files changed, 47 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c3c6175f/llap-common/src/java/org/apache/hadoop/hive/llap/LlapDaemonInfo.java
----------------------------------------------------------------------
diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/LlapDaemonInfo.java b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapDaemonInfo.java
index fa29b59..dae10c8 100644
--- a/llap-common/src/java/org/apache/hadoop/hive/llap/LlapDaemonInfo.java
+++ b/llap-common/src/java/org/apache/hadoop/hive/llap/LlapDaemonInfo.java
@@ -25,12 +25,13 @@ public enum LlapDaemonInfo {
private static final class LlapDaemonInfoHolder {
public LlapDaemonInfoHolder(int numExecutors, long executorMemory, long cacheSize,
- boolean isDirectCache, boolean isLlapIo) {
+ boolean isDirectCache, boolean isLlapIo, final String pid) {
this.numExecutors = numExecutors;
this.executorMemory = executorMemory;
this.cacheSize = cacheSize;
this.isDirectCache = isDirectCache;
this.isLlapIo = isLlapIo;
+ this.PID = pid;
}
final int numExecutors;
@@ -38,6 +39,7 @@ public enum LlapDaemonInfo {
final long cacheSize;
final boolean isDirectCache;
final boolean isLlapIo;
+ final String PID;
}
// add more variables as required
@@ -51,13 +53,14 @@ public enum LlapDaemonInfo {
long ioMemoryBytes = HiveConf.getSizeVar(daemonConf, ConfVars.LLAP_IO_MEMORY_MAX_SIZE);
boolean isDirectCache = HiveConf.getBoolVar(daemonConf, ConfVars.LLAP_ALLOCATOR_DIRECT);
boolean isLlapIo = HiveConf.getBoolVar(daemonConf, HiveConf.ConfVars.LLAP_IO_ENABLED, true);
- initialize(appName, numExecutors, executorMemoryBytes, ioMemoryBytes, isDirectCache, isLlapIo);
+ String pid = System.getenv("JVM_PID");
+ initialize(appName, numExecutors, executorMemoryBytes, ioMemoryBytes, isDirectCache, isLlapIo, pid);
}
public static void initialize(String appName, int numExecutors, long executorMemoryBytes,
- long ioMemoryBytes, boolean isDirectCache, boolean isLlapIo) {
+ long ioMemoryBytes, boolean isDirectCache, boolean isLlapIo, final String pid) {
INSTANCE.dataRef.set(new LlapDaemonInfoHolder(numExecutors, executorMemoryBytes, ioMemoryBytes,
- isDirectCache, isLlapIo));
+ isDirectCache, isLlapIo, pid));
}
public boolean isLlap() {
@@ -89,4 +92,7 @@ public enum LlapDaemonInfo {
return dataRef.get().isLlapIo;
}
+ public String getPID() {
+ return dataRef.get().PID;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c3c6175f/llap-server/bin/runLlapDaemon.sh
----------------------------------------------------------------------
diff --git a/llap-server/bin/runLlapDaemon.sh b/llap-server/bin/runLlapDaemon.sh
index 82c2cc5..5a0c10e 100755
--- a/llap-server/bin/runLlapDaemon.sh
+++ b/llap-server/bin/runLlapDaemon.sh
@@ -127,6 +127,6 @@ LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS} -Dllap.daemon.log.file=${LLAP_DAEMON_LOG_F
LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS} -Dllap.daemon.root.logger=${LLAP_DAEMON_LOGGER}"
LLAP_DAEMON_OPTS="${LLAP_DAEMON_OPTS} -Dllap.daemon.log.level=${LLAP_DAEMON_LOG_LEVEL}"
+export JVM_PID="$$"
exec "$JAVA" -Dproc_llapdaemon -Xms${LLAP_DAEMON_HEAPSIZE}m -Xmx${LLAP_DAEMON_HEAPSIZE}m ${LLAP_DAEMON_OPTS} -classpath "$CLASSPATH" $CLASS "$@"
-
http://git-wip-us.apache.org/repos/asf/hive/blob/c3c6175f/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
index cfca3f7..68ef200 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java
@@ -14,8 +14,6 @@
package org.apache.hadoop.hive.llap.daemon.impl;
-import org.apache.hadoop.hive.llap.LlapOutputFormatService;
-
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryPoolMXBean;
@@ -42,6 +40,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.llap.DaemonId;
import org.apache.hadoop.hive.llap.LlapDaemonInfo;
+import org.apache.hadoop.hive.llap.LlapOutputFormatService;
import org.apache.hadoop.hive.llap.LlapUtil;
import org.apache.hadoop.hive.llap.configuration.LlapDaemonConfiguration;
import org.apache.hadoop.hive.llap.daemon.ContainerRunner;
@@ -91,7 +90,6 @@ import com.google.common.primitives.Ints;
public class LlapDaemon extends CompositeService implements ContainerRunner, LlapDaemonMXBean {
private static final Logger LOG = LoggerFactory.getLogger(LlapDaemon.class);
-
private final Configuration shuffleHandlerConf;
private final SecretManager secretManager;
private final LlapProtocolServerImpl server;
@@ -245,7 +243,7 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
pauseMonitor.start();
String displayNameJvm = "LlapDaemonJvmMetrics-" + hostName;
String sessionId = MetricsUtils.getUUID();
- LlapDaemonJvmMetrics.create(displayNameJvm, sessionId);
+ LlapDaemonJvmMetrics.create(displayNameJvm, sessionId, daemonConf);
String displayName = "LlapDaemonExecutorMetrics-" + hostName;
daemonConf.set("llap.daemon.metrics.sessionid", sessionId);
String[] strIntervals = HiveConf.getTrimmedStringsVar(daemonConf,
@@ -539,7 +537,7 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla
llapDaemon.init(daemonConf);
llapDaemon.start();
- LOG.info("Started LlapDaemon");
+ LOG.info("Started LlapDaemon with PID: {}", LlapDaemonInfo.INSTANCE.getPID());
// Relying on the RPC threads to keep the service alive.
} catch (Throwable t) {
// TODO Replace this with a ExceptionHandler / ShutdownHook
http://git-wip-us.apache.org/repos/asf/hive/blob/c3c6175f/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonJvmInfo.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonJvmInfo.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonJvmInfo.java
index efbddaa..11ac5a4 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonJvmInfo.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonJvmInfo.java
@@ -38,6 +38,8 @@ public enum LlapDaemonJvmInfo implements MetricsInfo {
LlapDaemonMappedBufferMemoryUsed("Estimate of memory that JVM is using for mapped byte buffers in bytes"),
LlapDaemonOpenFileDescriptorCount("Number of open file descriptors"),
LlapDaemonMaxFileDescriptorCount("Maximum number of file descriptors"),
+ LlapDaemonResidentSetSize("Resident memory (RSS) used by llap daemon process in bytes"),
+ LlapDaemonVirtualMemorySize("Virtual memory (VMEM) used by llap daemon process in bytes")
;
private final String desc;
http://git-wip-us.apache.org/repos/asf/hive/blob/c3c6175f/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonJvmMetrics.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonJvmMetrics.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonJvmMetrics.java
index cfb8729..be25846 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonJvmMetrics.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonJvmMetrics.java
@@ -26,6 +26,8 @@ import static org.apache.hadoop.hive.llap.metrics.LlapDaemonJvmInfo.LlapDaemonMa
import static org.apache.hadoop.hive.llap.metrics.LlapDaemonJvmInfo.LlapDaemonMappedBufferTotalCapacity;
import static org.apache.hadoop.hive.llap.metrics.LlapDaemonJvmInfo.LlapDaemonMaxFileDescriptorCount;
import static org.apache.hadoop.hive.llap.metrics.LlapDaemonJvmInfo.LlapDaemonOpenFileDescriptorCount;
+import static org.apache.hadoop.hive.llap.metrics.LlapDaemonJvmInfo.LlapDaemonResidentSetSize;
+import static org.apache.hadoop.hive.llap.metrics.LlapDaemonJvmInfo.LlapDaemonVirtualMemorySize;
import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName;
import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId;
@@ -34,12 +36,17 @@ import java.lang.management.ManagementFactory;
import java.lang.management.OperatingSystemMXBean;
import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.llap.LlapDaemonInfo;
+import org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.MetricsSource;
import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.annotation.Metrics;
import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
import com.sun.management.UnixOperatingSystemMXBean;
@@ -51,24 +58,34 @@ public class LlapDaemonJvmMetrics implements MetricsSource {
private final String name;
private final String sessionId;
private final MetricsRegistry registry;
+ private final ResourceCalculatorProcessTree processTree;
+ private final String daemonPid = LlapDaemonInfo.INSTANCE.getPID();
- private LlapDaemonJvmMetrics(String displayName, String sessionId) {
+ private LlapDaemonJvmMetrics(String displayName, String sessionId, final Configuration conf) {
this.name = displayName;
this.sessionId = sessionId;
+ Class<? extends ResourceCalculatorProcessTree> clazz = conf.getClass(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE,
+ null, ResourceCalculatorProcessTree.class);
+ this.processTree = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree("" + daemonPid, clazz, conf);
+ if (processTree != null) {
+ this.processTree.setConf(conf);
+ }
this.registry = new MetricsRegistry("LlapDaemonJvmRegistry");
this.registry.tag(ProcessName, MetricsUtils.METRICS_PROCESS_NAME).tag(SessionId, sessionId);
}
- public static LlapDaemonJvmMetrics create(String displayName, String sessionId) {
+ public static LlapDaemonJvmMetrics create(String displayName, String sessionId,
+ final Configuration conf) {
MetricsSystem ms = LlapMetricsSystem.instance();
- return ms.register(displayName, "LlapDaemon JVM Metrics", new LlapDaemonJvmMetrics(displayName, sessionId));
+ return ms.register(displayName, "LlapDaemon JVM Metrics",
+ new LlapDaemonJvmMetrics(displayName, sessionId, conf));
}
@Override
public void getMetrics(MetricsCollector collector, boolean b) {
MetricsRecordBuilder rb = collector.addRecord(LlapDaemonJVMMetrics)
.setContext("jvm")
- .tag(ProcessName, MetricsUtils.METRICS_PROCESS_NAME)
+ .tag(ProcessName, MetricsUtils.METRICS_PROCESS_NAME + "(PID: " + daemonPid + ")")
.tag(SessionId, sessionId);
getJvmMetrics(rb);
}
@@ -100,6 +117,12 @@ public class LlapDaemonJvmMetrics implements MetricsSource {
openFileHandles = ((UnixOperatingSystemMXBean) os).getOpenFileDescriptorCount();
maxFileHandles = ((UnixOperatingSystemMXBean) os).getMaxFileDescriptorCount();
}
+ long rss = 0;
+ long vmem = 0;
+ if (processTree != null) {
+ rss = processTree.getRssMemorySize();
+ vmem = processTree.getVirtualMemorySize();
+ }
rb.addGauge(LlapDaemonDirectBufferCount, directBufferCount)
.addGauge(LlapDaemonDirectBufferTotalCapacity, directBufferTotalCapacity)
.addGauge(LlapDaemonDirectBufferMemoryUsed, directBufferMemoryUsed)
@@ -107,7 +130,9 @@ public class LlapDaemonJvmMetrics implements MetricsSource {
.addGauge(LlapDaemonMappedBufferTotalCapacity, mappedBufferTotalCapacity)
.addGauge(LlapDaemonMappedBufferMemoryUsed, mappedBufferMemoryUsed)
.addGauge(LlapDaemonOpenFileDescriptorCount, openFileHandles)
- .addGauge(LlapDaemonMaxFileDescriptorCount, maxFileHandles);
+ .addGauge(LlapDaemonMaxFileDescriptorCount, maxFileHandles)
+ .addGauge(LlapDaemonResidentSetSize, rss)
+ .addGauge(LlapDaemonVirtualMemorySize, vmem);
}
public String getName() {
http://git-wip-us.apache.org/repos/asf/hive/blob/c3c6175f/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
----------------------------------------------------------------------
diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
index 6f1305e..6af230e 100644
--- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
+++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/MiniLlapCluster.java
@@ -121,7 +121,7 @@ public class MiniLlapCluster extends AbstractService {
this.ioBytesPerService = ioBytesPerService;
LlapDaemonInfo.initialize("mini-llap-cluster", numExecutorsPerService, execMemoryPerService,
- ioBytesPerService, ioIsDirect, llapIoEnabled);
+ ioBytesPerService, ioIsDirect, llapIoEnabled, "-1");
// Setup Local Dirs
localDirs = new String[numLocalDirs];