You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2017/07/11 07:44:20 UTC
hive git commit: HIVE-17067: LLAP: Add http endpoint to provide
system level configurations (Prasanth Jayachandran reviewed by Siddharth Seth,
Gopal V)
Repository: hive
Updated Branches:
refs/heads/master 37f84ca09 -> 723768513
HIVE-17067: LLAP: Add http endpoint to provide system level configurations (Prasanth Jayachandran reviewed by Siddharth Seth, Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/72376851
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/72376851
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/72376851
Branch: refs/heads/master
Commit: 7237685136acd5a5d4cc6d6c7edb703b98924279
Parents: 37f84ca
Author: Prasanth Jayachandran <pr...@apache.org>
Authored: Tue Jul 11 00:44:10 2017 -0700
Committer: Prasanth Jayachandran <pr...@apache.org>
Committed: Tue Jul 11 00:44:10 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hive/http/HttpServer.java | 1 +
.../java/org/apache/hive/http/StackServlet.java | 2 +-
llap-server/bin/llapDaemon.sh | 3 +
.../daemon/services/impl/LlapWebServices.java | 1 +
.../impl/SystemConfigurationServlet.java | 196 +++++++++++++++++++
5 files changed, 202 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/72376851/common/src/java/org/apache/hive/http/HttpServer.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hive/http/HttpServer.java b/common/src/java/org/apache/hive/http/HttpServer.java
index 7368a91..0624a7e 100644
--- a/common/src/java/org/apache/hive/http/HttpServer.java
+++ b/common/src/java/org/apache/hive/http/HttpServer.java
@@ -33,6 +33,7 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.common.base.Preconditions;
+
import org.apache.commons.math3.util.Pair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
http://git-wip-us.apache.org/repos/asf/hive/blob/72376851/common/src/java/org/apache/hive/http/StackServlet.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hive/http/StackServlet.java b/common/src/java/org/apache/hive/http/StackServlet.java
index 610b391..3345466 100644
--- a/common/src/java/org/apache/hive/http/StackServlet.java
+++ b/common/src/java/org/apache/hive/http/StackServlet.java
@@ -74,7 +74,7 @@ public class StackServlet extends HttpServlet {
Thread.State state = info.getThreadState();
stream.println(" State: " + state);
stream.println(" Blocked count: " + info.getBlockedCount());
- stream.println(" Wtaited count: " + info.getWaitedCount());
+ stream.println(" Waited count: " + info.getWaitedCount());
if (contention) {
stream.println(" Blocked time: " + info.getBlockedTime());
stream.println(" Waited time: " + info.getWaitedTime());
http://git-wip-us.apache.org/repos/asf/hive/blob/72376851/llap-server/bin/llapDaemon.sh
----------------------------------------------------------------------
diff --git a/llap-server/bin/llapDaemon.sh b/llap-server/bin/llapDaemon.sh
index 566bbc8..4945473 100755
--- a/llap-server/bin/llapDaemon.sh
+++ b/llap-server/bin/llapDaemon.sh
@@ -115,6 +115,9 @@ case $startStop in
export LLAP_DAEMON_LOGFILE=${LLAP_DAEMON_LOG_BASE}.log
nohup nice -n $LLAP_DAEMON_NICENESS "$LLAP_DAEMON_BIN_HOME"/runLlapDaemon.sh run >> "$logOut" 2>&1 < /dev/null &
echo $! > $pid
+ # capture the ulimit output
+ echo "ulimit -a" >> $logOut
+ ulimit -a >> $logOut 2>&1
;;
(stop)
http://git-wip-us.apache.org/repos/asf/hive/blob/72376851/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java
index e896df2..ebc3437 100644
--- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/LlapWebServices.java
@@ -103,6 +103,7 @@ public class LlapWebServices extends AbstractService {
this.http.addServlet("status", "/status", LlapStatusServlet.class);
this.http.addServlet("peers", "/peers", LlapPeerRegistryServlet.class);
this.http.addServlet("iomem", "/iomem", LlapIoMemoryServlet.class);
+ this.http.addServlet("system", "/system", SystemConfigurationServlet.class);
} catch (IOException e) {
LOG.warn("LLAP web service failed to come up", e);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/72376851/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/SystemConfigurationServlet.java
----------------------------------------------------------------------
diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/SystemConfigurationServlet.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/SystemConfigurationServlet.java
new file mode 100644
index 0000000..aab1248
--- /dev/null
+++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/services/impl/SystemConfigurationServlet.java
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.llap.daemon.services.impl;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.hadoop.hive.llap.LlapDaemonInfo;
+import org.apache.hadoop.util.Shell;
+import org.apache.hive.http.HttpServer;
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonGenerator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.netty.util.NetUtil;
+
+/**
+ * A servlet to print system specific configurations that are not exposed via JMX.
+ * Currently it exposes
+ * - kernel configs
+ * - network configs
+ * - memory configs
+ *
+ * /system?refresh=true will run sysctl command again
+ */
+public class SystemConfigurationServlet extends HttpServlet {
+ private static final long serialVersionUID = 1L;
+ private static final Logger LOG = LoggerFactory.getLogger(SystemConfigurationServlet.class);
+ private static final String ACCESS_CONTROL_ALLOW_METHODS = "Access-Control-Allow-Methods";
+ private static final String ALLOWED_METHODS = "GET";
+ private static final String ACCESS_CONTROL_ALLOW_ORIGIN = "Access-Control-Allow-Origin";
+ private static final String CONTENT_TYPE_JSON_UTF8 = "application/json; charset=utf8";
+ protected transient JsonFactory jsonFactory;
+ private static final String SYSCTL_KV_SEPARATOR = Shell.MAC ? ":" : "=";
+ private AtomicReference<String> sysctlOutRef;
+ private String FAILED = "failed";
+
+ @Override
+ public void init() throws ServletException {
+ this.jsonFactory = new JsonFactory();
+ this.sysctlOutRef = new AtomicReference<>(null);
+ }
+
+ @Override
+ public void doGet(HttpServletRequest request, HttpServletResponse response)
+ throws ServletException, IOException {
+ JsonGenerator jg = null;
+ PrintWriter writer = null;
+ if (!HttpServer.isInstrumentationAccessAllowed(getServletContext(),
+ request, response)) {
+ response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
+ return;
+ }
+ setResponseHeader(response);
+ boolean refresh = Boolean.parseBoolean(request.getParameter("refresh"));
+ try {
+ writer = response.getWriter();
+ jg = jsonFactory.createJsonGenerator(writer);
+ jg.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET);
+ jg.useDefaultPrettyPrinter();
+ jg.writeStartObject();
+
+ jg.writeObjectField("pid", LlapDaemonInfo.INSTANCE.getPID());
+ jg.writeObjectField("os.name", System.getProperty("os.name"));
+ if (Shell.WINDOWS) {
+ jg.writeObjectField("net.core.somaxconn", NetUtil.SOMAXCONN);
+ } else {
+ String sysctlCmd = "sysctl -a";
+ try {
+ if (sysctlOutRef.get() == null || refresh) {
+ LOG.info("Reading kernel configs via sysctl..");
+ String sysctlOutput = Shell.execCommand(sysctlCmd.split("\\s+"));
+ sysctlOutRef.set(sysctlOutput);
+ }
+ } catch (IOException e) {
+ LOG.warn("Unable to execute '{}' command", sysctlCmd, e);
+ sysctlOutRef.set(FAILED); // failures will not be retried (to avoid fork + exec running sysctl command)
+ jg.writeObjectField("sysctl", FAILED);
+ jg.writeObjectField("sysctl-failure-reason", e.getMessage());
+ }
+
+ if (sysctlOutRef.get() != null && !sysctlOutRef.get().equals(FAILED)) {
+ String[] lines = sysctlOutRef.get().split("\\r?\\n");
+ for (String line : lines) {
+ int sepIdx = line.indexOf(SYSCTL_KV_SEPARATOR);
+ String key = sepIdx == -1 ? line.trim() : line.substring(0, sepIdx).trim();
+ String value = sepIdx == -1 ? null : line.substring(sepIdx + 1).trim().replaceAll("\t", " ");
+ if (!key.isEmpty()) {
+ jg.writeObjectField(key, value);
+ }
+ }
+ }
+
+ if (!Shell.MAC) {
+ // Red Hat/CentOS: /sys/kernel/mm/redhat_transparent_hugepage/defrag
+ // Ubuntu/Debian, OEL, SLES: /sys/kernel/mm/transparent_hugepage/defrag
+ String thpFileName = "/sys/kernel/mm/transparent_hugepage/enabled";
+ String thpFileStr = PrivilegedFileReader.read(thpFileName);
+ if (thpFileStr == null) {
+ LOG.warn("Unable to read contents of {}", thpFileName);
+ thpFileName = "/sys/kernel/mm/redhat_transparent_hugepage/enabled";
+ thpFileStr = PrivilegedFileReader.read(thpFileName);
+ }
+
+ if (thpFileStr != null) {
+ // Format: "always madvise [never]"
+ int strIdx = thpFileStr.indexOf('[');
+ int endIdx = thpFileStr.indexOf(']');
+ jg.writeObjectField(thpFileName, thpFileStr.substring(strIdx + 1, endIdx));
+ } else {
+ LOG.warn("Unable to read contents of {}", thpFileName);
+ }
+
+ String thpDefragFileName = "/sys/kernel/mm/transparent_hugepage/enabled";
+ String thpDefragFileStr = PrivilegedFileReader.read(thpDefragFileName);
+ if (thpDefragFileStr == null) {
+ LOG.warn("Unable to read contents of {}", thpDefragFileName);
+ thpDefragFileName = "/sys/kernel/mm/redhat_transparent_hugepage/enabled";
+ thpDefragFileStr = PrivilegedFileReader.read(thpDefragFileName);
+ }
+
+ if (thpDefragFileStr != null) {
+ // Format: "always madvise [never]"
+ int strIdx = thpDefragFileStr.indexOf('[');
+ int endIdx = thpDefragFileStr.indexOf(']');
+ jg.writeObjectField(thpDefragFileName, thpDefragFileStr.substring(strIdx + 1, endIdx));
+ } else {
+ LOG.warn("Unable to read contents of {}", thpDefragFileName);
+ }
+ }
+ }
+
+ jg.writeEndObject();
+ response.setStatus(HttpServletResponse.SC_OK);
+ } catch (Exception e) {
+ LOG.error("Caught exception while processing llap /system web service request", e);
+ response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ } finally {
+ if (jg != null) {
+ jg.close();
+ }
+ if (writer != null) {
+ writer.close();
+ }
+ }
+ }
+
+ private void setResponseHeader(final HttpServletResponse response) {
+ response.setContentType(CONTENT_TYPE_JSON_UTF8);
+ response.setHeader(ACCESS_CONTROL_ALLOW_METHODS, ALLOWED_METHODS);
+ response.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, "*");
+ }
+
+ private static class PrivilegedFileReader {
+ public static String read(String filename) {
+ String result = AccessController.doPrivileged(new PrivilegedAction<String>() {
+ @Override
+ public String run() {
+ String fileString = null;
+ try {
+ fileString = new String(Files.readAllBytes(Paths.get(filename)), StandardCharsets.UTF_8);
+ } catch (Exception e) {
+ LOG.warn("Unable to read file: {}", filename, e);
+ }
+ return fileString;
+ }
+ });
+ return result;
+ }
+ }
+}