You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by am...@apache.org on 2018/03/15 00:34:32 UTC
asterixdb git commit: [NO ISSUE][OTH] Log open file count on
unexpected HttpServer close
Repository: asterixdb
Updated Branches:
refs/heads/master d6d42c861 -> 57df8db1b
[NO ISSUE][OTH] Log open file count on unexpected HttpServer close
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Recently, there has been some cases where the Http server
repeatedly drops its listener. One of the possible causes
for this is a leak in open file descriptor. To determine if
this is the case, we log the open files count when that happens.
Change-Id: Ie43f3392268b3994839d1f98f4de9fe669c7be62
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2483
Sonar-Qube: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/57df8db1
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/57df8db1
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/57df8db1
Branch: refs/heads/master
Commit: 57df8db1b4188940d2f3c8632dcdf1a4c10d0229
Parents: d6d42c8
Author: Abdullah Alamoudi <ba...@gmail.com>
Authored: Wed Mar 14 15:25:29 2018 -0700
Committer: abdullah alamoudi <ba...@gmail.com>
Committed: Wed Mar 14 17:26:57 2018 -0700
----------------------------------------------------------------------
.../control/common/heartbeat/HeartbeatData.java | 2 +-
.../hyracks/control/common/utils/MXHelper.java | 48 ---------
.../control/nc/NodeControllerService.java | 8 +-
.../nc/heartbeat/HeartbeatComputeTask.java | 8 +-
.../apache/hyracks/http/server/HttpServer.java | 2 +
.../java/org/apache/hyracks/util/MXHelper.java | 103 +++++++++++++++++++
6 files changed, 114 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/57df8db1/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java
index d09d0b4..76b68bb 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/heartbeat/HeartbeatData.java
@@ -18,7 +18,7 @@
*/
package org.apache.hyracks.control.common.heartbeat;
-import static org.apache.hyracks.control.common.utils.MXHelper.gcMXBeans;
+import static org.apache.hyracks.util.MXHelper.gcMXBeans;
import java.io.DataInput;
import java.io.DataOutput;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/57df8db1/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/MXHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/MXHelper.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/MXHelper.java
deleted file mode 100644
index cfe8c21..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/utils/MXHelper.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.control.common.utils;
-
-import java.lang.management.GarbageCollectorMXBean;
-import java.lang.management.ManagementFactory;
-import java.lang.management.MemoryMXBean;
-import java.lang.management.OperatingSystemMXBean;
-import java.lang.management.RuntimeMXBean;
-import java.lang.management.ThreadMXBean;
-import java.util.Collections;
-import java.util.List;
-
-public class MXHelper {
- public static final MemoryMXBean memoryMXBean;
- public static final List<GarbageCollectorMXBean> gcMXBeans;
- public static final ThreadMXBean threadMXBean;
- public static final RuntimeMXBean runtimeMXBean;
- public static final OperatingSystemMXBean osMXBean;
-
- static {
- memoryMXBean = ManagementFactory.getMemoryMXBean();
- gcMXBeans = Collections.unmodifiableList(ManagementFactory.getGarbageCollectorMXBeans());
- threadMXBean = ManagementFactory.getThreadMXBean();
- runtimeMXBean = ManagementFactory.getRuntimeMXBean();
- osMXBean = ManagementFactory.getOperatingSystemMXBean();
- }
-
- private MXHelper() {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/57df8db1/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
index 6c3a641..0756210 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
@@ -18,10 +18,10 @@
*/
package org.apache.hyracks.control.nc;
-import static org.apache.hyracks.control.common.utils.MXHelper.gcMXBeans;
-import static org.apache.hyracks.control.common.utils.MXHelper.memoryMXBean;
-import static org.apache.hyracks.control.common.utils.MXHelper.osMXBean;
-import static org.apache.hyracks.control.common.utils.MXHelper.runtimeMXBean;
+import static org.apache.hyracks.util.MXHelper.gcMXBeans;
+import static org.apache.hyracks.util.MXHelper.memoryMXBean;
+import static org.apache.hyracks.util.MXHelper.osMXBean;
+import static org.apache.hyracks.util.MXHelper.runtimeMXBean;
import java.io.File;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/57df8db1/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/heartbeat/HeartbeatComputeTask.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/heartbeat/HeartbeatComputeTask.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/heartbeat/HeartbeatComputeTask.java
index 4965b85..dfa5ff3 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/heartbeat/HeartbeatComputeTask.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/heartbeat/HeartbeatComputeTask.java
@@ -18,10 +18,10 @@
*/
package org.apache.hyracks.control.nc.heartbeat;
-import static org.apache.hyracks.control.common.utils.MXHelper.gcMXBeans;
-import static org.apache.hyracks.control.common.utils.MXHelper.memoryMXBean;
-import static org.apache.hyracks.control.common.utils.MXHelper.osMXBean;
-import static org.apache.hyracks.control.common.utils.MXHelper.threadMXBean;
+import static org.apache.hyracks.util.MXHelper.gcMXBeans;
+import static org.apache.hyracks.util.MXHelper.memoryMXBean;
+import static org.apache.hyracks.util.MXHelper.osMXBean;
+import static org.apache.hyracks.util.MXHelper.threadMXBean;
import java.lang.management.GarbageCollectorMXBean;
import java.lang.management.MemoryUsage;
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/57df8db1/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
index d971a7c..41e0088 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
@@ -30,6 +30,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hyracks.http.api.IChannelClosedHandler;
import org.apache.hyracks.http.api.IServlet;
+import org.apache.hyracks.util.MXHelper;
import org.apache.hyracks.util.ThreadDumpUtil;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
@@ -251,6 +252,7 @@ public class HttpServer {
return;
}
LOGGER.log(Level.WARN, "{} has stopped unexpectedly. Starting server recovery", this);
+ MXHelper.logFileDescriptors();
triggerRecovery();
}
});
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/57df8db1/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java
new file mode 100644
index 0000000..2b65106
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/MXHelper.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.util;
+
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.lang.management.OperatingSystemMXBean;
+import java.lang.management.RuntimeMXBean;
+import java.lang.management.ThreadMXBean;
+import java.lang.reflect.Method;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class MXHelper {
+ private static final Logger LOGGER = LogManager.getLogger();
+ public static final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
+ public static final List<GarbageCollectorMXBean> gcMXBeans =
+ Collections.unmodifiableList(ManagementFactory.getGarbageCollectorMXBeans());
+ public static final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
+ public static final RuntimeMXBean runtimeMXBean = ManagementFactory.getRuntimeMXBean();
+ public static final OperatingSystemMXBean osMXBean = ManagementFactory.getOperatingSystemMXBean();
+ private static Method getOpenFileDescriptorCount;
+ private static Method getMaxFileDescriptorCount;
+
+ static {
+ try {
+ getOpenFileDescriptorCount = osMXBean.getClass().getMethod("getOpenFileDescriptorCount");
+ getMaxFileDescriptorCount = osMXBean.getClass().getDeclaredMethod("getMaxFileDescriptorCount");
+ getOpenFileDescriptorCount.setAccessible(true);
+ getMaxFileDescriptorCount.setAccessible(true);
+ } catch (Throwable th) { // NOSONAR: diagnostic code shouldn't cause server failure
+ getOpenFileDescriptorCount = null;
+ getMaxFileDescriptorCount = null;
+ LOGGER.log(Level.WARN, "Failed setting up the methods to get the number of file descriptors through {}",
+ osMXBean.getClass().getName(), th);
+ }
+ }
+
+ private MXHelper() {
+ }
+
+ public static boolean supportOpenFileCount() {
+ return getOpenFileDescriptorCount != null;
+ }
+
+ public static Integer getCurrentOpenFileCount() {
+ if (getOpenFileDescriptorCount == null) {
+ return -1;
+ }
+ try {
+ return (Integer) getOpenFileDescriptorCount.invoke(osMXBean);
+ } catch (Throwable e) { // NOSONAR
+ LOGGER.log(Level.WARN, "Failure invoking getOpenFileDescriptorCount", e);
+ return -1;
+ }
+ }
+
+ public static Integer getMaxOpenFileCount() {
+ if (getMaxFileDescriptorCount == null) {
+ return -1;
+ }
+ try {
+ return (Integer) getMaxFileDescriptorCount.invoke(osMXBean);
+ } catch (Throwable e) { // NOSONAR
+ LOGGER.log(Level.WARN, "Failure invoking getMaxFileDescriptorCount", e);
+ return -1;
+ }
+ }
+
+ public static void logFileDescriptors() {
+ try {
+ if (supportOpenFileCount()) {
+ LOGGER.log(Level.WARN,
+ "Number of open files by this process is {}. Max number of allowed open files is {} ",
+ MXHelper::getCurrentOpenFileCount, MXHelper::getMaxOpenFileCount);
+ }
+ } catch (Throwable th) { // NOSONAR: diagnostic code shouldn't cause server failure
+ LOGGER.log(Level.WARN, "Failed getting the count of open files", th);
+ }
+ }
+
+}