You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2011/12/12 23:14:28 UTC
svn commit: r1213466 - in
/hbase/trunk/src/test/java/org/apache/hadoop/hbase: ResourceChecker.java
ResourceCheckerJUnitRule.java
Author: stack
Date: Mon Dec 12 22:14:27 2011
New Revision: 1213466
URL: http://svn.apache.org/viewvc?rev=1213466&view=rev
Log:
HBASE-4965 Monitor the open file descriptors and the threads counters during the unit tests
Added:
hbase/trunk/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitRule.java
Added: hbase/trunk/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java?rev=1213466&view=auto
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java (added)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/ResourceChecker.java Mon Dec 12 22:14:27 2011
@@ -0,0 +1,164 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import com.sun.management.UnixOperatingSystemMXBean;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.OperatingSystemMXBean;
+
+
+/**
+ * Check the resources used:
+ * - threads
+ * - file descriptor
+ */
+public class ResourceChecker {
+ private static final Log LOG = LogFactory.getLog(ResourceChecker.class);
+
+ /**
+ * On unix, we know how to get the number of open file descriptor
+ */
+ private static class ResourceAnalyzer {
+ private static final OperatingSystemMXBean osStats;
+ private static final UnixOperatingSystemMXBean unixOsStats;
+
+ public long getThreadsCount() {
+ return Thread.getAllStackTraces().size();
+ }
+
+ public long getOpenFileDescriptorCount() {
+ if (unixOsStats == null) {
+ return 0;
+ } else {
+ return unixOsStats.getOpenFileDescriptorCount();
+ }
+ }
+
+ public long getMaxFileDescriptorCount() {
+ if (unixOsStats == null) {
+ return 0;
+ } else {
+ return unixOsStats.getMaxFileDescriptorCount();
+ }
+ }
+
+ static {
+ osStats =
+ ManagementFactory.getOperatingSystemMXBean();
+ if (osStats instanceof UnixOperatingSystemMXBean) {
+ unixOsStats = (UnixOperatingSystemMXBean) osStats;
+ } else {
+ unixOsStats = null;
+ }
+ }
+ }
+
+ private static final ResourceAnalyzer rc = new ResourceAnalyzer();
+
+ /**
+ * Maximum we set for the thread. Will get a warning in logs
+ * if we go other this limit
+ */
+ private static final long MAX_THREADS_COUNT = 500;
+
+ /**
+ * Maximum we set for the thread. Will get a warning in logs
+ * if we go other this limit
+ */
+ private static final long MAX_FILE_HANDLES_COUNT = 1024;
+
+
+ private long initialThreadsCount;
+ private long initialFileHandlesCount;
+
+ public boolean checkThreads(String tagLine) {
+ boolean isOk = true;
+
+ if (rc.getThreadsCount() > MAX_THREADS_COUNT) {
+ LOG.error(
+ tagLine + ": too many threads used. We use " +
+ rc.getThreadsCount() + " our max is " + MAX_THREADS_COUNT);
+ isOk = false;
+ }
+
+ return isOk;
+ }
+
+ public boolean check(String tagLine) {
+
+ boolean isOk = checkThreads(tagLine);
+ if (!checkFileHandles(tagLine)) isOk = false;
+
+ return isOk;
+ }
+
+ public ResourceChecker(String tagLine) {
+ init(tagLine);
+ }
+
+ public final void init(String tagLine) {
+ if (rc.getMaxFileDescriptorCount() < MAX_FILE_HANDLES_COUNT) {
+ LOG.error(
+ "Bad configuration: the operating systems file handles maximum is " +
+ rc.getMaxFileDescriptorCount() + " our is " + MAX_FILE_HANDLES_COUNT);
+ }
+
+ logInfo(tagLine);
+ initialThreadsCount = rc.getThreadsCount();
+ initialFileHandlesCount = rc.getOpenFileDescriptorCount();
+ check(tagLine);
+ }
+
+ public void logInfo(String tagLine) {
+ LOG.info(
+ tagLine + ": " +
+ rc.getThreadsCount() + " threads" +
+ (initialThreadsCount > 0 ?
+ " (was " + initialThreadsCount + "), " : ", ") +
+ rc.getOpenFileDescriptorCount() + " file descriptors" +
+ (initialFileHandlesCount > 0 ?
+ " (was " + initialFileHandlesCount + "). " : " ") +
+ (initialThreadsCount > 0 && rc.getThreadsCount() > initialThreadsCount ?
+ " -thread leak?- " : "") +
+ (initialFileHandlesCount > 0 &&
+ rc.getOpenFileDescriptorCount() > initialFileHandlesCount ?
+ " -file handle leak?- " : "")
+ );
+ }
+
+
+ public boolean checkFileHandles(String tagLine) {
+ boolean isOk = true;
+
+ if (rc.getOpenFileDescriptorCount() > MAX_FILE_HANDLES_COUNT) {
+ LOG.error(
+ tagLine + ": too many file handles used. We use " +
+ rc.getOpenFileDescriptorCount() + " our max is " +
+ MAX_FILE_HANDLES_COUNT);
+ isOk = false;
+ }
+
+ return isOk;
+ }
+}
Added: hbase/trunk/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitRule.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitRule.java?rev=1213466&view=auto
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitRule.java (added)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/ResourceCheckerJUnitRule.java Mon Dec 12 22:14:27 2011
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+
+/**
+ * Class that implements a JUnit rule to be called before and after each
+ * test method to check the resources used:
+ * - file descriptors
+ * - threads
+ * @see ResourceChecker
+ */
+public class ResourceCheckerJUnitRule extends org.junit.rules.TestWatcher {
+ private ResourceChecker cu;
+ private boolean endDone;
+
+ /**
+ * To be called before the test methods
+ * @param testName
+ */
+ private void start(String testName) {
+ cu = new ResourceChecker("before "+testName);
+ endDone = false;
+ }
+
+ /**
+ * To be called after the test methods
+ * @param testName
+ */
+ private void end(String testName) {
+ if (!endDone) {
+ endDone = true;
+ cu.logInfo("after " + testName);
+ cu.check("after "+testName);
+ }
+ }
+
+ /**
+ * Get the test name from the JUnit Description
+ * @param description
+ * @return the string for the short test name
+ */
+ private String descriptionToShortTestName(
+ org.junit.runner.Description description) {
+ final int toRemove = "org.apache.hadoop.hbase.".length();
+ return description.getTestClass().getName().substring(toRemove) +
+ "#" + description.getMethodName();
+ }
+
+ @Override
+ protected void succeeded(org.junit.runner.Description description) {
+ end(descriptionToShortTestName(description));
+ }
+
+ @Override
+ protected void failed(java.lang.Throwable e, org.junit.runner.Description description) {
+ end(descriptionToShortTestName(description));
+ }
+
+ @Override
+ protected void starting(org.junit.runner.Description description) {
+ start(descriptionToShortTestName(description));
+ }
+
+ @Override
+ protected void finished(org.junit.runner.Description description) {
+ end(descriptionToShortTestName(description));
+ }
+}
+