You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by cw...@apache.org on 2015/02/17 17:21:26 UTC
svn commit: r1660435 - in /uima/sandbox/uima-ducc/trunk:
uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/
uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/
uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metr...
Author: cwiklik
Date: Tue Feb 17 16:21:25 2015
New Revision: 1660435
URL: http://svn.apache.org/r1660435
Log:
UIMA-4251 Added support to collect and publish node CPU count and current load on a node
Modified:
uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/NodeAgent.java
uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/DefaultNodeMetricsProcessor.java
uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/LinuxNodeMetricsProcessor.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metrics/NodeCpuInfo.java
uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metrics/NodeMetrics.java
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/NodeAgent.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/NodeAgent.java?rev=1660435&r1=1660434&r2=1660435&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/NodeAgent.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/NodeAgent.java Tue Feb 17 16:21:25 2015
@@ -151,6 +151,8 @@ public class NodeAgent extends AbstractD
public int cpuClockRate = 100;
+ public int numProcessors=0;
+
// indicates whether or not this agent received at least one publication
// from the PM. This flag is used to determine if the agent should use
// rogue process detector. The detector will be used if this flag is true.
@@ -195,6 +197,9 @@ public class NodeAgent extends AbstractD
// fetch Page Size from the OS and cache it
pageSize = getOSPageSize();
+
+ numProcessors = getNodeProcessors();
+
logger.info("NodeAgent", null, "OS Page Size:" + pageSize);
cpuClockRate = getOSClockRate();
@@ -324,7 +329,9 @@ public class NodeAgent extends AbstractD
}
}
-
+ public int getNodeProcessors() {
+ return runOSCommand(new String[] { "/usr/bin/getconf", "_NPROCESSORS_ONLN" });
+ }
public int getOSPageSize() {
return runOSCommand(new String[] { "/usr/bin/getconf", "PAGESIZE" });
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/DefaultNodeMetricsProcessor.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/DefaultNodeMetricsProcessor.java?rev=1660435&r1=1660434&r2=1660435&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/DefaultNodeMetricsProcessor.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/DefaultNodeMetricsProcessor.java Tue Feb 17 16:21:25 2015
@@ -63,15 +63,17 @@ public class DefaultNodeMetricsProcessor
new DefaultNodeLoadAverageCollector();
Future<NodeLoadAverage> loadFuture = pool.submit(loadAvgCollector);
- NodeCpuCollector cpuCollector = new NodeCpuCollector();
- Future<NodeCpuInfo> cpuFuture = pool.submit(cpuCollector);
+// NodeCpuCollector cpuCollector = new NodeCpuCollector();
+// Future<NodeCpuInfo> cpuFuture = pool.submit(cpuCollector);
+ NodeCpuInfo cpuInfo = new NodeCpuInfo(agent.numProcessors);
+
NodeUsersCollector nodeUsersCollector = new NodeUsersCollector(agent, logger);
Future<TreeMap<String,NodeUsersInfo>> nuiFuture = pool.submit(nodeUsersCollector);
NodeMetrics nodeMetrics =
new NodeMetrics(agent.getIdentity(), nmiFuture.get(), loadFuture.get(),
- cpuFuture.get(), nuiFuture.get());
+ cpuInfo, nuiFuture.get());
//Node node = new DuccNode(new NodeIdentity(), nodeMetrics);
// jrc 2011-07-30 I think this needs to be agent.getIdentity(), not create a new identity.
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/LinuxNodeMetricsProcessor.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/LinuxNodeMetricsProcessor.java?rev=1660435&r1=1660434&r2=1660435&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/LinuxNodeMetricsProcessor.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/LinuxNodeMetricsProcessor.java Tue Feb 17 16:21:25 2015
@@ -102,8 +102,10 @@ public class LinuxNodeMetricsProcessor e
NodeLoadAverageCollector loadAvgCollector = new NodeLoadAverageCollector(
loadAvgFile, 5, 0);
Future<NodeLoadAverage> loadFuture = pool.submit(loadAvgCollector);
- NodeCpuCollector cpuCollector = new NodeCpuCollector();
- Future<NodeCpuInfo> cpuFuture = pool.submit(cpuCollector);
+// NodeCpuCollector cpuCollector = new NodeCpuCollector();
+// Future<NodeCpuInfo> cpuFuture = pool.submit(cpuCollector);
+ NodeCpuInfo cpuInfo = new NodeCpuInfo(agent.numProcessors);
+
e.getIn().setHeader("node", agent.getIdentity().getName());
NodeMemory memInfo = nmiFuture.get();
TreeMap<String, NodeUsersInfo> users = null;
@@ -122,7 +124,7 @@ public class LinuxNodeMetricsProcessor e
}
NodeMetrics nodeMetrics = new NodeMetrics(agent.getIdentity(), memInfo, loadFuture.get(),
- cpuFuture.get(), users);
+ cpuInfo, users);
Node node = new DuccNode(agent.getIdentity(), nodeMetrics, agent.useCgroups);
// Make the agent aware how much memory is available on the node. Do this once.
@@ -132,7 +134,10 @@ public class LinuxNodeMetricsProcessor e
((DuccNode)node).duccLingExists(agent.duccLingExists());
((DuccNode)node).runWithDuccLing(agent.runWithDuccLing());
- logger.info(methodName, null, "... Agent "+node.getNodeIdentity().getName()+" Posting Memory:"
+ logger.info(methodName, null, "... Agent "+node.getNodeIdentity().getName()+
+ " CPU Count:" + cpuInfo.getAvailableProcessors() +
+ " CPU Load:" +cpuInfo.getCurrentLoad() +
+ " Posting Memory:"
+ node.getNodeMetrics().getNodeMemory().getMemTotal()+
" Memory Free:"+node.getNodeMetrics().getNodeMemory().getMemFree()+
" Swap Total:"+node.getNodeMetrics().getNodeMemory().getSwapTotal()+
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metrics/NodeCpuInfo.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metrics/NodeCpuInfo.java?rev=1660435&r1=1660434&r2=1660435&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metrics/NodeCpuInfo.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metrics/NodeCpuInfo.java Tue Feb 17 16:21:25 2015
@@ -15,24 +15,74 @@
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
-*/
+ */
package org.apache.uima.ducc.common.node.metrics;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
import java.io.Serializable;
public class NodeCpuInfo implements Serializable {
- private static final long serialVersionUID = -4770810550678067034L;
- private String availableProcessors="0";
+ private static final long serialVersionUID = -4770810550678067034L;
+ private String availableProcessors = "0";
+ private String currentLoad = "0.0";
+
+ public NodeCpuInfo(int availableProcessors) {
+ setAvailableProcessors(String.valueOf(availableProcessors));
+ try {
+ currentLoad = String.valueOf(getCPULoad());
+ } catch( Exception e) {
+ e.printStackTrace();
+ }
+ }
+ public String getCurrentLoad() {
+ return currentLoad;
+ }
+ public String getAvailableProcessors() {
+ return availableProcessors;
+ }
- public NodeCpuInfo( int availableProcessors ) {
- setAvailableProcessors(String.valueOf(availableProcessors));
- }
- public String getAvailableProcessors() {
- return availableProcessors;
- }
-
- private void setAvailableProcessors(String availableProcessors) {
- this.availableProcessors = availableProcessors;
- }
-
+ private void setAvailableProcessors(String availableProcessors) {
+ this.availableProcessors = availableProcessors;
+ }
+
+ private double getCPULoad() throws Exception {
+ double cpu = 0.0;
+ InputStreamReader in = null;
+ String[] command = {
+ "/bin/sh",
+ "-c",
+ "/bin/grep 'cpu' /proc/stat | /bin/awk '{usage=($2+$4)*100/($2+$4+$5)} END {print usage}'" };
+ try {
+ ProcessBuilder pb = new ProcessBuilder();
+ pb.command(command);
+
+ pb.redirectErrorStream(true);
+ Process swapCollectorProcess = pb.start();
+ in = new InputStreamReader(swapCollectorProcess.getInputStream());
+ BufferedReader reader = new BufferedReader(in);
+ String line = null;
+ while ((line = reader.readLine()) != null) {
+ System.out.println(line);
+ try {
+ cpu = Double.parseDouble(line.trim());
+ } catch (NumberFormatException e) {
+ cpu = 0;
+ e.printStackTrace();
+ }
+ }
+ } catch (Exception e) {
+ throw e;
+ } finally {
+ if (in != null) {
+ try {
+ in.close();
+ } catch (Exception e) {
+ }
+
+ }
+ }
+
+ return cpu;
+ }
}
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metrics/NodeMetrics.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metrics/NodeMetrics.java?rev=1660435&r1=1660434&r2=1660435&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metrics/NodeMetrics.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metrics/NodeMetrics.java Tue Feb 17 16:21:25 2015
@@ -33,7 +33,7 @@ public class NodeMetrics implements Seri
private NodeIdentity nodeIdentity;
TreeMap<String,NodeUsersInfo> nodeUsersMap;
- public NodeMetrics(NodeIdentity nodeIdentity, NodeMemory nodeMemory, NodeLoadAverage nodeLoadAverage , NodeCpuInfo nodeCpu, TreeMap<String,NodeUsersInfo> userProcessMap ) {
+ public NodeMetrics(NodeIdentity nodeIdentity, NodeMemory nodeMemory, NodeLoadAverage nodeLoadAverage , NodeCpuInfo nodeCpu, TreeMap<String,NodeUsersInfo> userProcessMap) {
this.nodeIdentity = nodeIdentity;
setNodeMemory(nodeMemory);
setNodeLoadAverage(nodeLoadAverage);
@@ -45,7 +45,7 @@ public class NodeMetrics implements Seri
return nodeUsersMap;
}
- public NodeIdentity getNodeIdentity() {
+ public NodeIdentity getNodeIdentity() {
return nodeIdentity;
}
public NodeCpuInfo getNodeCpu() {