You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by cw...@apache.org on 2015/02/17 17:21:26 UTC

svn commit: r1660435 - in /uima/sandbox/uima-ducc/trunk: uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/ uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/ uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metr...

Author: cwiklik
Date: Tue Feb 17 16:21:25 2015
New Revision: 1660435

URL: http://svn.apache.org/r1660435
Log:
UIMA-4251 Added support to collect and publish node CPU count and current load on a node

Modified:
    uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/NodeAgent.java
    uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/DefaultNodeMetricsProcessor.java
    uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/LinuxNodeMetricsProcessor.java
    uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metrics/NodeCpuInfo.java
    uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metrics/NodeMetrics.java

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/NodeAgent.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/NodeAgent.java?rev=1660435&r1=1660434&r2=1660435&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/NodeAgent.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/NodeAgent.java Tue Feb 17 16:21:25 2015
@@ -151,6 +151,8 @@ public class NodeAgent extends AbstractD
 
   public int cpuClockRate = 100;
   
+  public int numProcessors=0;
+  
   //  indicates whether or not this agent received at least one publication
   //  from the PM. This flag is used to determine if the agent should use
   //  rogue process detector. The detector will be used if this flag is true.
@@ -195,6 +197,9 @@ public class NodeAgent extends AbstractD
     
     // fetch Page Size from the OS and cache it
     pageSize = getOSPageSize();
+    
+    numProcessors = getNodeProcessors();
+    
     logger.info("NodeAgent", null, "OS Page Size:" + pageSize);
 
     cpuClockRate = getOSClockRate();
@@ -324,7 +329,9 @@ public class NodeAgent extends AbstractD
 
     }
   }
-
+  public int getNodeProcessors() {
+	    return runOSCommand(new String[] { "/usr/bin/getconf", "_NPROCESSORS_ONLN" });
+  }
   public int getOSPageSize() {
     return runOSCommand(new String[] { "/usr/bin/getconf", "PAGESIZE" });
   }

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/DefaultNodeMetricsProcessor.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/DefaultNodeMetricsProcessor.java?rev=1660435&r1=1660434&r2=1660435&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/DefaultNodeMetricsProcessor.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/DefaultNodeMetricsProcessor.java Tue Feb 17 16:21:25 2015
@@ -63,15 +63,17 @@ public class DefaultNodeMetricsProcessor
 	            new DefaultNodeLoadAverageCollector();
 	    Future<NodeLoadAverage> loadFuture = pool.submit(loadAvgCollector);
 
-	    NodeCpuCollector cpuCollector = new NodeCpuCollector();
-	    Future<NodeCpuInfo> cpuFuture = pool.submit(cpuCollector);
+//	    NodeCpuCollector cpuCollector = new NodeCpuCollector();
+//	    Future<NodeCpuInfo> cpuFuture = pool.submit(cpuCollector);
 
+	    NodeCpuInfo cpuInfo = new NodeCpuInfo(agent.numProcessors);
+	    
 	    NodeUsersCollector nodeUsersCollector = new NodeUsersCollector(agent, logger);
 	    Future<TreeMap<String,NodeUsersInfo>> nuiFuture = pool.submit(nodeUsersCollector);
 
 	    NodeMetrics nodeMetrics = 
 	            new NodeMetrics(agent.getIdentity(), nmiFuture.get(), loadFuture.get(), 
-	                    cpuFuture.get(), nuiFuture.get());
+	                    cpuInfo, nuiFuture.get());
 
 	    //Node node = new DuccNode(new NodeIdentity(), nodeMetrics);
 	    // jrc 2011-07-30 I think this needs to be agent.getIdentity(), not create a new identity.

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/LinuxNodeMetricsProcessor.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/LinuxNodeMetricsProcessor.java?rev=1660435&r1=1660434&r2=1660435&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/LinuxNodeMetricsProcessor.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-agent/src/main/java/org/apache/uima/ducc/agent/processors/LinuxNodeMetricsProcessor.java Tue Feb 17 16:21:25 2015
@@ -102,8 +102,10 @@ public class LinuxNodeMetricsProcessor e
 			NodeLoadAverageCollector loadAvgCollector = new NodeLoadAverageCollector(
 					loadAvgFile, 5, 0);
 			Future<NodeLoadAverage> loadFuture = pool.submit(loadAvgCollector);
-			NodeCpuCollector cpuCollector = new NodeCpuCollector();
-			Future<NodeCpuInfo> cpuFuture = pool.submit(cpuCollector);
+//			NodeCpuCollector cpuCollector = new NodeCpuCollector();
+//			Future<NodeCpuInfo> cpuFuture = pool.submit(cpuCollector);
+		    NodeCpuInfo cpuInfo = new NodeCpuInfo(agent.numProcessors);
+		    
 			e.getIn().setHeader("node", agent.getIdentity().getName());
 			NodeMemory memInfo = nmiFuture.get();
 			TreeMap<String, NodeUsersInfo> users = null;
@@ -122,7 +124,7 @@ public class LinuxNodeMetricsProcessor e
 			}
 	    
             NodeMetrics nodeMetrics = new NodeMetrics(agent.getIdentity(), memInfo, loadFuture.get(),
-              cpuFuture.get(), users);
+              cpuInfo, users);
       
 			Node node = new DuccNode(agent.getIdentity(), nodeMetrics, agent.useCgroups);
 			// Make the agent aware how much memory is available on the node. Do this once.
@@ -132,7 +134,10 @@ public class LinuxNodeMetricsProcessor e
 						
 			((DuccNode)node).duccLingExists(agent.duccLingExists());
 			((DuccNode)node).runWithDuccLing(agent.runWithDuccLing());
-			logger.info(methodName, null, "... Agent "+node.getNodeIdentity().getName()+" Posting Memory:"
+			logger.info(methodName, null, "... Agent "+node.getNodeIdentity().getName()+
+					" CPU Count:" + cpuInfo.getAvailableProcessors() +
+					" CPU Load:" +cpuInfo.getCurrentLoad() +
+					" Posting Memory:"
 					+ node.getNodeMetrics().getNodeMemory().getMemTotal()+
 					" Memory Free:"+node.getNodeMetrics().getNodeMemory().getMemFree()+
 					" Swap Total:"+node.getNodeMetrics().getNodeMemory().getSwapTotal()+

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metrics/NodeCpuInfo.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metrics/NodeCpuInfo.java?rev=1660435&r1=1660434&r2=1660435&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metrics/NodeCpuInfo.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metrics/NodeCpuInfo.java Tue Feb 17 16:21:25 2015
@@ -15,24 +15,74 @@
  * KIND, either express or implied.  See the License for the
  * specific language governing permissions and limitations
  * under the License.
-*/
+ */
 package org.apache.uima.ducc.common.node.metrics;
 
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
 import java.io.Serializable;
 
 public class NodeCpuInfo implements Serializable {
-  private static final long serialVersionUID = -4770810550678067034L;
-  private String availableProcessors="0";
+	private static final long serialVersionUID = -4770810550678067034L;
+	private String availableProcessors = "0";
+    private String currentLoad = "0.0";
+    
+	public NodeCpuInfo(int availableProcessors) {
+		setAvailableProcessors(String.valueOf(availableProcessors));
+		try {
+			currentLoad = String.valueOf(getCPULoad());
+		} catch( Exception e) {
+			e.printStackTrace();
+		}
+	}
+    public String getCurrentLoad() {
+    	return currentLoad;
+    }
+	public String getAvailableProcessors() {
+		return availableProcessors;
+	}
 
-  public NodeCpuInfo( int availableProcessors ) {
-    setAvailableProcessors(String.valueOf(availableProcessors));
-  }
-  public String getAvailableProcessors() {
-    return availableProcessors;
-  }
-
-  private void setAvailableProcessors(String availableProcessors) {
-    this.availableProcessors = availableProcessors;
-  }
-  
+	private void setAvailableProcessors(String availableProcessors) {
+		this.availableProcessors = availableProcessors;
+	}
+
+	private double getCPULoad() throws Exception {
+		double cpu = 0.0;
+		InputStreamReader in = null;
+		String[] command = {
+				"/bin/sh",
+				"-c",
+				"/bin/grep 'cpu' /proc/stat | /bin/awk '{usage=($2+$4)*100/($2+$4+$5)} END {print usage}'" };
+		try {
+			ProcessBuilder pb = new ProcessBuilder();
+			pb.command(command);
+
+			pb.redirectErrorStream(true);
+			Process swapCollectorProcess = pb.start();
+			in = new InputStreamReader(swapCollectorProcess.getInputStream());
+			BufferedReader reader = new BufferedReader(in);
+			String line = null;
+			while ((line = reader.readLine()) != null) {
+				System.out.println(line);
+				try {
+					cpu = Double.parseDouble(line.trim());
+				} catch (NumberFormatException e) {
+					cpu = 0;
+					e.printStackTrace();
+				}
+			}
+		} catch (Exception e) {
+			throw e;
+		} finally {
+			if (in != null) {
+				try {
+					in.close();
+				} catch (Exception e) {
+				}
+
+			}
+		}
+
+		return cpu;
+	}
 }

Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metrics/NodeMetrics.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metrics/NodeMetrics.java?rev=1660435&r1=1660434&r2=1660435&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metrics/NodeMetrics.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-common/src/main/java/org/apache/uima/ducc/common/node/metrics/NodeMetrics.java Tue Feb 17 16:21:25 2015
@@ -33,7 +33,7 @@ public class NodeMetrics implements Seri
   private NodeIdentity nodeIdentity;
   TreeMap<String,NodeUsersInfo> nodeUsersMap;
   
-  public NodeMetrics(NodeIdentity nodeIdentity, NodeMemory nodeMemory, NodeLoadAverage nodeLoadAverage , NodeCpuInfo nodeCpu, TreeMap<String,NodeUsersInfo> userProcessMap ) {
+  public NodeMetrics(NodeIdentity nodeIdentity, NodeMemory nodeMemory, NodeLoadAverage nodeLoadAverage , NodeCpuInfo nodeCpu, TreeMap<String,NodeUsersInfo> userProcessMap) {
     this.nodeIdentity = nodeIdentity;
 	  setNodeMemory(nodeMemory);
     setNodeLoadAverage(nodeLoadAverage);
@@ -45,7 +45,7 @@ public class NodeMetrics implements Seri
     return nodeUsersMap;
   }
 
-  public NodeIdentity getNodeIdentity() {
+   public NodeIdentity getNodeIdentity() {
 	  return nodeIdentity;
   }
   public NodeCpuInfo getNodeCpu() {