You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uima.apache.org by ch...@apache.org on 2014/07/02 16:17:59 UTC
svn commit: r1607376 - in /uima/sandbox/uima-ducc/trunk: src/main/config/
src/main/resources/
uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/
uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/
Author: challngr
Date: Wed Jul 2 14:17:59 2014
New Revision: 1607376
URL: http://svn.apache.org/r1607376
Log:
UIMA-3990 First pass, integrated node visualization.
Added:
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/JobFragment.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/Markup.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/SpectrumColorMap.java
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/VisualizedHost.java
Modified:
uima/sandbox/uima-ducc/trunk/src/main/config/log4j.xml
uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties
uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerViz.java
Modified: uima/sandbox/uima-ducc/trunk/src/main/config/log4j.xml
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/config/log4j.xml?rev=1607376&r1=1607375&r2=1607376&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/config/log4j.xml (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/config/log4j.xml Wed Jul 2 14:17:59 2014
@@ -197,6 +197,11 @@
<appender-ref ref="wslog" />
</category>
+ <category name="org.apache.uima.ducc.ws.server.nodeviz" additivity="true">
+ <priority value="info"/>
+ <appender-ref ref="wslog" />
+ </category>
+
<category name="org.apache.uima.ducc.viz" additivity="true">
<priority value="debug"/>
<appender-ref ref="vizlog" />
Modified: uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties?rev=1607376&r1=1607375&r2=1607376&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties (original)
+++ uima/sandbox/uima-ducc/trunk/src/main/resources/ducc.properties Wed Jul 2 14:17:59 2014
@@ -201,6 +201,9 @@ ducc.ws.jsp.compilation.directory = /tmp
ducc.ws.login.enabled = false
# Specify precalculate machines (default is true)
ducc.ws.precalculate.machines = true
+# multihomed systems, viz may have an external face other than ${ducc.head}
+ducc.viz.public.hostname = ${ducc.head}
+
# ========== Web Server Configuration block ==========
# ========== Job Driver Configuration block ==========
Modified: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerViz.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerViz.java?rev=1607376&r1=1607375&r2=1607376&view=diff
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerViz.java (original)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/DuccHandlerViz.java Wed Jul 2 14:17:59 2014
@@ -28,6 +28,7 @@ import org.apache.uima.ducc.common.inter
import org.apache.uima.ducc.common.utils.DuccLogger;
import org.apache.uima.ducc.common.utils.DuccLoggerComponents;
import org.apache.uima.ducc.common.utils.id.DuccId;
+import org.apache.uima.ducc.ws.server.nodeviz.NodeViz;
import org.eclipse.jetty.server.Request;
public class DuccHandlerViz extends DuccAbstractHandler {
@@ -38,13 +39,22 @@ public class DuccHandlerViz extends Ducc
public final String vizNodes = duccContextViz+"-nodes";
+ NodeViz viz = null;
+
+ DuccHandlerViz()
+ {
+ viz = new NodeViz();
+ }
+
private void handleServletVizNodes(String target,Request baseRequest,HttpServletRequest request,HttpServletResponse response)
throws IOException, ServletException
{
String methodName = "handleServletVizNodes";
duccLogger.trace(methodName, jobid, messages.fetch("enter"));
- String data = "<html><p>"+methodName+" not yet implemented</p></html>";
+
+ //String data = "<html><p>"+methodName+" not yet implemented</p></html>";
+ String data = viz.getVisualization();
duccLogger.debug(methodName, jobid, data);
response.getWriter().println(data);
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/JobFragment.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/JobFragment.java?rev=1607376&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/JobFragment.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/JobFragment.java Wed Jul 2 14:17:59 2014
@@ -0,0 +1,54 @@
+package org.apache.uima.ducc.ws.server.nodeviz;
+
+import org.apache.uima.ducc.transport.event.common.IDuccTypes.DuccType;
+
+ /**
+ * Represents the number of processes for a given job, and the total qshares occupied.
+ */
+class JobFragment
+{
+
+ String user; // Owner of the job
+ String id; // DUCC id of the job
+ int nprocesses; // number of processes in the fragment (i.e. on the same node)
+ int qshares; // total qshares represented by the node
+ String service_endpoint; // for services only, the defined endpoint we we can link back to services page
+ int mem; // Actual memory requested
+ String color; // color to draw this
+ DuccType type; // Job, Service, Reservation, Pop.
+
+ JobFragment(String user, DuccType type, String id, int mem, int qshares, String service_endpoint)
+ {
+ this.user = user;
+ this.type = type;
+ this.id = id;
+ this.qshares = qshares;
+ this.mem = mem;
+ this.nprocesses = 1;
+ this.service_endpoint = service_endpoint;
+ }
+
+ void addShares(int qshares)
+ {
+ this.qshares += qshares;
+ this.nprocesses++;
+ }
+
+ boolean matches(String id)
+ {
+ return this.id.equals(id);
+ }
+
+ String getTitle() {
+ switch ( type ) {
+ case Reservation:
+ return id;
+ case Undefined:
+ return "";
+ default:
+ return id + " " + nprocesses + " processes";
+ }
+ }
+ String getUser() { return user; }
+}
+
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/Markup.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/Markup.java?rev=1607376&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/Markup.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/Markup.java Wed Jul 2 14:17:59 2014
@@ -0,0 +1,292 @@
+package org.apache.uima.ducc.ws.server.nodeviz;
+
+
+public class Markup
+{
+
+ private int XSCALE = 8;
+ private int YSCALE = 8;
+
+ private StringBuffer out;
+
+ public Markup()
+ {
+ this.out = new StringBuffer();
+ }
+
+ String close()
+ {
+ return out.toString();
+ }
+
+ void svgStart(float height, float width)
+ {
+ out.append("<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"" + (height * YSCALE) + "\" width=\"" + (width * XSCALE) + "\">" );
+ }
+
+ void svgEnd()
+ {
+ out.append("</svg>");
+ }
+
+ void divStart()
+ {
+ // out.append("<div style=\"border: 1px solid red;display:inline-block\">");
+ out.append("<div style=\"display:inline-block\">");
+ }
+
+ void divEnd()
+ {
+ out.append("</div>");
+ }
+
+ void tooltipStart(String label) {
+ out.append("<g>\n<title>" + label + "</title>");
+ }
+
+ void tooltipEnd() {
+ out.append("</g>");
+ }
+
+ void rect(float x, float y, float width, float height, String color, String borderColor, float strokeWidth, String newAttr)
+ {
+ if (newAttr == null) {
+ newAttr = "";
+ }
+ out.append("<rect x=\""
+ + (x*XSCALE)
+ + "\" y=\""
+ + (y*YSCALE)
+ + "\" width=\""
+ + (width*XSCALE)
+ + "\" height=\""
+ + (height*YSCALE)
+ + "\" fill=\""
+ + color
+ + "\" stroke=\""
+ + borderColor
+ + "\" stroke-width=\""
+ + (strokeWidth*XSCALE)
+ + "\""
+ + newAttr
+ + "/> ");
+ }
+
+ void text(float x, float y, String label, String color, float fontsize, String newAttr)
+ {
+ if (newAttr == null) {
+ newAttr = "";
+ }
+ out.append(
+ "<text x=\""
+ + (x*XSCALE)
+ + "\" y=\""
+ + (y*YSCALE)
+ + "\" font-family=\"verdana\" font-size=\""
+ + fontsize
+ + "\" fill=\""
+ + color
+ + "\""
+ + newAttr
+ + ">"
+ + label
+ + "</text>"
+ );
+ }
+
+ void hyperlinkStart(VisualizedHost host, JobFragment j)
+ {
+ switch ( j.type ) {
+ case Job:
+ out.append(
+ "<a xlink:href=http://" +
+ NodeViz.wshost +
+ ":" +
+ NodeViz.wsport +
+ "/job.details.jsp?id=" +
+ j.id +
+ ">"
+ );
+
+ break;
+ case Service:
+ out.append(
+ "<a xlink:href=http://" +
+ NodeViz.wshost +
+ ":" +
+ NodeViz.wsport +
+ "/service.details.jsp?name=" +
+ j.service_endpoint +
+ ">"
+ );
+
+ break;
+ case Pop:
+ out.append(
+ "<a xlink:href=http://" +
+ NodeViz.wshost +
+ ":" +
+ NodeViz.wsport +
+ "/reservations.details.jsp?id=" +
+ j.id +
+ ">"
+ );
+
+ break;
+ case Reservation:
+ out.append(
+ "<a xlink:href=http://" +
+ NodeViz.wshost +
+ ":" +
+ NodeViz.wsport +
+ "/reservations.jsp>"
+ );
+ break;
+ case Undefined:
+ out.append("<a>");
+ break;
+ }
+ }
+
+ void hyperlinkEnd()
+ {
+ out.append("</a>");
+ }
+
+ void titleForFragment(VisualizedHost h, JobFragment j)
+ {
+ String jobtype = "" + j.type;
+ switch ( j.type ) {
+ case Pop:
+ jobtype = "Managed Reservation";
+ case Job:
+ case Service:
+
+ out.append(
+ "<title>" +
+ jobtype +
+ " " +
+ j.user +
+ ":" +
+ j.id +
+ " runs " +
+ j.nprocesses +
+ " process(es) of " +
+ j.mem +
+ "GB on host " +
+ h.name +
+ "(" +
+ h.mem +
+ "GB)</title>"
+ );
+ break;
+ case Reservation:
+ out.append(
+ "<title>" +
+ jobtype +
+ " " +
+ j.user +
+ ":" +
+ j.id +
+ " on host " +
+ h.name +
+ "(" +
+ j.mem +
+ "GB)</title>"
+ );
+ break;
+ case Undefined:
+ out.append(
+ "<title>" +
+ j.qshares +
+ " unused shares (" +
+ (j.qshares * NodeViz.quantum) +
+ "GB) on " +
+ h.name +
+ "(" +
+ h.mem +
+ "GB)</title>"
+ );
+ break;
+ }
+ }
+
+ String patternedFill(JobFragment j)
+ {
+ String color = "rgb(" + SpectrumColorMap.neverDarkColorAlmostLight(j.user + " ")+")";
+
+ switch ( j.type ) {
+ case Job:
+ return color;
+ case Pop:
+ return popFill(j, color);
+ case Service:
+ return serviceFill(j, color);
+ case Reservation:
+ return reservationFill(j, color);
+ default:
+ return color;
+ }
+ }
+
+ String popFill(JobFragment j, String color)
+ {
+ String id = "patP" + j.id;
+
+ out.append("<pattern id=\"");
+ out.append(id);
+ out.append("\" patternUnits=\"userSpaceOnUse\" x=\"0\" y=\"0\" width=\"4\" height=\"4\">");
+ out.append("<g>");
+
+ out.append("<line x1=\"-2\" y1=\"4\" x2=\"4\" y2=\"-2\" stroke=\"");
+ out.append(color);
+ out.append("\" stroke-width=\"2\" />");
+
+ out.append("<line x1=\"0\" y1=\"6\" x2=\"6\" y2=\"0\" stroke=\"");
+ out.append(color);
+ out.append("\" stroke-width=\"2\" />");
+
+ out.append("</g>");
+ out.append("</pattern>");
+
+ return "url(#" + id + ")";
+ }
+
+ String serviceFill(JobFragment j, String color)
+ {
+ String id = "patS" + j.id;
+
+ out.append("<pattern id=\"");
+ out.append(id);
+ out.append("\" patternUnits=\"userSpaceOnUse\" x=\"0\" y=\"0\" width=\"4\" height=\"4\">");
+ out.append("<g>");
+
+ out.append("<line x1=\"0\" y1=\"-2\" x2=\"6\" y2=\"4\" stroke=\"");
+ out.append(color);
+ out.append("\" stroke-width=\"2\" />");
+
+ out.append("<line x1=\"-2\" y1=\"0\" x2=\"4\" y2=\"6\" stroke=\"");
+ out.append(color);
+ out.append("\" stroke-width=\"2\" />");
+
+ out.append("</g>");
+ out.append("</pattern>");
+
+ return "url(#" + id + ")";
+ }
+
+ String reservationFill(JobFragment j, String color)
+ {
+ String id = "patR" + j.id;
+
+ out.append("<pattern id=\"");
+ out.append(id);
+ out.append("\" patternUnits=\"userSpaceOnUse\" x=\"0\" y=\"0\" width=\"4\" height=\"4\">");
+ out.append("<g><rect x=\"0\" y=\"0\" width=\"3.7\" height=\"3.7\" style=\"fill:");
+ out.append(color);
+ out.append("; stroke:none\"/></g></pattern>");
+ return "url(#" + id + ")";
+ }
+
+}
+
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java?rev=1607376&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/NodeViz.java Wed Jul 2 14:17:59 2014
@@ -0,0 +1,296 @@
+package org.apache.uima.ducc.ws.server.nodeviz;
+
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentSkipListMap;
+
+import org.apache.uima.ducc.common.Node;
+import org.apache.uima.ducc.common.utils.DuccLogger;
+import org.apache.uima.ducc.common.utils.DuccLoggerComponents;
+import org.apache.uima.ducc.common.utils.SystemPropertyResolver;
+import org.apache.uima.ducc.common.utils.Version;
+import org.apache.uima.ducc.transport.event.OrchestratorStateDuccEvent;
+import org.apache.uima.ducc.transport.event.common.DuccWorkMap;
+import org.apache.uima.ducc.transport.event.common.IDuccProcess;
+import org.apache.uima.ducc.transport.event.common.IDuccProcessMap;
+import org.apache.uima.ducc.transport.event.common.IDuccReservation;
+import org.apache.uima.ducc.transport.event.common.IDuccReservationMap;
+import org.apache.uima.ducc.transport.event.common.IDuccSchedulingInfo;
+import org.apache.uima.ducc.transport.event.common.IDuccStandardInfo;
+import org.apache.uima.ducc.transport.event.common.IDuccTypes.DuccType;
+import org.apache.uima.ducc.transport.event.common.IDuccWork;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkExecutable;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkReservation;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkService;
+import org.apache.uima.ducc.transport.event.common.IDuccWorkService.ServiceDeploymentType;
+import org.apache.uima.ducc.transport.event.common.IProcessState.ProcessState;
+import org.apache.uima.ducc.ws.DuccMachinesData;
+import org.apache.uima.ducc.ws.IListenerOrchestrator;
+import org.apache.uima.ducc.ws.MachineInfo;
+import org.apache.uima.ducc.ws.server.DuccListeners;
+
+public class NodeViz
+ implements IListenerOrchestrator
+{
+ private static DuccLogger logger = DuccLoggerComponents.getWsLogger(NodeViz.class.getName());
+
+ private DuccMachinesData machineData; // handle to static machine information
+
+ private long lastUpdate = 0; // time of last orchestrator state
+ private String visualization; // cached visualization
+ private long update_interval = 60000; // Only gen a new viz every 'this long'
+
+ private String version = "1.1.0";
+ static int quantum = 4;
+ static String wshost = "";
+ static String wsport = "42133";
+
+ public NodeViz()
+ {
+ String methodName = "NodeViz";
+
+ update_interval = SystemPropertyResolver.getLongProperty("ducc.viz.update.interval", update_interval);
+ quantum = SystemPropertyResolver.getIntProperty("ducc.rm.share.quantum", quantum);
+
+ wshost = SystemPropertyResolver.getStringProperty("ducc.ws.node", System.getProperty("ducc.head"));
+ wshost = SystemPropertyResolver.getStringProperty("ducc.viz.public.hostname", wshost);
+
+ wsport = SystemPropertyResolver.getStringProperty("ducc.ws.port", wsport);
+
+ logger.info(methodName, null, "------------------------------------------------------------------------------------");
+ logger.info(methodName, null, "Nodeviz starting:");
+ logger.info(methodName, null, " DUCC home : ", System.getProperty("DUCC_HOME"));
+ logger.info(methodName, null, " ActiveMQ URL : ", System.getProperty("ducc.broker.url"));
+ logger.info(methodName, null, "Using Share Quantum : ", quantum);
+ logger.info(methodName, null, "Viz update Interval : ", update_interval);
+ logger.info(methodName, null, "Web Server Host : ", wshost);
+ logger.info(methodName, null, "Web Server Port : ", wsport);
+ logger.info(methodName, null, "");
+ logger.info(methodName, null, " JVM : ", System.getProperty("java.vendor") +
+ " "+ System.getProperty("java.version"));
+ logger.info(methodName, null, " JAVA_HOME : ", System.getProperty("java.home"));
+ logger.info(methodName, null, " JVM Path : ", System.getProperty("ducc.jvm"));
+ logger.info(methodName, null, " JMX URL : ", System.getProperty("ducc.jmx.url"));
+ logger.info(methodName, null, "");
+ logger.info(methodName, null, " OS Architecture : ", System.getProperty("os.arch"));
+ logger.info(methodName, null, "");
+ logger.info(methodName, null, " DUCC Version : ", Version.version());
+ logger.info(methodName, null, " Viz Version : ", version);
+ logger.info(methodName, null, "------------------------------------------------------------------------------------");
+
+ DuccListeners.getInstance().register(this);
+ machineData = DuccMachinesData.getInstance();
+
+ visualization = "<html><p>Waiting for node updates ...</p></html>";
+ }
+
+ public String getVisualization()
+ {
+ String methodName = "getVisualization";
+ logger.debug(methodName, null, "Request for visualization");
+ return visualization;
+ }
+
+ public void generateVisualization(OrchestratorStateDuccEvent ev)
+ {
+ String methodName = "generateVisualization";
+ Map<String, VisualizedHost> hosts = new HashMap<String, VisualizedHost>();
+
+ DuccWorkMap jobmap = ev.getWorkMap();
+
+ int job_shares = 0;
+ int service_shares = 0;
+ int pop_shares = 0;
+ int reservation_shares = 0;
+
+ // first step, generate the viz from the OR map which seems to have everything we need
+ // next stop, walk the machines list and generate empty node for any machine in that list
+ // that had no work on it
+ // lext step, walk the machines data and overlay a graphic for any node that is 'down' or 'defined'
+ // move 'down' hosts with jobs on them to the front
+ // move all onther 'not up' hosts to the end
+ // finally, walk the list and make them render
+
+ for ( IDuccWork w : jobmap.values() ) {
+ DuccType type = w.getDuccType();
+ String service_endpoint = null;
+ // If it looks service-y and os of deployment type 'other' it's a Pop.
+ if ( type == DuccType.Service ) {
+ IDuccWorkService dws = (IDuccWorkService) w;
+ if ( dws.getServiceDeploymentType() == ServiceDeploymentType.other) {
+ type = DuccType.Pop;
+ } else {
+ service_endpoint = dws.getServiceEndpoint();
+ }
+ }
+
+ if ( ! w.isSchedulable() ) {
+ logger.debug(methodName, w.getDuccId(), "Ignoring unschedulable work:", w.getDuccType(), ":", w.getStateObject());
+ continue;
+ }
+
+ IDuccStandardInfo si = w.getStandardInfo();
+ IDuccSchedulingInfo sti = w.getSchedulingInfo();
+
+ String user = si.getUser();
+ String duccid = Long.toString(w.getDuccId().getFriendly());
+ int jobmem = Integer.parseInt(sti.getShareMemorySize());
+ int qshares = jobmem / quantum;
+ if ( jobmem % quantum != 0 ) qshares++;
+
+ switch ( type ) {
+ case Job:
+ case Pop:
+ case Service:
+ {
+ IDuccWorkExecutable de = (IDuccWorkExecutable) w;
+ IDuccProcessMap pm = de.getProcessMap();
+ logger.debug(methodName, w.getDuccId(), "Receive:", type, w.getStateObject(), "processes[", pm.size() + "]");
+
+ for ( IDuccProcess proc : pm.values() ) {
+ String pid = proc.getPID();
+ ProcessState state = proc.getProcessState();
+ Node n = proc.getNode();
+
+ logger.debug(methodName, w.getDuccId(), (n == null ? "N/A" : n.getNodeIdentity().getName()), "Process[", pid, "] state [", state, "] is complete[", proc.isComplete(), "]");
+ if ( proc.isComplete() ) {
+ continue;
+ }
+
+ switch ( type ) {
+ case Job:
+ job_shares += qshares;
+ break;
+ case Pop:
+ pop_shares += qshares;
+ break;
+ case Service:
+ service_shares += qshares;
+ break;
+ }
+
+ if ( n != null ) {
+ String key = n.getNodeIdentity().getName();
+ VisualizedHost vh = hosts.get(key);
+ if ( vh == null ) {
+ vh = new VisualizedHost(n, quantum);
+ hosts.put(key, vh);
+ }
+
+ vh.addWork(type, user, duccid, jobmem, qshares, service_endpoint);
+ }
+ }
+ }
+ break;
+
+ case Reservation:
+ {
+ IDuccWorkReservation de = (IDuccWorkReservation) w;
+ IDuccReservationMap rm = de.getReservationMap();
+
+ logger.debug(methodName, w.getDuccId(), "Receive:", type, w.getStateObject(), "processes[", rm.size(), "] Completed:", w.isCompleted());
+ reservation_shares += qshares;
+
+ for ( IDuccReservation r: rm.values()) {
+ Node n = r.getNode();
+ if ( n == null ) {
+ logger.debug(methodName, w.getDuccId(), "Node [N/A] mem[N/A");
+ } else {
+ String key = n.getNodeIdentity().getName();
+ VisualizedHost vh = hosts.get(key);
+ if ( vh == null ) {
+ vh = new VisualizedHost(n, quantum);
+ hosts.put(key, vh);
+ }
+ vh.addWork(type, user, duccid, jobmem, qshares, null);
+ }
+ }
+ }
+ break;
+
+ default:
+ logger.warn(methodName, w.getDuccId(), "Received work of type ?", w.getDuccType());
+ break;
+ }
+ }
+
+ logger.debug(methodName, null, "Generateing visualizaiton");
+ ConcurrentSkipListMap<String,MachineInfo> m = machineData.getMachines();
+
+
+ for (String s : m.keySet()) {
+ if ( ! hosts.containsKey(s) ) {
+ VisualizedHost vh = new VisualizedHost(m.get(s), quantum);
+ hosts.put(s, vh);
+ }
+ }
+
+ int total_shares = 0;
+ int total_ram = 0;
+ Markup markup = new Markup();
+ VisualizedHost[] sorted = hosts.values().toArray(new VisualizedHost[hosts.size()]);
+ Arrays.sort(sorted, new HostSorter());
+ for ( VisualizedHost vh : sorted ) {
+ vh.toSvg(markup);
+ total_shares += vh.countShares();
+ total_ram += vh.countRam();
+ }
+ String page = markup.close();
+
+ int unoccupied_shares = total_shares - (job_shares + pop_shares + service_shares + reservation_shares);
+
+ visualization =
+ "<html>" +
+ "<table width=\"100%\">" +
+ "<caption>" +
+ "<b>Shares of size " + quantum + "GB: </b>" + total_shares +
+ ", <b>Jobs: </b>" + job_shares +
+ ", <b>Services: </b>" + service_shares +
+ ", <b>Managed Reservations: </b>" + pop_shares +
+ ", <b>Reservations: </b>" + reservation_shares +
+ ", <b>Unoccupied: </b>" + unoccupied_shares +
+ "<br><i><small>" +
+ "<b>RAM Total:</b> " + total_ram +
+ "GB, <b>For shares:</b> " + (total_shares * quantum) +
+ "GB, <b>Jobs:</b> " + (job_shares * quantum) +
+ "GB, <b>Services:</b> " + (service_shares * quantum) +
+ "GB, <b>Managed Reservations:</b> " + (pop_shares * quantum) +
+ "GB, <b>Reservations:</b> " + (reservation_shares * quantum) +
+ "GB, <b>Unoccupied:</b> " + (unoccupied_shares * quantum) +
+ "</small></i></caption>" +
+ "</table>" +
+ page +
+ "</html>";
+ logger.info(methodName, null, "Size of node visualization:", visualization.length());
+ hosts = null;
+ }
+
+ public void update(OrchestratorStateDuccEvent ev)
+ {
+ String methodName = "update";
+ logger.debug(methodName, null, "Received Orchestrator Event");
+ long currentUpdate = System.currentTimeMillis();
+
+ // if ( currentUpdate - lastUpdate > update_interval ) {
+ if ( true ) { // for debug, never skip
+ generateVisualization(ev);
+ lastUpdate = currentUpdate;
+ } else {
+ logger.debug(methodName, null, "Skipping visualization");
+ }
+ }
+
+ private static class HostSorter
+ implements Comparator<VisualizedHost>
+ {
+ public int compare(VisualizedHost h1, VisualizedHost h2)
+ {
+ if ( h1.equals(h2) ) return 0;
+ if ( h2.shares == h1.shares ) return h2.name.compareTo(h1.name);
+ return h2.shares - h1.shares;
+ }
+ }
+
+}
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/SpectrumColorMap.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/SpectrumColorMap.java?rev=1607376&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/SpectrumColorMap.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/SpectrumColorMap.java Wed Jul 2 14:17:59 2014
@@ -0,0 +1,119 @@
+package org.apache.uima.ducc.ws.server.nodeviz;
+
+public class SpectrumColorMap {
+
+ /*
+ * calculate colors for index colind
+ */
+ public static String color (int colind) {
+ colind = Math.abs(colind) % 512;
+ int i1 = (colind % 8) * 32;
+ int i2 = ((colind / 8) % 8) * 32;
+ int i3 = ((colind / 64) % 8) * 32;
+ return i1 + "," + i2 + "," + i3;
+ }
+
+ /*
+ * returns the regular color;
+ * however, if the color is too dark, we increase the brightness by 2
+ */
+ public static String neverDarkColor (int colind) {
+ colind = Math.abs(colind) % 512;
+ int i1 = (colind % 8) * 32;
+ int i2 = ((colind / 8) % 8) * 32;
+ int i3 = ((colind / 64) % 8) * 32;
+ if (i1+i2+i3 < 60) {
+ i1 *=2 ; i2 *=2; i3 *=2;
+ }
+ return i1 + "," + i2 + "," + i3;
+//
+ }
+
+ /*
+ * returns the bit lighter than the regular color;
+ * however, if the color is too dark, we increase the brightness by 2
+ */
+ public static String neverDarkColorAlmostLight (int colind) {
+ colind = Math.abs(colind) % 512;
+ int i1 = (colind % 8) * 28 + 44;
+ int i2 = ((colind / 8) % 8) * 28 + 44;
+ int i3 = ((colind / 64) % 8) * 28 + 44;
+ if (i1+i2+i3 < 60) {
+ i1 *=2 ; i2 *=2; i3 *=2;
+ }
+ return i1 + "," + i2 + "," + i3;
+//
+ }
+
+ public static String darkColor (int colind) {
+ colind = Math.abs(colind) % 512;
+ int i1 = (colind % 8) * 20;
+ int i2 = ((colind / 8) % 8) * 20;
+ int i3 = ((colind / 64) % 8) * 20;
+ return i1 + "," + i2 + "," + i3;
+ }
+
+ public static String lightColor (int colind) {
+ colind = Math.abs(colind) % 512;
+ int i1 = (colind % 8) * 24 +87;
+ int i2 = ((colind / 8) % 8) * 24 +87;
+ int i3 = ((colind / 64) % 8) * 24 +87;
+ return i1 + "," + i2 + "," + i3;
+ }
+ public static String veryLightColor (int colind) {
+ colind = Math.abs(colind) % 512;
+ int i1 = (colind % 8) * 8 + 199;
+ int i2 = ((colind / 8) % 8) * 8 + 199;
+ int i3 = ((colind / 64) % 8) * 8 + 199;
+ return i1 + "," + i2 + "," + i3;
+ }
+
+ /*
+ * calculate colors for a String s
+ */
+
+ public static String color (String s) {
+ if (s == null)
+ return color(0);
+ else {
+ return color(s.hashCode());
+ }
+ }
+
+ public static String neverDarkColor (String s) {
+ if (s == null)
+ return color(0);
+ else {
+ return neverDarkColor(s.hashCode());
+ }
+ }
+
+ public static String neverDarkColorAlmostLight (String s) {
+ if (s == null)
+ return color(0);
+ else {
+ return neverDarkColorAlmostLight(s.hashCode());
+ }
+ }
+
+ public static String lightColor (String s) {
+ if (s == null)
+ return lightColor(0);
+ else
+ return lightColor(s.hashCode()); //is a prime
+ }
+
+ public static String darkColor (String s) {
+ if (s == null)
+ return darkColor(0);
+ else
+ return darkColor(s.hashCode()); //is a prime
+ }
+
+ public static String veryLightColor (String s) {
+ if (s == null)
+ return veryLightColor(0);
+ else
+ return veryLightColor(s.hashCode()); //is a prime
+ }
+}
Added: uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/VisualizedHost.java
URL: http://svn.apache.org/viewvc/uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/VisualizedHost.java?rev=1607376&view=auto
==============================================================================
--- uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/VisualizedHost.java (added)
+++ uima/sandbox/uima-ducc/trunk/uima-ducc-web/src/main/java/org/apache/uima/ducc/ws/server/nodeviz/VisualizedHost.java Wed Jul 2 14:17:59 2014
@@ -0,0 +1,201 @@
+package org.apache.uima.ducc.ws.server.nodeviz;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.uima.ducc.common.Node;
+import org.apache.uima.ducc.common.utils.DuccLogger;
+import org.apache.uima.ducc.common.utils.DuccLoggerComponents;
+import org.apache.uima.ducc.transport.event.common.IDuccTypes.DuccType;
+import org.apache.uima.ducc.ws.MachineInfo;
+
+class VisualizedHost
+{
+
+ private static DuccLogger logger = DuccLoggerComponents.getWsLogger(VisualizedHost.class.getName());
+ private static FragmentSorter sorter = new FragmentSorter();
+
+ // TODO:
+ // Some goodies - aliens, swap, heartbeat info, pubsize, expired, status
+ String name; // host name
+ String ip; // host ip
+ int mem; // actual mem as reported by agent
+ int shares; // q shares available on this host (constant)
+
+ int shares_free; // shares not used by jobs
+ int mem_reservable; // schedulable (reservable) memory on this hosts
+
+ int quantum; // RM scheduling quantum
+
+ List<JobFragment> fragments = new ArrayList<JobFragment>();
+
+ /**
+ * Generate host from OR state which contains info about the work on a host.
+ */
+ VisualizedHost(Node n, int quantum)
+ {
+ this.quantum = quantum;
+ this.name = n.getNodeIdentity().getName();
+ this.ip = n.getNodeIdentity().getIp();
+
+ // mem from OR pub is in KB. must convert to GB
+ this.mem = (int) n.getNodeMetrics().getNodeMemory().getMemTotal() / ( 1024 * 1024 );
+
+ this.shares = (mem / quantum);
+ this.shares_free = shares;
+ this.mem_reservable = shares * quantum;
+ }
+
+ /**
+ * Generate host from agent publications because OR state only has host info for hosts with work on them.
+ */
+ VisualizedHost(MachineInfo info, int quantum)
+ {
+ this.quantum = quantum;
+
+ this.name = info.getName();
+ this.ip = info.getIp();
+
+ String ns = info.getSharesTotal();
+ if ( ns == "" || ns == null ) {
+ this.mem = 0;
+ this.shares = 0;
+ this.mem_reservable = 0;
+ } else {
+ this.mem = Integer.parseInt(info.getMemTotal());
+ this.shares = (mem / quantum);
+ this.mem_reservable = shares * quantum;
+ }
+
+ this.shares_free = shares;
+ }
+
+ int countShares()
+ {
+ return shares;
+ }
+
+ int countRam()
+ {
+ return mem;
+ }
+
+ void addWork(DuccType type, String user, String duccid, int jobmem, int qshares, String service_endpoint)
+ {
+ String methodName = "addWork";
+
+ // The job list is going to be short almost always, so cost of linear search will be less than the overhead
+ // of maintaining a map - on the order of fewer than 5-6 items in worst case. It's rare to see more than
+ // 2-3 in real life. If this should change so it's common to have more than about 10 elements in the list
+ // we should switch to a map.
+
+ logger.debug(methodName, null, name, "Set", qshares, "qshares for", name, type, duccid, ": mem", mem, "free qshares", shares_free, "from OR publication.");
+
+ // if ( type == DuccType.Reservation ) qshares = shares_free; // Trust the RM and the Force, Luke
+
+ if ( shares_free - qshares < 0 ) {
+ logger.warn(methodName, null, name, "SHARES FREE WENT NEGATIVE for", type, duccid, user, "qshares", qshares, "mem", mem, "shares_free", shares_free);
+ return;
+ } else {
+ shares_free -= qshares;
+ }
+
+ boolean found = false;
+ for ( JobFragment j : fragments ) {
+ if ( j.matches(duccid) ) {
+ j.addShares(qshares);
+ logger.debug(methodName, null, name, "Update job fragment for", user, "with", qshares, "qshares", "total qshares", j.qshares);
+ found = true;
+ break;
+ }
+ }
+ if ( ! found ) {
+ logger.debug(methodName, null, name, "Create new job fragment for", user, "with", qshares, "qshare, type", type);
+ JobFragment j = new JobFragment(user, type, duccid, jobmem, qshares, service_endpoint);
+ fragments.add(j);
+ }
+ }
+
+// String getIp() { return ip; }
+// int getMem() { return mem; }
+// int getShares() { return shares; }
+
+ float TITLE_ADJUSTMENT = 2f; // Amount of space to add to each square at top to hold nodename
+ void toSvg(Markup m)
+ {
+ String methodName = "toSvg "; // (extra spaces so logs line up better)
+
+ if ( shares == 0 ) return;
+ if ( shares_free > 0 ) addWork(DuccType.Undefined, "", "", 0, shares_free, null);
+
+ float size = (float) Math.sqrt(mem);
+ logger.debug(methodName, null, name, "size =", size);
+
+ m.divStart();
+ m.svgStart(size + TITLE_ADJUSTMENT, size + .2f); // a bit taller than needed to make room for label
+ // a bit wider, for horizontal spacing
+
+ m.rect(0f, TITLE_ADJUSTMENT, size, size, "white", "none", .1f, "");
+
+ m.tooltipStart(name + " (" + mem + "GB)");
+ m.text(0f, TITLE_ADJUSTMENT - .1f, name, "black", 10, "");
+ m.tooltipEnd();
+
+ Collections.sort(fragments, sorter);
+ float height_one_share = size / shares;
+ float top = 0f + TITLE_ADJUSTMENT; // the top of the box
+ logger.debug(methodName, null, name, "Draw", fragments.size(), "rectangles, box size", size, "share height", height_one_share);
+ for (JobFragment j : fragments ) {
+
+ /**
+ * Structure of this block. Remembering that the link tag is a block level tag.
+ *
+ * <a link-to-ws page for job>
+ * <title> tooltip stuff for the job fragment </title>
+ * <rect> rectangle for job fragment </rect>
+ * <text> text for job fragment (job id) </text>
+ * </a>
+ */
+ if ( top > size ) {
+ logger.warn(methodName, null, name, "Box overflow. Size", size, "top", top);
+ }
+
+ float height = j.qshares * height_one_share;
+ logger.debug(methodName, null, name, "Draw box of height", height, "for", j.type, j.id, "shares", j.qshares);
+
+ String fill = m.patternedFill(j);
+
+ m.hyperlinkStart(this, j);
+
+ m.titleForFragment(this, j);
+
+ if ( j.type == DuccType.Undefined ) {
+ m.rect(0, top, size, height, "", "black", .1f, "");
+ } else {
+ m.rect(0, top, size, height, fill, "black", .1f, "");
+ }
+ m.text(.1f, top + 1.5f, j.id, "white", 12, "");
+
+ m.hyperlinkEnd();
+
+ top += height;
+ }
+
+ m.svgEnd();
+ m.divEnd();
+ }
+
+ static private class FragmentSorter
+ implements Comparator<JobFragment>
+ {
+ public int compare(JobFragment f1, JobFragment f2)
+ {
+ if ( f1.type == DuccType.Undefined && f2.type != DuccType.Undefined) return 1;
+ if ( f1.type != DuccType.Undefined && f2.type == DuccType.Undefined) return -1;
+ return f2.qshares - f1.qshares;
+ }
+ }
+
+}