You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hama.apache.org by ed...@apache.org on 2011/09/16 15:05:11 UTC
svn commit: r1171537 - in /incubator/hama/trunk: ./
core/src/main/java/org/apache/hama/bsp/
core/src/main/java/org/apache/hama/util/ core/src/main/webapp/bspmaster/
core/src/test/java/org/apache/hama/bsp/
Author: edwardyoon
Date: Fri Sep 16 13:05:11 2011
New Revision: 1171537
URL: http://svn.apache.org/viewvc?rev=1171537&view=rev
Log:
Web Interface does not update Superstep Count
Modified:
incubator/hama/trunk/CHANGES.txt
incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/BSPMaster.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/ClusterStatus.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/LocalBSPRunner.java
incubator/hama/trunk/core/src/main/java/org/apache/hama/util/BSPServletUtil.java
incubator/hama/trunk/core/src/main/webapp/bspmaster/machines.jsp
incubator/hama/trunk/core/src/test/java/org/apache/hama/bsp/TestClusterStatus.java
Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=1171537&r1=1171536&r2=1171537&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Fri Sep 16 13:05:11 2011
@@ -9,6 +9,7 @@ Release 0.4 - Unreleased
BUG FIXES
+ HAMA-436: Web Interface does not update Superstep Count (Thomas Jungblut)
HAMA-429: Groom statuses should be reported periodically (ChiaHung Lin via edwardyoon)
HAMA-421: Maven build issues using proxy (Joe Crobak via edwardyoon)
Modified: incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/BSPMaster.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/BSPMaster.java?rev=1171537&r1=1171536&r2=1171537&view=diff
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/BSPMaster.java (original)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/BSPMaster.java Fri Sep 16 13:05:11 2011
@@ -72,7 +72,7 @@ public class BSPMaster implements JobSub
private ZooKeeper zk = null;
private String bspRoot = null;
-
+
/**
* Constants for BSPMaster's status.
*/
@@ -343,7 +343,7 @@ public class BSPMaster implements JobSub
LOG.error("Fail to register GroomServer " + status.getGroomName(), e);
return false;
}
- LOG.info(status.getGroomName()+" is added.");
+ LOG.info(status.getGroomName() + " is added.");
return true;
}
@@ -426,7 +426,7 @@ public class BSPMaster implements JobSub
BSPMaster result = new BSPMaster(conf, identifier);
result.taskScheduler.setGroomServerManager(result);
result.taskScheduler.start();
-
+
// init zk root and child nodes
result.initZK(conf);
@@ -440,8 +440,8 @@ public class BSPMaster implements JobSub
*/
private void initZK(HamaConfiguration conf) {
try {
- zk = new ZooKeeper(QuorumPeer.getZKQuorumServersString(conf), conf
- .getInt(Constants.ZOOKEEPER_SESSION_TIMEOUT, 1200000), this);
+ zk = new ZooKeeper(QuorumPeer.getZKQuorumServersString(conf),
+ conf.getInt(Constants.ZOOKEEPER_SESSION_TIMEOUT, 1200000), this);
} catch (IOException e) {
LOG.error("Exception during reinitialization!", e);
}
@@ -499,7 +499,7 @@ public class BSPMaster implements JobSub
for (String node : zk.getChildren("/" + string, this)) {
zk.delete("/" + string + "/" + node, 0);
}
-
+
zk.delete("/" + string, 0);
} catch (InterruptedException e) {
e.printStackTrace();
@@ -507,7 +507,7 @@ public class BSPMaster implements JobSub
e.printStackTrace();
}
}
-
+
public static InetSocketAddress getAddress(Configuration conf) {
String hamaMasterStr = conf.get("bsp.master.address", "localhost");
int defaultPort = conf.getInt("bsp.master.port", 40000);
@@ -594,16 +594,16 @@ public class BSPMaster implements JobSub
@Override
public ClusterStatus getClusterStatus(boolean detailed) {
- Map<String, String> groomsMap = null;
+ Map<String, GroomServerStatus> groomsMap = null;
// give the caller a snapshot of the cluster status
int numGroomServers = groomServers.size();
if (detailed) {
- groomsMap = new HashMap<String, String>();
+ groomsMap = new HashMap<String, GroomServerStatus>();
for (Map.Entry<GroomServerStatus, GroomProtocol> entry : groomServers
.entrySet()) {
GroomServerStatus s = entry.getKey();
- groomsMap.put(s.getGroomName(), s.getGroomHostName());
+ groomsMap.put(s.getGroomName(), s);
}
}
Modified: incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/ClusterStatus.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/ClusterStatus.java?rev=1171537&r1=1171536&r2=1171537&view=diff
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/ClusterStatus.java (original)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/ClusterStatus.java Fri Sep 16 13:05:11 2011
@@ -20,10 +20,9 @@ package org.apache.hama.bsp;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
@@ -31,56 +30,58 @@ import org.apache.hadoop.io.WritableUtil
/**
* Status information on the current state of the BSP cluster.
*
- * <p><code>ClusterStatus</code> provides clients with information such as:
+ * <p>
+ * <code>ClusterStatus</code> provides clients with information such as:
* <ol>
- * <li>
- * Size of the cluster.
- * </li>
- * <li>
- * Name of the grooms.
- * </li>
- * <li>
- * Task capacity of the cluster.
- * </li>
- * <li>
- * The number of currently running bsp tasks.
- * </li>
- * <li>
- * State of the <code>BSPMaster</code>.
- * </li>
- * </ol></p>
+ * <li>
+ * Size of the cluster.</li>
+ * <li>
+ * Name of the grooms.</li>
+ * <li>
+ * Task capacity of the cluster.</li>
+ * <li>
+ * The number of currently running bsp tasks.</li>
+ * <li>
+ * State of the <code>BSPMaster</code>.</li>
+ * </ol>
+ * </p>
*
- * <p>Clients can query for the latest <code>ClusterStatus</code>, via
- * {@link BSPJobClient#getClusterStatus(boolean)}.</p>
+ * <p>
+ * Clients can query for the latest <code>ClusterStatus</code>, via
+ * {@link BSPJobClient#getClusterStatus(boolean)}.
+ * </p>
*
* @see BSPMaster
*/
public class ClusterStatus implements Writable {
private int numActiveGrooms;
- private Map<String, String> activeGrooms = new HashMap<String, String>();
+ private Map<String, GroomServerStatus> activeGrooms = new HashMap<String, GroomServerStatus>();
+ private Map<String, String> cachedActiveGroomNames = null;
private int tasks;
private int maxTasks;
private BSPMaster.State state;
-
+
/**
*
*/
- public ClusterStatus() {}
-
- public ClusterStatus(int grooms, int tasks, int maxTasks, BSPMaster.State state) {
+ public ClusterStatus() {
+ }
+
+ public ClusterStatus(int grooms, int tasks, int maxTasks,
+ BSPMaster.State state) {
this.numActiveGrooms = grooms;
this.tasks = tasks;
this.maxTasks = maxTasks;
this.state = state;
}
-
- public ClusterStatus(Map<String, String> activeGrooms, int tasks, int maxTasks,
- BSPMaster.State state) {
+
+ public ClusterStatus(Map<String, GroomServerStatus> activeGrooms, int tasks,
+ int maxTasks, BSPMaster.State state) {
this(activeGrooms.size(), tasks, maxTasks, state);
this.activeGrooms = activeGrooms;
}
-
+
/**
* Get the number of groom servers in the cluster.
*
@@ -89,16 +90,34 @@ public class ClusterStatus implements Wr
public int getGroomServers() {
return numActiveGrooms;
}
-
+
/**
* Get the names of groom servers, and their hostnames, in the cluster.
*
* @return the active groom servers in the cluster.
- */
+ */
public Map<String, String> getActiveGroomNames() {
+ if (cachedActiveGroomNames == null) {
+ if (activeGrooms != null) {
+ Map<String, String> map = new HashMap<String, String>();
+ for (Entry<String, GroomServerStatus> entry : activeGrooms.entrySet()) {
+ map.put(entry.getKey(), entry.getValue().getGroomHostName());
+ }
+ cachedActiveGroomNames = map;
+ }
+ }
+ return cachedActiveGroomNames;
+ }
+
+ /**
+ * Get the names of groom servers, and their current status in the cluster.
+ *
+ * @return the active groom servers in the cluster.
+ */
+ public Map<String, GroomServerStatus> getActiveGroomServerStatus() {
return activeGrooms;
}
-
+
/**
* Get the number of currently running tasks in the cluster.
*
@@ -107,7 +126,7 @@ public class ClusterStatus implements Wr
public int getTasks() {
return tasks;
}
-
+
/**
* Get the maximum capacity for running tasks in the cluster.
*
@@ -116,20 +135,20 @@ public class ClusterStatus implements Wr
public int getMaxTasks() {
return maxTasks;
}
-
+
/**
- * Get the current state of the <code>BSPMaster</code>,
- * as {@link BSPMaster.State}
+ * Get the current state of the <code>BSPMaster</code>, as
+ * {@link BSPMaster.State}
*
* @return the current state of the <code>BSPMaster</code>.
*/
public BSPMaster.State getBSPMasterState() {
return state;
}
-
- //////////////////////////////////////////////
+
+ // ////////////////////////////////////////////
// Writable
- //////////////////////////////////////////////
+ // ////////////////////////////////////////////
@Override
public void write(DataOutput out) throws IOException {
if (activeGrooms.isEmpty()) {
@@ -139,33 +158,30 @@ public class ClusterStatus implements Wr
out.writeInt(activeGrooms.size());
out.writeBoolean(true);
- String[] groomNames = activeGrooms.keySet().toArray(new String[0]);
- List<String> peerNames = new ArrayList<String>();
-
- for (String groomName : groomNames) {
- peerNames.add(activeGrooms.get(groomName));
+ for (Entry<String, GroomServerStatus> entry : activeGrooms.entrySet()) {
+ out.writeUTF(entry.getKey());
+ entry.getValue().write(out);
}
- WritableUtils.writeCompressedStringArray(out, groomNames);
- WritableUtils.writeCompressedStringArray(out, peerNames.toArray(new String[0]));
}
out.writeInt(tasks);
out.writeInt(maxTasks);
WritableUtils.writeEnum(out, state);
}
-
+
@Override
public void readFields(DataInput in) throws IOException {
numActiveGrooms = in.readInt();
boolean groomListFollows = in.readBoolean();
if (groomListFollows) {
- String[] groomNames = WritableUtils.readCompressedStringArray(in);
- String[] peerNames = WritableUtils.readCompressedStringArray(in);
- activeGrooms = new HashMap<String, String>(groomNames.length);
+ activeGrooms = new HashMap<String, GroomServerStatus>(numActiveGrooms);
- for (int i = 0; i < groomNames.length; i++) {
- activeGrooms.put(groomNames[i], peerNames[i]);
+ for (int i = 0; i < numActiveGrooms; i++) {
+ final String groomName = in.readUTF();
+ final GroomServerStatus status = new GroomServerStatus();
+ status.readFields(in);
+ activeGrooms.put(groomName, status);
}
}
Modified: incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/LocalBSPRunner.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/LocalBSPRunner.java?rev=1171537&r1=1171536&r2=1171537&view=diff
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/LocalBSPRunner.java (original)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/LocalBSPRunner.java Fri Sep 16 13:05:11 2011
@@ -18,6 +18,7 @@
package org.apache.hama.bsp;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
@@ -58,7 +59,6 @@ public class LocalBSPRunner implements J
protected static CyclicBarrier barrier;
static {
- threadPoolSize = Runtime.getRuntime().availableProcessors();
barrier = new CyclicBarrier(threadPoolSize);
threadPool = (ThreadPoolExecutor) Executors
.newFixedThreadPool(threadPoolSize);
@@ -125,9 +125,9 @@ public class LocalBSPRunner implements J
@Override
public ClusterStatus getClusterStatus(boolean detailed) throws IOException {
- Map<String, String> map = new HashMap<String, String>();
+ Map<String, GroomServerStatus> map = new HashMap<String, GroomServerStatus>();
for (Entry<String, LocalGroom> entry : localGrooms.entrySet()) {
- map.put(entry.getKey(), entry.getValue().getPeerName());
+ map.put(entry.getKey(), new GroomServerStatus(entry.getKey(), new ArrayList<TaskStatus>(0), 0, 0, "", entry.getKey()));
}
return new ClusterStatus(map, threadPoolSize, threadPoolSize, State.RUNNING);
}
Modified: incubator/hama/trunk/core/src/main/java/org/apache/hama/util/BSPServletUtil.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/util/BSPServletUtil.java?rev=1171537&r1=1171536&r2=1171537&view=diff
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/util/BSPServletUtil.java (original)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/util/BSPServletUtil.java Fri Sep 16 13:05:11 2011
@@ -18,10 +18,14 @@
package org.apache.hama.util;
import java.io.IOException;
-import java.util.Date;
import java.util.Calendar;
+import java.util.Date;
+import java.util.Map.Entry;
import org.apache.hadoop.util.ServletUtil;
+import org.apache.hama.bsp.BSPMaster;
+import org.apache.hama.bsp.ClusterStatus;
+import org.apache.hama.bsp.GroomServerStatus;
import org.apache.hama.bsp.JobStatus;
public class BSPServletUtil extends ServletUtil {
@@ -59,14 +63,15 @@ public class BSPServletUtil extends Serv
sb.append("<tr><th>Jobid</th>" + "<th>User</th>" + "<th>Name</th>"
+ "<th>SuperStep</th>" + "<th>Starttime</th>" + "</tr>\n");
for (JobStatus status : jobs) {
- sb.append("<tr><td><a href=\"bspjob.jsp?jobid="+status.getJobID()+ "\">");
+ sb.append("<tr><td><a href=\"bspjob.jsp?jobid=" + status.getJobID()
+ + "\">");
sb.append(status.getJobID());
sb.append("</a></td><td>");
sb.append(status.getUsername());
sb.append("</td><td>");
sb.append(status.getName());
sb.append("</td><td>");
- sb.append(status.getSuperstepCount());
+ sb.append(status.progress());
sb.append("</td><td>");
sb.append(new Date(status.getStartTime()));
sb.append("</td></tr>\n");
@@ -79,4 +84,32 @@ public class BSPServletUtil extends Serv
return sb.toString();
}
+ public static String generateGroomsTable(String type, ClusterStatus status,
+ BSPMaster master) throws IOException {
+ StringBuilder sb = new StringBuilder();
+ sb.append("<center>\n");
+ sb.append("<table border=\"2\" cellpadding=\"5\" cellspacing=\"2\">\n");
+ sb.append("<tr><td align=\"center\" colspan=\"6\"><b>Groom Servers</b></td></tr>\n");
+ sb.append("<tr><td><b>Name</b></td>"
+ + "<td><b>Host</b></td>"
+ + "<td><b># maximum tasks</b></td><td><b># current running tasks</b></td>" +
+ "<td><b># current failures</b></td>" +
+ "<td><b>Last seen</b></td>" +
+ "</tr>\n");
+ for (Entry<String, GroomServerStatus> entry : status
+ .getActiveGroomServerStatus().entrySet()) {
+ sb.append("<tr><td>");
+ sb.append(entry.getKey() + "</td><td>");
+ sb.append(entry.getValue().getGroomHostName() + "</td>" + "<td>"
+ + entry.getValue().getMaxTasks() + "</td><td>");
+ sb.append(entry.getValue().countTasks() + "</td><td>");
+ sb.append(entry.getValue().getFailures() + "</td><td>");
+ sb.append(entry.getValue().getLastSeen() + "</td>");
+ sb.append("</tr>\n");
+ }
+ sb.append("</table>\n");
+ sb.append("</center>\n");
+ return sb.toString();
+ }
+
}
Modified: incubator/hama/trunk/core/src/main/webapp/bspmaster/machines.jsp
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/webapp/bspmaster/machines.jsp?rev=1171537&r1=1171536&r2=1171537&view=diff
==============================================================================
--- incubator/hama/trunk/core/src/main/webapp/bspmaster/machines.jsp (original)
+++ incubator/hama/trunk/core/src/main/webapp/bspmaster/machines.jsp Fri Sep 16 13:05:11 2011
@@ -26,23 +26,6 @@
String trackerName = tracker.getBSPMasterName();
String type = request.getParameter("type");
%>
-<%!public void generateGroomsTable(JspWriter out, String type,
- ClusterStatus status, BSPMaster master) throws IOException {
- out.print("<center>\n");
- out.print("<table border=\"2\" cellpadding=\"5\" cellspacing=\"2\">\n");
- out.print("<tr><td align=\"center\" colspan=\"6\"><b>Groom Servers</b></td></tr>\n");
- out.print("<tr><td><b>Name</b></td>" + "<td><b>Host</b></td>"
- + "<td><b># running tasks</b></td></tr>\n");
- for (Map.Entry<String, String> entry : status.getActiveGroomNames()
- .entrySet()) {
- out.print("<tr><td><a href=\"http://");
- out.print(entry.getKey() + ":" + master.getHttpPort() + "/\">");
- out.print(entry.getValue() + "</a></td><td>");
- out.print(entry.getValue() + "</td>" + "<td>" + 1 + "</td></tr>\n");
- }
- out.print("</table>\n");
- out.print("</center>\n");
- }%>
<html>
@@ -53,9 +36,6 @@
<h2>Grooms</h2>
<%
- generateGroomsTable(out, type, status, tracker);
-%>
-
-<%
+ out.println(BSPServletUtil.generateGroomsTable(type, status, tracker));
out.println(BSPServletUtil.htmlFooter());
%>
\ No newline at end of file
Modified: incubator/hama/trunk/core/src/test/java/org/apache/hama/bsp/TestClusterStatus.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/test/java/org/apache/hama/bsp/TestClusterStatus.java?rev=1171537&r1=1171536&r2=1171537&view=diff
==============================================================================
--- incubator/hama/trunk/core/src/test/java/org/apache/hama/bsp/TestClusterStatus.java (original)
+++ incubator/hama/trunk/core/src/test/java/org/apache/hama/bsp/TestClusterStatus.java Fri Sep 16 13:05:11 2011
@@ -18,16 +18,16 @@
package org.apache.hama.bsp;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Random;
import junit.framework.TestCase;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hama.bsp.BSPMaster;
-import org.apache.hama.bsp.ClusterStatus;
public class TestClusterStatus extends TestCase {
Random rnd = new Random();
@@ -41,13 +41,13 @@ public class TestClusterStatus extends T
DataInputBuffer in = new DataInputBuffer();
ClusterStatus status1;
- Map<String, String> grooms = new HashMap<String, String>();
+ Map<String, GroomServerStatus> grooms = new HashMap<String, GroomServerStatus>();
for (int i = 0; i < 10; i++) {
int num = rnd.nextInt();
String groomName = "groom_" + num;
String peerName = "peerhost:" + num;
- grooms.put(groomName, peerName);
+ grooms.put(groomName, new GroomServerStatus(peerName, new ArrayList<TaskStatus>(0), 25, 2));
}
int tasks = rnd.nextInt(100);
@@ -61,11 +61,16 @@ public class TestClusterStatus extends T
ClusterStatus status2 = new ClusterStatus();
status2.readFields(in);
+
+ for(Entry<String, GroomServerStatus> entry : status2.getActiveGroomServerStatus().entrySet()){
+ assertEquals(entry.getValue().getMaxTasks(),2);
+ assertEquals(entry.getValue().getFailures(),25);
+ }
- Map<String, String> grooms_s = new HashMap<String, String>(status1
- .getActiveGroomNames());
- Map<String, String> grooms_o = new HashMap<String, String>(status2
- .getActiveGroomNames());
+ Map<String, String> grooms_s = new HashMap<String, String>(
+ status1.getActiveGroomNames());
+ Map<String, String> grooms_o = new HashMap<String, String>(
+ status2.getActiveGroomNames());
assertEquals(status1.getGroomServers(), status2.getGroomServers());