You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ma...@apache.org on 2012/10/04 02:31:24 UTC
svn commit: r1393852 - in /giraph/trunk: CHANGELOG
src/main/java/org/apache/giraph/graph/BspService.java
src/main/java/org/apache/giraph/graph/BspServiceMaster.java
src/main/java/org/apache/giraph/zk/ZooKeeperExt.java
Author: maja
Date: Thu Oct 4 00:31:23 2012
New Revision: 1393852
URL: http://svn.apache.org/viewvc?rev=1393852&view=rev
Log:
GIRAPH-342: Recursive ZooKeeper calls should call progress, dynamic ZooKeeper can skip delete
Modified:
giraph/trunk/CHANGELOG
giraph/trunk/src/main/java/org/apache/giraph/graph/BspService.java
giraph/trunk/src/main/java/org/apache/giraph/graph/BspServiceMaster.java
giraph/trunk/src/main/java/org/apache/giraph/zk/ZooKeeperExt.java
Modified: giraph/trunk/CHANGELOG
URL: http://svn.apache.org/viewvc/giraph/trunk/CHANGELOG?rev=1393852&r1=1393851&r2=1393852&view=diff
==============================================================================
--- giraph/trunk/CHANGELOG (original)
+++ giraph/trunk/CHANGELOG Thu Oct 4 00:31:23 2012
@@ -1,6 +1,9 @@
Giraph Change Log
Release 0.2.0 - unreleased
+ GIRAPH-342: Recursive ZooKeeper calls should call progress, dynamic
+ ZooKeeper can skip delete (aching via majakabiljo)
+
GIRAPH-351: Fail job early when there is no input (aching via ereisman)
GIRAPH-212: Security is busted since GIRAPH-168. (ekoontz via
Modified: giraph/trunk/src/main/java/org/apache/giraph/graph/BspService.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/main/java/org/apache/giraph/graph/BspService.java?rev=1393852&r1=1393851&r2=1393852&view=diff
==============================================================================
--- giraph/trunk/src/main/java/org/apache/giraph/graph/BspService.java (original)
+++ giraph/trunk/src/main/java/org/apache/giraph/graph/BspService.java Thu Oct 4 00:31:23 2012
@@ -338,7 +338,8 @@ public abstract class BspService<I exten
", " + getTaskPartition() + " on " + serverPortList);
}
try {
- this.zk = new ZooKeeperExt(serverPortList, sessionMsecTimeout, this);
+ this.zk = new ZooKeeperExt(
+ serverPortList, sessionMsecTimeout, this, context);
connectedEvent.waitForever();
this.fs = FileSystem.get(getConfiguration());
} catch (IOException e) {
Modified: giraph/trunk/src/main/java/org/apache/giraph/graph/BspServiceMaster.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/main/java/org/apache/giraph/graph/BspServiceMaster.java?rev=1393852&r1=1393851&r2=1393852&view=diff
==============================================================================
--- giraph/trunk/src/main/java/org/apache/giraph/graph/BspServiceMaster.java (original)
+++ giraph/trunk/src/main/java/org/apache/giraph/graph/BspServiceMaster.java Thu Oct 4 00:31:23 2012
@@ -1455,11 +1455,13 @@ public class BspServiceMaster<I extends
}
// At this point, all processes have acknowledged the cleanup,
- // and the master can do any final cleanup
+ // and the master can do any final cleanup if the ZooKeeper service was
+ // provided (not dynamically started) and we don't want to keep the data
try {
- if (!getConfiguration().getBoolean(
- GiraphConfiguration.KEEP_ZOOKEEPER_DATA,
- GiraphConfiguration.KEEP_ZOOKEEPER_DATA_DEFAULT)) {
+ if (getConfiguration().get(GiraphConfiguration.ZOOKEEPER_LIST) != null &&
+ !getConfiguration().getBoolean(
+ GiraphConfiguration.KEEP_ZOOKEEPER_DATA,
+ GiraphConfiguration.KEEP_ZOOKEEPER_DATA_DEFAULT)) {
if (LOG.isInfoEnabled()) {
LOG.info("cleanupZooKeeper: Removing the following path " +
"and all children - " + basePath);
Modified: giraph/trunk/src/main/java/org/apache/giraph/zk/ZooKeeperExt.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/main/java/org/apache/giraph/zk/ZooKeeperExt.java?rev=1393852&r1=1393851&r2=1393852&view=diff
==============================================================================
--- giraph/trunk/src/main/java/org/apache/giraph/zk/ZooKeeperExt.java (original)
+++ giraph/trunk/src/main/java/org/apache/giraph/zk/ZooKeeperExt.java Thu Oct 4 00:31:23 2012
@@ -20,6 +20,7 @@ package org.apache.giraph.zk;
import java.io.IOException;
+import org.apache.hadoop.util.Progressable;
import org.apache.log4j.Logger;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.CreateMode;
@@ -43,9 +44,11 @@ public class ZooKeeperExt extends ZooKee
private static final Logger LOG = Logger.getLogger(ZooKeeperExt.class);
/** Length of the ZK sequence number */
private static final int SEQUENCE_NUMBER_LENGTH = 10;
+ /** Ensure we have progress */
+ private final Progressable progressable;
/**
- * Constructor to connect to ZooKeeper
+ * Constructor to connect to ZooKeeper, does not make progress
*
* @param connectString Comma separated host:port pairs, each corresponding
* to a zk server. e.g.
@@ -62,9 +65,35 @@ public class ZooKeeperExt extends ZooKee
* @throws IOException
*/
public ZooKeeperExt(String connectString,
+ int sessionTimeout,
+ Watcher watcher) throws IOException {
+ this(connectString, sessionTimeout, watcher, null);
+ }
+
+ /**
+ * Constructor to connect to ZooKeeper, make progress
+ *
+ * @param connectString Comma separated host:port pairs, each corresponding
+ * to a zk server. e.g.
+ * "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" If the optional
+ * chroot suffix is used the example would look
+ * like: "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002/app/a"
+ * where the client would be rooted at "/app/a" and all paths
+ * would be relative to this root - ie getting/setting/etc...
+ * "/foo/bar" would result in operations being run on
+ * "/app/a/foo/bar" (from the server perspective).
+ * @param sessionTimeout Session timeout in milliseconds
+ * @param watcher A watcher object which will be notified of state changes,
+ * may also be notified for node events
+ * @param progressable Makes progress for longer operations
+ * @throws IOException
+ */
+ public ZooKeeperExt(String connectString,
int sessionTimeout,
- Watcher watcher) throws IOException {
+ Watcher watcher,
+ Progressable progressable) throws IOException {
super(connectString, sessionTimeout, watcher);
+ this.progressable = progressable;
}
/**
@@ -106,6 +135,9 @@ public class ZooKeeperExt extends ZooKee
int pos = path.indexOf("/", 1);
for (; pos != -1; pos = path.indexOf("/", pos + 1)) {
try {
+ if (progressable != null) {
+ progressable.progress();
+ }
create(
path.substring(0, pos), null, acl, CreateMode.PERSISTENT);
} catch (KeeperException.NodeExistsException e) {
@@ -246,6 +278,9 @@ public class ZooKeeperExt extends ZooKee
List<String> childList = getChildren(path, false);
for (String child : childList) {
+ if (progressable != null) {
+ progressable.progress();
+ }
deleteExt(path + "/" + child, -1, true);
}