You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zookeeper.apache.org by iv...@apache.org on 2012/11/29 15:19:01 UTC
svn commit: r1415176 - in /zookeeper/bookkeeper/trunk: ./
hedwig-server/src/main/java/org/apache/hedwig/admin/
hedwig-server/src/main/java/org/apache/hedwig/admin/console/
hedwig-server/src/main/java/org/apache/hedwig/server/meta/
hedwig-server/src/tes...
Author: ivank
Date: Thu Nov 29 14:18:59 2012
New Revision: 1415176
URL: http://svn.apache.org/viewvc?rev=1415176&view=rev
Log:
BOOKKEEPER-457: Create a format command for Hedwig to cleanup its metadata. (sijie via ivank)
Modified:
zookeeper/bookkeeper/trunk/CHANGES.txt
zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/admin/HedwigAdmin.java
zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/admin/console/HedwigCommands.java
zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/admin/console/HedwigConsole.java
zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/meta/FactoryLayout.java
zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/meta/MetadataManagerFactory.java
zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/meta/ZkMetadataManagerFactory.java
zookeeper/bookkeeper/trunk/hedwig-server/src/test/java/org/apache/hedwig/server/meta/TestMetadataManagerFactory.java
zookeeper/bookkeeper/trunk/hedwig-server/src/test/java/org/apache/hedwig/server/persistence/TestBookKeeperPersistenceManager.java
Modified: zookeeper/bookkeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/CHANGES.txt?rev=1415176&r1=1415175&r2=1415176&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/CHANGES.txt (original)
+++ zookeeper/bookkeeper/trunk/CHANGES.txt Thu Nov 29 14:18:59 2012
@@ -234,6 +234,8 @@ Trunk (unreleased changes)
BOOKKEEPER-479: Fix apache-rat issues in tree (ivank via fpj)
+ BOOKKEEPER-457: Create a format command for Hedwig to cleanup its metadata. (sijie via ivank)
+
hedwig-client:
BOOKKEEPER-306: Change C++ client to use gtest for testing (ivank via sijie)
Modified: zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/admin/HedwigAdmin.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/admin/HedwigAdmin.java?rev=1415176&r1=1415175&r2=1415176&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/admin/HedwigAdmin.java (original)
+++ zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/admin/HedwigAdmin.java Thu Nov 29 14:18:59 2012
@@ -42,6 +42,7 @@ import org.apache.hedwig.protocol.PubSub
import org.apache.hedwig.protocol.PubSubProtocol.SubscriptionData;
import org.apache.hedwig.server.common.ServerConfiguration;
import org.apache.hedwig.server.meta.MetadataManagerFactory;
+import org.apache.hedwig.server.meta.FactoryLayout;
import org.apache.hedwig.server.meta.SubscriptionDataManager;
import org.apache.hedwig.server.meta.TopicOwnershipManager;
import org.apache.hedwig.server.meta.TopicPersistenceManager;
@@ -224,6 +225,15 @@ public class HedwigAdmin {
}
/**
+ * Return metadata manager factory.
+ *
+ * @return metadata manager factory instance.
+ */
+ public MetadataManagerFactory getMetadataManagerFactory() {
+ return mmFactory;
+ }
+
+ /**
* Return bookeeper passwd used in hedwig admin
*
* @return bookeeper passwd
@@ -520,4 +530,20 @@ public class HedwigAdmin {
return syncObj.value;
}
+
+ /**
+ * Format metadata for Hedwig.
+ */
+ public void format() throws Exception {
+ // format metadata first
+ mmFactory.format(serverConf, zk);
+ LOG.info("Formatted Hedwig metadata successfully.");
+ // remove metadata layout
+ FactoryLayout.deleteLayout(zk, serverConf);
+ LOG.info("Removed old factory layout.");
+ // create new metadata manager factory and write new metadata layout
+ MetadataManagerFactory.createMetadataManagerFactory(serverConf, zk,
+ serverConf.getMetadataManagerFactoryClass());
+ LOG.info("Created new factory layout.");
+ }
}
Modified: zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/admin/console/HedwigCommands.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/admin/console/HedwigCommands.java?rev=1415176&r1=1415175&r2=1415176&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/admin/console/HedwigCommands.java (original)
+++ zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/admin/console/HedwigCommands.java Thu Nov 29 14:18:59 2012
@@ -260,6 +260,16 @@ public final class HedwigCommands {
" hello",
};
+ /* FORMAT: format metadata for Hedwig */
+ static final String FORMAT = "format";
+ static final String FORMAT_DESC = "format metadata for Hedwig";
+ static final String[] FORMAT_USAGE = new String[] {
+ "usage: format [-force]",
+ "",
+ " [-force] : Format metadata for Hedwig w/o confirmation.",
+ };
+
+
//
// List other useful commands
//
@@ -314,6 +324,7 @@ public final class HedwigCommands {
CMD_SHOW (SHOW, SHOW_DESC, SHOW_USAGE),
CMD_DESCRIBE (DESCRIBE, DESCRIBE_DESC, DESCRIBE_USAGE),
CMD_READTOPIC (READTOPIC, READTOPIC_DESC, READTOPIC_USAGE),
+ CMD_FORMAT (FORMAT, FORMAT_DESC, FORMAT_USAGE),
CMD_SET (SET, SET_DESC, SET_USAGE),
CMD_HISTORY (HISTORY, HISTORY_DESC, HISTORY_USAGE),
CMD_REDO (REDO, REDO_DESC, REDO_USAGE),
@@ -387,6 +398,7 @@ public final class HedwigCommands {
addCommand(COMMAND.CMD_DESCRIBE);
addCommand(COMMAND.CMD_READTOPIC);
+ addCommand(COMMAND.CMD_FORMAT);
addCommand(COMMAND.CMD_SET);
addCommand(COMMAND.CMD_HISTORY);
addCommand(COMMAND.CMD_REDO);
Modified: zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/admin/console/HedwigConsole.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/admin/console/HedwigConsole.java?rev=1415176&r1=1415175&r2=1415176&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/admin/console/HedwigConsole.java (original)
+++ zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/admin/console/HedwigConsole.java Thu Nov 29 14:18:59 2012
@@ -672,6 +672,31 @@ public class HedwigConsole {
}
+ class FormatCmd implements MyCommand {
+
+ @Override
+ public boolean runCmd(String[] args) throws Exception {
+ boolean force = false;
+ if (args.length >= 2 && "-force".equals(args[1])) {
+ force = true;
+ }
+ boolean doFormat = true;
+ System.out.println("You ask to format hedwig metadata stored in "
+ + admin.getMetadataManagerFactory().getClass().getName() + ".");
+ if (!force) {
+ doFormat = continueOrQuit();
+ }
+ if (doFormat) {
+ admin.format();
+ System.out.println("Formatted hedwig metadata successfully.");
+ } else {
+ System.out.println("Given up formatting hedwig metadata.");
+ }
+ return true;
+ }
+
+ }
+
protected Map<String, MyCommand> buildMyCommands() {
Map<String, MyCommand> cmds =
new HashMap<String, MyCommand>();
@@ -694,6 +719,7 @@ public class HedwigConsole {
cmds.put(SHOW, new ShowCmd());
cmds.put(DESCRIBE, new DescribeCmd());
cmds.put(READTOPIC, new ReadTopicCmd());
+ cmds.put(FORMAT, new FormatCmd());
return cmds;
}
@@ -879,7 +905,7 @@ public class HedwigConsole {
}
protected boolean continueOrQuit() throws IOException {
- System.out.println("Press <Return> for more, or Q to cancel ...");
+ System.out.println("Press <Return> to continue, or Q to cancel ...");
int ch;
if (null != console) {
ch = console.readCharacter(CONTINUE_OR_QUIT);
Modified: zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/meta/FactoryLayout.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/meta/FactoryLayout.java?rev=1415176&r1=1415175&r2=1415176&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/meta/FactoryLayout.java (original)
+++ zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/meta/FactoryLayout.java Thu Nov 29 14:18:59 2012
@@ -146,4 +146,21 @@ public class FactoryLayout {
return new FactoryLayout(meta);
}
+
+ /**
+ * Remove the factory layout from ZooKeeper.
+ *
+ * @param zk
+ * ZooKeeper instance
+ * @param cfg
+ * Server configuration object
+ * @throws KeeperException
+ * @throws InterruptedException
+ */
+ public static void deleteLayout(ZooKeeper zk, ServerConfiguration cfg)
+ throws KeeperException, InterruptedException {
+ String factoryLayoutPath = getFactoryLayoutPath(new StringBuilder(), cfg);
+ zk.delete(factoryLayoutPath, -1);
+ }
+
}
Modified: zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/meta/MetadataManagerFactory.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/meta/MetadataManagerFactory.java?rev=1415176&r1=1415175&r2=1415176&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/meta/MetadataManagerFactory.java (original)
+++ zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/meta/MetadataManagerFactory.java Thu Nov 29 14:18:59 2012
@@ -100,6 +100,16 @@ public abstract class MetadataManagerFac
public abstract TopicOwnershipManager newTopicOwnershipManager();
/**
+ * Format the metadata for Hedwig.
+ *
+ * @param cfg
+ * Configuration instance
+ * @param zk
+ * ZooKeeper instance
+ */
+ public abstract void format(ServerConfiguration cfg, ZooKeeper zk) throws IOException;
+
+ /**
* Create new Metadata Manager Factory.
*
* @param conf
@@ -118,38 +128,11 @@ public abstract class MetadataManagerFac
} catch (Exception e) {
throw new IOException("Failed to get metadata manager factory class from configuration : ", e);
}
-
- MetadataManagerFactory managerFactory;
-
// check that the configured manager is
// compatible with the existing layout
FactoryLayout layout = FactoryLayout.readLayout(zk, conf);
if (layout == null) { // no existing layout
- // use default manager if no one provided
- if (factoryClass == null) {
- factoryClass = ZkMetadataManagerFactory.class;
- }
- try {
- managerFactory = ReflectionUtils.newInstance(factoryClass);
- } catch (Throwable t) {
- throw new IOException("Fail to instantiate metadata manager factory : " + factoryClass, t);
- }
- ManagerMeta managerMeta = ManagerMeta.newBuilder()
- .setManagerImpl(factoryClass.getName())
- .setManagerVersion(managerFactory.getCurrentVersion())
- .build();
- layout = new FactoryLayout(managerMeta);
- try {
- layout.store(zk, conf);
- } catch (KeeperException.NodeExistsException nee) {
- FactoryLayout layout2 = FactoryLayout.readLayout(zk, conf);
- if (!layout2.equals(layout)) {
- throw new IOException("Contention writing to layout to zookeeper, "
- + " other layout " + layout2 + " is incompatible with our "
- + "layout " + layout);
- }
- }
- return managerFactory.initialize(conf, zk, layout.getManagerMeta().getManagerVersion());
+ return createMetadataManagerFactory(conf, zk, factoryClass);
}
LOG.debug("read meta layout {}", layout);
@@ -172,6 +155,7 @@ public abstract class MetadataManagerFac
}
}
// instantiate the metadata manager factory
+ MetadataManagerFactory managerFactory;
try {
managerFactory = ReflectionUtils.newInstance(factoryClass);
} catch (Throwable t) {
@@ -179,4 +163,51 @@ public abstract class MetadataManagerFac
}
return managerFactory.initialize(conf, zk, layout.getManagerMeta().getManagerVersion());
}
+
+ /**
+ * Create metadata manager factory and write factory layout to ZooKeeper.
+ *
+ * @param cfg
+ * Server Configuration object.
+ * @param zk
+ * ZooKeeper instance.
+ * @param factoryClass
+ * Metadata Manager Factory Class.
+ * @return metadata manager factory instance.
+ * @throws IOException
+ * @throws KeeperException
+ * @throws InterruptedException
+ */
+ public static MetadataManagerFactory createMetadataManagerFactory(
+ ServerConfiguration cfg, ZooKeeper zk,
+ Class<? extends MetadataManagerFactory> factoryClass)
+ throws IOException, KeeperException, InterruptedException {
+ // use default manager if no one provided
+ if (factoryClass == null) {
+ factoryClass = ZkMetadataManagerFactory.class;
+ }
+
+ MetadataManagerFactory managerFactory;
+ try {
+ managerFactory = ReflectionUtils.newInstance(factoryClass);
+ } catch (Throwable t) {
+ throw new IOException("Fail to instantiate metadata manager factory : " + factoryClass, t);
+ }
+ ManagerMeta managerMeta = ManagerMeta.newBuilder()
+ .setManagerImpl(factoryClass.getName())
+ .setManagerVersion(managerFactory.getCurrentVersion())
+ .build();
+ FactoryLayout layout = new FactoryLayout(managerMeta);
+ try {
+ layout.store(zk, cfg);
+ } catch (KeeperException.NodeExistsException nee) {
+ FactoryLayout layout2 = FactoryLayout.readLayout(zk, cfg);
+ if (!layout2.equals(layout)) {
+ throw new IOException("Contention writing to layout to zookeeper, "
+ + " other layout " + layout2 + " is incompatible with our "
+ + "layout " + layout);
+ }
+ }
+ return managerFactory.initialize(cfg, zk, layout.getManagerMeta().getManagerVersion());
+ }
}
Modified: zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/meta/ZkMetadataManagerFactory.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/meta/ZkMetadataManagerFactory.java?rev=1415176&r1=1415175&r2=1415176&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/meta/ZkMetadataManagerFactory.java (original)
+++ zookeeper/bookkeeper/trunk/hedwig-server/src/main/java/org/apache/hedwig/server/meta/ZkMetadataManagerFactory.java Thu Nov 29 14:18:59 2012
@@ -29,8 +29,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.KeeperException.Code;
+import org.apache.zookeeper.ZKUtil;
+import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.data.Stat;
import com.google.protobuf.ByteString;
@@ -824,4 +825,17 @@ public class ZkMetadataManagerFactory ex
}, ctx);
}
}
+
+ @Override
+ public void format(ServerConfiguration cfg, ZooKeeper zk) throws IOException {
+ try {
+ ZKUtil.deleteRecursive(zk, cfg.getZkTopicsPrefix(new StringBuilder()).toString());
+ } catch (KeeperException.NoNodeException e) {
+ logger.debug("Hedwig root node doesn't exist in zookeeper to delete");
+ } catch (KeeperException ke) {
+ throw new IOException(ke);
+ } catch (InterruptedException ie) {
+ throw new IOException(ie);
+ }
+ }
}
Modified: zookeeper/bookkeeper/trunk/hedwig-server/src/test/java/org/apache/hedwig/server/meta/TestMetadataManagerFactory.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/hedwig-server/src/test/java/org/apache/hedwig/server/meta/TestMetadataManagerFactory.java?rev=1415176&r1=1415175&r2=1415176&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/hedwig-server/src/test/java/org/apache/hedwig/server/meta/TestMetadataManagerFactory.java (original)
+++ zookeeper/bookkeeper/trunk/hedwig-server/src/test/java/org/apache/hedwig/server/meta/TestMetadataManagerFactory.java Thu Nov 29 14:18:59 2012
@@ -96,6 +96,10 @@ public class TestMetadataManagerFactory
public TopicOwnershipManager newTopicOwnershipManager() {
return null;
}
+
+ public void format(ServerConfiguration cfg, ZooKeeper zk) throws IOException {
+ // do nothing
+ }
}
private void writeFactoryLayout(ServerConfiguration conf,
Modified: zookeeper/bookkeeper/trunk/hedwig-server/src/test/java/org/apache/hedwig/server/persistence/TestBookKeeperPersistenceManager.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/hedwig-server/src/test/java/org/apache/hedwig/server/persistence/TestBookKeeperPersistenceManager.java?rev=1415176&r1=1415175&r2=1415176&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/hedwig-server/src/test/java/org/apache/hedwig/server/persistence/TestBookKeeperPersistenceManager.java (original)
+++ zookeeper/bookkeeper/trunk/hedwig-server/src/test/java/org/apache/hedwig/server/persistence/TestBookKeeperPersistenceManager.java Thu Nov 29 14:18:59 2012
@@ -167,6 +167,11 @@ public class TestBookKeeperPersistenceMa
public TopicOwnershipManager newTopicOwnershipManager() {
return factory.newTopicOwnershipManager();
}
+
+ @Override
+ public void format(ServerConfiguration cfg, ZooKeeper zk) throws IOException {
+ factory.format(cfg, zk);
+ }
}
public TestBookKeeperPersistenceManager(boolean removeStartSeqId) {