svn commit: r1207495 [3/3] - in /zookeeper/bookkeeper/trunk: ./ bookkeeper-server/conf/ bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/ bookkeeper-server/src/main/java/org/apache/bookkeeper/client/ bookkeeper-server/src/main/java/org/apac...

[iv...@apache.org]

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BaseTestCase.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BaseTestCase.java?rev=1207495&r1=1207494&r2=1207495&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BaseTestCase.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BaseTestCase.java Mon Nov 28 18:25:18 2011
@@ -36,6 +36,7 @@ import org.apache.bookkeeper.conf.Server
 import org.apache.bookkeeper.proto.BookieServer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.Watcher;
@@ -140,9 +141,11 @@ public abstract class BaseTestCase exten
                 server.start();
                 bs.add(server);
             }
-            zkc.close();
+
             baseClientConf.setZkServers("127.0.0.1");
-            bkc = new BookKeeperTestClient(baseClientConf);
+            if (numBookies > 0) {
+                bkc = new BookKeeperTestClient(baseClientConf);
+            }
         } catch(Exception e) {
             LOG.error("Error setting up", e);
             throw e;
@@ -168,14 +171,16 @@ public abstract class BaseTestCase exten
      * @throws InterruptedException
      * @throws IOException
      */
-    protected void restartBookies() throws InterruptedException, IOException {
+    protected void restartBookies() 
+            throws InterruptedException, IOException, KeeperException {
         restartBookies(null);
     }
 
     /**
      * Restart bookie servers add new configuration settings
      */
-    protected void restartBookies(ServerConfiguration newConf) throws InterruptedException, IOException {
+    protected void restartBookies(ServerConfiguration newConf)
+            throws InterruptedException, IOException, KeeperException {
         // shut down bookie server
         for (BookieServer server : bs) {
             server.shutdown();
@@ -209,6 +214,10 @@ public abstract class BaseTestCase exten
             server.shutdown();
         }
 
+        if (zkc != null) {
+            zkc.close();
+        }
+
         for (File f : tmpDirs) {
             cleanUpDir(f);
         }

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ConcurrentLedgerTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ConcurrentLedgerTest.java?rev=1207495&r1=1207494&r2=1207495&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ConcurrentLedgerTest.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/ConcurrentLedgerTest.java Mon Nov 28 18:25:18 2011
@@ -54,7 +54,7 @@ public class ConcurrentLedgerTest extend
 
     @Override
     @Before
-    public void setUp() throws IOException {
+    public void setUp() throws Exception {
         String txnDirName = System.getProperty("txnDir");
         if (txnDirName != null) {
             txnDir = new File(txnDirName);

Modified: zookeeper/bookkeeper/trunk/doc/bookkeeperConfig.textile
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/doc/bookkeeperConfig.textile?rev=1207495&r1=1207494&r2=1207495&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/doc/bookkeeperConfig.textile (original)
+++ zookeeper/bookkeeper/trunk/doc/bookkeeperConfig.textile Mon Nov 28 18:25:18 2011
@@ -72,4 +72,46 @@ bq. NOTE: keeping number of old journal 
 
 h1. ZooKeeper Metadata
 
-p. For BookKeeper, we require a ZooKeeper installation to store metadata, and to pass the list of ZooKeeper servers as parameter to the constructor of the BookKeeper class ( @org.apache.bookkeeper.client,BookKeeper@ ). To setup ZooKeeper, please check the "ZooKeeper documentation":index.html. 
\ No newline at end of file
+p. For BookKeeper, we require a ZooKeeper installation to store metadata, and to pass the list of ZooKeeper servers as parameter to the constructor of the BookKeeper class ( @org.apache.bookkeeper.client,BookKeeper@ ). To setup ZooKeeper, please check the "ZooKeeper documentation":index.html. 
+
+p. BookKeeper provides two mechnaisms to organize its metadata in ZooKeeper.
+
+* FlatLedgerManager : All ledger metadata are placed as children in a single zookeeper path.
+* HierarchicalLedgerManager : All ledger metadata are partitioned into 2-level znodes.
+
+h2. Flat Ledger Manager
+
+p. All ledgers' metadata are put in a single zookeeper path, created using zookeeper sequential node, which can ensure uniqueness of ledger id. Each ledger node is prefixed with 'L'.
+
+p. Bookie server manages its owned active ledgers in a hash map. So it is easy for bookie server to find what ledgers are deleted from zookeeper and garbage collect them. And its garbage collection flow is described as below:
+
+* Fetch all existing ledgers from zookeeper (*zkActiveLedgers*).
+* Fetch all ledgers currently active within the Bookie (*bkActiveLedgers*).
+* Loop over *bkActiveLedgers* to find those ledgers which do not exist in *zkActiveLedgers* and garbage collect them.
+
+h2. Hierarchical Ledger Manager
+
+p. Hierarchical Ledger Manager first obtains a global unique id from ZooKeeper using a EPHEMERAL_SEQUENTIAL znode.
+
+p. Since ZooKeeper sequential counter has a format of %10d -- that is 10 digits with 0 (zero) padding, i.e. "<path>0000000001", HierarchicalLedgerManager splits the generated id into 3 parts :
+
+ @ {level1 (2 digits)}{level2 (4 digits)}{level3 (4 digits)}
+
+p. These 3 parts are used to form the actual ledger node path used to store ledger metadata:
+
+ @ {ledgers_root_path}/{level1}/{level2}/L{level3}
+
+p. E.g. Ledger 0000000001 is split into 3 parts 00, 0000, 00001, which is stored in znode /{ledgers_root_path}/00/0000/L0001. So each znode could have at most 10000 ledgers, which avoids the problem of the child list being larger than the maximum ZooKeeper packet size.
+
+p. Bookie server manages its active ledgers in a sorted map, which simplifies access to active ledgers in a particular (level1, level2) partition.
+
+p. Garbage collection in bookie server is processed node by node as follows:
+
+* Fetching all level1 nodes, by calling zk#getChildren(ledgerRootPath).
+** For each level1 nodes, fetching their level2 nodes :
+** For each partition (level1, level2) :
+*** Fetch all existed ledgers from zookeeper belonging to partition (level1, level2) (*zkActiveLedgers*).
+*** Fetch all ledgers currently active in the bookie which belong to partition (level1, level2) (*bkActiveLedgers*).
+*** Loop over *bkActiveLedgers* to find those ledgers which do not exist in *zkActiveLedgers*, and garbage collect them.
+
+bq. NOTE: Hierarchical Ledger Manager is more suitable to manage large number of ledgers existed in BookKeeper.