You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zookeeper.apache.org by iv...@apache.org on 2013/02/12 11:16:45 UTC

svn commit: r1445085 - in /zookeeper/bookkeeper/trunk: ./ bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/ bookkeeper-server/src/test/java/org/apache/bookkeeper/test/

Author: ivank
Date: Tue Feb 12 10:16:44 2013
New Revision: 1445085

URL: http://svn.apache.org/r1445085
Log:
BOOKKEEPER-555: Make BookieServer use Netty rather than a custom IO server (ivank)

Added:
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieNettyServer.java
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieRequestHandler.java
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ResponseBuilder.java
Removed:
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/NIOServerFactory.java
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/NIOServerFactoryTest.java
Modified:
    zookeeper/bookkeeper/trunk/CHANGES.txt
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtocol.java
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServerBean.java
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieZKExpireTest.java

Modified: zookeeper/bookkeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/CHANGES.txt?rev=1445085&r1=1445084&r2=1445085&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/CHANGES.txt (original)
+++ zookeeper/bookkeeper/trunk/CHANGES.txt Tue Feb 12 10:16:44 2013
@@ -16,6 +16,8 @@ Trunk (unreleased changes)
 
     IMPROVEMENTS:
 
+      BOOKKEEPER-555: Make BookieServer use Netty rather than a custom IO server (ivank)
+
       BOOKKEEPER-526: multiple threads for delivery manager (sijie via ivank)
 
       BOOKKEEPER-544: Modify hedwig server tests to allow client testcases to start/stop them as part of their tests (mridul via ivank)

Added: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieNettyServer.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieNettyServer.java?rev=1445085&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieNettyServer.java (added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieNettyServer.java Tue Feb 12 10:16:44 2013
@@ -0,0 +1,189 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.bookkeeper.proto;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.bookkeeper.bookie.Bookie;
+import org.apache.bookkeeper.bookie.BookieException;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.zookeeper.KeeperException;
+
+import org.jboss.netty.bootstrap.ServerBootstrap;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelFactory;
+import org.jboss.netty.channel.ChannelPipeline;
+import org.jboss.netty.channel.ChannelPipelineFactory;
+import org.jboss.netty.channel.Channels;
+import org.jboss.netty.channel.group.ChannelGroup;
+import org.jboss.netty.channel.group.ChannelGroupFuture;
+import org.jboss.netty.channel.group.DefaultChannelGroup;
+import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
+import org.jboss.netty.handler.codec.frame.LengthFieldBasedFrameDecoder;
+import org.jboss.netty.handler.codec.frame.LengthFieldPrepender;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Netty server for serving bookie requests
+ */
+class BookieNettyServer {
+    static Logger LOG = LoggerFactory.getLogger(BookieNettyServer.class);
+
+    final static int maxMessageSize = 0xfffff;
+    final ServerConfiguration conf;
+    final ChannelFactory serverChannelFactory;
+    final Bookie bookie;
+    final ChannelGroup allChannels = new CleanupChannelGroup();
+    final AtomicBoolean isRunning = new AtomicBoolean(false);
+    Object suspensionLock = new Object();
+    boolean suspended = false;
+
+    InetSocketAddress localAddress = null;
+
+    BookieNettyServer(ServerConfiguration conf, Bookie bookie)
+            throws IOException, KeeperException, InterruptedException, BookieException  {
+        this.conf = conf;
+        this.bookie = bookie;
+
+        ThreadFactoryBuilder tfb = new ThreadFactoryBuilder();
+        String base = "bookie-" + conf.getBookiePort() + "-netty";
+        serverChannelFactory = new NioServerSocketChannelFactory(
+                Executors.newCachedThreadPool(tfb.setNameFormat(base + "-boss-%d").build()),
+                Executors.newCachedThreadPool(tfb.setNameFormat(base + "-worker-%d").build()));
+
+    }
+
+    boolean isRunning() {
+        return isRunning.get();
+    }
+
+    @VisibleForTesting
+    void suspendProcessing() {
+        synchronized (suspensionLock) {
+            suspended = true;
+            allChannels.setReadable(false).awaitUninterruptibly();
+        }
+    }
+
+    @VisibleForTesting
+    void resumeProcessing() {
+        synchronized (suspensionLock) {
+            suspended = false;
+            allChannels.setReadable(true).awaitUninterruptibly();
+            suspensionLock.notifyAll();
+        }
+    }
+
+    InetSocketAddress getLocalAddress() {
+        if (localAddress != null) {
+            return localAddress;
+        } else {
+            return new InetSocketAddress(conf.getBookiePort());
+        }
+    }
+
+    void start() {
+        ServerBootstrap bootstrap = new ServerBootstrap(serverChannelFactory);
+        bootstrap.setPipelineFactory(new BookiePipelineFactory());
+        bootstrap.setOption("child.tcpNoDelay", conf.getServerTcpNoDelay());
+        bootstrap.setOption("child.soLinger", 2);
+
+        Channel listen = bootstrap.bind(new InetSocketAddress(conf.getBookiePort()));
+
+        assert(listen.getLocalAddress() instanceof InetSocketAddress);
+        localAddress = (InetSocketAddress)listen.getLocalAddress();
+        allChannels.add(listen);
+        isRunning.set(true);
+    }
+
+    void shutdown() {
+        isRunning.set(false);
+        allChannels.close().awaitUninterruptibly();
+        serverChannelFactory.releaseExternalResources();
+    }
+
+    private class BookiePipelineFactory implements ChannelPipelineFactory {
+        public ChannelPipeline getPipeline() throws Exception {
+            synchronized (suspensionLock) {
+                while (suspended) {
+                    suspensionLock.wait();
+                }
+            }
+            ChannelPipeline pipeline = Channels.pipeline();
+            pipeline.addLast("lengthbaseddecoder",
+                             new LengthFieldBasedFrameDecoder(maxMessageSize, 0, 4, 0, 4));
+            pipeline.addLast("lengthprepender", new LengthFieldPrepender(4));
+
+            pipeline.addLast("bookieProtoDecoder", new BookieProtoEncoding.Decoder());
+            pipeline.addLast("bookieProtoEncoder", new BookieProtoEncoding.Encoder());
+            pipeline.addLast("bookieRequestHandler", new BookieRequestHandler(conf, bookie,
+                                                                              allChannels));
+            return pipeline;
+        }
+    }
+
+    private static class CleanupChannelGroup extends DefaultChannelGroup {
+        private AtomicBoolean closed = new AtomicBoolean(false);
+
+        CleanupChannelGroup() {
+            super("BookieChannelGroup");
+        }
+
+        @Override
+        public boolean add(Channel channel) {
+            boolean ret = super.add(channel);
+            if (closed.get()) {
+                channel.close();
+            }
+            return ret;
+        }
+
+        @Override
+        public ChannelGroupFuture close() {
+            closed.set(true);
+            return super.close();
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (!(o instanceof CleanupChannelGroup)) {
+                return false;
+            }
+            CleanupChannelGroup other = (CleanupChannelGroup)o;
+            return other.closed.get() == closed.get()
+                && super.equals(other);
+        }
+
+        @Override
+        public int hashCode() {
+            return super.hashCode() * 17 + (closed.get() ? 1 : 0);
+        }
+    }
+}

Added: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java?rev=1445085&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java (added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtoEncoding.java Tue Feb 12 10:16:44 2013
@@ -0,0 +1,118 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.bookkeeper.proto;
+
+import org.jboss.netty.buffer.ChannelBuffers;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelHandlerContext;
+
+import org.apache.bookkeeper.proto.BookieProtocol.PacketHeader;
+import org.jboss.netty.handler.codec.oneone.OneToOneEncoder;
+import org.jboss.netty.handler.codec.oneone.OneToOneDecoder;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class BookieProtoEncoding {
+    static Logger LOG = LoggerFactory.getLogger(BookieProtoEncoding.class);
+
+    public static class Decoder extends OneToOneDecoder {
+        @Override
+        public Object decode(ChannelHandlerContext ctx, Channel channel, Object msg)
+                throws Exception {
+            if (!(msg instanceof ChannelBuffer)) {
+                return msg;
+            }
+            ChannelBuffer packet = (ChannelBuffer)msg;
+
+            PacketHeader h = PacketHeader.fromInt(packet.readInt());
+
+            // packet format is different between ADDENTRY and READENTRY
+            long ledgerId = -1;
+            long entryId = BookieProtocol.INVALID_ENTRY_ID;
+            byte[] masterKey = null;
+            short flags = h.getFlags();
+
+            ServerStats.getInstance().incrementPacketsReceived();
+
+            switch (h.getOpCode()) {
+            case BookieProtocol.ADDENTRY:
+                // first read master key
+                masterKey = new byte[BookieProtocol.MASTER_KEY_LENGTH];
+                packet.readBytes(masterKey, 0, BookieProtocol.MASTER_KEY_LENGTH);
+
+                ChannelBuffer bb = packet.duplicate();
+                ledgerId = bb.readLong();
+                entryId = bb.readLong();
+
+                return new BookieProtocol.AddRequest(h.getVersion(), ledgerId, entryId,
+                        flags, masterKey, packet.toByteBuffer().slice());
+            case BookieProtocol.READENTRY:
+                ledgerId = packet.readLong();
+                entryId = packet.readLong();
+
+                if ((flags & BookieProtocol.FLAG_DO_FENCING) == BookieProtocol.FLAG_DO_FENCING
+                    && h.getVersion() >= 2) {
+                    masterKey = new byte[BookieProtocol.MASTER_KEY_LENGTH];
+                    packet.readBytes(masterKey, 0, BookieProtocol.MASTER_KEY_LENGTH);
+                    return new BookieProtocol.ReadRequest(h.getVersion(), ledgerId, entryId, flags, masterKey);
+                } else {
+                    return new BookieProtocol.ReadRequest(h.getVersion(), ledgerId, entryId, flags);
+                }
+            }
+            return msg;
+        }
+    }
+
+    public static class Encoder extends OneToOneEncoder {
+        @Override
+        public Object encode(ChannelHandlerContext ctx, Channel channel, Object msg)
+                throws Exception {
+            if (!(msg instanceof BookieProtocol.Response)) {
+                return msg;
+            }
+            BookieProtocol.Response r = (BookieProtocol.Response)msg;
+            ChannelBuffer buf = ctx.getChannel().getConfig().getBufferFactory()
+                .getBuffer(24);
+            buf.writeInt(new PacketHeader(r.getProtocolVersion(),
+                                          r.getOpCode(), (short)0).toInt());
+            buf.writeInt(r.getErrorCode());
+            buf.writeLong(r.getLedgerId());
+            buf.writeLong(r.getEntryId());
+
+            ServerStats.getInstance().incrementPacketsSent();
+
+            if (msg instanceof BookieProtocol.ReadResponse) {
+                BookieProtocol.ReadResponse rr = (BookieProtocol.ReadResponse)r;
+                return ChannelBuffers.wrappedBuffer(buf,
+                        ChannelBuffers.wrappedBuffer(rr.getData()));
+            } else if ((msg instanceof BookieProtocol.AddResponse)
+                       || (msg instanceof BookieProtocol.ErrorResponse)) {
+                return buf;
+            } else {
+                LOG.error("Cannot encode unknown response type {}", msg.getClass().getName());
+                return msg;
+            }
+        }
+    }
+
+}

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtocol.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtocol.java?rev=1445085&r1=1445084&r2=1445085&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtocol.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieProtocol.java Tue Feb 12 10:16:44 2013
@@ -21,6 +21,8 @@ package org.apache.bookkeeper.proto;
  *
  */
 
+import java.nio.ByteBuffer;
+
 /**
  * The packets of the Bookie protocol all have a 4-byte integer indicating the
  * type of request or response at the very beginning of the packet followed by a
@@ -173,4 +175,160 @@ public interface BookieProtocol {
     public static final short FLAG_NONE = 0x0;
     public static final short FLAG_DO_FENCING = 0x0001;
     public static final short FLAG_RECOVERY_ADD = 0x0002;
+
+    static class Request {
+
+        final byte protocolVersion;
+        final byte opCode;
+        final long ledgerId;
+        final long entryId;
+        final short flags;
+        final byte[] masterKey;
+
+        protected Request(byte protocolVersion, byte opCode, long ledgerId,
+                          long entryId, short flags) {
+            this(protocolVersion, opCode, ledgerId, entryId, flags, null);
+        }
+
+        protected Request(byte protocolVersion, byte opCode, long ledgerId,
+                          long entryId, short flags, byte[] masterKey) {
+            this.protocolVersion = protocolVersion;
+            this.opCode = opCode;
+            this.ledgerId = ledgerId;
+            this.entryId = entryId;
+            this.flags = flags;
+            this.masterKey = masterKey;
+        }
+
+        byte getProtocolVersion() {
+            return protocolVersion;
+        }
+
+        byte getOpCode() {
+            return opCode;
+        }
+
+        long getLedgerId() {
+            return ledgerId;
+        }
+
+        long getEntryId() {
+            return entryId;
+        }
+
+        boolean hasMasterKey() {
+            return masterKey != null;
+        }
+
+        byte[] getMasterKey() {
+            assert hasMasterKey();
+            return masterKey;
+        }
+
+        @Override
+        public String toString() {
+            return String.format("Op(%d)[Ledger:%d,Entry:%d]", opCode, ledgerId, entryId);
+        }
+    }
+
+    static class AddRequest extends Request {
+        final ByteBuffer data;
+
+        AddRequest(byte protocolVersion, long ledgerId, long entryId,
+                   short flags, byte[] masterKey, ByteBuffer data) {
+            super(protocolVersion, ADDENTRY, ledgerId, entryId, flags, masterKey);
+            this.data = data;
+        }
+
+        ByteBuffer getData() {
+            return data;
+        }
+
+        boolean isRecoveryAdd() {
+            return (flags & FLAG_RECOVERY_ADD) == FLAG_RECOVERY_ADD;
+        }
+    }
+
+    static class ReadRequest extends Request {
+        ReadRequest(byte protocolVersion, long ledgerId, long entryId, short flags) {
+            super(protocolVersion, READENTRY, ledgerId, entryId, flags);
+        }
+
+        ReadRequest(byte protocolVersion, long ledgerId, long entryId,
+                    short flags, byte[] masterKey) {
+            super(protocolVersion, READENTRY, ledgerId, entryId, flags, masterKey);
+        }
+
+        boolean isFencingRequest() {
+            return (flags & FLAG_DO_FENCING) == FLAG_DO_FENCING;
+        }
+    }
+
+    static class Response {
+        final byte protocolVersion;
+        final byte opCode;
+        final int errorCode;
+        final long ledgerId;
+        final long entryId;
+
+        protected Response(byte protocolVersion, byte opCode,
+                           int errorCode, long ledgerId, long entryId) {
+            this.protocolVersion = protocolVersion;
+            this.opCode = opCode;
+            this.errorCode = errorCode;
+            this.ledgerId = ledgerId;
+            this.entryId = entryId;
+        }
+
+        byte getProtocolVersion() {
+            return protocolVersion;
+        }
+
+        byte getOpCode() {
+            return opCode;
+        }
+
+        long getLedgerId() {
+            return ledgerId;
+        }
+
+        long getEntryId() {
+            return entryId;
+        }
+
+        int getErrorCode() {
+            return errorCode;
+        }
+
+        @Override
+        public String toString() {
+            return String.format("Op(%d)[Ledger:%d,Entry:%d]", opCode, ledgerId, entryId);
+        }
+    }
+
+    static class ReadResponse extends Response {
+        final ByteBuffer data;
+
+        ReadResponse(byte protocolVersion, long ledgerId, long entryId, ByteBuffer data) {
+            super(protocolVersion, READENTRY, EOK, ledgerId, entryId);
+            this.data = data;
+        }
+
+        ByteBuffer getData() {
+            return data;
+        }
+    }
+
+    static class AddResponse extends Response {
+        AddResponse(byte protocolVersion, long ledgerId, long entryId) {
+            super(protocolVersion, ADDENTRY, EOK, ledgerId, entryId);
+        }
+    }
+
+    static class ErrorResponse extends Response {
+        ErrorResponse(byte protocolVersion, byte opCode, int errorCode,
+                      long ledgerId, long entryId) {
+            super(protocolVersion, opCode, errorCode, ledgerId, entryId);
+        }
+    }
 }

Added: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieRequestHandler.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieRequestHandler.java?rev=1445085&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieRequestHandler.java (added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieRequestHandler.java Tue Feb 12 10:16:44 2013
@@ -0,0 +1,303 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.bookkeeper.proto;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.bookkeeper.util.MathUtils;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.bookie.Bookie;
+import org.apache.bookkeeper.bookie.BookieException;
+
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelStateEvent;
+import org.jboss.netty.channel.ExceptionEvent;
+import org.jboss.netty.channel.MessageEvent;
+import org.jboss.netty.channel.SimpleChannelHandler;
+import org.jboss.netty.channel.group.ChannelGroup;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Serverside handler for bookkeeper requests
+ */
+class BookieRequestHandler extends SimpleChannelHandler
+    implements BookkeeperInternalCallbacks.WriteCallback {
+
+    static Logger LOG = LoggerFactory.getLogger(BookieRequestHandler.class);
+    private final Bookie bookie;
+    private final ChannelGroup allChannels;
+
+    private final BKStats bkStats = BKStats.getInstance();
+    private final boolean statsEnabled;
+
+    BookieRequestHandler(ServerConfiguration conf, Bookie bookie, ChannelGroup allChannels) {
+        this.bookie = bookie;
+        this.allChannels = allChannels;
+        this.statsEnabled = conf.isStatisticsEnabled();
+    }
+
+    @Override
+    public void channelOpen(ChannelHandlerContext ctx,
+                            ChannelStateEvent e)
+            throws Exception {
+        allChannels.add(ctx.getChannel());
+    }
+
+    @Override
+    public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
+        Throwable throwable = e.getCause();
+        LOG.error("Unhandled exception occurred in I/O thread or handler", throwable);
+    }
+
+    @Override
+    public void channelConnected(ChannelHandlerContext ctx, ChannelStateEvent e)
+            throws Exception {
+        LOG.debug("Channel connected {}", e);
+    }
+
+    public void channelDisconnected(ChannelHandlerContext ctx, ChannelStateEvent e)
+            throws Exception {
+        LOG.debug("Channel disconnected {}", e);
+    }
+
+    @Override
+    public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
+        if (!(e.getMessage() instanceof BookieProtocol.Request)) {
+            ctx.sendUpstream(e);
+            return;
+        }
+        BookieProtocol.Request r = (BookieProtocol.Request)e.getMessage();
+
+        Channel c = ctx.getChannel();
+
+        if (r.getProtocolVersion() < BookieProtocol.LOWEST_COMPAT_PROTOCOL_VERSION
+            || r.getProtocolVersion() > BookieProtocol.CURRENT_PROTOCOL_VERSION) {
+            LOG.error("Invalid protocol version, expected something between "
+                      + BookieProtocol.LOWEST_COMPAT_PROTOCOL_VERSION
+                      + " & " + BookieProtocol.CURRENT_PROTOCOL_VERSION
+                      + ". got " + r.getProtocolVersion());
+            c.write(ResponseBuilder.buildErrorResponse(BookieProtocol.EBADVERSION, r));
+            return;
+        }
+
+        switch (r.getOpCode()) {
+        case BookieProtocol.ADDENTRY:
+            handleAdd(r, c);
+            break;
+        case BookieProtocol.READENTRY:
+            handleRead(r, c);
+            break;
+        default:
+            LOG.error("Unknown op type {}, sending error", r.getOpCode());
+            c.write(ResponseBuilder.buildErrorResponse(BookieProtocol.EBADREQ, r));
+            if (statsEnabled) {
+                bkStats.getOpStats(BKStats.STATS_UNKNOWN).incrementFailedOps();
+            }
+            break;
+        }
+    }
+
+    class AddCtx {
+        final Channel c;
+        final BookieProtocol.AddRequest r;
+        final long startTime;
+
+        AddCtx(Channel c, BookieProtocol.AddRequest r) {
+            this.c = c;
+            this.r = r;
+
+            if (statsEnabled) {
+                startTime = MathUtils.now();
+            } else {
+                startTime = 0;
+            }
+        }
+    }
+
+    private void handleAdd(BookieProtocol.Request r, Channel c) {
+        assert (r instanceof BookieProtocol.AddRequest);
+        BookieProtocol.AddRequest add = (BookieProtocol.AddRequest)r;
+
+        if (bookie.isReadOnly()) {
+            LOG.warn("BookieServer is running as readonly mode,"
+                     + " so rejecting the request from the client!");
+            c.write(ResponseBuilder.buildErrorResponse(BookieProtocol.EREADONLY, add));
+            if (statsEnabled) {
+                bkStats.getOpStats(BKStats.STATS_ADD).incrementFailedOps();
+            }
+            return;
+        }
+
+        int rc = BookieProtocol.EOK;
+        try {
+            if (add.isRecoveryAdd()) {
+                bookie.recoveryAddEntry(add.getData(), this, new AddCtx(c, add),
+                                        add.getMasterKey());
+            } else {
+                bookie.addEntry(add.getData(), this, new AddCtx(c, add), add.getMasterKey());
+            }
+        } catch (IOException e) {
+            LOG.error("Error writing " + add, e);
+            rc = BookieProtocol.EIO;
+        } catch (BookieException.LedgerFencedException lfe) {
+            LOG.error("Attempt to write to fenced ledger", lfe);
+            rc = BookieProtocol.EFENCED;
+        } catch (BookieException e) {
+            LOG.error("Unauthorized access to ledger " + add.getLedgerId(), e);
+            rc = BookieProtocol.EUA;
+        }
+        if (rc != BookieProtocol.EOK) {
+            c.write(ResponseBuilder.buildErrorResponse(rc, add));
+            if (statsEnabled) {
+                bkStats.getOpStats(BKStats.STATS_ADD).incrementFailedOps();
+            }
+        }
+    }
+
+    @Override
+    public void writeComplete(int rc, long ledgerId, long entryId,
+                              InetSocketAddress addr, Object ctx) {
+        assert(ctx instanceof AddCtx);
+        AddCtx addctx = (AddCtx)ctx;
+        addctx.c.write(ResponseBuilder.buildAddResponse(addctx.r));
+
+        if (statsEnabled) {
+            // compute the latency
+            if (0 == rc) {
+                // for add operations, we compute latency in writeComplete callbacks.
+                long elapsedTime = MathUtils.now() - addctx.startTime;
+                bkStats.getOpStats(BKStats.STATS_ADD).updateLatency(elapsedTime);
+            } else {
+                bkStats.getOpStats(BKStats.STATS_ADD).incrementFailedOps();
+            }
+        }
+    }
+
+    private void handleRead(BookieProtocol.Request r, Channel c) {
+        assert (r instanceof BookieProtocol.ReadRequest);
+        BookieProtocol.ReadRequest read = (BookieProtocol.ReadRequest)r;
+
+        LOG.debug("Received new read request: {}", r);
+        int errorCode = BookieProtocol.EIO;
+        long startTime = 0;
+        if (statsEnabled) {
+            startTime = MathUtils.now();
+        }
+        ByteBuffer data = null;
+        try {
+            Future<Boolean> fenceResult = null;
+            if (read.isFencingRequest()) {
+                LOG.warn("Ledger " + r.getLedgerId() + " fenced by " + c.getRemoteAddress());
+
+                if (read.hasMasterKey()) {
+                    fenceResult = bookie.fenceLedger(read.getLedgerId(), read.getMasterKey());
+                } else {
+                    LOG.error("Password not provided, Not safe to fence {}", read.getLedgerId());
+                    if (statsEnabled) {
+                        bkStats.getOpStats(BKStats.STATS_READ).incrementFailedOps();
+                    }
+                    throw BookieException.create(BookieException.Code.UnauthorizedAccessException);
+                }
+            }
+            data = bookie.readEntry(r.getLedgerId(), r.getEntryId());
+            LOG.debug("##### Read entry ##### {}", data.remaining());
+            if (null != fenceResult) {
+                // TODO:
+                // currently we don't have readCallback to run in separated read
+                // threads. after BOOKKEEPER-429 is complete, we could improve
+                // following code to make it not wait here
+                //
+                // For now, since we only try to wait after read entry. so writing
+                // to journal and read entry are executed in different thread
+                // it would be fine.
+                try {
+                    Boolean fenced = fenceResult.get(1000, TimeUnit.MILLISECONDS);
+                    if (null == fenced || !fenced) {
+                        // if failed to fence, fail the read request to make it retry.
+                        errorCode = BookieProtocol.EIO;
+                        data = null;
+                    } else {
+                        errorCode = BookieProtocol.EOK;
+                    }
+                } catch (InterruptedException ie) {
+                    LOG.error("Interrupting fence read entry " + read, ie);
+                    errorCode = BookieProtocol.EIO;
+                    data = null;
+                } catch (ExecutionException ee) {
+                    LOG.error("Failed to fence read entry " + read, ee);
+                    errorCode = BookieProtocol.EIO;
+                    data = null;
+                } catch (TimeoutException te) {
+                    LOG.error("Timeout to fence read entry " + read, te);
+                    errorCode = BookieProtocol.EIO;
+                    data = null;
+                }
+            } else {
+                errorCode = BookieProtocol.EOK;
+            }
+        } catch (Bookie.NoLedgerException e) {
+            if (LOG.isTraceEnabled()) {
+                LOG.error("Error reading " + read, e);
+            }
+            errorCode = BookieProtocol.ENOLEDGER;
+        } catch (Bookie.NoEntryException e) {
+            if (LOG.isTraceEnabled()) {
+                LOG.error("Error reading " + read, e);
+            }
+            errorCode = BookieProtocol.ENOENTRY;
+        } catch (IOException e) {
+            if (LOG.isTraceEnabled()) {
+                LOG.error("Error reading " + read, e);
+            }
+            errorCode = BookieProtocol.EIO;
+        } catch (BookieException e) {
+            LOG.error("Unauthorized access to ledger " + read.getLedgerId(), e);
+            errorCode = BookieProtocol.EUA;
+        }
+
+        LOG.trace("Read entry rc = {} for {}",
+                  new Object[] { errorCode, read});
+        if (errorCode == BookieProtocol.EOK) {
+            assert data != null;
+
+            c.write(ResponseBuilder.buildReadResponse(data, read));
+            if (statsEnabled) {
+                long elapsedTime = MathUtils.now() - startTime;
+                bkStats.getOpStats(BKStats.STATS_READ).updateLatency(elapsedTime);
+            }
+        } else {
+            c.write(ResponseBuilder.buildErrorResponse(errorCode, read));
+           if (statsEnabled) {
+               bkStats.getOpStats(BKStats.STATS_READ).incrementFailedOps();
+           }
+        }
+    }
+}

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java?rev=1445085&r1=1445084&r2=1445085&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java Tue Feb 12 10:16:44 2013
@@ -25,11 +25,6 @@ import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.MalformedURLException;
 import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
 
 import org.apache.zookeeper.KeeperException;
 
@@ -38,22 +33,18 @@ import org.apache.bookkeeper.bookie.Book
 import org.apache.bookkeeper.bookie.ExitCode;
 import org.apache.bookkeeper.conf.ServerConfiguration;
 import org.apache.bookkeeper.jmx.BKMBeanRegistry;
-import org.apache.bookkeeper.proto.NIOServerFactory.Cnxn;
 import org.apache.bookkeeper.replication.AutoRecoveryMain;
 import org.apache.bookkeeper.replication.ReplicationException.CompatibilityException;
 import org.apache.bookkeeper.replication.ReplicationException.UnavailableException;
-import org.apache.bookkeeper.util.MathUtils;
 
 import com.google.common.annotations.VisibleForTesting;
 
-import static org.apache.bookkeeper.proto.BookieProtocol.PacketHeader;
 import org.apache.commons.configuration.ConfigurationException;
 import org.apache.commons.cli.BasicParser;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.HelpFormatter;
 import org.apache.commons.cli.ParseException;
-import org.apache.commons.codec.binary.Hex;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -62,9 +53,9 @@ import org.slf4j.LoggerFactory;
  * Implements the server-side part of the BookKeeper protocol.
  *
  */
-public class BookieServer implements NIOServerFactory.PacketProcessor, BookkeeperInternalCallbacks.WriteCallback {
+public class BookieServer {
     final ServerConfiguration conf;
-    NIOServerFactory nioServerFactory;
+    BookieNettyServer nettyServer;
     private volatile boolean running = false;
     Bookie bookie;
     DeathWatcher deathWatcher;
@@ -73,8 +64,6 @@ public class BookieServer implements NIO
     int exitCode = ExitCode.OK;
 
     // operation stats
-    final BKStats bkStats = BKStats.getInstance();
-    final boolean isStatsEnabled;
     protected BookieServerBean jmxBkServerBean;
     private AutoRecoveryMain autoRecoveryMain = null;
     private boolean isAutoRecoveryDaemonEnabled;
@@ -84,11 +73,11 @@ public class BookieServer implements NIO
             UnavailableException, CompatibilityException {
         this.conf = conf;
         this.bookie = newBookie(conf);
+        this.nettyServer = new BookieNettyServer(this.conf, this.bookie);
         isAutoRecoveryDaemonEnabled = conf.isAutoRecoveryDaemonEnabled();
         if (isAutoRecoveryDaemonEnabled) {
             this.autoRecoveryMain = new AutoRecoveryMain(conf);
         }
-        isStatsEnabled = conf.isStatisticsEnabled();
     }
 
     protected Bookie newBookie(ServerConfiguration conf)
@@ -97,14 +86,12 @@ public class BookieServer implements NIO
     }
 
     public void start() throws IOException, UnavailableException {
-        nioServerFactory = new NIOServerFactory(conf, this);
-
         this.bookie.start();
         if (isAutoRecoveryDaemonEnabled && this.autoRecoveryMain != null) {
             this.autoRecoveryMain.start();
         }
+        this.nettyServer.start();
 
-        nioServerFactory.start();
         running = true;
         deathWatcher = new DeathWatcher(conf);
         deathWatcher.start();
@@ -117,7 +104,7 @@ public class BookieServer implements NIO
         try {
             return Bookie.getBookieAddress(conf);
         } catch (UnknownHostException uhe) {
-            return nioServerFactory.getLocalAddress();
+            return nettyServer.getLocalAddress();
         }
     }
 
@@ -131,7 +118,7 @@ public class BookieServer implements NIO
      */
     @VisibleForTesting
     public void suspendProcessing() {
-        nioServerFactory.suspendProcessing();
+        nettyServer.suspendProcessing();
     }
 
     /**
@@ -139,14 +126,14 @@ public class BookieServer implements NIO
      */
     @VisibleForTesting
     public void resumeProcessing() {
-        nioServerFactory.resumeProcessing();
+        nettyServer.resumeProcessing();
     }
 
     public synchronized void shutdown() {
         if (!running) {
             return;
         }
-        nioServerFactory.shutdown();
+        this.nettyServer.shutdown();
         exitCode = bookie.shutdown();
         if (isAutoRecoveryDaemonEnabled && this.autoRecoveryMain != null) {
             this.autoRecoveryMain.shutdown();
@@ -182,7 +169,7 @@ public class BookieServer implements NIO
     }
 
     public boolean isRunning() {
-        return bookie.isRunning() && nioServerFactory.isRunning() && running;
+        return bookie.isRunning() && nettyServer.isRunning() && running;
     }
 
     /**
@@ -204,17 +191,8 @@ public class BookieServer implements NIO
                 && this.autoRecoveryMain.isAutoRecoveryRunning();
     }
 
-    /**
-     * Whether nio server is running?
-     *
-     * @return true if nio server is running, otherwise return false
-     */
-    public boolean isNioServerRunning() {
-        return nioServerFactory.isRunning();
-    }
-
     public void join() throws InterruptedException {
-        nioServerFactory.join();
+        bookie.join();
     }
 
     public int getExitCode() {
@@ -242,7 +220,6 @@ public class BookieServer implements NIO
                     // do nothing
                 }
                 if (!isBookieRunning()
-                        || !isNioServerRunning()
                         || (isAutoRecoveryDaemonEnabled && !isAutoRecoveryRunning())) {
                     shutdown();
                     break;
@@ -376,254 +353,4 @@ public class BookieServer implements NIO
             System.exit(ExitCode.SERVER_EXCEPTION);
         }
     }
-
-    public void processPacket(ByteBuffer packet, Cnxn src) {
-        PacketHeader h = PacketHeader.fromInt(packet.getInt());
-
-        boolean success = false;
-        int statType = BKStats.STATS_UNKNOWN;
-        long startTime = 0;
-        if (isStatsEnabled) {
-            startTime = MathUtils.now();
-        }
-
-        // packet format is different between ADDENTRY and READENTRY
-        long ledgerId = -1;
-        long entryId = BookieProtocol.INVALID_ENTRY_ID;
-        byte[] masterKey = null;
-        switch (h.getOpCode()) {
-        case BookieProtocol.ADDENTRY:
-            // first read master key
-            masterKey = new byte[BookieProtocol.MASTER_KEY_LENGTH];
-            packet.get(masterKey, 0, BookieProtocol.MASTER_KEY_LENGTH);
-            ByteBuffer bb = packet.duplicate();
-            ledgerId = bb.getLong();
-            entryId = bb.getLong();
-            break;
-        case BookieProtocol.READENTRY:
-            ledgerId = packet.getLong();
-            entryId = packet.getLong();
-            break;
-        default:
-            LOG.warn("Received bad request (opcode:{})", h.getOpCode());
-            src.sendResponse(buildResponse(BookieProtocol.EBADREQ, h.getVersion(),
-                                           h.getOpCode(), ledgerId, entryId));
-            break;
-        }
-
-        if (h.getVersion() < BookieProtocol.LOWEST_COMPAT_PROTOCOL_VERSION
-            || h.getVersion() > BookieProtocol.CURRENT_PROTOCOL_VERSION) {
-            LOG.error("Invalid protocol version, expected something between "
-                      + BookieProtocol.LOWEST_COMPAT_PROTOCOL_VERSION 
-                      + " & " + BookieProtocol.CURRENT_PROTOCOL_VERSION
-                    + ". got " + h.getVersion());
-            src.sendResponse(buildResponse(BookieProtocol.EBADVERSION, 
-                                           h.getVersion(), h.getOpCode(), ledgerId, entryId));
-            return;
-        }
-        short flags = h.getFlags();
-        switch (h.getOpCode()) {
-        case BookieProtocol.ADDENTRY:
-            statType = BKStats.STATS_ADD;
-
-            if (bookie.isReadOnly()) {
-                LOG.warn("BookieServer is running as readonly mode,"
-                        + " so rejecting the request from the client!");
-                src.sendResponse(buildResponse(BookieProtocol.EREADONLY,
-                        h.getVersion(), h.getOpCode(), ledgerId, entryId));
-                break;
-            }
-
-            try {
-                TimedCnxn tsrc = new TimedCnxn(src, startTime);
-                if ((flags & BookieProtocol.FLAG_RECOVERY_ADD) == BookieProtocol.FLAG_RECOVERY_ADD) {
-                    bookie.recoveryAddEntry(packet.slice(), this, tsrc, masterKey);
-                } else {
-                    bookie.addEntry(packet.slice(), this, tsrc, masterKey);
-                }
-                success = true;
-            } catch (IOException e) {
-                LOG.error("Error writing " + entryId + "@" + ledgerId, e);
-                src.sendResponse(buildResponse(BookieProtocol.EIO, h.getVersion(), h.getOpCode(), ledgerId, entryId));
-            } catch (BookieException.LedgerFencedException lfe) {
-                LOG.error("Attempt to write to fenced ledger", lfe);
-                src.sendResponse(buildResponse(BookieProtocol.EFENCED, h.getVersion(), h.getOpCode(), ledgerId, entryId));
-            } catch (BookieException e) {
-                LOG.error("Unauthorized access to ledger " + ledgerId, e);
-                src.sendResponse(buildResponse(BookieProtocol.EUA, h.getVersion(), h.getOpCode(), ledgerId, entryId));
-            }
-            break;
-        case BookieProtocol.READENTRY:
-            statType = BKStats.STATS_READ;
-            ByteBuffer[] rsp = new ByteBuffer[2];
-            LOG.debug("Received new read request: {}, {}", ledgerId, entryId);
-            int errorCode = BookieProtocol.EIO;
-            try {
-                Future<Boolean> fenceResult = null;
-                if ((flags & BookieProtocol.FLAG_DO_FENCING) == BookieProtocol.FLAG_DO_FENCING) {
-                    LOG.warn("Ledger " + ledgerId + " fenced by " + src.getPeerName());
-                    if (h.getVersion() >= 2) {
-                        masterKey = new byte[BookieProtocol.MASTER_KEY_LENGTH];
-                        packet.get(masterKey, 0, BookieProtocol.MASTER_KEY_LENGTH);
-
-                        fenceResult = bookie.fenceLedger(ledgerId, masterKey);
-                    } else {
-                        LOG.error("Password not provided, Not safe to fence {}", ledgerId);
-                        throw BookieException.create(BookieException.Code.UnauthorizedAccessException);
-                    }
-                }
-                rsp[1] = bookie.readEntry(ledgerId, entryId);
-                LOG.debug("##### Read entry ##### {}", rsp[1].remaining());
-                if (null != fenceResult) {
-                    // TODO:
-                    // currently we don't have readCallback to run in separated read
-                    // threads. after BOOKKEEPER-429 is complete, we could improve
-                    // following code to make it not wait here
-                    //
-                    // For now, since we only try to wait after read entry. so writing
-                    // to journal and read entry are executed in different thread
-                    // it would be fine.
-                    try {
-                        Boolean fenced = fenceResult.get(1000, TimeUnit.MILLISECONDS);
-                        if (null == fenced || !fenced) {
-                            // if failed to fence, fail the read request to make it retry.
-                            errorCode = BookieProtocol.EIO;
-                            success = false;
-                            rsp[1] = null;
-                        } else {
-                            errorCode = BookieProtocol.EOK;
-                            success = true;
-                        }
-                    } catch (InterruptedException ie) {
-                        LOG.error("Interrupting fence read entry (lid:" + ledgerId
-                                  + ", eid:" + entryId + ") :", ie);
-                        errorCode = BookieProtocol.EIO;
-                        success = false;
-                        rsp[1] = null;
-                    } catch (ExecutionException ee) {
-                        LOG.error("Failed to fence read entry (lid:" + ledgerId
-                                  + ", eid:" + entryId + ") :", ee);
-                        errorCode = BookieProtocol.EIO;
-                        success = false;
-                        rsp[1] = null;
-                    } catch (TimeoutException te) {
-                        LOG.error("Timeout to fence read entry (lid:" + ledgerId
-                                  + ", eid:" + entryId + ") :", te);
-                        errorCode = BookieProtocol.EIO;
-                        success = false;
-                        rsp[1] = null;
-                    }
-                } else {
-                    errorCode = BookieProtocol.EOK;
-                    success = true;
-                }
-            } catch (Bookie.NoLedgerException e) {
-                if (LOG.isTraceEnabled()) {
-                    LOG.error("Error reading " + entryId + "@" + ledgerId, e);
-                }
-                errorCode = BookieProtocol.ENOLEDGER;
-            } catch (Bookie.NoEntryException e) {
-                if (LOG.isTraceEnabled()) {
-                    LOG.error("Error reading " + entryId + "@" + ledgerId, e);
-                }
-                errorCode = BookieProtocol.ENOENTRY;
-            } catch (IOException e) {
-                if (LOG.isTraceEnabled()) {
-                    LOG.error("Error reading " + entryId + "@" + ledgerId, e);
-                }
-                errorCode = BookieProtocol.EIO;
-            } catch (BookieException e) {
-                LOG.error("Unauthorized access to ledger " + ledgerId, e);
-                errorCode = BookieProtocol.EUA;
-            }
-            rsp[0] = buildResponse(errorCode, h.getVersion(), h.getOpCode(), ledgerId, entryId);
-
-            if (LOG.isTraceEnabled()) {
-                LOG.trace("Read entry rc = " + errorCode + " for " + entryId + "@" + ledgerId);
-            }
-            if (rsp[1] == null) {
-                // We haven't filled in entry data, so we have to send back
-                // the ledger and entry ids here
-                rsp[1] = ByteBuffer.allocate(16);
-                rsp[1].putLong(ledgerId);
-                rsp[1].putLong(entryId);
-                rsp[1].flip();
-            }
-            if (LOG.isTraceEnabled()) {
-                byte[] content = new byte[rsp[1].remaining()];
-                rsp[1].duplicate().get(content);
-                LOG.trace("Sending response for: {}, content: {}", entryId, Hex.encodeHexString(content));
-            } else {
-                LOG.debug("Sending response for: {}, length: {}", entryId, rsp[1].remaining());
-            }
-            src.sendResponse(rsp);
-            break;
-        default:
-            src.sendResponse(buildResponse(BookieProtocol.EBADREQ, h.getVersion(), h.getOpCode(), ledgerId, entryId));
-        }
-        if (isStatsEnabled) {
-            if (success) {
-                // for add operations, we compute latency in writeComplete callbacks.
-                if (statType != BKStats.STATS_ADD) {
-                    long elapsedTime = MathUtils.now() - startTime;
-                    bkStats.getOpStats(statType).updateLatency(elapsedTime);
-                }
-            } else {
-                bkStats.getOpStats(statType).incrementFailedOps();
-            }
-        }
-    }
-
-    private ByteBuffer buildResponse(int errorCode, byte version, byte opCode, long ledgerId, long entryId) {
-        ByteBuffer rsp = ByteBuffer.allocate(24);
-        rsp.putInt(new PacketHeader(version, 
-                                    opCode, (short)0).toInt());
-        rsp.putInt(errorCode);
-        rsp.putLong(ledgerId);
-        rsp.putLong(entryId);
-
-        rsp.flip();
-        return rsp;
-    }
-
-    public void writeComplete(int rc, long ledgerId, long entryId, InetSocketAddress addr, Object ctx) {
-        TimedCnxn tcnxn = (TimedCnxn) ctx;
-        Cnxn src = tcnxn.cnxn;
-        long startTime = tcnxn.time;
-        ByteBuffer bb = ByteBuffer.allocate(24);
-        bb.putInt(new PacketHeader(BookieProtocol.CURRENT_PROTOCOL_VERSION, 
-                                   BookieProtocol.ADDENTRY, (short)0).toInt());
-        bb.putInt(rc);
-        bb.putLong(ledgerId);
-        bb.putLong(entryId);
-        bb.flip();
-        if (LOG.isTraceEnabled()) {
-            LOG.trace("Add entry rc = " + rc + " for " + entryId + "@" + ledgerId);
-        }
-        src.sendResponse(new ByteBuffer[] { bb });
-        if (isStatsEnabled) {
-            // compute the latency
-            if (0 == rc) {
-                // for add operations, we compute latency in writeComplete callbacks.
-                long elapsedTime = MathUtils.now() - startTime;
-                bkStats.getOpStats(BKStats.STATS_ADD).updateLatency(elapsedTime);
-            } else {
-                bkStats.getOpStats(BKStats.STATS_ADD).incrementFailedOps();                
-            }
-        }
-    }
-
-    /**
-     * A cnxn wrapper for time
-     */
-    static class TimedCnxn {
-        Cnxn cnxn;
-        long time;
-
-        public TimedCnxn(Cnxn cnxn, long startTime) {
-            this.cnxn = cnxn;
-            this.time = startTime;
-        }
-    }
-
 }

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServerBean.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServerBean.java?rev=1445085&r1=1445084&r2=1445085&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServerBean.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServerBean.java Tue Feb 12 10:16:44 2013
@@ -18,7 +18,6 @@
 
 package org.apache.bookkeeper.proto;
 
-import java.net.InetAddress;
 import java.net.UnknownHostException;
 
 import org.apache.bookkeeper.util.StringUtils;
@@ -26,7 +25,6 @@ import org.apache.bookkeeper.bookie.Book
 import org.apache.bookkeeper.conf.ServerConfiguration;
 import org.apache.bookkeeper.jmx.BKMBeanInfo;
 import org.apache.bookkeeper.proto.BKStats;
-import org.apache.bookkeeper.proto.BKStats.OpStats;
 import org.apache.bookkeeper.proto.BKStats.OpStatData;
 
 /**
@@ -66,12 +64,12 @@ public class BookieServerBean implements
 
     @Override
     public OpStatData getAddStats() {
-        return bks.bkStats.getOpStats(BKStats.STATS_ADD).toOpStatData();
+        return BKStats.getInstance().getOpStats(BKStats.STATS_ADD).toOpStatData();
     }
 
     @Override
     public OpStatData getReadStats() {
-        return bks.bkStats.getOpStats(BKStats.STATS_READ).toOpStatData();
+        return BKStats.getInstance().getOpStats(BKStats.STATS_READ).toOpStatData();
     }
 
     @Override

Added: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ResponseBuilder.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ResponseBuilder.java?rev=1445085&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ResponseBuilder.java (added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/ResponseBuilder.java Tue Feb 12 10:16:44 2013
@@ -0,0 +1,40 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.bookkeeper.proto;
+
+import java.nio.ByteBuffer;
+
+class ResponseBuilder {
+    static BookieProtocol.Response buildErrorResponse(int errorCode, BookieProtocol.Request r) {
+        return new BookieProtocol.ErrorResponse(r.getProtocolVersion(), r.getOpCode(),
+                                                errorCode, r.getLedgerId(), r.getEntryId());
+    }
+
+    static BookieProtocol.Response buildAddResponse(BookieProtocol.Request r) {
+        return new BookieProtocol.AddResponse(r.getProtocolVersion(), r.getLedgerId(),
+                                              r.getEntryId());
+    }
+
+    static BookieProtocol.Response buildReadResponse(ByteBuffer data, BookieProtocol.Request r) {
+        return new BookieProtocol.ReadResponse(r.getProtocolVersion(),
+                                               r.getLedgerId(), r.getEntryId(), data);
+    }
+}

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java?rev=1445085&r1=1445084&r2=1445085&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieInitializationTest.java Tue Feb 12 10:16:44 2013
@@ -29,6 +29,7 @@ import java.net.InetAddress;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
+import org.jboss.netty.channel.ChannelException;
 import junit.framework.Assert;
 
 import org.apache.bookkeeper.bookie.Bookie;
@@ -229,9 +230,11 @@ public class BookieInitializationTest {
             BookieServer bs2 = new BookieServer(conf);
             bs2.start();
             fail("Should throw BindException, as the bk server is already running!");
-        } catch (BindException be) {
-            Assert.assertTrue("BKServer allowed duplicate startups!", be
-                    .getMessage().contains("Address already in use"));
+        } catch (ChannelException ce) {
+            Assert.assertTrue("Should be caused by a bind exception",
+                              ce.getCause() instanceof BindException);
+            Assert.assertTrue("BKServer allowed duplicate startups!",
+                    ce.getCause().getMessage().contains("Address already in use"));
         }
     }
 

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieZKExpireTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieZKExpireTest.java?rev=1445085&r1=1445084&r2=1445085&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieZKExpireTest.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BookieZKExpireTest.java Tue Feb 12 10:16:44 2013
@@ -92,7 +92,6 @@ public class BookieZKExpireTest extends 
             // allow watcher thread to run
             secondsToWait = 20;
             while (server.isBookieRunning()
-                   || server.isNioServerRunning()
                    || server.isRunning()) {
                 Thread.sleep(1000);
                 if (secondsToWait-- <= 0) {
@@ -100,7 +99,6 @@ public class BookieZKExpireTest extends 
                 }
             }
             assertFalse("Bookie should have shutdown on losing zk session", server.isBookieRunning());
-            assertFalse("Nio Server should have shutdown on losing zk session", server.isNioServerRunning());
             assertFalse("Bookie Server should have shutdown on losing zk session", server.isRunning());
         } finally {
             server.shutdown();