You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@distributedlog.apache.org by si...@apache.org on 2017/06/12 15:45:26 UTC
[19/30] incubator-distributedlog git commit: DL-205: Remove
StatusCode dependency on DLException
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-protocol/pom.xml
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-protocol/pom.xml b/distributedlog-proxy-protocol/pom.xml
new file mode 100644
index 0000000..0f6a85f
--- /dev/null
+++ b/distributedlog-proxy-protocol/pom.xml
@@ -0,0 +1,130 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.distributedlog</groupId>
+ <artifactId>distributedlog</artifactId>
+ <version>0.5.0-incubating-SNAPSHOT</version>
+ </parent>
+ <artifactId>distributedlog-proxy-protocol</artifactId>
+ <name>Apache DistributedLog :: Proxy Protocol</name>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.distributedlog</groupId>
+ <artifactId>distributedlog-protocol</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ <version>${libthrift.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>scrooge-core_2.11</artifactId>
+ <version>${scrooge.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>finagle-core_2.11</artifactId>
+ <version>${finagle.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>finagle-thrift_2.11</artifactId>
+ <version>${finagle.version}</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.twitter</groupId>
+ <artifactId>scrooge-maven-plugin</artifactId>
+ <version>${scrooge-maven-plugin.version}</version>
+ <configuration>
+ <language>java</language>
+ <thriftOpts>
+ <thriftOpt>--finagle</thriftOpt>
+ </thriftOpts>
+ </configuration>
+ <executions>
+ <execution>
+ <id>thrift-sources</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>${maven-jar-plugin.version}</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <excludeFilterFile>${basedir}/src/main/resources/findbugsExclude.xml</excludeFilterFile>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <version>${maven-checkstyle-plugin.version}</version>
+ <dependencies>
+ <dependency>
+ <groupId>com.puppycrawl.tools</groupId>
+ <artifactId>checkstyle</artifactId>
+ <version>${puppycrawl.checkstyle.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.distributedlog</groupId>
+ <artifactId>distributedlog-build-tools</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <configuration>
+ <configLocation>distributedlog/checkstyle.xml</configLocation>
+ <suppressionsLocation>distributedlog/suppressions.xml</suppressionsLocation>
+ <consoleOutput>true</consoleOutput>
+ <failOnViolation>true</failOnViolation>
+ <includeResources>false</includeResources>
+ <includeTestSourceDirectory>true</includeTestSourceDirectory>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>test-compile</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-protocol/src/main/java/org/apache/distributedlog/protocol/util/ProtocolUtils.java
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-protocol/src/main/java/org/apache/distributedlog/protocol/util/ProtocolUtils.java b/distributedlog-proxy-protocol/src/main/java/org/apache/distributedlog/protocol/util/ProtocolUtils.java
new file mode 100644
index 0000000..1f91968
--- /dev/null
+++ b/distributedlog-proxy-protocol/src/main/java/org/apache/distributedlog/protocol/util/ProtocolUtils.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.distributedlog.protocol.util;
+
+import static com.google.common.base.Charsets.UTF_8;
+
+import org.apache.distributedlog.DLSN;
+import java.util.zip.CRC32;
+import org.apache.distributedlog.exceptions.DLException;
+import org.apache.distributedlog.exceptions.OwnershipAcquireFailedException;
+import org.apache.distributedlog.thrift.service.ResponseHeader;
+
+/**
+ * With CRC embedded in the application, we have to keep track of per api crc. Ideally this
+ * would be done by thrift.
+ */
+public class ProtocolUtils {
+
+ // For request payload checksum
+ private static final ThreadLocal<CRC32> requestCRC = new ThreadLocal<CRC32>() {
+ @Override
+ protected CRC32 initialValue() {
+ return new CRC32();
+ }
+ };
+
+ /**
+ * Generate crc32 for WriteOp.
+ */
+ public static Long writeOpCRC32(String stream, byte[] payload) {
+ CRC32 crc = requestCRC.get();
+ try {
+ crc.update(stream.getBytes(UTF_8));
+ crc.update(payload);
+ return crc.getValue();
+ } finally {
+ crc.reset();
+ }
+ }
+
+ /**
+ * Generate crc32 for TruncateOp.
+ */
+ public static Long truncateOpCRC32(String stream, DLSN dlsn) {
+ CRC32 crc = requestCRC.get();
+ try {
+ crc.update(stream.getBytes(UTF_8));
+ crc.update(dlsn.serializeBytes());
+ return crc.getValue();
+ } finally {
+ crc.reset();
+ }
+ }
+
+ /**
+ * Generate crc32 for any op which only passes a stream name.
+ */
+ public static Long streamOpCRC32(String stream) {
+ CRC32 crc = requestCRC.get();
+ try {
+ crc.update(stream.getBytes(UTF_8));
+ return crc.getValue();
+ } finally {
+ crc.reset();
+ }
+ }
+
+ public static DLException exception(ResponseHeader response) {
+ String errMsg;
+ switch (response.getCode()) {
+ case FOUND:
+ if (response.isSetErrMsg()) {
+ errMsg = response.getErrMsg();
+ } else {
+ errMsg = "Request is redirected to " + response.getLocation();
+ }
+ return new OwnershipAcquireFailedException(errMsg, response.getLocation());
+ case SUCCESS:
+ throw new IllegalArgumentException("Can't instantiate an exception for success response.");
+ default:
+ if (response.isSetErrMsg()) {
+ errMsg = response.getErrMsg();
+ } else {
+ errMsg = response.getCode().name();
+ }
+ return new DLException(response.getCode().getValue(), errMsg);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-protocol/src/main/java/org/apache/distributedlog/protocol/util/package-info.java
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-protocol/src/main/java/org/apache/distributedlog/protocol/util/package-info.java b/distributedlog-proxy-protocol/src/main/java/org/apache/distributedlog/protocol/util/package-info.java
new file mode 100644
index 0000000..311f6c1
--- /dev/null
+++ b/distributedlog-proxy-protocol/src/main/java/org/apache/distributedlog/protocol/util/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * defines the protocol related utilities.
+ */
+package org.apache.distributedlog.protocol.util;
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-protocol/src/main/resources/findbugsExclude.xml
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-protocol/src/main/resources/findbugsExclude.xml b/distributedlog-proxy-protocol/src/main/resources/findbugsExclude.xml
new file mode 100644
index 0000000..05ee085
--- /dev/null
+++ b/distributedlog-proxy-protocol/src/main/resources/findbugsExclude.xml
@@ -0,0 +1,23 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+//-->
+<FindBugsFilter>
+ <Match>
+ <!-- generated code, we can't be held responsible for findbugs in it //-->
+ <Class name="~org\.apache\.distributedlog\.thrift.*" />
+ </Match>
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-protocol/src/main/thrift/service.thrift
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-protocol/src/main/thrift/service.thrift b/distributedlog-proxy-protocol/src/main/thrift/service.thrift
new file mode 100644
index 0000000..45e1449
--- /dev/null
+++ b/distributedlog-proxy-protocol/src/main/thrift/service.thrift
@@ -0,0 +1,203 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+namespace java org.apache.distributedlog.thrift.service
+
+/* Response stats codes */
+enum StatusCode {
+ /* 2xx: action requested by the client was received, understood, accepted and processed successfully. */
+
+ /* standard response for successful requests. */
+ SUCCESS = 200,
+
+ /* 3xx: client must take additional action to complete the request. */
+
+ /* client closed. */
+ CLIENT_CLOSED = 301,
+ /* found the stream in a different server, a redirection is required by client. */
+ FOUND = 302,
+
+ /* 4xx: client seems to have erred. */
+
+ /* request is denied for some reason */
+ REQUEST_DENIED = 403,
+ /* request record too large */
+ TOO_LARGE_RECORD = 413,
+
+ /* 5xx: server failed to fulfill an apparently valid request. */
+
+ /* Generic error message, given when no more specific message is suitable. */
+ INTERNAL_SERVER_ERROR = 500,
+ /* Not implemented */
+ NOT_IMPLEMENTED = 501,
+ /* Already Closed Exception */
+ ALREADY_CLOSED = 502,
+ /* Service is currently unavailable (because it is overloaded or down for maintenance). */
+ SERVICE_UNAVAILABLE = 503,
+ /* Locking exception */
+ LOCKING_EXCEPTION = 504,
+ /* ZooKeeper Errors */
+ ZOOKEEPER_ERROR = 505,
+ /* Metadata exception */
+ METADATA_EXCEPTION = 506,
+ /* BK Transmit Error */
+ BK_TRANSMIT_ERROR = 507,
+ /* Flush timeout */
+ FLUSH_TIMEOUT = 508,
+ /* Log empty */
+ LOG_EMPTY = 509,
+ /* Log not found */
+ LOG_NOT_FOUND = 510,
+ /* Truncated Transactions */
+ TRUNCATED_TRANSACTION = 511,
+ /* End of Stream */
+ END_OF_STREAM = 512,
+ /* Transaction Id Out of Order */
+ TRANSACTION_OUT_OF_ORDER = 513,
+ /* Write exception */
+ WRITE_EXCEPTION = 514,
+ /* Stream Unavailable */
+ STREAM_UNAVAILABLE = 515,
+ /* Write cancelled exception */
+ WRITE_CANCELLED_EXCEPTION = 516,
+ /* over-capacity/backpressure */
+ OVER_CAPACITY = 517,
+ /** stream exists but is not ready (recovering etc.).
+ the difference between NOT_READY and UNAVAILABLE is that UNAVAILABLE
+ indicates the stream is no longer owned by the proxy and we should
+ redirect. NOT_READY indicates the stream exist at the proxy but isn't
+ eady for writes. */
+ STREAM_NOT_READY = 518,
+ /* Region Unavailable */
+ REGION_UNAVAILABLE = 519,
+ /* Invalid Enveloped Entry */
+ INVALID_ENVELOPED_ENTRY = 520,
+ /* Unsupported metadata version */
+ UNSUPPORTED_METADATA_VERSION = 521,
+ /* Log Already Exists */
+ LOG_EXISTS = 522,
+ /* Checksum failed on the request */
+ CHECKSUM_FAILED = 523,
+ /* Overcapacity: too many streams */
+ TOO_MANY_STREAMS = 524,
+ /* Log Segment Not Found */
+ LOG_SEGMENT_NOT_FOUND = 525,
+ /* End of Log Segment */
+ END_OF_LOG_SEGMENT = 526,
+ /* Log Segment Is Truncated */
+ LOG_SEGMENT_IS_TRUNCATED = 527,
+
+ /* 6xx: unexpected */
+
+ UNEXPECTED = 600,
+ INTERRUPTED = 601,
+ INVALID_STREAM_NAME = 602,
+ ILLEGAL_STATE = 603,
+
+ /* 10xx: reader exceptions */
+
+ RETRYABLE_READ = 1000,
+ LOG_READ_ERROR = 1001,
+ /* Read cancelled exception */
+ READ_CANCELLED_EXCEPTION = 1002,
+}
+
+/* Response Header */
+struct ResponseHeader {
+ 1: required StatusCode code;
+ 2: optional string errMsg;
+ 3: optional string location;
+}
+
+/* Write Response */
+struct WriteResponse {
+ 1: required ResponseHeader header;
+ 2: optional string dlsn;
+}
+
+/* Bulk write response */
+struct BulkWriteResponse {
+ 1: required ResponseHeader header;
+ 2: optional list<WriteResponse> writeResponses;
+}
+
+/* Write Context */
+struct WriteContext {
+ 1: optional set<string> triedHosts;
+ 2: optional i64 crc32;
+ 3: optional bool isRecordSet;
+}
+
+/* HeartBeat Options */
+struct HeartbeatOptions {
+ 1: optional bool sendHeartBeatToReader;
+}
+
+/* Server Status */
+enum ServerStatus {
+ /* service is writing and accepting new streams */
+ WRITE_AND_ACCEPT = 100,
+ /* service is only writing to old streams, not accepting new streams */
+ WRITE_ONLY = 200,
+ /* service is shutting down, will not write */
+ DOWN = 300,
+}
+
+/* Server Info */
+struct ServerInfo {
+ 1: optional map<string, string> ownerships;
+ 2: optional ServerStatus serverStatus;
+}
+
+/* Client Info */
+struct ClientInfo {
+ 1: optional string streamNameRegex;
+ 2: optional bool getOwnerships;
+}
+
+service DistributedLogService {
+
+ /* Deprecated */
+ ServerInfo handshake();
+
+ ServerInfo handshakeWithClientInfo(ClientInfo clientInfo);
+
+ /* Deprecated */
+ WriteResponse heartbeat(string stream, WriteContext ctx);
+
+ WriteResponse heartbeatWithOptions(string stream, WriteContext ctx, HeartbeatOptions options);
+
+ /* Deprecated */
+ WriteResponse write(string stream, binary data);
+
+ WriteResponse writeWithContext(string stream, binary data, WriteContext ctx);
+
+ BulkWriteResponse writeBulkWithContext(string stream, list<binary> data, WriteContext ctx);
+
+ WriteResponse truncate(string stream, string dlsn, WriteContext ctx);
+
+ WriteResponse release(string stream, WriteContext ctx);
+
+ WriteResponse create(string stream, WriteContext ctx);
+
+ WriteResponse delete(string stream, WriteContext ctx);
+
+ WriteResponse getOwner(string stream, WriteContext ctx);
+
+ /* Admin Methods */
+ void setAcceptNewStream(bool enabled);
+}
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/bin/bundle
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-server/bin/bundle b/distributedlog-proxy-server/bin/bundle
new file mode 100755
index 0000000..1b1904e
--- /dev/null
+++ b/distributedlog-proxy-server/bin/bundle
@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+set -e
+
+BASEDIR=$(dirname "$0")
+DLOG_ROOT="${BASEDIR}/../.."
+
+cd "${DLOG_ROOT}"
+
+bash scripts/bundle service
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/bin/common.sh
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-server/bin/common.sh b/distributedlog-proxy-server/bin/common.sh
new file mode 100755
index 0000000..2b13157
--- /dev/null
+++ b/distributedlog-proxy-server/bin/common.sh
@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+if [ $(uname) == "Linux" ]; then
+ # check if net.ipv6.bindv6only is set to 1
+ bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null)
+ if [ -n "${bindv6only}" ] && [ "${bindv6only}" -eq "1" ]; then
+ echo "Error: \"net.ipv6.bindv6only\" is set to 1 - Java networking could be broken"
+ echo "For more info (the following page also applies to DistributedLog): http://wiki.apache.org/hadoop/HadoopIPv6"
+ exit 1
+ fi
+fi
+
+# See the following page for extensive details on setting
+# up the JVM to accept JMX remote management:
+# http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
+# by default we allow local JMX connections
+if [ -z "${JMXLOCALONLY}" ]; then
+ JMXLOCALONLY=false
+fi
+
+if [ -z "${JMXDISABLE}" ]; then
+ echo "JMX enabled by default" >&2
+ # for some reason these two options are necessary on jdk6 on Ubuntu
+ # accord to the docs they are not necessary, but otw jconsole cannot
+ # do a local attach
+ JMX_ARGS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=${JMXLOCALONLY}"
+else
+ echo "JMX disabled by user request" >&2
+fi
+
+echo "DLOG_HOME => ${DLOG_HOME}"
+DEFAULT_LOG_CONF="${DLOG_HOME}/conf/log4j.properties"
+
+[ -f "${DLOG_HOME}/conf/dlogenv.sh" ] && source "${DLOG_HOME}/conf/dlogenv.sh"
+
+# exclude tests jar
+RELEASE_JAR=$(ls ${DLOG_HOME}/distributedlog-*.jar 2> /dev/null | grep -v 'tests\|javadoc\|sources' | tail -1)
+if [ $? == 0 ]; then
+ DLOG_JAR="${RELEASE_JAR}"
+fi
+
+# exclude tests jar
+BUILT_JAR=$(ls ${DLOG_HOME}/target/distributedlog-*.jar 2> /dev/null | grep -v 'tests\|javadoc\|sources' | tail -1)
+
+if [ -e "${BUILD_JAR}" ] && [ -e "${DLOG_JAR}" ]; then
+ echo "\nCouldn't find dlog jar.";
+ echo "Make sure you've run 'mvn package'\n";
+ exit 1;
+elif [ -e "${BUILT_JAR}" ]; then
+ DLOG_JAR="${BUILT_JAR}"
+fi
+
+add_maven_deps_to_classpath() {
+ MVN="mvn"
+ if [ -n "${MAVEN_HOME}" ]; then
+ MVN="${MAVEN_HOME}/bin/mvn"
+ fi
+
+ # Need to generate classpath from maven pom. This is costly so generate it
+ # and cache it. Save the file into our target dir so a mvn clean will get
+ # clean it up and force us create a new one.
+ f="${PWD}/${DLOG_HOME}/target/cached_classpath.txt"
+ if [ ! -f "${f}" ]; then
+ "${MVN}" -f "${DLOG_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null
+ fi
+ DLOG_CLASSPATH="${CLASSPATH}":$(cat "${f}")
+}
+
+if [ -d "${DLOG_HOME}/lib" ]; then
+ for i in ${DLOG_HOME}/lib/*.jar; do
+ DLOG_CLASSPATH="${DLOG_CLASSPATH}:${i}"
+ done
+else
+ add_maven_deps_to_classpath
+fi
+
+# if no args specified, exit
+if [ $# = 0 ]; then
+ exit 1
+fi
+
+if [ -z "${DLOG_LOG_CONF}" ]; then
+ DLOG_LOG_CONF="${DEFAULT_LOG_CONF}"
+fi
+
+DLOG_CLASSPATH="${DLOG_JAR}:${DLOG_CLASSPATH}:${DLOG_EXTRA_CLASSPATH}"
+if [ -n "${DLOG_LOG_CONF}" ]; then
+ DLOG_CLASSPATH="$(dirname ${DLOG_LOG_CONF}):${DLOG_CLASSPATH}"
+ OPTS="${OPTS} -Dlog4j.configuration=$(basename ${DLOG_LOG_CONF})"
+fi
+OPTS="-cp ${DLOG_CLASSPATH} ${OPTS} ${DLOG_EXTRA_OPTS}"
+
+OPTS="${OPTS} ${DLOG_EXTRA_OPTS}"
+
+# Disable ipv6 as it can cause issues
+OPTS="${OPTS} -Djava.net.preferIPv4Stack=true"
+
+# log directory & file
+DLOG_ROOT_LOGGER=${DLOG_ROOT_LOGGER:-"INFO,R"}
+DLOG_LOG_DIR=${DLOG_LOG_DIR:-"$DLOG_HOME/logs"}
+DLOG_LOG_FILE=${DLOG_LOG_FILE:-"dlog.log"}
+
+#Configure log configuration system properties
+OPTS="$OPTS -Ddlog.root.logger=${DLOG_ROOT_LOGGER}"
+OPTS="$OPTS -Ddlog.log.dir=${DLOG_LOG_DIR}"
+OPTS="$OPTS -Ddlog.log.file=${DLOG_LOG_FILE}"
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/bin/dlog
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-server/bin/dlog b/distributedlog-proxy-server/bin/dlog
new file mode 100755
index 0000000..99cad40
--- /dev/null
+++ b/distributedlog-proxy-server/bin/dlog
@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+set -e
+
+BASEDIR=$(dirname "$0")
+DLOG_HOME="${BASEDIR}/.."
+
+usage() {
+ cat <<EOF
+Usage: runner <command>
+where command is one of:
+ local Run distributedlog sandbox
+ example Run distributedlog example
+ tool Run distributedlog tool
+ proxy_tool Run distributedlog proxy tool to interact with proxies
+ balancer Run distributedlog balancer
+ admin Run distributedlog admin tool
+ zkshell Run zookeeper shell
+ bkshell Run bookkeeper shell
+ help This help message
+
+or command is the full name of a class with a defined main() method.
+
+Environment variables:
+ DLOG_LOG_CONF Log4j configuration file (default $DEFAULT_LOG_CONF)
+ DLOG_EXTRA_OPTS Extra options to be passed to the jvm
+ DLOG_EXTRA_CLASSPATH Add extra paths to the dlog classpath
+
+These variable can also be set in conf/dlogenv.sh
+EOF
+}
+
+source "${DLOG_HOME}"/bin/common.sh
+
+# get arguments
+COMMAND=$1
+shift
+
+case "${COMMAND}" in
+ local)
+ exec java $OPTS $JMX_ARGS org.apache.distributedlog.LocalDLMEmulator $@
+ ;;
+ tool)
+ exec java $OPTS org.apache.distributedlog.tools.Tool org.apache.distributedlog.tools.DistributedLogTool $@
+ ;;
+ proxy_tool)
+ exec java $OPTS org.apache.distributedlog.tools.Tool org.apache.distributedlog.service.tools.ProxyTool $@
+ ;;
+ balancer)
+ exec java $OPTS org.apache.distributedlog.tools.Tool org.apache.distributedlog.service.balancer.BalancerTool $@
+ ;;
+ admin)
+ exec java $OPTS org.apache.distributedlog.tools.Tool org.apache.distributedlog.admin.DistributedLogAdmin $@
+ ;;
+ zkshell)
+ exec java $OPTS org.apache.zookeeper.ZooKeeperMain -server $@
+ ;;
+ bkshell)
+ ENTRY_FORMATTER_ARG="-DentryFormatterClass=${ENTRY_FORMATTER_CLASS:-org.apache.bookkeeper.util.StringEntryFormatter}"
+ exec java $OPTS $ENTRY_FORMATTER_ARG org.apache.bookkeeper.bookie.BookieShell -conf $BOOKIE_CONF $@
+ ;;
+ help)
+ usage
+ ;;
+ *)
+ exec java $OPTS $COMMAND $@
+ ;;
+esac
+
+
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/bin/dlog-daemon.sh
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-server/bin/dlog-daemon.sh b/distributedlog-proxy-server/bin/dlog-daemon.sh
new file mode 100755
index 0000000..fdd87df
--- /dev/null
+++ b/distributedlog-proxy-server/bin/dlog-daemon.sh
@@ -0,0 +1,200 @@
+#!/usr/bin/env bash
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+usage() {
+ cat <<EOF
+Usage: dlog-daemon.sh (start|stop) <service> <args...>
+where service is one of:
+ zookeeper Run the zookeeper server
+ bookie Run the bookie server
+ bookie-rereplicator Run the bookie rereplicator
+ writeproxy Run the write proxy server
+ writeproxy-monitor Run the write proxy monitor
+
+where argument is one of:
+ -force (accepted only with stop service): Decides whether to stop the process forcefully if not stopped by normal shutdown
+EOF
+}
+
+BINDIR=$(dirname "$0")
+DL_HOME=$(cd $BINDIR/..;pwd)
+
+if [ -f "${DL_HOME}/conf/dlogenv.sh" ]; then
+ . "${DL_HOME}/conf/dlogenv.sh"
+fi
+
+SERVICE_PORT=${SERVICE_PORT:-"0"}
+SERVICE_ARGS=""
+
+# DLOG logging configuration
+DLOG_LOG_DIR=${DLOG_LOG_DIR:-"$DL_HOME/logs"}
+DLOG_ROOT_LOGGER=${DLOG_ROOT_LOGGER:-'INFO,R'}
+
+# Process Control Parameters
+DLOG_STOP_TIMEOUT=${DLOG_STOP_TIMEOUT:-30}
+DLOG_PID_DIR=${DLOG_PID_DIR:-$DL_HOME/pids}
+
+if [ $# -lt 2 ]; then
+ echo "Error: not enough arguments provided."
+ usage
+ exit 1
+fi
+
+command=$1
+shift
+
+service=$1
+shift
+
+service_class=$service
+case $service in
+ (zookeeper)
+ service_class="org.apache.zookeeper.server.quorum.QuorumPeerMain"
+ DLOG_ROOT_LOGGER=${ZK_ROOT_LOGGER:-'INFO,R'}
+ ;;
+ (bookie)
+ service_class="org.apache.bookkeeper.proto.BookieServer"
+ DLOG_ROOT_LOGGER=${BK_ROOT_LOGGER:-'INFO,R'}
+ ;;
+ (bookie-rereplicator)
+ service_class="org.apache.bookkeeper.replication.AutoRecoveryMain"
+ DLOG_ROOT_LOGGER=${BK_ROOT_LOGGER:-'INFO,R'}
+ ;;
+ (writeproxy)
+ service_class="org.apache.distributedlog.service.DistributedLogServerApp"
+ DLOG_ROOT_LOGGER=${WP_ROOT_LOGGER:-'INFO,R'}
+ WP_CONF_FILE=${WP_CONF_FILE:-"$DL_HOME/conf/write_proxy.conf"}
+ WP_SERVICE_PORT=${WP_SERVICE_PORT:-'4181'}
+ WP_STATS_PORT=${WP_STATS_PORT:-'9000'}
+ WP_STATS_PROVIDER=${WP_STATS_PROVIDER:-'org.apache.bookkeeper.stats.CodahaleMetricsServletProvider'}
+ WP_SHARD_ID=${WP_SHARD_ID:-'0'}
+ WP_NAMESPACE=${WP_NAMESPACE:-'distributedlog://127.0.0.1:2181/messaging/distributedlog/mynamespace'}
+ SERVICE_PORT=${WP_SERVICE_PORT}
+ SERVICE_ARGS="--conf ${WP_CONF_FILE} --uri ${WP_NAMESPACE} --shard-id ${WP_SHARD_ID} --port ${WP_SERVICE_PORT} --stats-port ${WP_STATS_PORT} --stats-provider ${WP_STATS_PROVIDER} --announce --thriftmux"
+ DLOG_EXTRA_OPTS="${DLOG_EXTRA_OPTS} -DcodahaleStatsHttpPort=${WP_STATS_PORT} -Dserver_port=${WP_SERVICE_PORT} -Dserver_shard=${WP_SHARD_ID}"
+ ;;
+ (writeproxy-monitor)
+ ;;
+ (*)
+ echo "Error: unknown service name $service"
+ usage
+ exit 1
+ ;;
+esac
+
+echo "doing $command $service ..."
+
+export DLOG_LOG_DIR=$DLOG_LOG_DIR
+export DLOG_ROOT_LOGGER=$DLOG_ROOT_LOGGER
+export DLOG_LOG_FILE=dlog-$service-$HOSTNAME-$SERVICE_PORT.log
+export DLOG_EXTRA_OPTS=$DLOG_EXTRA_OPTS
+
+pid=$DLOG_PID_DIR/dlog-$service-$SERVICE_PORT.pid
+out=$DLOG_LOG_DIR/dlog-$service-$HOSTNAME-$SERVICE_PORT.out
+logfile=$DLOG_LOG_DIR/$DLOG_LOG_FILE
+
+rotate_out_log () {
+ log=$1;
+ num=5;
+ if [ -n "$2" ]; then
+ num=$2
+ fi
+ if [ -f "$log" ]; then # rotate logs
+ while [ $num -gt 1 ]; do
+ prev=`expr $num - 1`
+ [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num"
+ num=$prev
+ done
+ mv "$log" "$log.$num";
+ fi
+}
+
+mkdir -p "$DLOG_LOG_DIR"
+mkdir -p "$DLOG_PID_DIR"
+
+case $command in
+ (start)
+ if [ -f $pid ]; then
+ if kill -0 $(cat $pid) > /dev/null 2>&1; then
+ echo $service running as process $(cat $pid). Stop it first.
+ exit 1
+ fi
+ fi
+
+ rotate_out_log $out
+ echo starting $service, logging to $logfile
+ dlog=$DL_HOME/bin/dlog
+ nohup $dlog $service_class ${SERVICE_ARGS} "$@" > "$out" 2>&1 < /dev/null &
+ echo $! > $pid
+ sleep 1; head $out
+ sleep 2;
+ if ! ps -p $! > /dev/null ; then
+ exit 1
+ fi
+ ;;
+
+ (stop)
+ if [ -f $pid ]; then
+ TARGET_PID=$(cat $pid)
+ if kill -0 $TARGET_PID > /dev/null 2>&1; then
+ echo stopping $service
+ kill $TARGET_PID
+
+ count=0
+ location=$DLOG_LOG_DIR
+ while ps -p $TARGET_PID > /dev/null; do
+ echo "Shutdown is in progress... Please wait..."
+ sleep 1
+ count=$(expr $count + 1)
+ if [ "$count" = "$DLOG_STOP_TIMEOUT" ]; then
+ break
+ fi
+ done
+ if [ "$count" != "$DLOG_STOP_TIMEOUT" ]; then
+ echo "Shutdown completed."
+ fi
+ if kill -0 $TARGET_PID > /dev/null 2>&1; then
+ fileName=$location/$service.out
+ $JAVA_HOME/bin/jstack $TARGET_PID > $fileName
+ echo Thread dumps are taken for analysis at $fileName
+ if [ "$1" == "-force" ]; then
+ echo forcefully stopping $service
+ kill -9 $TARGET_PID >/dev/null 2>&1
+ echo Successfully stopped the process
+ else
+ echo "WARNNING : $service is not stopped completely."
+ exit 1
+ fi
+ fi
+ else
+ echo no $service to stop
+ fi
+ rm $pid
+ else
+ echo no $service to stop
+ fi
+ ;;
+
+ (*)
+ usage
+ echo $supportedargs
+ exit 1
+ ;;
+esac
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/bin/dlog-env.sh
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-server/bin/dlog-env.sh b/distributedlog-proxy-server/bin/dlog-env.sh
new file mode 100644
index 0000000..50a1960
--- /dev/null
+++ b/distributedlog-proxy-server/bin/dlog-env.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# we need the DLog URI to be set
+if [[ -z "${DISTRIBUTEDLOG_URI}" ]]; then
+ echo "Environment variable DISTRIBUTEDLOG_URI is not set."
+ exit 1
+fi
+
+# add the jars from current dir to the class path (should be distributedlog-service)
+for i in ./*.jar; do
+ CLASSPATH="$i:${CLASSPATH}"
+done
+
+# add all the jar from lib/ to the class path
+for i in ./lib/*.jar; do
+ CLASSPATH="$i:${CLASSPATH}"
+done
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/bin/dlog-start.sh
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-server/bin/dlog-start.sh b/distributedlog-proxy-server/bin/dlog-start.sh
new file mode 100755
index 0000000..c9c2192
--- /dev/null
+++ b/distributedlog-proxy-server/bin/dlog-start.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+DLOGBIN="${BASH_SOURCE-$0}"
+DLOGBIN="$(dirname "${DLOGBIN}")"
+DLOGBINDIR="$(cd "${DLOGBIN}"; pwd)"
+
+. "${DLOGBINDIR}"/dlog-env.sh
+
+java -cp "${CLASSPATH}" \
+ -Dlog4j.configuration=conf/log4j.properties \
+ -DstatsHttpPort=9000 -DstatsExport=true \
+ -Dserver_shard=0 \
+ org.apache.distributedlog.service.DistributedLogServerApp \
+ --port 8000 \
+ --uri "${DISTRIBUTEDLOG_URI}" \
+ --conf conf/distributedlog.conf
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/bin/zk-server-start.sh
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-server/bin/zk-server-start.sh b/distributedlog-proxy-server/bin/zk-server-start.sh
new file mode 100644
index 0000000..f222756
--- /dev/null
+++ b/distributedlog-proxy-server/bin/zk-server-start.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ];
+then
+ echo "USAGE: $0 [-daemon] zookeeper.properties"
+ exit 1
+fi
+BASE_DIR=$(dirname $0)
+
+if [ "x$KAFKA_LOG4J_OPTS" = "x" ]; then
+ export KAFKA_LOG4J_OPTS="-Dlog4j.configuration=file:$BASE_DIR/../config/log4j.properties"
+fi
+
+if [ "x$KAFKA_HEAP_OPTS" = "x" ]; then
+ export KAFKA_HEAP_OPTS="-Xmx512M -Xms512M"
+fi
+
+EXTRA_ARGS="-name zookeeper -loggc"
+
+COMMAND=$1
+case $COMMAND in
+ -daemon)
+ EXTRA_ARGS="-daemon "$EXTRA_ARGS
+ shift
+ ;;
+ *)
+ ;;
+esac
+
+exec $BASE_DIR/kafka-run-class.sh $EXTRA_ARGS org.apache.zookeeper.server.quorum.QuorumPeerMain "$@"
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/conf/bookie.conf.template
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-server/conf/bookie.conf.template b/distributedlog-proxy-server/conf/bookie.conf.template
new file mode 100644
index 0000000..5ca89d0
--- /dev/null
+++ b/distributedlog-proxy-server/conf/bookie.conf.template
@@ -0,0 +1,183 @@
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+## Bookie settings
+
+# Port that bookie server listen on
+bookiePort=3181
+
+# TODO: change the journal directory
+# Directory Bookkeeper outputs its write ahead log
+journalDirectory=/tmp/data/bk/journal
+
+# TODO: change the ledgers directory
+# Directory Bookkeeper outputs ledger snapshots
+ledgerDirectories=/tmp/data/bk/ledgers
+
+# TODO: change the index directory
+# Directory in which index files will be stored.
+indexDirectories=/tmp/data/bk/ledgers
+
+# Ledger Manager Class
+# What kind of ledger manager is used to manage how ledgers are stored, managed
+# and garbage collected. Try to read 'BookKeeper Internals' for detail info.
+ledgerManagerType=hierarchical
+
+# Root zookeeper path to store ledger metadata
+# This parameter is used by zookeeper-based ledger manager as a root znode to
+# store all ledgers.
+zkLedgersRootPath=/messaging/bookkeeper/ledgers
+
+# Max file size of entry logger, in bytes
+# A new entry log file will be created when the old one reaches the file size limitation
+logSizeLimit=1073741823
+
+# Max file size of journal file, in mega bytes
+# A new journal file will be created when the old one reaches the file size limitation
+#
+journalMaxSizeMB=2048
+
+# Max number of old journal file to kept
+# Keep a number of old journal files would help data recovery in specia case
+#
+journalMaxBackups=5
+
+# How long the interval to trigger next garbage collection, in milliseconds
+# Since garbage collection is running in background, too frequent gc
+# will heart performance. It is better to give a higher number of gc
+# interval if there is enough disk capacity.
+# gc per 1 hour (aligning with most DL rolling interval)
+gcInitialWaitTime=600000
+gcWaitTime=3600000
+# do minor compaction per 2 hours
+minorCompactionInterval=7200
+minorCompactionThreshold=0.2
+# disable major compaction
+majorCompactionInterval=0
+# reduce major compaction threshold to a low value to prevent bad force compaction behavior
+majorCompactionThreshold=0.3
+# Compaction Rate & Max Outstanding
+compactionRate=10737418
+compactionMaxOutstandingRequests=10737418
+
+# How long the interval to flush ledger index pages to disk, in milliseconds
+# Flushing index files will introduce much random disk I/O.
+# If separating journal dir and ledger dirs each on different devices,
+# flushing would not affect performance. But if putting journal dir
+# and ledger dirs on same device, performance degrade significantly
+# on too frequent flushing. You can consider increment flush interval
+# to get better performance, but you need to pay more time on bookie
+# server restart after failure.
+#
+flushInterval=1000
+
+# Interval to watch whether bookie is dead or not, in milliseconds
+#
+# bookieDeathWatchInterval=1000
+
+## zookeeper client settings
+
+# A list of one of more servers on which zookeeper is running.
+# The server list can be comma separated values, for example:
+# zkServers=zk1:2181,zk2:2181,zk3:2181
+zkServers=localhost:2181
+
+# ZooKeeper client session timeout in milliseconds
+# Bookie server will exit if it received SESSION_EXPIRED because it
+# was partitioned off from ZooKeeper for more than the session timeout
+# JVM garbage collection, disk I/O will cause SESSION_EXPIRED.
+# Increment this value could help avoiding this issue
+zkTimeout=30000
+
+## NIO Server settings
+
+# This settings is used to enabled/disabled Nagle's algorithm, which is a means of
+# improving the efficiency of TCP/IP networks by reducing the number of packets
+# that need to be sent over the network.
+# If you are sending many small messages, such that more than one can fit in
+# a single IP packet, setting server.tcpnodelay to false to enable Nagle algorithm
+# can provide better performance.
+# Default value is true.
+#
+serverTcpNoDelay=true
+
+## ledger cache settings
+
+# Max number of ledger index files could be opened in bookie server
+# If number of ledger index files reaches this limitation, bookie
+# server started to swap some ledgers from memory to disk.
+# Too frequent swap will affect performance. You can tune this number
+# to gain performance according your requirements.
+openFileLimit=20000
+
+# Size of a index page in ledger cache, in bytes
+# A larger index page can improve performance writing page to disk,
+# which is efficent when you have small number of ledgers and these
+# ledgers have similar number of entries.
+# If you have large number of ledgers and each ledger has fewer entries,
+# smaller index page would improve memory usage.
+pageSize=8192
+
+# How many index pages provided in ledger cache
+# If number of index pages reaches this limitation, bookie server
+# starts to swap some ledgers from memory to disk. You can increment
+# this value when you found swap became more frequent. But make sure
+# pageLimit*pageSize should not more than JVM max memory limitation,
+# otherwise you would got OutOfMemoryException.
+# In general, incrementing pageLimit, using smaller index page would
+# gain bettern performance in lager number of ledgers with fewer entries case
+# If pageLimit is -1, bookie server will use 1/3 of JVM memory to compute
+# the limitation of number of index pages.
+pageLimit=131072
+
+#If all ledger directories configured are full, then support only read requests for clients.
+#If "readOnlyModeEnabled=true" then on all ledger disks full, bookie will be converted
+#to read-only mode and serve only read requests. Otherwise the bookie will be shutdown.
+readOnlyModeEnabled=true
+
+# Bookie Journal Settings
+writeBufferSizeBytes=262144
+journalFlushWhenQueueEmpty=false
+journalRemoveFromPageCache=true
+journalAdaptiveGroupWrites=true
+journalMaxGroupWaitMSec=4
+journalBufferedEntriesThreshold=180
+journalBufferedWritesThreshold=131072
+journalMaxGroupedEntriesToCommit=200
+journalPreAllocSizeMB=4
+
+# Sorted Ledger Storage Settings
+sortedLedgerStorageEnabled=true
+skipListSizeLimit=67108864
+skipListArenaChunkSize=2097152
+skipListArenaMaxAllocSize=131072
+fileInfoCacheInitialCapacity=10000
+fileInfoMaxIdleTime=3600
+
+# Bookie Threads Settings (NOTE: change this to align the cpu cores)
+numAddWorkerThreads=4
+numJournalCallbackThreads=4
+numReadWorkerThreads=4
+numLongPollWorkerThreads=4
+
+# stats
+statsProviderClass=org.apache.bookkeeper.stats.CodahaleMetricsServletProvider
+# Exporting codahale stats
+codahaleStatsHttpPort=9001
+useHostNameAsBookieID=true
+allowLoopback=true
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/conf/distributedlog.conf
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-server/conf/distributedlog.conf b/distributedlog-proxy-server/conf/distributedlog.conf
new file mode 100644
index 0000000..dac71ac
--- /dev/null
+++ b/distributedlog-proxy-server/conf/distributedlog.conf
@@ -0,0 +1,125 @@
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+########################
+# ZooKeeper Client Settings
+########################
+
+# zookeeper settings
+zkSessionTimeoutSeconds=30
+zkNumRetries=0
+zkRetryStartBackoffMillis=100
+zkRetryMaxBackoffMillis=200
+# bkc zookeeper settings
+bkcZKSessionTimeoutSeconds=60
+bkcZKNumRetries=20
+bkcZKRetryStartBackoffMillis=100
+bkcZKRetryMaxBackoffMillis=200
+
+########################
+# BookKeeper Client Settings
+########################
+
+# bookkeeper client timeouts
+bkcWriteTimeoutSeconds=10
+bkcReadTimeoutSeconds=1
+bkcNumWorkerThreads=16
+# bkcNumIOThreads=16
+bkc.numChannelsPerBookie=1
+bkc.enableTaskExecutionStats=true
+bkc.connectTimeoutMillis=1000
+bkc.enablePerHostStats=true
+
+########################
+# DL Settings
+########################
+
+# lock timeout
+lockTimeoutSeconds=0
+# dl worker threads
+numWorkerThreads=16
+
+### Recovery Related Settings
+
+# recover log segments in background
+recoverLogSegmentsInBackground=true
+# disable max id in proxy
+maxIdSanityCheck=true
+# use allocator pool for proxy
+enableLedgerAllocatorPool=false
+# ledger allocator pool size
+ledgerAllocatorPoolCoreSize=20
+# check stream exists or not
+createStreamIfNotExists=true
+# encode dc id in version
+encodeDCIDInVersion=true
+# logSegmentNameVersion
+logSegmentNameVersion=1
+
+### Write Performance Related Settings
+
+# ensemble size
+ensemble-size=3
+write-quorum-size=3
+ack-quorum-size=2
+bkc.ensemblePlacementPolicy=org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy
+bkc.delayEnsembleChange=true
+
+# sync settings
+# buffer size is large because when we rewrite we perform a very large write to persist
+# all queue state at once (up to max queue memory size, ex. 16MB). the write will be
+# throttled if it takes too long, which can hurt performance, so important to optimize
+# for this case.
+output-buffer-size=512000
+enableImmediateFlush=false
+periodicFlushFrequencyMilliSeconds=6
+logFlushTimeoutSeconds=120
+
+### Ledger Rolling Related Settings
+
+# retention policy
+retention-size=0
+# rolling ledgers (disable time rolling/enable size rolling)
+rolling-interval=0
+
+# max logsegment bytes=2GB
+# much larger than max journal size, effectively never roll and let drpc do it
+maxLogSegmentBytes=2147483648
+
+# rolling concurrency
+logSegmentRollingConcurrency=1
+# disable sanityCheckDelete
+sanityCheckDelete=false
+ledgerAllocatorPoolName=drpc-alloc-pool
+
+### Readahead settings
+
+enableReadAhead=true
+ReadAheadBatchSize=10
+ReadAheadMaxEntries=100
+ReadAheadWaitTime=10
+
+### Rate limit
+
+rpsSoftWriteLimit=1
+rpsHardWriteLimit=5
+rpsHardServiceLimit=15
+
+### Config
+
+dynamicConfigReloadIntervalSec=5
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/conf/dlogenv.sh
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-server/conf/dlogenv.sh b/distributedlog-proxy-server/conf/dlogenv.sh
new file mode 100644
index 0000000..345e60f
--- /dev/null
+++ b/distributedlog-proxy-server/conf/dlogenv.sh
@@ -0,0 +1,75 @@
+#!/bin/sh
+#
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+##################
+# General
+##################
+
+# Log4j configuration file
+# DLOG_LOG_CONF=
+
+# Extra options to be passed to the jvm
+# DLOG_EXTRA_OPTS=
+
+# Add extra paths to the dlog classpath
+# DLOG_EXTRA_CLASSPATH=
+
+# Configure the root logger
+# DLOG_ROOT_LOGGER=
+
+# Configure the log dir
+# DLOG_LOG_DIR=
+
+# Configure the log file
+# DLOG_LOG_FILE=
+
+#################
+# ZooKeeper
+#################
+
+# Configure zookeeper root logger
+# ZK_ROOT_LOGGER=
+
+#################
+# Bookie
+#################
+
+# Configure bookie root logger
+# BK_ROOT_LOGGER=
+
+#################
+# Write Proxy
+#################
+
+# Configure write proxy root logger
+# WP_ROOT_LOGGER=
+
+# write proxy configuration file
+# WP_CONF_FILE=${DL_HOME}/conf/write_proxy.conf
+
+# port and stats port
+# WP_SERVICE_PORT=4181
+# WP_STATS_PORT=9000
+
+# shard id
+# WP_SHARD_ID=0
+
+# write proxy namespace
+# WP_NAMESPACE=distributedlog://127.0.0.1:2181/messaging/distributedlog/mynamespace
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/conf/log4j.properties
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-server/conf/log4j.properties b/distributedlog-proxy-server/conf/log4j.properties
new file mode 100644
index 0000000..73b4cfa
--- /dev/null
+++ b/distributedlog-proxy-server/conf/log4j.properties
@@ -0,0 +1,60 @@
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+#
+# DistributedLog Logging Configuration
+#
+
+# Default values
+dlog.root.logger=INFO, R
+dlog.log.dir=logs
+dlog.log.file=dlog.log
+
+log4j.rootLogger=${dlog.root.logger}
+log4j.logger.org.apache.zookeeper=INFO
+log4j.logger.org.apache.bookkeeper=INFO
+
+# redirect executor output to executors.log since slow op warnings can be quite verbose
+log4j.logger.com.twitter.distributedlog.util.MonitoredFuturePool=INFO, Executors
+log4j.logger.com.twitter.distributedlog.util.MonitoredScheduledThreadPoolExecutor=INFO, Executors
+log4j.logger.org.apache.bookkeeper.util.SafeRunnable=INFO, Executors
+log4j.additivity.com.twitter.distributedlog.util.MonitoredFuturePool=false
+log4j.additivity.com.twitter.distributedlog.util.MonitoredScheduledThreadPoolExecutor=false
+log4j.additivity.org.apache.bookkeeper.util.SafeRunnable=false
+
+log4j.appender.Executors=org.apache.log4j.RollingFileAppender
+log4j.appender.Executors.Threshold=INFO
+log4j.appender.Executors.File=${dlog.log.dir}/executors.log
+log4j.appender.Executors.MaxFileSize=20MB
+log4j.appender.Executors.MaxBackupIndex=5
+log4j.appender.Executors.layout=org.apache.log4j.PatternLayout
+log4j.appender.Executors.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n
+
+log4j.appender.R=org.apache.log4j.RollingFileAppender
+log4j.appender.R.Threshold=INFO
+log4j.appender.R.File=${dlog.log.dir}/${dlog.log.file}
+log4j.appender.R.MaxFileSize=20MB
+log4j.appender.R.MaxBackupIndex=50
+log4j.appender.R.layout=org.apache.log4j.PatternLayout
+log4j.appender.R.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n
+
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Target=System.err
+log4j.appender.stderr.Threshold=INFO
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} - %-5p - [%t:%C{1}@%L] - %m%n
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/conf/write_proxy.conf
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-server/conf/write_proxy.conf b/distributedlog-proxy-server/conf/write_proxy.conf
new file mode 100644
index 0000000..7f5351a
--- /dev/null
+++ b/distributedlog-proxy-server/conf/write_proxy.conf
@@ -0,0 +1,143 @@
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+########################
+# ZooKeeper Client Settings
+########################
+
+# zookeeper settings
+zkSessionTimeoutSeconds=1
+zkNumRetries=0
+zkRetryStartBackoffMillis=100
+zkRetryMaxBackoffMillis=200
+# bkc zookeeper settings
+bkcZKSessionTimeoutSeconds=60
+bkcZKNumRetries=20
+bkcZKRetryStartBackoffMillis=100
+bkcZKRetryMaxBackoffMillis=200
+
+########################
+# BookKeeper Client Settings
+########################
+
+# bookkeeper client timeouts
+bkcWriteTimeoutSeconds=2
+bkcReadTimeoutSeconds=2
+bkcNumWorkerThreads=32
+bkc.numChannelsPerBookie=1
+bkc.enableTaskExecutionStats=true
+bkc.connectTimeoutMillis=200
+bkc.enableParallelRecoveryRead=true
+bkc.recoveryReadBatchSize=5
+bkc.enablePerHostStats=true
+
+########################
+# DL Settings
+########################
+
+# Metadata Settings
+
+# ledger metadata version that supports sequence id
+ledger-metadata-layout=5
+
+# lock timeout
+lockTimeoutSeconds=0
+# dl worker threads
+numWorkerThreads=32
+
+### Recovery Related Settings
+
+# recover log segments in background
+recoverLogSegmentsInBackground=false
+# disable max id in proxy
+maxIdSanityCheck=false
+# use allocator pool for proxy
+enableLedgerAllocatorPool=true
+# ledger allocator pool path
+ledgerAllocatorPoolPath=.write_proxy_allocation_pool
+# ledger allocator pool size
+ledgerAllocatorPoolCoreSize=40
+# check stream exists or not
+createStreamIfNotExists=true
+# encode dc id in version
+encodeDCIDInVersion=true
+# logSegmentNameVersion
+logSegmentNameVersion=1
+
+### Write Performance Related Settings
+
+# ensemble size
+ensemble-size=3
+write-quorum-size=3
+ack-quorum-size=2
+bkc.ensemblePlacementPolicy=org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy
+bkc.delayEnsembleChange=true
+bkc.writeRequestToChannelAsync=true
+
+# enable immediate flush
+enableImmediateFlush=true
+# 0k output buffer
+output-buffer-size=0
+# disable periodical flush
+periodicFlushFrequencyMilliSeconds=0
+enableTaskExecutionStats=true
+taskExecutionWarnTimeMicros=100000
+
+### Ledger Rolling Related Settings
+
+# retention policy
+retention-size=4
+# rolling ledgers (enable time rolling): 120 minutes = 2 hours
+rolling-interval=120
+# max logsegment bytes : 2GB
+maxLogSegmentBytes=2147483648
+# rolling concurrency
+logSegmentRollingConcurrency=1
+# disable sanityCheckDelete
+sanityCheckDelete=false
+# compression codec
+compressionType=lz4
+
+### Per Stream Stats
+enablePerStreamStat=true
+
+########################
+# DL Settings
+########################
+
+# proxy server settings
+server_mode=DURABLE
+serviceTimeoutMs=60000
+streamProbationTimeoutMs=120000
+server_threads=16
+server_dlsn_version=1
+server_enable_perstream_stat=true
+server_graceful_shutdown_period_ms=20000
+
+# write limits
+perWriterOutstandingWriteLimit=-1
+globalOutstandingWriteLimit=-1
+outstandingWriteLimitDarkmode=false
+
+# bytes per second limit applied at the host level (50MBps on 1Gib machines)
+bpsHardServiceLimit=52428800
+# bytes per second limit after which no new streams may be acquired (65MBps on 1Gib machines)
+bpsStreamAcquireServiceLimit=47185920
+
+# limit the maximum number of streams
+maxAcquiredPartitionsPerProxy=-1
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/conf/zookeeper.conf.template
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-server/conf/zookeeper.conf.template b/distributedlog-proxy-server/conf/zookeeper.conf.template
new file mode 100644
index 0000000..3c0546e
--- /dev/null
+++ b/distributedlog-proxy-server/conf/zookeeper.conf.template
@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The number of milliseconds of each tick
+tickTime=2000
+
+# the port at which the clients will connect
+clientPort=2181
+
+# The number of ticks that the initial
+# synchronization phase can take
+initLimit=10
+
+# The number of ticks that can pass between
+# sending a request and getting an acknowledgement
+syncLimit=30
+
+# the directory where the snapshot is stored.
+dataDir=/tmp/data/zookeeper
+
+# where txlog are written
+dataLogDir=/tmp/data/zookeeper/txlog
+
+# the port at which the admin will listen
+adminPort=9990
+zookeeper.admin.enableServer=true
+
+# limit on queued clients - default: 1000
+globalOutstandingLimit=1000
+
+# number of transactions before snapshots are taken - default: 100000
+snapCount=100000
+
+# max # of clients - 0==unlimited
+maxClientCnxns=25
+
+# Election implementation to use. A value of "0" corresponds to the original
+# UDP-based version, "1" corresponds to the non-authenticated UDP-based
+# version of fast leader election, "2" corresponds to the authenticated
+# UDP-based version of fast leader election, and "3" corresponds to TCP-based
+# version of fast leader election. Currently, only 0 and 3 are supported,
+# 3 being the default
+electionAlg=3
+
+# Leader accepts client connections. Default value is "yes". The leader
+# machine coordinates updates. For higher update throughput at thes slight
+# expense of read throughput the leader can be configured to not accept
+# clients and focus on coordination.
+leaderServes=yes
+
+# Skips ACL checks. This results in a boost in throughput, but opens up full
+# access to the data tree to everyone.
+skipACL=no
+
+# Purge txn logs every hour. Before 3.4.x this was done with an external cron
+# job, now we can do it internally.
+autopurge.purgeInterval=1
+
+# Prior to version 3.4 ZooKeeper has always used NIO directly, however in
+# versions 3.4 and later Netty is supported as an option to NIO (replaces).
+# serverCnxnFactory=org.apache.zookeeper.server.NIOServerCnxnFactory
+
+standaloneEnabled=false
+# ZooKeeper Dynamic Reconfiguration
+# See: https://zookeeper.apache.org/doc/trunk/zookeeperReconfig.html
+#
+# standaloneEnabled=false
+# dynamicConfigFile=/path/to/zoo.cfg.dynamic
+#
+server.1=127.0.0.1:2710:3710:participant;0.0.0.0:2181
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/pom.xml
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-server/pom.xml b/distributedlog-proxy-server/pom.xml
new file mode 100644
index 0000000..d7cbd56
--- /dev/null
+++ b/distributedlog-proxy-server/pom.xml
@@ -0,0 +1,275 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.distributedlog</groupId>
+ <artifactId>distributedlog</artifactId>
+ <version>0.5.0-incubating-SNAPSHOT</version>
+ </parent>
+ <artifactId>distributedlog-proxy-server</artifactId>
+ <name>Apache DistributedLog :: Proxy Service</name>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.distributedlog</groupId>
+ <artifactId>distributedlog-proxy-client</artifactId>
+ <version>${project.parent.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.distributedlog</groupId>
+ <artifactId>distributedlog-proxy-protocol</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.distributedlog</groupId>
+ <artifactId>distributedlog-core</artifactId>
+ <version>${project.parent.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>finagle-ostrich4_2.11</artifactId>
+ <version>${finagle.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>finagle-thriftmux_2.11</artifactId>
+ <version>${finagle.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.distributedlog</groupId>
+ <artifactId>distributedlog-core</artifactId>
+ <version>${project.parent.version}</version>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>${zookeeper.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>${junit.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>${mockito.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.bookkeeper.stats</groupId>
+ <artifactId>codahale-metrics-provider</artifactId>
+ <version>${bookkeeper.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-servlets</artifactId>
+ <version>${codahale.metrics.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ <version>${jetty.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>libthrift</artifactId>
+ <version>${libthrift.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>scrooge-core_2.11</artifactId>
+ <version>${scrooge.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-test</artifactId>
+ <version>${curator.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.distributedlog</groupId>
+ <artifactId>distributedlog-protocol</artifactId>
+ <version>${project.parent.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>com.twitter</groupId>
+ <artifactId>scrooge-maven-plugin</artifactId>
+ <version>${scrooge-maven-plugin.version}</version>
+ <configuration>
+ <language>java</language>
+ </configuration>
+ <executions>
+ <execution>
+ <id>thrift-sources</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven-assembly-plugin.version}</version>
+ <configuration>
+ <descriptors>
+ <descriptor>../src/assemble/bin.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>${maven-surefire-plugin.version}</version>
+ <configuration>
+ <redirectTestOutputToFile>true</redirectTestOutputToFile>
+ <argLine>-Xmx3G -Djava.net.preferIPv4Stack=true -XX:MaxDirectMemorySize=2G</argLine>
+ <forkMode>always</forkMode>
+ <forkedProcessTimeoutInSeconds>1800</forkedProcessTimeoutInSeconds>
+ <properties>
+ <property>
+ <name>listener</name>
+ <value>org.apache.distributedlog.TimedOutTestsListener</value>
+ </property>
+ </properties>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <excludeFilterFile>${basedir}/src/main/resources/findbugsExclude.xml</excludeFilterFile>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${basedir}/lib</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <version>${maven-checkstyle-plugin.version}</version>
+ <dependencies>
+ <dependency>
+ <groupId>com.puppycrawl.tools</groupId>
+ <artifactId>checkstyle</artifactId>
+ <version>${puppycrawl.checkstyle.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.distributedlog</groupId>
+ <artifactId>distributedlog-build-tools</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ <configuration>
+ <configLocation>distributedlog/checkstyle.xml</configLocation>
+ <suppressionsLocation>distributedlog/suppressions.xml</suppressionsLocation>
+ <consoleOutput>true</consoleOutput>
+ <failOnViolation>true</failOnViolation>
+ <includeResources>false</includeResources>
+ <includeTestSourceDirectory>true</includeTestSourceDirectory>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>test-compile</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <profiles>
+ <profile>
+ <id>twitter-ostrich-provider</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.bookkeeper.stats</groupId>
+ <artifactId>twitter-ostrich-provider</artifactId>
+ <version>${bookkeeper.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.twitter</groupId>
+ <artifactId>ostrich_2.10</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.twitter</groupId>
+ <artifactId>ostrich_2.9.2</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsServletProvider.java
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsServletProvider.java b/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsServletProvider.java
new file mode 100644
index 0000000..8db3e90
--- /dev/null
+++ b/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/CodahaleMetricsServletProvider.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bookkeeper.stats;
+
+import com.codahale.metrics.health.HealthCheckRegistry;
+import org.apache.commons.configuration.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Extend the codahale metrics provider to run servlets.
+ */
+public class CodahaleMetricsServletProvider extends CodahaleMetricsProvider {
+
+ private static final Logger logger = LoggerFactory.getLogger(CodahaleMetricsServletProvider.class);
+
+ ServletReporter servletReporter = null;
+ private final HealthCheckRegistry healthCheckRegistry = new HealthCheckRegistry();
+
+ @Override
+ public void start(Configuration conf) {
+ super.start(conf);
+ Integer httpPort = conf.getInteger("codahaleStatsHttpPort", null);
+ if (null != httpPort) {
+ servletReporter = new ServletReporter(
+ getMetrics(),
+ healthCheckRegistry,
+ httpPort);
+ try {
+ servletReporter.start();
+ } catch (Exception e) {
+ logger.warn("Encountered error on starting the codahale metrics servlet", e);
+ }
+ }
+ }
+
+ @Override
+ public void stop() {
+ if (null != servletReporter) {
+ try {
+ servletReporter.stop();
+ } catch (Exception e) {
+ logger.error("Encountered error on stopping the codahale metrics servlet", e);
+ }
+ }
+ super.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/HealthCheckServletContextListener.java
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/HealthCheckServletContextListener.java b/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/HealthCheckServletContextListener.java
new file mode 100644
index 0000000..348787a
--- /dev/null
+++ b/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/HealthCheckServletContextListener.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bookkeeper.stats;
+
+import com.codahale.metrics.health.HealthCheckRegistry;
+import com.codahale.metrics.servlets.HealthCheckServlet;
+
+/**
+ * Health Check Servlet Listener.
+ */
+public class HealthCheckServletContextListener extends HealthCheckServlet.ContextListener {
+
+ private final HealthCheckRegistry healthCheckRegistry;
+
+ public HealthCheckServletContextListener(HealthCheckRegistry healthCheckRegistry) {
+ this.healthCheckRegistry = healthCheckRegistry;
+ }
+
+ @Override
+ protected HealthCheckRegistry getHealthCheckRegistry() {
+ return healthCheckRegistry;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/MetricsServletContextListener.java
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/MetricsServletContextListener.java b/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/MetricsServletContextListener.java
new file mode 100644
index 0000000..15279fe
--- /dev/null
+++ b/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/MetricsServletContextListener.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bookkeeper.stats;
+
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.servlets.MetricsServlet;
+
+/**
+ * A servlet to report metrics.
+ */
+public class MetricsServletContextListener extends MetricsServlet.ContextListener {
+
+ private final MetricRegistry metricRegistry;
+
+ public MetricsServletContextListener(MetricRegistry metricRegistry) {
+ this.metricRegistry = metricRegistry;
+ }
+
+ @Override
+ protected MetricRegistry getMetricRegistry() {
+ return metricRegistry;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/ServletReporter.java
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/ServletReporter.java b/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/ServletReporter.java
new file mode 100644
index 0000000..267f75a
--- /dev/null
+++ b/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/ServletReporter.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.bookkeeper.stats;
+
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.health.HealthCheckRegistry;
+import com.codahale.metrics.servlets.AdminServlet;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+
+/**
+ * Starts a jetty server on a configurable port to export stats.
+ */
+public class ServletReporter {
+
+ private final MetricRegistry metricRegistry;
+ private final HealthCheckRegistry healthCheckRegistry;
+ private final int port;
+ private final Server jettyServer;
+
+ public ServletReporter(MetricRegistry metricRegistry,
+ HealthCheckRegistry healthCheckRegistry,
+ int port) {
+ this.metricRegistry = metricRegistry;
+ this.healthCheckRegistry = healthCheckRegistry;
+ this.port = port;
+ this.jettyServer = new Server(port);
+ }
+
+ public void start() throws Exception {
+ ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
+ context.setContextPath("/");
+ jettyServer.setHandler(context);
+
+ context.addEventListener(new HealthCheckServletContextListener(healthCheckRegistry));
+ context.addEventListener(new MetricsServletContextListener(metricRegistry));
+ context.addServlet(new ServletHolder(new AdminServlet()), "/*");
+
+ jettyServer.start();
+ }
+
+ public void stop() throws Exception {
+ jettyServer.stop();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-distributedlog/blob/c44e0278/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/package-info.java
----------------------------------------------------------------------
diff --git a/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/package-info.java b/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/package-info.java
new file mode 100644
index 0000000..5bdb3ce
--- /dev/null
+++ b/distributedlog-proxy-server/src/main/java/org/apache/bookkeeper/stats/package-info.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Extension of {@link org.apache.bookkeeper.stats.CodahaleMetricsProvider}.
+ */
+package org.apache.bookkeeper.stats;