You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@bookkeeper.apache.org by GitBox <gi...@apache.org> on 2018/01/11 15:09:26 UTC
[GitHub] sijie closed pull request #946: ISSUE #945: Enabling BookieShell for LocalBookKeeper
sijie closed pull request #946: ISSUE #945: Enabling BookieShell for LocalBookKeeper
URL: https://github.com/apache/bookkeeper/pull/946
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/bookkeeper-server/bin/bookkeeper b/bookkeeper-server/bin/bookkeeper
index b3510e3d8..ca2d279c3 100755
--- a/bookkeeper-server/bin/bookkeeper
+++ b/bookkeeper-server/bin/bookkeeper
@@ -56,6 +56,8 @@ DEFAULT_LOG_CONF=$BK_HOME/conf/log4j.properties
source $BK_HOME/conf/bkenv.sh
+LOCALBOOKIES_CONFIG_DIR="${LOCALBOOKIES_CONFIG_DIR:-/tmp/localbookies-config}"
+
# Check for the java to use
if [[ -z $JAVA_HOME ]]; then
JAVA=$(which java)
@@ -184,6 +186,16 @@ shift
if [ $COMMAND == "shell" ]; then
DEFAULT_LOG_CONF=$BK_HOME/conf/log4j.shell.properties
+ if [[ $1 == "-localbookie" ]]; then
+ if [[ $2 == *:* ]];
+ then
+ BOOKIE_CONF=$LOCALBOOKIES_CONFIG_DIR/$2.conf
+ shift 2
+ else
+ BOOKIE_CONF=$LOCALBOOKIES_CONFIG_DIR/baseconf.conf
+ shift
+ fi
+ fi
fi
if [ -z "$BOOKIE_CONF" ]; then
diff --git a/bookkeeper-server/conf/bkenv.sh b/bookkeeper-server/conf/bkenv.sh
index c85b295c2..2eaeecebb 100644
--- a/bookkeeper-server/conf/bkenv.sh
+++ b/bookkeeper-server/conf/bkenv.sh
@@ -48,3 +48,6 @@
#Entry formatter class to format entries.
#ENTRY_FORMATTER_CLASS=
+
+# this default config dir should match the 'localBookiesConfigDirectory' config value in the conf file of LocalBookKeeper
+#LOCALBOOKIES_CONFIG_DIR=/tmp/localbookies-config
\ No newline at end of file
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
index 780852af2..e34d76c98 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
@@ -2559,7 +2559,7 @@ public void setConf(CompositeConfiguration conf) throws Exception {
}
private void printShellUsage() {
- System.err.println("Usage: bookkeeper shell [-ledgeridformat <hex/long/uuid>] "
+ System.err.println("Usage: bookkeeper shell [-localbookie [<host:port>]] [-ledgeridformat <hex/long/uuid>] "
+ "[-entryformat <hex/string>] [-conf configuration] <command>");
System.err.println("where command is one of:");
List<String> commandNames = new ArrayList<String>();
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java
index 7347ba5cc..d4eccccbf 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java
@@ -24,11 +24,14 @@
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
+import java.io.PrintWriter;
import java.net.InetAddress;
import java.net.Socket;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
+import org.apache.bookkeeper.bookie.Bookie;
import org.apache.bookkeeper.bookie.BookieException;
import org.apache.bookkeeper.client.BKException;
import org.apache.bookkeeper.conf.AbstractConfiguration;
@@ -61,12 +64,13 @@ public LocalBookKeeper() {
}
public LocalBookKeeper(int numberOfBookies) {
- this(numberOfBookies, 5000);
+ this(numberOfBookies, 5000, defaultLocalBookiesConfigDir);
}
- public LocalBookKeeper(int numberOfBookies, int initialPort) {
+ public LocalBookKeeper(int numberOfBookies, int initialPort, String localBookiesConfigDirName) {
this.numberOfBookies = numberOfBookies;
this.initialPort = initialPort;
+ this.localBookiesConfigDir = new File(localBookiesConfigDirName);
LOG.info("Running {} bookie(s) on zkServer {}.", this.numberOfBookies);
}
@@ -74,6 +78,7 @@ public LocalBookKeeper(int numberOfBookies, int initialPort) {
private static int zooKeeperDefaultPort = 2181;
private static int zkSessionTimeOut = 5000;
private static Integer bookieDefaultInitialPort = 5000;
+ private static String defaultLocalBookiesConfigDir = "/tmp/localbookies-config";
//BookKeeper variables
File journalDirs[];
@@ -81,6 +86,7 @@ public LocalBookKeeper(int numberOfBookies, int initialPort) {
ServerConfiguration bsConfs[];
Integer initialPort = 5000;
+ File localBookiesConfigDir;
/**
* @param maxCC
* Max Concurrency of Client
@@ -172,6 +178,15 @@ private void runBookies(ServerConfiguration baseConf, List<File> tempDirs, Strin
bs = new BookieServer[numberOfBookies];
bsConfs = new ServerConfiguration[numberOfBookies];
+ if (localBookiesConfigDir.exists() && localBookiesConfigDir.isFile()) {
+ throw new IOException("Unable to create LocalBookiesConfigDir, since there is a file at "
+ + localBookiesConfigDir.getAbsolutePath());
+ }
+ if (!localBookiesConfigDir.exists() && !localBookiesConfigDir.mkdirs()) {
+ throw new IOException(
+ "Unable to create LocalBookiesConfigDir - " + localBookiesConfigDir.getAbsolutePath());
+ }
+
for (int i = 0; i < numberOfBookies; i++) {
if (null == baseConf.getJournalDirNameWithoutDefault()) {
journalDirs[i] = IOUtils.createTempDir("localbookkeeper" + Integer.toString(i), dirSuffix);
@@ -210,7 +225,7 @@ private void runBookies(ServerConfiguration baseConf, List<File> tempDirs, Strin
}
}
- bsConfs[i] = new ServerConfiguration(baseConf);
+ bsConfs[i] = new ServerConfiguration((ServerConfiguration) baseConf.clone());
// If the caller specified ephemeral ports then use ephemeral ports for all
// the bookies else use numBookie ports starting at initialPort
@@ -225,13 +240,35 @@ private void runBookies(ServerConfiguration baseConf, List<File> tempDirs, Strin
+ zooKeeperDefaultPort);
}
-
bsConfs[i].setJournalDirName(journalDirs[i].getPath());
bsConfs[i].setLedgerDirNames(ledgerDirs);
+ // write config into file before start so we can know what's wrong if start failed
+ String fileName = Bookie.getBookieAddress(bsConfs[i]).toString() + ".conf";
+ serializeLocalBookieConfig(bsConfs[i], fileName);
+
bs[i] = new BookieServer(bsConfs[i]);
bs[i].start();
}
+
+ /*
+ * baseconf.conf is needed because for executing any BookieShell command
+ * of Metadata/Zookeeper Operation nature we need a valid conf file
+ * having correct zk details and this could be used for running any such
+ * bookieshell commands if bookieid is not provided as parameter to
+ * bookkeeper shell operation. for eg:
+ * "./bookkeeper shell localbookie listbookies -rw". But for execution
+ * shell command of bookie Operation nature we need to provide bookieid,
+ * for eg "./bookkeeper shell -localbookie 10.3.27.190:5000 lastmark",
+ * so this shell command would use '10.3.27.190:5000.conf' file
+ */
+ ServerConfiguration baseConfWithCorrectZKServers = new ServerConfiguration(
+ (ServerConfiguration) baseConf.clone());
+ if (null == baseConf.getZkServers()) {
+ baseConfWithCorrectZKServers
+ .setZkServers(InetAddress.getLocalHost().getHostAddress() + ":" + zooKeeperDefaultPort);
+ }
+ serializeLocalBookieConfig(baseConfWithCorrectZKServers, "baseconf.conf");
}
public static void startLocalBookies(String zkHost,
@@ -243,7 +280,7 @@ public static void startLocalBookies(String zkHost,
ServerConfiguration conf = new ServerConfiguration();
startLocalBookiesInternal(
conf, zkHost, zkPort, numBookies, shouldStartZK,
- initialBookiePort, true, "test", null);
+ initialBookiePort, true, "test", null, defaultLocalBookiesConfigDir);
}
public static void startLocalBookies(String zkHost,
@@ -255,7 +292,7 @@ public static void startLocalBookies(String zkHost,
throws Exception {
startLocalBookiesInternal(
conf, zkHost, zkPort, numBookies, shouldStartZK,
- initialBookiePort, true, "test", null);
+ initialBookiePort, true, "test", null, defaultLocalBookiesConfigDir);
}
public static void startLocalBookies(String zkHost,
@@ -268,7 +305,7 @@ public static void startLocalBookies(String zkHost,
ServerConfiguration conf = new ServerConfiguration();
startLocalBookiesInternal(
conf, zkHost, zkPort, numBookies, shouldStartZK,
- initialBookiePort, true, dirSuffix, null);
+ initialBookiePort, true, dirSuffix, null, defaultLocalBookiesConfigDir);
}
static void startLocalBookiesInternal(ServerConfiguration conf,
@@ -279,9 +316,10 @@ static void startLocalBookiesInternal(ServerConfiguration conf,
int initialBookiePort,
boolean stopOnExit,
String dirSuffix,
- String zkDataDir)
+ String zkDataDir,
+ String localBookiesConfigDirName)
throws Exception {
- LocalBookKeeper lb = new LocalBookKeeper(numBookies, initialBookiePort);
+ LocalBookKeeper lb = new LocalBookKeeper(numBookies, initialBookiePort, localBookiesConfigDirName);
ZooKeeperServerShim zks = null;
File zkTmpDir = null;
@@ -291,6 +329,13 @@ static void startLocalBookiesInternal(ServerConfiguration conf,
File zkDataDirFile = null;
if (zkDataDir != null) {
zkDataDirFile = new File(zkDataDir);
+ if (zkDataDirFile.exists() && zkDataDirFile.isFile()) {
+ throw new IOException("Unable to create zkDataDir, since there is a file at "
+ + zkDataDirFile.getAbsolutePath());
+ }
+ if (!zkDataDirFile.exists() && !zkDataDirFile.mkdirs()) {
+ throw new IOException("Unable to create zkDataDir - " + zkDataDirFile.getAbsolutePath());
+ }
}
zkTmpDir = IOUtils.createTempDir("zookeeper", dirSuffix, zkDataDirFile);
zkTmpDir.deleteOnExit();
@@ -329,6 +374,38 @@ static void startLocalBookiesInternal(ServerConfiguration conf,
}
}
+ /**
+ * Serializes the config object to the specified file in localBookiesConfigDir.
+ *
+ * @param localBookieConfig
+ * config object which has to be serialized
+ * @param fileName
+ * name of the file
+ * @throws IOException
+ */
+ private void serializeLocalBookieConfig(ServerConfiguration localBookieConfig, String fileName) throws IOException {
+ File localBookieConfFile = new File(localBookiesConfigDir, fileName);
+ if (localBookieConfFile.exists() && !localBookieConfFile.delete()) {
+ throw new IOException(
+ "Unable to delete the existing LocalBookieConfigFile - " + localBookieConfFile.getAbsolutePath());
+ }
+ if (!localBookieConfFile.createNewFile()) {
+ throw new IOException("Unable to create new File - " + localBookieConfFile.getAbsolutePath());
+ }
+ Iterator<String> keys = localBookieConfig.getKeys();
+ try (PrintWriter writer = new PrintWriter(localBookieConfFile, "UTF-8")) {
+ while (keys.hasNext()) {
+ String key = keys.next().toString();
+ String[] values = localBookieConfig.getStringArray(key);
+ StringBuilder concatenatedValue = new StringBuilder(values[0]);
+ for (int i = 1; i < values.length; i++) {
+ concatenatedValue.append("," + values[i]);
+ }
+ writer.println(key + "=" + concatenatedValue.toString());
+ }
+ }
+ }
+
public static void main(String[] args) throws Exception, SecurityException {
if (args.length < 1) {
usage();
@@ -355,17 +432,22 @@ public static void main(String[] args) throws Exception, SecurityException {
zkDataDir = args[2];
}
+ String localBookiesConfigDirName = defaultLocalBookiesConfigDir;
+ if (args.length >= 4) {
+ localBookiesConfigDirName = args[3];
+ }
+
startLocalBookiesInternal(
conf, zooKeeperDefaultHost, zooKeeperDefaultPort,
numBookies, true, bookieDefaultInitialPort,
false, "test",
- zkDataDir);
+ zkDataDir, localBookiesConfigDirName);
}
private static void usage() {
System.err.println(
"Usage: LocalBookKeeper number-of-bookies [path to bookie config] "
- + "[path to create ZK data directory at]");
+ + "[path to create ZK data directory at] [path to LocalBookiesConfigDir]");
}
public static boolean waitForServerUp(String hp, long timeout) {
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services