You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@bookkeeper.apache.org by GitBox <gi...@apache.org> on 2017/12/25 13:08:35 UTC
[GitHub] jiazhai closed pull request #894: ISSUE #893: more options for UpdateCookieCmd
jiazhai closed pull request #894: ISSUE #893: more options for UpdateCookieCmd
URL: https://github.com/apache/bookkeeper/pull/894
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
index aacd7a21f..10ef55da0 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/BookieShell.java
@@ -49,6 +49,7 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -86,6 +87,7 @@
import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.GenericCallback;
import org.apache.bookkeeper.replication.AuditorElector;
import org.apache.bookkeeper.stats.NullStatsLogger;
+import org.apache.bookkeeper.util.BookKeeperConstants;
import org.apache.bookkeeper.util.DiskChecker;
import org.apache.bookkeeper.util.EntryFormatter;
import org.apache.bookkeeper.util.IOUtils;
@@ -99,6 +101,8 @@
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.MissingArgumentException;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.configuration.CompositeConfiguration;
@@ -106,6 +110,7 @@
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.mutable.MutableBoolean;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooKeeper;
import org.slf4j.Logger;
@@ -141,7 +146,6 @@
static final String CMD_LISTBOOKIES = "listbookies";
static final String CMD_LISTFILESONDISC = "listfilesondisc";
static final String CMD_UPDATECOOKIE = "updatecookie";
- static final String CMD_EXPANDSTORAGE = "expandstorage";
static final String CMD_UPDATELEDGER = "updateledgers";
static final String CMD_DELETELEDGER = "deleteledger";
static final String CMD_BOOKIEINFO = "bookieinfo";
@@ -1319,7 +1323,7 @@ public int runCmd(CommandLine cmdLine) throws Exception {
List<File> journalFiles = listFilesAndSort(journalDirs, "txn");
System.out.println("--------- Printing the list of Journal Files ---------");
for (File journalFile : journalFiles) {
- System.out.println(journalFile.getName());
+ System.out.println(journalFile.getCanonicalPath());
}
System.out.println();
}
@@ -1328,7 +1332,7 @@ public int runCmd(CommandLine cmdLine) throws Exception {
List<File> ledgerFiles = listFilesAndSort(ledgerDirs, "log");
System.out.println("--------- Printing the list of EntryLog/Ledger Files ---------");
for (File ledgerFile : ledgerFiles) {
- System.out.println(ledgerFile.getName());
+ System.out.println(ledgerFile.getCanonicalPath());
}
System.out.println();
}
@@ -1337,7 +1341,7 @@ public int runCmd(CommandLine cmdLine) throws Exception {
List<File> indexFiles = listFilesAndSort(indexDirs, "idx");
System.out.println("--------- Printing the list of Index Files ---------");
for (File indexFile : indexFiles) {
- System.out.println(indexFile.getName());
+ System.out.println(indexFile.getCanonicalPath());
}
}
return 0;
@@ -1593,10 +1597,23 @@ int runCmd(CommandLine cmdLine) throws Exception {
*/
class UpdateCookieCmd extends MyCommand {
Options opts = new Options();
+ private static final String BOOKIEID = "bookieId";
+ private static final String EXPANDSTORAGE = "expandstorage";
+ private static final String LIST = "list";
+ private static final String DELETE = "delete";
+ private static final String HOSTNAME = "hostname";
+ private static final String IP = "ip";
+ private static final String FORCE = "force";
UpdateCookieCmd() {
super(CMD_UPDATECOOKIE);
- opts.addOption("b", "bookieId", true, "Bookie Id");
+ opts.addOption("b", BOOKIEID, true, "Bookie Id");
+ opts.addOption("e", EXPANDSTORAGE, false, "Expand Storage");
+ opts.addOption("l", LIST, false, "List paths of all the cookies present locally and on zookkeeper");
+ @SuppressWarnings("static-access")
+ Option deleteOption = OptionBuilder.withLongOpt(DELETE).hasOptionalArgs(1)
+ .withDescription("Delete cookie both locally and in ZooKeeper").create("d");
+ opts.addOption(deleteOption);
}
@Override
@@ -1606,36 +1623,69 @@ Options getOptions() {
@Override
String getDescription() {
- return "Update bookie id in cookie.";
+ return "Command to update cookie"
+ + "bookieId - Update bookie id in cookie\n"
+ + "expandstorage - Add new empty ledger/index directories."
+ + " Update the directories info in the conf file before running the command\n"
+ + "list - list the local cookie files path and ZK cookiePath "
+ + "delete - Delete cookies locally and in zookeeper";
}
@Override
String getUsage() {
- return "updatecookie -bookieId <hostname|ip>";
+ return "updatecookie [-bookieId <hostname|ip>] [-expandstorage] [-list] [-delete <force>]";
}
@Override
int runCmd(CommandLine cmdLine) throws Exception {
- final String bookieId = cmdLine.getOptionValue("bookieId");
- if (StringUtils.isBlank(bookieId)) {
- LOG.error("Invalid argument list!");
+ int retValue = -1;
+ Option[] options = cmdLine.getOptions();
+ if (options.length != 1) {
+ LOG.error("Invalid command!");
this.printUsage();
return -1;
}
- if (!StringUtils.equals(bookieId, "hostname") && !StringUtils.equals(bookieId, "ip")) {
- LOG.error("Invalid option value:" + bookieId);
+ Option thisCommandOption = options[0];
+ if (thisCommandOption.getLongOpt().equals(BOOKIEID)) {
+ final String bookieId = cmdLine.getOptionValue(BOOKIEID);
+ if (StringUtils.isBlank(bookieId)) {
+ LOG.error("Invalid argument list!");
+ this.printUsage();
+ return -1;
+ }
+ if (!StringUtils.equals(bookieId, HOSTNAME) && !StringUtils.equals(bookieId, IP)) {
+ LOG.error("Invalid option value:" + bookieId);
+ this.printUsage();
+ return -1;
+ }
+ boolean useHostName = getOptionalValue(bookieId, HOSTNAME);
+ if (!bkConf.getUseHostNameAsBookieID() && useHostName) {
+ LOG.error(
+ "Expects config useHostNameAsBookieID=true as the option value passed is 'hostname'");
+ return -1;
+ } else if (bkConf.getUseHostNameAsBookieID() && !useHostName) {
+ LOG.error("Expects configuration useHostNameAsBookieID=false as the option value passed is 'ip'");
+ return -1;
+ }
+ retValue = updateBookieIdInCookie(bookieId, useHostName);
+ } else if (thisCommandOption.getLongOpt().equals(EXPANDSTORAGE)) {
+ bkConf.setAllowStorageExpansion(true);
+ return expandStorage();
+ } else if (thisCommandOption.getLongOpt().equals(LIST)) {
+ return listOrDeleteCookies(false, false);
+ } else if (thisCommandOption.getLongOpt().equals(DELETE)) {
+ boolean force = false;
+ String optionValue = thisCommandOption.getValue();
+ if (!StringUtils.isEmpty(optionValue) && optionValue.equals(FORCE)) {
+ force = true;
+ }
+ return listOrDeleteCookies(true, force);
+ } else {
+ LOG.error("Invalid command!");
this.printUsage();
return -1;
}
- boolean useHostName = getOptionalValue(bookieId, "hostname");
- if (!bkConf.getUseHostNameAsBookieID() && useHostName) {
- LOG.error("Expects configuration useHostNameAsBookieID=true as the option value passed is 'hostname'");
- return -1;
- } else if (bkConf.getUseHostNameAsBookieID() && !useHostName) {
- LOG.error("Expects configuration useHostNameAsBookieID=false as the option value passed is 'ip'");
- return -1;
- }
- return updateBookieIdInCookie(bookieId, useHostName);
+ return retValue;
}
private int updateBookieIdInCookie(final String bookieId, final boolean useHostname) throws BookieException,
@@ -1715,6 +1765,30 @@ private int updateBookieIdInCookie(final String bookieId, final boolean useHostn
return 0;
}
+ private int expandStorage() throws InterruptedException {
+ try (RegistrationManager rm = new ZKRegistrationManager()) {
+ rm.initialize(bkConf, () -> { }, NullStatsLogger.INSTANCE);
+
+ List<File> allLedgerDirs = Lists.newArrayList();
+ allLedgerDirs.addAll(Arrays.asList(ledgerDirectories));
+ if (indexDirectories != ledgerDirectories) {
+ allLedgerDirs.addAll(Arrays.asList(indexDirectories));
+ }
+
+ try {
+ Bookie.checkEnvironmentWithStorageExpansion(
+ bkConf, rm, Arrays.asList(journalDirectories), allLedgerDirs);
+ } catch (BookieException e) {
+ LOG.error("Exception while updating cookie for storage expansion", e);
+ return -1;
+ }
+ } catch (BookieException e) {
+ LOG.error("Exception while establishing RegistrationManager connection.", e);
+ return -1;
+ }
+ return 0;
+ }
+
private boolean verifyCookie(Cookie oldCookie, File dir) throws IOException {
try {
Cookie cookie = Cookie.readFromDirectory(dir);
@@ -1724,63 +1798,85 @@ private boolean verifyCookie(Cookie oldCookie, File dir) throws IOException {
}
return true;
}
- }
-
- /**
- * Expand the storage directories owned by a bookie.
- */
- class ExpandStorageCmd extends MyCommand {
- Options opts = new Options();
-
- ExpandStorageCmd() {
- super(CMD_EXPANDSTORAGE);
- }
- @Override
- Options getOptions() {
- return opts;
- }
+ private int listOrDeleteCookies(boolean delete, boolean force)
+ throws IOException, BookieException {
+ BookieSocketAddress bookieAddress = Bookie.getBookieAddress(bkConf);
+ File[] journalDirs = bkConf.getJournalDirs();
+ File[] ledgerDirs = bkConf.getLedgerDirs();
+ File[] indexDirs = bkConf.getIndexDirs();
+ File[] allDirs = ArrayUtils.addAll(journalDirs, ledgerDirs);
+ if (indexDirs != null) {
+ allDirs = ArrayUtils.addAll(allDirs, indexDirs);
+ }
- @Override
- String getDescription() {
- return "Add new empty ledger/index directories. Update the directories"
- + "info in the conf file before running the command.";
- }
+ File[] allCurDirs = Bookie.getCurrentDirectories(allDirs);
+ List<File> allVersionFiles = new LinkedList<File>();
+ File versionFile;
+ for (File curDir : allCurDirs) {
+ versionFile = new File(curDir, BookKeeperConstants.VERSION_FILENAME);
+ if (versionFile.exists()) {
+ allVersionFiles.add(versionFile);
+ }
+ }
- @Override
- String getUsage() {
- return "expandstorage";
- }
+ if (!allVersionFiles.isEmpty()) {
+ if (delete) {
+ boolean confirm = force;
+ if (!confirm) {
+ confirm = IOUtils.confirmPrompt("Are you sure you want to delete Cookies locally?");
+ }
+ if (confirm) {
+ for (File verFile : allVersionFiles) {
+ if (!verFile.delete()) {
+ LOG.error(
+ "Failed to delete Local cookie file {}. So aborting deletecookie of Bookie: {}",
+ verFile, bookieAddress);
+ return -1;
+ }
+ }
+ LOG.info("Deleted Local Cookies of Bookie: {}", bookieAddress);
+ } else {
+ LOG.info("Skipping deleting local Cookies of Bookie: {}", bookieAddress);
+ }
+ } else {
+ LOG.info("Listing local Cookie Files of Bookie: {}", bookieAddress);
+ for (File verFile : allVersionFiles) {
+ LOG.info(verFile.getCanonicalPath());
+ }
+ }
+ } else {
+ LOG.info("No local cookies for Bookie: {}", bookieAddress);
+ }
- @Override
- int runCmd(CommandLine cmdLine) {
- ServerConfiguration conf = new ServerConfiguration(bkConf);
- try (RegistrationManager rm = new ZKRegistrationManager()) {
+ try (ZKRegistrationManager rm = new ZKRegistrationManager()) {
+ Versioned<Cookie> cookie = null;
try {
- rm.initialize(bkConf, () -> {
- }, NullStatsLogger.INSTANCE);
- } catch (BookieException e) {
- LOG.error("Exception while establishing zookeeper connection.", e);
- return -1;
+ rm.initialize(bkConf, () -> { }, NullStatsLogger.INSTANCE);
+ cookie = Cookie.readFromRegistrationManager(rm, bookieAddress);
+ } catch (CookieNotFoundException nne) {
+ LOG.info("No cookie for {} in ZooKeeper", bookieAddress);
+ return 0;
}
- List<File> allLedgerDirs = Lists.newArrayList();
- allLedgerDirs.addAll(Arrays.asList(ledgerDirectories));
- if (indexDirectories != ledgerDirectories) {
- allLedgerDirs.addAll(Arrays.asList(indexDirectories));
- }
+ if (delete) {
+ boolean confirm = force;
+ if (!confirm) {
+ confirm = IOUtils.confirmPrompt("Are you sure you want to delete Cookies from Zookeeper?");
+ }
- try {
- conf.setAllowStorageExpansion(true);
- Bookie.checkEnvironmentWithStorageExpansion(conf, rm,
- Lists.newArrayList(journalDirectories), allLedgerDirs);
- } catch (BookieException e) {
- LOG.error(
- "Exception while updating cookie for storage expansion", e);
- return -1;
+ if (confirm) {
+ cookie.getValue().deleteFromRegistrationManager(rm, bkConf, cookie.getVersion());
+ LOG.info("Deleted Cookie from Zookeeper for Bookie: {}", bookieAddress);
+ } else {
+ LOG.info("Skipping deleting cookie from ZooKeeper for Bookie: {}", bookieAddress);
+ }
+ } else {
+ LOG.info("{} bookie's Cookie path in Zookeeper: {} ", bookieAddress,
+ rm.getCookiePath(bookieAddress.toString()));
}
- return 0;
}
+ return 0;
}
}
@@ -2370,7 +2466,6 @@ int runCmd(CommandLine cmdLine) throws Exception {
commands.put(CMD_LISTBOOKIES, new ListBookiesCmd());
commands.put(CMD_LISTFILESONDISC, new ListDiskFilesCmd());
commands.put(CMD_UPDATECOOKIE, new UpdateCookieCmd());
- commands.put(CMD_EXPANDSTORAGE, new ExpandStorageCmd());
commands.put(CMD_UPDATELEDGER, new UpdateLedgerCmd());
commands.put(CMD_DELETELEDGER, new DeleteLedgerCmd());
commands.put(CMD_BOOKIEINFO, new BookieInfoCmd());
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/discover/RegistrationManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/discover/RegistrationManager.java
index 2b274d752..23b0139c8 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/discover/RegistrationManager.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/discover/RegistrationManager.java
@@ -104,5 +104,4 @@ RegistrationManager initialize(ServerConfiguration conf,
* @throws BookieException when fail to remove cookie
*/
void removeCookie(String bookieId, Version version) throws BookieException;
-
}
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/discover/ZKRegistrationManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/discover/ZKRegistrationManager.java
index dbe924d07..172b69678 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/discover/ZKRegistrationManager.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/discover/ZKRegistrationManager.java
@@ -165,7 +165,13 @@ public void close() {
}
}
- private String getCookiePath(String bookieId) {
+ /**
+ * Returns the CookiePath of the bookie in the ZooKeeper.
+ *
+ * @param bookieId bookie id
+ * @return
+ */
+ public String getCookiePath(String bookieId) {
return this.cookiePath + "/" + bookieId;
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services