You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by jx...@apache.org on 2018/07/13 20:50:03 UTC
[1/2] helix git commit: Support current state clean up tool
Repository: helix
Updated Branches:
refs/heads/master c0d5792b7 -> 5529c4d59
Support current state clean up tool
Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/7d695e99
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/7d695e99
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/7d695e99
Branch: refs/heads/master
Commit: 7d695e990eef089b27e929e5ad11da425adab312
Parents: c0d5792
Author: Junkai Xue <jx...@linkedin.com>
Authored: Mon Jun 11 16:11:19 2018 -0700
Committer: Junkai Xue <jx...@linkedin.com>
Committed: Fri Jul 13 13:44:33 2018 -0700
----------------------------------------------------------------------
helix-core/pom.xml | 4 +
.../tools/commandtools/CurrentStateCleanUp.java | 150 +++++++++++++++++++
.../tools/commandtools/ExampleParticipant.java | 16 +-
.../helix/tools/commandtools/ToolsUtil.java | 24 +++
4 files changed, 179 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/helix/blob/7d695e99/helix-core/pom.xml
----------------------------------------------------------------------
diff --git a/helix-core/pom.xml b/helix-core/pom.xml
index 45f4d0a..2051051 100644
--- a/helix-core/pom.xml
+++ b/helix-core/pom.xml
@@ -224,6 +224,10 @@ under the License.
<name>start-participants</name>
</program>
<program>
+ <mainClass>org.apache.helix.tools.commandtools.CurrentStateCleanUp</mainClass>
+ <name>cleanup-currentstate</name>
+ </program>
+ <program>
<mainClass>org.apache.helix.tools.commandtools.LocalZKServer</mainClass>
<name>start-standalone-zookeeper</name>
</program>
http://git-wip-us.apache.org/repos/asf/helix/blob/7d695e99/helix-core/src/main/java/org/apache/helix/tools/commandtools/CurrentStateCleanUp.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/tools/commandtools/CurrentStateCleanUp.java b/helix-core/src/main/java/org/apache/helix/tools/commandtools/CurrentStateCleanUp.java
new file mode 100644
index 0000000..a8d2bae
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/tools/commandtools/CurrentStateCleanUp.java
@@ -0,0 +1,150 @@
+package org.apache.helix.tools.commandtools;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.I0Itec.zkclient.DataUpdater;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.OptionGroup;
+import org.apache.commons.cli.Options;
+import org.apache.helix.AccessOption;
+import org.apache.helix.HelixDataAccessor;
+import org.apache.helix.HelixDefinedState;
+import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerFactory;
+import org.apache.helix.InstanceType;
+import org.apache.helix.PropertyKey;
+import org.apache.helix.ZNRecord;
+import org.apache.helix.model.CurrentState;
+import org.apache.helix.model.LiveInstance;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CurrentStateCleanUp {
+ private static final Logger LOG = LoggerFactory.getLogger(CurrentStateCleanUp.class);
+
+ public static final String zkServer = "zkSvr";
+ public static final String cluster = "cluster";
+ public static final String instances = "instances";
+ public static final String help = "help";
+
+ private static Options parseCommandLineOptions() {
+ Option helpOption =
+ OptionBuilder.withLongOpt(help).withDescription("Prints command-line options info")
+ .create();
+
+ Option zkServerOption =
+ OptionBuilder.withLongOpt(zkServer).withDescription("Provide zookeeper address").create();
+ zkServerOption.setArgs(1);
+ zkServerOption.setRequired(true);
+ zkServerOption.setArgName("ZookeeperServerAddress(Required)");
+
+ Option clusterOption =
+ OptionBuilder.withLongOpt(cluster).withDescription("Provide cluster name").create();
+ clusterOption.setArgs(1);
+ clusterOption.setRequired(true);
+ clusterOption.setArgName("Cluster name (Required)");
+
+ Option instancesOption = OptionBuilder.withLongOpt(instances)
+ .withDescription("Provide instance names, separated by ','").create();
+ instancesOption.setArgs(1);
+ instancesOption.setRequired(true);
+ instancesOption.setArgName("Instance names (Required)");
+
+ OptionGroup optionGroup = new OptionGroup();
+ optionGroup.addOption(zkServerOption);
+
+ Options options = new Options();
+ options.addOption(helpOption);
+ options.addOption(clusterOption);
+ options.addOption(instancesOption);
+
+ options.addOptionGroup(optionGroup);
+
+ return options;
+ }
+
+ public static void cleanupCurrentStatesForCluster(String zkConnectString, String clusterName,
+ List<String> instanceNames) throws Exception {
+ HelixManager manager = HelixManagerFactory
+ .getZKHelixManager(clusterName, "Administorator", InstanceType.ADMINISTRATOR,
+ zkConnectString);
+ manager.connect();
+ try {
+ HelixDataAccessor accessor = manager.getHelixDataAccessor();
+ List<LiveInstance> liveInstances =
+ accessor.getChildValues(accessor.keyBuilder().liveInstances());
+ Map<String, LiveInstance> liveInstanceMap = new HashMap<>();
+ for (LiveInstance liveInstance : liveInstances) {
+ liveInstanceMap.put(liveInstance.getInstanceName(), liveInstance);
+ }
+
+ DataUpdater<ZNRecord> updater = new DataUpdater<ZNRecord>() {
+ @Override
+ public ZNRecord update(ZNRecord currentData) {
+ Set<String> partitionToRemove = new HashSet<>();
+ for (String partition : currentData.getMapFields().keySet()) {
+ if (currentData.getMapField(partition).get("CURRENT_STATE")
+ .equals(HelixDefinedState.DROPPED.name())) {
+ partitionToRemove.add(partition);
+ }
+ }
+ currentData.getMapFields().keySet().removeAll(partitionToRemove);
+ return currentData;
+ }
+ };
+
+ for (String instanceName : instanceNames) {
+ if (liveInstanceMap.containsKey(instanceName)) {
+ LOG.info(
+ String.format("Processing cleaning current state for instance: %s", instanceName));
+ List<String> currentStateNames = accessor.getChildNames(accessor.keyBuilder()
+ .currentStates(instanceName, liveInstanceMap.get(instanceName).getSessionId()));
+ for (String currentStateName : currentStateNames) {
+ PropertyKey key = accessor.keyBuilder()
+ .currentState(instanceName, liveInstanceMap.get(instanceName).getSessionId(),
+ currentStateName);
+ accessor.getBaseDataAccessor().update(key.getPath(), updater, AccessOption.PERSISTENT);
+ CurrentState currentState = accessor.getProperty(key);
+ if (currentState.getPartitionStateMap().size() == 0) {
+ accessor.getBaseDataAccessor().remove(key.getPath(), AccessOption.PERSISTENT);
+ LOG.info(String
+ .format("Remove current state for instance: %s, resource %s", instanceName,
+ currentStateName));
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ manager.disconnect();
+ }
+ }
+
+ public static void printUsage(Options cliOptions) {
+ HelpFormatter helpFormatter = new HelpFormatter();
+ helpFormatter.setWidth(1000);
+ helpFormatter.printHelp("java " + CurrentStateCleanUp.class.getName(), cliOptions);
+ }
+
+ public static void main(String[] args) throws Exception {
+ CommandLine cmd = ToolsUtil.processCommandLineArgs(args, parseCommandLineOptions());
+ String zkConnectString = cmd.getOptionValue(zkServer);
+ String clusterName = cmd.getOptionValue(cluster);
+ String instance = cmd.getOptionValue(instances);
+ List<String> instanceNames = Arrays.asList(instance.split(","));
+
+ LOG.info(String
+ .format("Starting cleaning current state with ZK: %s, cluster: %s", zkConnectString,
+ clusterName));
+
+ cleanupCurrentStatesForCluster(zkConnectString, clusterName, instanceNames);
+ }
+}
http://git-wip-us.apache.org/repos/asf/helix/blob/7d695e99/helix-core/src/main/java/org/apache/helix/tools/commandtools/ExampleParticipant.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/tools/commandtools/ExampleParticipant.java b/helix-core/src/main/java/org/apache/helix/tools/commandtools/ExampleParticipant.java
index 3085075..d12e414 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/commandtools/ExampleParticipant.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/commandtools/ExampleParticipant.java
@@ -155,24 +155,10 @@ public class ExampleParticipant {
helpFormatter.printHelp("java " + ExampleParticipant.class.getName(), cliOptions);
}
- public static CommandLine processCommandLineArgs(String[] cliArgs) throws Exception {
- CommandLineParser cliParser = new GnuParser();
- Options cliOptions = constructCommandLineOptions();
- try {
- return cliParser.parse(cliOptions, cliArgs);
- } catch (ParseException pe) {
- System.err.println("CommandLineClient: failed to parse command-line options: "
- + pe.toString());
- printUsage(cliOptions);
- System.exit(1);
- }
- return null;
- }
-
public static void main(String[] args) throws Exception {
int delay = 0;
- CommandLine cmd = processCommandLineArgs(args);
+ CommandLine cmd = ToolsUtil.processCommandLineArgs(args, constructCommandLineOptions());
String zkConnectString = cmd.getOptionValue(zkServer);
String clusterName = cmd.getOptionValue(cluster);
String instanceNames = cmd.getOptionValue(instances);
http://git-wip-us.apache.org/repos/asf/helix/blob/7d695e99/helix-core/src/main/java/org/apache/helix/tools/commandtools/ToolsUtil.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/tools/commandtools/ToolsUtil.java b/helix-core/src/main/java/org/apache/helix/tools/commandtools/ToolsUtil.java
new file mode 100644
index 0000000..0229585
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/tools/commandtools/ToolsUtil.java
@@ -0,0 +1,24 @@
+package org.apache.helix.tools.commandtools;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+import static org.apache.helix.tools.commandtools.IntegrationTestUtil.*;
+
+public class ToolsUtil {
+ public static CommandLine processCommandLineArgs(String[] cliArgs, Options cliOptions) throws Exception {
+ CommandLineParser cliParser = new GnuParser();
+ try {
+ return cliParser.parse(cliOptions, cliArgs);
+ } catch (ParseException pe) {
+ System.err.println("CommandLineClient: failed to parse command-line options: "
+ + pe.toString());
+ printUsage(cliOptions);
+ System.exit(1);
+ }
+ return null;
+ }
+}
[2/2] helix git commit: Fix test TestEntropyFreeNodeBounce
Posted by jx...@apache.org.
Fix test TestEntropyFreeNodeBounce
Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/5529c4d5
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/5529c4d5
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/5529c4d5
Branch: refs/heads/master
Commit: 5529c4d594004df136b37475b4437e075ccbf25a
Parents: 7d695e9
Author: Junkai Xue <jx...@linkedin.com>
Authored: Thu Jun 14 17:42:18 2018 -0700
Committer: Junkai Xue <jx...@linkedin.com>
Committed: Fri Jul 13 13:45:58 2018 -0700
----------------------------------------------------------------------
.../apache/helix/integration/TestEntropyFreeNodeBounce.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/helix/blob/5529c4d5/helix-core/src/test/java/org/apache/helix/integration/TestEntropyFreeNodeBounce.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestEntropyFreeNodeBounce.java b/helix-core/src/test/java/org/apache/helix/integration/TestEntropyFreeNodeBounce.java
index a879320..da72ee0 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/TestEntropyFreeNodeBounce.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/TestEntropyFreeNodeBounce.java
@@ -19,8 +19,10 @@ package org.apache.helix.integration;
* under the License.
*/
+import java.util.ArrayList;
import java.util.Date;
+import java.util.List;
import org.apache.helix.BaseDataAccessor;
import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixDataAccessor;
@@ -95,6 +97,7 @@ public class TestEntropyFreeNodeBounce extends ZkUnitTestBase {
BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor);
PropertyKey.Builder keyBuilder = accessor.keyBuilder();
+ List<HelixManager> participantToClose = new ArrayList<>();
// do the test
try {
@@ -113,6 +116,7 @@ public class TestEntropyFreeNodeBounce extends ZkUnitTestBase {
participant.disconnect();
Thread.sleep(1000);
participant = createParticipant(clusterName, participant.getInstanceName());
+ participantToClose.add(participant);
participant.connect();
Thread.sleep(1000);
helixAdmin.enableCluster(clusterName, true);
@@ -125,7 +129,7 @@ public class TestEntropyFreeNodeBounce extends ZkUnitTestBase {
} finally {
// clean up
controller.syncStop();
- for (HelixManager participant : participants) {
+ for (HelixManager participant : participantToClose) {
participant.disconnect();
}
TestHelper.dropCluster(clusterName, _gZkClient);