You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by ka...@apache.org on 2014/04/17 20:12:20 UTC
[1/2] git commit: [HELIX-433] Fix how FULL_AUTO responds to untagging
Repository: helix
Updated Branches:
refs/heads/helix-0.6.x 38ba9130f -> 4aa54eb42
[HELIX-433] Fix how FULL_AUTO responds to untagging
Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/9a1c205d
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/9a1c205d
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/9a1c205d
Branch: refs/heads/helix-0.6.x
Commit: 9a1c205d986ca6a60609b2467d264222f01daa16
Parents: 44d4d17
Author: Kanak Biscuitwala <ka...@apache.org>
Authored: Wed Apr 16 16:55:17 2014 -0700
Committer: Kanak Biscuitwala <ka...@apache.org>
Committed: Wed Apr 16 16:55:17 2014 -0700
----------------------------------------------------------------------
.../strategy/AutoRebalanceStrategy.java | 2 +
.../integration/TestFullAutoNodeTagging.java | 107 +++++++++++++++++++
2 files changed, 109 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/helix/blob/9a1c205d/helix-core/src/main/java/org/apache/helix/controller/strategy/AutoRebalanceStrategy.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/strategy/AutoRebalanceStrategy.java b/helix-core/src/main/java/org/apache/helix/controller/strategy/AutoRebalanceStrategy.java
index 203612a..768286b 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/strategy/AutoRebalanceStrategy.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/strategy/AutoRebalanceStrategy.java
@@ -418,6 +418,7 @@ public class AutoRebalanceStrategy {
int count = countStateReplicas();
for (String partition : currentMapping.keySet()) {
Map<String, String> nodeStateMap = currentMapping.get(partition);
+ nodeStateMap.keySet().retainAll(_nodeMap.keySet());
for (String nodeId : nodeStateMap.keySet()) {
Node node = _nodeMap.get(nodeId);
boolean skip = false;
@@ -487,6 +488,7 @@ public class AutoRebalanceStrategy {
int count = countStateReplicas();
for (String partition : currentMapping.keySet()) {
Map<String, String> nodeStateMap = currentMapping.get(partition);
+ nodeStateMap.keySet().retainAll(_nodeMap.keySet());
for (String nodeId : nodeStateMap.keySet()) {
Node node = _nodeMap.get(nodeId);
node.currentlyAssigned = node.currentlyAssigned + 1;
http://git-wip-us.apache.org/repos/asf/helix/blob/9a1c205d/helix-core/src/test/java/org/apache/helix/integration/TestFullAutoNodeTagging.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestFullAutoNodeTagging.java b/helix-core/src/test/java/org/apache/helix/integration/TestFullAutoNodeTagging.java
index 0acd775..021f63a 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/TestFullAutoNodeTagging.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/TestFullAutoNodeTagging.java
@@ -28,6 +28,7 @@ import org.apache.helix.HelixAdmin;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.PropertyKey;
import org.apache.helix.TestHelper;
+import org.apache.helix.TestHelper.Verifier;
import org.apache.helix.ZNRecord;
import org.apache.helix.ZkUnitTestBase;
import org.apache.helix.integration.manager.ClusterControllerManager;
@@ -57,6 +58,112 @@ import com.google.common.collect.Sets;
public class TestFullAutoNodeTagging extends ZkUnitTestBase {
private static final Logger LOG = Logger.getLogger(TestFullAutoNodeTagging.class);
+ @Test
+ public void testUntag() throws Exception {
+ final int NUM_PARTICIPANTS = 2;
+ final int NUM_PARTITIONS = 4;
+ final int NUM_REPLICAS = 1;
+ final String RESOURCE_NAME = "TestResource0";
+ final String TAG = "ASSIGNABLE";
+
+ String className = TestHelper.getTestClassName();
+ String methodName = TestHelper.getTestMethodName();
+ final String clusterName = className + "_" + methodName;
+ System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
+
+ // Set up cluster
+ TestHelper.setupCluster(clusterName, ZK_ADDR, 12918, // participant port
+ "localhost", // participant name prefix
+ "TestResource", // resource name prefix
+ 1, // resources
+ NUM_PARTITIONS, // partitions per resource
+ NUM_PARTICIPANTS, // number of nodes
+ NUM_REPLICAS, // replicas
+ "OnlineOffline", RebalanceMode.FULL_AUTO, // use FULL_AUTO mode to test node tagging
+ true); // do rebalance
+
+ // Tag the resource
+ final HelixAdmin helixAdmin = new ZKHelixAdmin(_gZkClient);
+ IdealState idealState = helixAdmin.getResourceIdealState(clusterName, RESOURCE_NAME);
+ idealState.setInstanceGroupTag(TAG);
+ helixAdmin.setResourceIdealState(clusterName, RESOURCE_NAME, idealState);
+
+ // Get a data accessor
+ final HelixDataAccessor accessor =
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
+ final PropertyKey.Builder keyBuilder = accessor.keyBuilder();
+
+ // Tag the participants
+ for (int i = 0; i < NUM_PARTICIPANTS; i++) {
+ final String instanceName = "localhost_" + (12918 + i);
+ helixAdmin.addInstanceTag(clusterName, instanceName, TAG);
+ }
+
+ // Start controller
+ ClusterControllerManager controller =
+ new ClusterControllerManager(ZK_ADDR, clusterName, "controller");
+ controller.syncStart();
+
+ // Start participants
+ MockParticipantManager[] participants = new MockParticipantManager[NUM_PARTICIPANTS];
+ for (int i = 0; i < NUM_PARTICIPANTS; i++) {
+ final String instanceName = "localhost_" + (12918 + i);
+
+ participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
+ participants[i].syncStart();
+ }
+
+ // Verify that there are NUM_PARTITIONS partitions in the external view, each having
+ // NUM_REPLICAS replicas, where all assigned replicas are to tagged nodes, and they are all
+ // ONLINE.
+ Verifier v = new Verifier() {
+ @Override
+ public boolean verify() throws Exception {
+ ExternalView externalView =
+ pollForProperty(ExternalView.class, accessor, keyBuilder.externalView(RESOURCE_NAME),
+ true);
+ if (externalView == null) {
+ return false;
+ }
+ Set<String> taggedInstances =
+ Sets.newHashSet(helixAdmin.getInstancesInClusterWithTag(clusterName, TAG));
+ Set<String> partitionSet = externalView.getPartitionSet();
+ if (partitionSet.size() != NUM_PARTITIONS) {
+ return false;
+ }
+ for (String partitionName : partitionSet) {
+ Map<String, String> stateMap = externalView.getStateMap(partitionName);
+ if (stateMap.size() != NUM_REPLICAS) {
+ return false;
+ }
+ for (String participantName : stateMap.keySet()) {
+ if (!taggedInstances.contains(participantName)) {
+ return false;
+ }
+ String state = stateMap.get(participantName);
+ if (!state.equalsIgnoreCase("ONLINE")) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+ };
+
+ // Run the verifier for both nodes tagged
+ boolean initialResult = TestHelper.verify(v, 10 * 1000);
+ Assert.assertTrue(initialResult);
+
+ // Untag a node
+ helixAdmin.removeInstanceTag(clusterName, "localhost_12918", TAG);
+
+ // Verify again
+ boolean finalResult = TestHelper.verify(v, 10 * 1000);
+ Assert.assertTrue(finalResult);
+
+ System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
+ }
+
/**
* Ensure that no assignments happen when there are no tagged nodes, but the resource is tagged
*/
[2/2] git commit: Merge branch 'helix-0.6.x' of
https://git-wip-us.apache.org/repos/asf/helix into helix-0.6.x
Posted by ka...@apache.org.
Merge branch 'helix-0.6.x' of https://git-wip-us.apache.org/repos/asf/helix into helix-0.6.x
Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/4aa54eb4
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/4aa54eb4
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/4aa54eb4
Branch: refs/heads/helix-0.6.x
Commit: 4aa54eb427d441fbd4445ddc842300c4800a17c1
Parents: 9a1c205 38ba913
Author: Kanak Biscuitwala <ka...@apache.org>
Authored: Thu Apr 17 11:11:35 2014 -0700
Committer: Kanak Biscuitwala <ka...@apache.org>
Committed: Thu Apr 17 11:11:35 2014 -0700
----------------------------------------------------------------------
.../integration/TestPreferenceListAsQueue.java | 487 +++++++++++++++++++
1 file changed, 487 insertions(+)
----------------------------------------------------------------------