You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by ji...@apache.org on 2019/10/28 22:33:16 UTC
[helix] 38/50: Fixing rebalance cache issue and stablize the tests.
(#510)
This is an automated email from the ASF dual-hosted git repository.
jiajunwang pushed a commit to branch wagedRebalancer
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 102a1af44065cb1f5d305584bae68d3def10b3b4
Author: Jiajun Wang <18...@users.noreply.github.com>
AuthorDate: Mon Oct 14 16:02:14 2019 -0700
Fixing rebalance cache issue and stablize the tests. (#510)
1. Fix the DelayedAutoRebalancer Cache issue that ClusterConfig change won't trigger rebalance. The current workaround in our code blocks the WAGED rebalancer logic. So we need to fix it while merging the WAGED rebalancer code.
2. Refine the ResourceChangeDetector's usage in the WAGED rebalancer so as to avoid unnecessary global rebalance.
3. Extend the StrictMatchExternalViewVerifier so it can be used to test the WAGED rebalance feature.
---
.../changedetector/ResourceChangeDetector.java | 11 ++--
.../changedetector/ResourceChangeSnapshot.java | 11 ++++
.../ResourceControllerDataProvider.java | 3 +-
.../rebalancer/waged/WagedRebalancer.java | 8 ++-
.../StrictMatchExternalViewVerifier.java | 60 ++++++++++++++++------
.../ClusterVerifiers/ZkHelixClusterVerifier.java | 17 ------
.../main/java/org/apache/helix/util/HelixUtil.java | 29 +++++++++--
.../changedetector/TestResourceChangeDetector.java | 13 +++--
.../TestCrushAutoRebalanceNonRack.java | 9 ++--
.../rebalancer/CrushRebalancers/TestNodeSwap.java | 4 +-
.../PartitionMigration/TestWagedExpandCluster.java | 2 +-
.../TestWagedRebalancerMigration.java | 3 +-
.../rebalancer/TestMixedModeAutoRebalance.java | 1 -
.../rebalancer/TestZeroReplicaAvoidance.java | 3 +-
.../WagedRebalancer/TestDelayedWagedRebalance.java | 4 +-
...tDelayedWagedRebalanceWithDisabledInstance.java | 4 +-
.../TestDelayedWagedRebalanceWithRackaware.java | 4 +-
.../TestMixedModeWagedRebalance.java | 3 +-
.../rebalancer/WagedRebalancer/TestNodeSwap.java | 6 +--
.../WagedRebalancer/TestWagedRebalance.java | 4 +-
.../TestWagedRebalanceFaultZone.java | 4 +-
.../apache/helix/tools/TestClusterVerifier.java | 26 ++++++----
...chExternalViewVerifier.ComputeIdealMapping.json | 14 +++--
23 files changed, 158 insertions(+), 85 deletions(-)
diff --git a/helix-core/src/main/java/org/apache/helix/controller/changedetector/ResourceChangeDetector.java b/helix-core/src/main/java/org/apache/helix/controller/changedetector/ResourceChangeDetector.java
index a154af2..1df61f8 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/changedetector/ResourceChangeDetector.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/changedetector/ResourceChangeDetector.java
@@ -23,6 +23,7 @@ import com.google.common.collect.Sets;
import org.apache.helix.HelixConstants;
import org.apache.helix.HelixProperty;
import org.apache.helix.controller.dataproviders.ResourceControllerDataProvider;
+import org.apache.helix.model.ClusterConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -112,10 +113,12 @@ public class ResourceChangeDetector implements ChangeDetector {
case LIVE_INSTANCE:
return snapshot.getLiveInstances();
case CLUSTER_CONFIG:
- // In the case of ClusterConfig, we return an empty map
- // This is to allow the caller to iterate on the change types without throwing an exception or
- // leaving a warn log for ClusterConfig changes
- return Collections.emptyMap();
+ ClusterConfig config = snapshot.getClusterConfig();
+ if (config == null) {
+ return Collections.emptyMap();
+ } else {
+ return Collections.singletonMap(config.getClusterName(), config);
+ }
default:
LOG.warn(
"ResourceChangeDetector cannot determine propertyMap for the given ChangeType: {}. Returning an empty map.",
diff --git a/helix-core/src/main/java/org/apache/helix/controller/changedetector/ResourceChangeSnapshot.java b/helix-core/src/main/java/org/apache/helix/controller/changedetector/ResourceChangeSnapshot.java
index cbc3539..6351eb9 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/changedetector/ResourceChangeSnapshot.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/changedetector/ResourceChangeSnapshot.java
@@ -23,8 +23,10 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+
import org.apache.helix.HelixConstants;
import org.apache.helix.controller.dataproviders.ResourceControllerDataProvider;
+import org.apache.helix.model.ClusterConfig;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.InstanceConfig;
import org.apache.helix.model.LiveInstance;
@@ -47,6 +49,7 @@ class ResourceChangeSnapshot {
private Map<String, IdealState> _idealStateMap;
private Map<String, ResourceConfig> _resourceConfigMap;
private Map<String, LiveInstance> _liveInstances;
+ private ClusterConfig _clusterConfig;
/**
* Default constructor that constructs an empty snapshot.
@@ -57,10 +60,12 @@ class ResourceChangeSnapshot {
_idealStateMap = new HashMap<>();
_resourceConfigMap = new HashMap<>();
_liveInstances = new HashMap<>();
+ _clusterConfig = null;
}
/**
* Constructor using controller cache (ResourceControllerDataProvider).
+ *
* @param dataProvider
*/
ResourceChangeSnapshot(ResourceControllerDataProvider dataProvider) {
@@ -69,6 +74,7 @@ class ResourceChangeSnapshot {
_idealStateMap = new HashMap<>(dataProvider.getIdealStates());
_resourceConfigMap = new HashMap<>(dataProvider.getResourceConfigMap());
_liveInstances = new HashMap<>(dataProvider.getLiveInstances());
+ _clusterConfig = dataProvider.getClusterConfig();
}
/**
@@ -81,6 +87,7 @@ class ResourceChangeSnapshot {
_idealStateMap = new HashMap<>(cache._idealStateMap);
_resourceConfigMap = new HashMap<>(cache._resourceConfigMap);
_liveInstances = new HashMap<>(cache._liveInstances);
+ _clusterConfig = cache._clusterConfig;
}
Set<HelixConstants.ChangeType> getChangedTypes() {
@@ -102,4 +109,8 @@ class ResourceChangeSnapshot {
Map<String, LiveInstance> getLiveInstances() {
return _liveInstances;
}
+
+ ClusterConfig getClusterConfig() {
+ return _clusterConfig;
+ }
}
diff --git a/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java b/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
index 9e1550a..1631d50 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/dataproviders/ResourceControllerDataProvider.java
@@ -124,7 +124,8 @@ public class ResourceControllerDataProvider extends BaseControllerDataProvider {
if (changedTypes.contains(HelixConstants.ChangeType.IDEAL_STATE)
|| changedTypes.contains(HelixConstants.ChangeType.LIVE_INSTANCE)
|| changedTypes.contains(HelixConstants.ChangeType.INSTANCE_CONFIG)
- || changedTypes.contains(HelixConstants.ChangeType.RESOURCE_CONFIG)) {
+ || changedTypes.contains(HelixConstants.ChangeType.RESOURCE_CONFIG)
+ || changedTypes.contains((HelixConstants.ChangeType.CLUSTER_CONFIG))) {
clearCachedResourceAssignments();
}
diff --git a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/waged/WagedRebalancer.java b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/waged/WagedRebalancer.java
index 53c9840..f39d3cb 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/waged/WagedRebalancer.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/waged/WagedRebalancer.java
@@ -19,7 +19,6 @@ package org.apache.helix.controller.rebalancer.waged;
* under the License.
*/
-import com.google.common.collect.ImmutableSet;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -28,6 +27,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
+
+import com.google.common.collect.ImmutableSet;
import org.apache.helix.HelixConstants;
import org.apache.helix.HelixException;
import org.apache.helix.HelixManager;
@@ -216,6 +217,11 @@ public class WagedRebalancer {
itemKeys.addAll(getChangeDetector().getRemovalsByType(changeType));
return itemKeys;
}));
+ // Filter for the items that have content changed.
+ clusterChanges =
+ clusterChanges.entrySet().stream().filter(changeEntry -> !changeEntry.getValue().isEmpty())
+ .collect(Collectors
+ .toMap(changeEntry -> changeEntry.getKey(), changeEntry -> changeEntry.getValue()));
// Perform Global Baseline Calculation
if (clusterChanges.keySet().stream()
diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/StrictMatchExternalViewVerifier.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/StrictMatchExternalViewVerifier.java
index f3bca9e..4b427fa 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/StrictMatchExternalViewVerifier.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/StrictMatchExternalViewVerifier.java
@@ -23,7 +23,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -56,19 +55,34 @@ public class StrictMatchExternalViewVerifier extends ZkHelixClusterVerifier {
private final Set<String> _resources;
private final Set<String> _expectLiveInstances;
+ private final boolean _isDeactivatedNodeAware;
+ @Deprecated
public StrictMatchExternalViewVerifier(String zkAddr, String clusterName, Set<String> resources,
Set<String> expectLiveInstances) {
+ this(zkAddr, clusterName, resources, expectLiveInstances, false);
+ }
+
+ @Deprecated
+ public StrictMatchExternalViewVerifier(HelixZkClient zkClient, String clusterName,
+ Set<String> resources, Set<String> expectLiveInstances) {
+ this(zkClient, clusterName, resources, expectLiveInstances, false);
+ }
+
+ private StrictMatchExternalViewVerifier(String zkAddr, String clusterName, Set<String> resources,
+ Set<String> expectLiveInstances, boolean isDeactivatedNodeAware) {
super(zkAddr, clusterName);
_resources = resources;
_expectLiveInstances = expectLiveInstances;
+ _isDeactivatedNodeAware = isDeactivatedNodeAware;
}
- public StrictMatchExternalViewVerifier(HelixZkClient zkClient, String clusterName,
- Set<String> resources, Set<String> expectLiveInstances) {
+ private StrictMatchExternalViewVerifier(HelixZkClient zkClient, String clusterName,
+ Set<String> resources, Set<String> expectLiveInstances, boolean isDeactivatedNodeAware) {
super(zkClient, clusterName);
_resources = resources;
_expectLiveInstances = expectLiveInstances;
+ _isDeactivatedNodeAware = isDeactivatedNodeAware;
}
public static class Builder {
@@ -77,6 +91,8 @@ public class StrictMatchExternalViewVerifier extends ZkHelixClusterVerifier {
private Set<String> _expectLiveInstances;
private String _zkAddr;
private HelixZkClient _zkClient;
+ // For backward compatibility, set the default isDeactivatedNodeAware to be false.
+ private boolean _isDeactivatedNodeAware = false;
public StrictMatchExternalViewVerifier build() {
if (_clusterName == null || (_zkAddr == null && _zkClient == null)) {
@@ -85,10 +101,10 @@ public class StrictMatchExternalViewVerifier extends ZkHelixClusterVerifier {
if (_zkClient != null) {
return new StrictMatchExternalViewVerifier(_zkClient, _clusterName, _resources,
- _expectLiveInstances);
+ _expectLiveInstances, _isDeactivatedNodeAware);
}
return new StrictMatchExternalViewVerifier(_zkAddr, _clusterName, _resources,
- _expectLiveInstances);
+ _expectLiveInstances, _isDeactivatedNodeAware);
}
public Builder(String clusterName) {
@@ -139,6 +155,15 @@ public class StrictMatchExternalViewVerifier extends ZkHelixClusterVerifier {
_zkClient = zkClient;
return this;
}
+
+ public boolean getDeactivatedNodeAwareness() {
+ return _isDeactivatedNodeAware;
+ }
+
+ public Builder setDeactivatedNodeAwareness(boolean isDeactivatedNodeAware) {
+ _isDeactivatedNodeAware = isDeactivatedNodeAware;
+ return this;
+ }
}
@Override
@@ -251,12 +276,11 @@ public class StrictMatchExternalViewVerifier extends ZkHelixClusterVerifier {
+ "is enabled."));
}
for (String partition : idealState.getPartitionSet()) {
- if (idealState.getInstanceStateMap(partition) == null || idealState
- .getInstanceStateMap(partition).isEmpty()) {
+ if (idealState.getPreferenceList(partition) == null || idealState.getPreferenceList(partition).isEmpty()) {
return false;
}
}
- idealPartitionState = idealState.getRecord().getMapFields();
+ idealPartitionState = computeIdealPartitionState(dataCache, idealState);
break;
case SEMI_AUTO:
case USER_DEFINED:
@@ -279,17 +303,21 @@ public class StrictMatchExternalViewVerifier extends ZkHelixClusterVerifier {
String stateModelDefName = idealState.getStateModelDefRef();
StateModelDefinition stateModelDef = cache.getStateModelDef(stateModelDefName);
- Map<String, Map<String, String>> idealPartitionState =
- new HashMap<String, Map<String, String>>();
-
- Set<String> liveEnabledInstances = new HashSet<String>(cache.getLiveInstances().keySet());
- liveEnabledInstances.removeAll(cache.getDisabledInstances());
+ Map<String, Map<String, String>> idealPartitionState = new HashMap<>();
for (String partition : idealState.getPartitionSet()) {
List<String> preferenceList = AbstractRebalancer
- .getPreferenceList(new Partition(partition), idealState, liveEnabledInstances);
- Map<String, String> idealMapping =
- HelixUtil.computeIdealMapping(preferenceList, stateModelDef, liveEnabledInstances);
+ .getPreferenceList(new Partition(partition), idealState, cache.getEnabledLiveInstances());
+ Map<String, String> idealMapping;
+ if (_isDeactivatedNodeAware) {
+ idealMapping = HelixUtil
+ .computeIdealMapping(preferenceList, stateModelDef, cache.getLiveInstances().keySet(),
+ cache.getDisabledInstancesForPartition(idealState.getResourceName(), partition));
+ } else {
+ idealMapping = HelixUtil
+ .computeIdealMapping(preferenceList, stateModelDef, cache.getEnabledLiveInstances(),
+ Collections.emptySet());
+ }
idealPartitionState.put(partition, idealMapping);
}
diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ZkHelixClusterVerifier.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ZkHelixClusterVerifier.java
index 07c7493..839f3b7 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ZkHelixClusterVerifier.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterVerifiers/ZkHelixClusterVerifier.java
@@ -19,7 +19,6 @@ package org.apache.helix.tools.ClusterVerifiers;
* under the License.
*/
-import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.I0Itec.zkclient.IZkChildListener;
@@ -33,7 +32,6 @@ import org.apache.helix.manager.zk.ZkBaseDataAccessor;
import org.apache.helix.manager.zk.ZkClient;
import org.apache.helix.manager.zk.client.DedicatedZkClientFactory;
import org.apache.helix.manager.zk.client.HelixZkClient;
-import org.apache.helix.model.ResourceConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -162,10 +160,6 @@ public abstract class ZkHelixClusterVerifier
long start = System.currentTimeMillis();
boolean success;
do {
- // Add a rebalance invoker in case some callbacks got buried - sometimes callbacks get
- // processed even before changes get fully written to ZK.
- invokeRebalance(_accessor);
-
success = verifyState();
if (success) {
return true;
@@ -295,15 +289,4 @@ public abstract class ZkHelixClusterVerifier
public String getClusterName() {
return _clusterName;
}
-
- /**
- * Invoke a cluster rebalance in case some callbacks get ignored. This is for Helix integration
- * testing purposes only.
- */
- public static synchronized void invokeRebalance(HelixDataAccessor accessor) {
- String dummyName = UUID.randomUUID().toString();
- ResourceConfig dummyConfig = new ResourceConfig(dummyName);
- accessor.updateProperty(accessor.keyBuilder().resourceConfig(dummyName), dummyConfig);
- accessor.removeProperty(accessor.keyBuilder().resourceConfig(dummyName));
- }
}
diff --git a/helix-core/src/main/java/org/apache/helix/util/HelixUtil.java b/helix-core/src/main/java/org/apache/helix/util/HelixUtil.java
index 5e7f7b4..bfda60e 100644
--- a/helix-core/src/main/java/org/apache/helix/util/HelixUtil.java
+++ b/helix-core/src/main/java/org/apache/helix/util/HelixUtil.java
@@ -29,6 +29,8 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;
+
+import com.google.common.base.Joiner;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.PropertyType;
import org.apache.helix.controller.dataproviders.ResourceControllerDataProvider;
@@ -43,8 +45,6 @@ import org.apache.helix.model.StateModelDefinition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Joiner;
-
public final class HelixUtil {
static private Logger LOG = LoggerFactory.getLogger(HelixUtil.class);
@@ -201,23 +201,42 @@ public final class HelixUtil {
*/
public static Map<String, String> computeIdealMapping(List<String> preferenceList,
StateModelDefinition stateModelDef, Set<String> liveAndEnabled) {
+ return computeIdealMapping(preferenceList, stateModelDef, liveAndEnabled,
+ Collections.emptySet());
+ }
+
+ /**
+ * compute the ideal mapping for resource in Full-Auto and Semi-Auto based on its preference list
+ */
+ public static Map<String, String> computeIdealMapping(List<String> preferenceList,
+ StateModelDefinition stateModelDef, Set<String> liveInstanceSet,
+ Set<String> disabledInstancesForPartition) {
Map<String, String> idealStateMap = new HashMap<String, String>();
if (preferenceList == null) {
return idealStateMap;
}
+ for (String instance : preferenceList) {
+ if (disabledInstancesForPartition.contains(instance) && liveInstanceSet.contains(instance)) {
+ idealStateMap.put(instance, stateModelDef.getInitialState());
+ }
+ }
+
+ Set<String> liveAndEnabledInstances = new HashSet<>(liveInstanceSet);
+ liveAndEnabledInstances.removeAll(disabledInstancesForPartition);
+
List<String> statesPriorityList = stateModelDef.getStatesPriorityList();
Set<String> assigned = new HashSet<String>();
for (String state : statesPriorityList) {
- int stateCount = AbstractRebalancer.getStateCount(state, stateModelDef, liveAndEnabled.size(),
- preferenceList.size());
+ int stateCount = AbstractRebalancer
+ .getStateCount(state, stateModelDef, liveAndEnabledInstances.size(), preferenceList.size());
for (String instance : preferenceList) {
if (stateCount <= 0) {
break;
}
- if (!assigned.contains(instance)) {
+ if (!assigned.contains(instance) && liveAndEnabledInstances.contains(instance)) {
idealStateMap.put(instance, state);
assigned.add(instance);
stateCount--;
diff --git a/helix-core/src/test/java/org/apache/helix/controller/changedetector/TestResourceChangeDetector.java b/helix-core/src/test/java/org/apache/helix/controller/changedetector/TestResourceChangeDetector.java
index 3ef41e4..445add4 100644
--- a/helix-core/src/test/java/org/apache/helix/controller/changedetector/TestResourceChangeDetector.java
+++ b/helix-core/src/test/java/org/apache/helix/controller/changedetector/TestResourceChangeDetector.java
@@ -47,7 +47,7 @@ public class TestResourceChangeDetector extends ZkTestBase {
// since we don't provide the names of changed fields for ClusterConfig
private static final ChangeType[] RESOURCE_CHANGE_TYPES = {
ChangeType.IDEAL_STATE, ChangeType.INSTANCE_CONFIG, ChangeType.LIVE_INSTANCE,
- ChangeType.RESOURCE_CONFIG
+ ChangeType.RESOURCE_CONFIG, ChangeType.CLUSTER_CONFIG
};
private static final String CLUSTER_NAME = TestHelper.getTestClassName();
@@ -121,7 +121,7 @@ public class TestResourceChangeDetector extends ZkTestBase {
* @throws Exception
*/
@Test
- public void testResourceChangeDetectorInit() throws Exception {
+ public void testResourceChangeDetectorInit() {
_dataProvider.refresh(_dataAccessor);
_resourceChangeDetector.updateSnapshots(_dataProvider);
@@ -135,6 +135,9 @@ public class TestResourceChangeDetector extends ZkTestBase {
// Check that the right amount of instances show up as added
checkDetectionCounts(ChangeType.LIVE_INSTANCE, NUM_NODES, 0, 0);
checkDetectionCounts(ChangeType.INSTANCE_CONFIG, NUM_NODES, 0, 0);
+
+ // Check that the right amount of cluster config item show up
+ checkDetectionCounts(ChangeType.CLUSTER_CONFIG, 1, 0, 0);
}
/**
@@ -308,7 +311,11 @@ public class TestResourceChangeDetector extends ZkTestBase {
checkChangeTypes(ChangeType.CLUSTER_CONFIG);
// Check the counts for other types
for (ChangeType type : RESOURCE_CHANGE_TYPES) {
- checkDetectionCounts(type, 0, 0, 0);
+ if (type == ChangeType.CLUSTER_CONFIG) {
+ checkDetectionCounts(type, 0, 1, 0);
+ } else {
+ checkDetectionCounts(type, 0, 0, 0);
+ }
}
}
diff --git a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/CrushRebalancers/TestCrushAutoRebalanceNonRack.java b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/CrushRebalancers/TestCrushAutoRebalanceNonRack.java
index ce4f1dd..5b7d083 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/CrushRebalancers/TestCrushAutoRebalanceNonRack.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/CrushRebalancers/TestCrushAutoRebalanceNonRack.java
@@ -26,6 +26,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+
import org.apache.helix.ConfigAccessor;
import org.apache.helix.controller.rebalancer.strategy.CrushEdRebalanceStrategy;
import org.apache.helix.controller.rebalancer.strategy.CrushRebalanceStrategy;
@@ -149,7 +150,7 @@ public class TestCrushAutoRebalanceNonRack extends ZkStandAloneCMTestBase {
HelixClusterVerifier _clusterVerifier =
new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
- .setResources(_allDBs).build();
+ .setDeactivatedNodeAwareness(true).setResources(_allDBs).build();
Assert.assertTrue(_clusterVerifier.verify(5000));
for (String db : _allDBs) {
IdealState is =
@@ -181,7 +182,7 @@ public class TestCrushAutoRebalanceNonRack extends ZkStandAloneCMTestBase {
HelixClusterVerifier _clusterVerifier =
new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
- .setResources(_allDBs).build();
+ .setDeactivatedNodeAwareness(true).setResources(_allDBs).build();
Assert.assertTrue(_clusterVerifier.verify(5000));
for (String db : _allDBs) {
IdealState is =
@@ -217,7 +218,7 @@ public class TestCrushAutoRebalanceNonRack extends ZkStandAloneCMTestBase {
HelixClusterVerifier _clusterVerifier =
new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
- .setResources(_allDBs).build();
+ .setDeactivatedNodeAwareness(true).setResources(_allDBs).build();
Assert.assertTrue(_clusterVerifier.verify(5000));
for (String db : _allDBs) {
IdealState is =
@@ -261,7 +262,7 @@ public class TestCrushAutoRebalanceNonRack extends ZkStandAloneCMTestBase {
Thread.sleep(300);
ZkHelixClusterVerifier _clusterVerifier =
new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
- .setResources(_allDBs).build();
+ .setDeactivatedNodeAwareness(true).setResources(_allDBs).build();
Assert.assertTrue(_clusterVerifier.verifyByPolling());
for (String db : _allDBs) {
IdealState is =
diff --git a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/CrushRebalancers/TestNodeSwap.java b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/CrushRebalancers/TestNodeSwap.java
index b71580e..7f4c669 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/CrushRebalancers/TestNodeSwap.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/CrushRebalancers/TestNodeSwap.java
@@ -148,7 +148,7 @@ public class TestNodeSwap extends ZkTestBase {
HelixClusterVerifier _clusterVerifier =
new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
- .setResources(_allDBs).build();
+ .setDeactivatedNodeAwareness(true).setResources(_allDBs).build();
Assert.assertTrue(_clusterVerifier.verify(5000));
Map<String, ExternalView> record = new HashMap<>();
@@ -192,7 +192,7 @@ public class TestNodeSwap extends ZkTestBase {
Thread.sleep(2000);
_clusterVerifier = new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
- .setResources(_allDBs).build();
+ .setDeactivatedNodeAwareness(true).setResources(_allDBs).build();
Assert.assertTrue(_clusterVerifier.verify(5000));
for (String db : _allDBs) {
diff --git a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/PartitionMigration/TestWagedExpandCluster.java b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/PartitionMigration/TestWagedExpandCluster.java
index d303e87..01db9eb 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/PartitionMigration/TestWagedExpandCluster.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/PartitionMigration/TestWagedExpandCluster.java
@@ -38,7 +38,7 @@ public class TestWagedExpandCluster extends TestExpandCluster {
dbNames.add("Test-DB-" + i++);
}
return new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setResources(dbNames)
- .setZkAddr(ZK_ADDR).build();
+ .setDeactivatedNodeAwareness(true).setZkAddr(ZK_ADDR).build();
}
protected Map<String, IdealState> createTestDBs(long delayTime) {
diff --git a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/PartitionMigration/TestWagedRebalancerMigration.java b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/PartitionMigration/TestWagedRebalancerMigration.java
index 1a13496..3bfdc37 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/PartitionMigration/TestWagedRebalancerMigration.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/PartitionMigration/TestWagedRebalancerMigration.java
@@ -85,7 +85,8 @@ public class TestWagedRebalancerMigration extends TestPartitionMigrationBase {
Thread.sleep(2000);
ZkHelixClusterVerifier clusterVerifier =
new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME)
- .setResources(Collections.singleton(db)).setZkAddr(ZK_ADDR).build();
+ .setResources(Collections.singleton(db)).setZkAddr(ZK_ADDR)
+ .setDeactivatedNodeAwareness(true).build();
Assert.assertTrue(clusterVerifier.verifyByPolling());
_migrationVerifier =
diff --git a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestMixedModeAutoRebalance.java b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestMixedModeAutoRebalance.java
index 33dab8d..2f13d8b 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestMixedModeAutoRebalance.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestMixedModeAutoRebalance.java
@@ -211,7 +211,6 @@ public class TestMixedModeAutoRebalance extends ZkTestBase {
}
return false;
}, 2000);
- Assert.assertTrue(_clusterVerifier.verify(3000));
ExternalView ev =
_gSetupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME, DB_NAME);
diff --git a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestZeroReplicaAvoidance.java b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestZeroReplicaAvoidance.java
index 7090cbf..58d0dce 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestZeroReplicaAvoidance.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/TestZeroReplicaAvoidance.java
@@ -165,7 +165,8 @@ public class TestZeroReplicaAvoidance extends ZkTestBase
createResourceWithWagedRebalance(CLUSTER_NAME, db, stateModel, partition, replica, replica);
}
ZkHelixClusterVerifier clusterVerifier =
- new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR).build();
+ new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
+ .setDeactivatedNodeAwareness(true).build();
Assert.assertTrue(clusterVerifier.verifyByPolling(50000L, 100L));
_startListen = true;
diff --git a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestDelayedWagedRebalance.java b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestDelayedWagedRebalance.java
index 8587f40..713c095 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestDelayedWagedRebalance.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestDelayedWagedRebalance.java
@@ -42,7 +42,7 @@ public class TestDelayedWagedRebalance extends TestDelayedAutoRebalance {
dbNames.add("Test-DB-" + i++);
}
return new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setResources(dbNames)
- .setZkAddr(ZK_ADDR).build();
+ .setDeactivatedNodeAwareness(true).setZkAddr(ZK_ADDR).build();
}
// create test DBs, wait it converged and return externalviews
@@ -55,7 +55,7 @@ public class TestDelayedWagedRebalance extends TestDelayedAutoRebalance {
_minActiveReplica);
_testDBs.add(db);
}
- Thread.sleep(100);
+ Thread.sleep(DEFAULT_REBALANCE_PROCESSING_WAIT_TIME);
Assert.assertTrue(_clusterVerifier.verifyByPolling());
for (String db : _testDBs) {
ExternalView ev =
diff --git a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestDelayedWagedRebalanceWithDisabledInstance.java b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestDelayedWagedRebalanceWithDisabledInstance.java
index fab254c..bcb2260 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestDelayedWagedRebalanceWithDisabledInstance.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestDelayedWagedRebalanceWithDisabledInstance.java
@@ -43,7 +43,7 @@ public class TestDelayedWagedRebalanceWithDisabledInstance
dbNames.add("Test-DB-" + i++);
}
return new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setResources(dbNames)
- .setZkAddr(ZK_ADDR).build();
+ .setDeactivatedNodeAwareness(true).setZkAddr(ZK_ADDR).build();
}
// create test DBs, wait it converged and return externalviews
@@ -56,7 +56,7 @@ public class TestDelayedWagedRebalanceWithDisabledInstance
_minActiveReplica);
_testDBs.add(db);
}
- Thread.sleep(100);
+ Thread.sleep(DEFAULT_REBALANCE_PROCESSING_WAIT_TIME);
Assert.assertTrue(_clusterVerifier.verifyByPolling());
for (String db : _testDBs) {
ExternalView ev =
diff --git a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestDelayedWagedRebalanceWithRackaware.java b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestDelayedWagedRebalanceWithRackaware.java
index 4791e6e..e0adf72 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestDelayedWagedRebalanceWithRackaware.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestDelayedWagedRebalanceWithRackaware.java
@@ -42,7 +42,7 @@ public class TestDelayedWagedRebalanceWithRackaware extends TestDelayedAutoRebal
dbNames.add("Test-DB-" + i++);
}
return new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setResources(dbNames)
- .setZkAddr(ZK_ADDR).build();
+ .setDeactivatedNodeAwareness(true).setZkAddr(ZK_ADDR).build();
}
// create test DBs, wait it converged and return externalviews
@@ -55,7 +55,7 @@ public class TestDelayedWagedRebalanceWithRackaware extends TestDelayedAutoRebal
_minActiveReplica);
_testDBs.add(db);
}
- Thread.sleep(100);
+ Thread.sleep(DEFAULT_REBALANCE_PROCESSING_WAIT_TIME);
Assert.assertTrue(_clusterVerifier.verifyByPolling());
for (String db : _testDBs) {
ExternalView ev =
diff --git a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestMixedModeWagedRebalance.java b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestMixedModeWagedRebalance.java
index 7087dfc..00a31e2 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestMixedModeWagedRebalance.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestMixedModeWagedRebalance.java
@@ -20,6 +20,7 @@ package org.apache.helix.integration.rebalancer.WagedRebalancer;
*/
import java.util.Collections;
+
import org.apache.helix.integration.rebalancer.TestMixedModeAutoRebalance;
import org.apache.helix.model.BuiltInStateModelDefinitions;
import org.apache.helix.tools.ClusterVerifiers.StrictMatchExternalViewVerifier;
@@ -44,7 +45,7 @@ public class TestMixedModeWagedRebalance extends TestMixedModeAutoRebalance {
protected ZkHelixClusterVerifier getClusterVerifier() {
return new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
- .setResources(Collections.singleton(DB_NAME)).build();
+ .setDeactivatedNodeAwareness(true).setResources(Collections.singleton(DB_NAME)).build();
}
protected void createResource(String stateModel, int numPartition,
diff --git a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestNodeSwap.java b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestNodeSwap.java
index 360e495..cbc9fea 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestNodeSwap.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestNodeSwap.java
@@ -116,7 +116,7 @@ public class TestNodeSwap extends ZkTestBase {
Thread.sleep(1000);
_clusterVerifier = new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
- .setResources(_allDBs).build();
+ .setDeactivatedNodeAwareness(true).setResources(_allDBs).build();
Assert.assertTrue(_clusterVerifier.verify(5000));
}
@@ -177,7 +177,7 @@ public class TestNodeSwap extends ZkTestBase {
Thread.sleep(2000);
_clusterVerifier = new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
- .setResources(_allDBs).build();
+ .setDeactivatedNodeAwareness(true).setResources(_allDBs).build();
Assert.assertTrue(_clusterVerifier.verify(5000));
// Since only one node temporary down, the same partitions will be moved to the newly added node.
@@ -265,7 +265,7 @@ public class TestNodeSwap extends ZkTestBase {
Thread.sleep(2000);
_clusterVerifier = new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
- .setResources(_allDBs).build();
+ .setDeactivatedNodeAwareness(true).setResources(_allDBs).build();
Assert.assertTrue(_clusterVerifier.verify(5000));
for (String db : _allDBs) {
diff --git a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalance.java b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalance.java
index 37c1229..4920414 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalance.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalance.java
@@ -417,7 +417,7 @@ public class TestWagedRebalance extends ZkTestBase {
private void validate(int expectedReplica) {
HelixClusterVerifier _clusterVerifier =
new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
- .setResources(_allDBs).build();
+ .setDeactivatedNodeAwareness(true).setResources(_allDBs).build();
Assert.assertTrue(_clusterVerifier.verify(5000));
for (String db : _allDBs) {
IdealState is =
@@ -457,7 +457,7 @@ public class TestWagedRebalance extends ZkTestBase {
Thread.sleep(100);
ZkHelixClusterVerifier _clusterVerifier =
new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
- .setResources(_allDBs).build();
+ .setDeactivatedNodeAwareness(true).setResources(_allDBs).build();
Assert.assertTrue(_clusterVerifier.verifyByPolling());
}
diff --git a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalanceFaultZone.java b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalanceFaultZone.java
index 84c6ac4..0a4c232 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalanceFaultZone.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/rebalancer/WagedRebalancer/TestWagedRebalanceFaultZone.java
@@ -305,7 +305,7 @@ public class TestWagedRebalanceFaultZone extends ZkTestBase {
private void validate(int expectedReplica) {
ZkHelixClusterVerifier _clusterVerifier =
new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
- .setResources(_allDBs).build();
+ .setDeactivatedNodeAwareness(true).setResources(_allDBs).build();
Assert.assertTrue(_clusterVerifier.verifyByPolling());
for (String db : _allDBs) {
@@ -351,7 +351,7 @@ public class TestWagedRebalanceFaultZone extends ZkTestBase {
Thread.sleep(100);
ZkHelixClusterVerifier _clusterVerifier =
new StrictMatchExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR)
- .setResources(_allDBs).build();
+ .setDeactivatedNodeAwareness(true).setResources(_allDBs).build();
Assert.assertTrue(_clusterVerifier.verifyByPolling());
}
diff --git a/helix-core/src/test/java/org/apache/helix/tools/TestClusterVerifier.java b/helix-core/src/test/java/org/apache/helix/tools/TestClusterVerifier.java
index e1ecdee..60dec1f 100644
--- a/helix-core/src/test/java/org/apache/helix/tools/TestClusterVerifier.java
+++ b/helix-core/src/test/java/org/apache/helix/tools/TestClusterVerifier.java
@@ -19,9 +19,10 @@ package org.apache.helix.tools;
* under the License.
*/
+import java.util.Arrays;
+
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import java.util.Arrays;
import org.apache.helix.ConfigAccessor;
import org.apache.helix.HelixAdmin;
import org.apache.helix.TestHelper;
@@ -149,7 +150,8 @@ public class TestClusterVerifier extends ZkUnitTestBase {
HelixClusterVerifier strictMatchVerifier =
new StrictMatchExternalViewVerifier.Builder(_clusterName)
- .setResources(Sets.newHashSet(RESOURCES)).setZkClient(_gZkClient).build();
+ .setResources(Sets.newHashSet(RESOURCES)).setZkClient(_gZkClient)
+ .setDeactivatedNodeAwareness(true).build();
Assert.assertTrue(strictMatchVerifier.verify(10000));
// Disable partition for 1 instance, then Full-Auto ExternalView should not match IdealState.
@@ -168,19 +170,20 @@ public class TestClusterVerifier extends ZkUnitTestBase {
_participants[0].syncStop();
Thread.sleep(1000);
- // Semi-Auto ExternalView should not match IdealState
+ // Semi-Auto ExternalView matching
for (String resource : SEMI_AUTO_RESOURCES) {
System.out.println("Un-verify resource: " + resource);
strictMatchVerifier =
new StrictMatchExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient)
- .setResources(Sets.newHashSet(resource)).build();
- Assert.assertFalse(strictMatchVerifier.verify(3000));
+ .setResources(Sets.newHashSet(resource)).setDeactivatedNodeAwareness(true).build();
+ Assert.assertTrue(strictMatchVerifier.verify(3000));
}
- // Full-Auto still match, because preference list wouldn't contain non-live instances
+ // Full-Auto ExternalView matching
strictMatchVerifier =
new StrictMatchExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient)
- .setResources(Sets.newHashSet(FULL_AUTO_RESOURCES)).build();
+ .setResources(Sets.newHashSet(FULL_AUTO_RESOURCES)).setDeactivatedNodeAwareness(true)
+ .build();
Assert.assertTrue(strictMatchVerifier.verify(10000));
}
@@ -208,7 +211,7 @@ public class TestClusterVerifier extends ZkUnitTestBase {
Assert.assertTrue(verifier.verifyByPolling());
verifier = new StrictMatchExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient)
- .setResources(Sets.newHashSet(testDB)).build();
+ .setResources(Sets.newHashSet(testDB)).setDeactivatedNodeAwareness(true).build();
Assert.assertTrue(verifier.verifyByPolling());
// But the full cluster verification should fail
@@ -216,8 +219,8 @@ public class TestClusterVerifier extends ZkUnitTestBase {
new BestPossibleExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient).build();
Assert.assertFalse(verifier.verify(3000));
- verifier =
- new StrictMatchExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient).build();
+ verifier = new StrictMatchExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient)
+ .setDeactivatedNodeAwareness(true).build();
Assert.assertFalse(verifier.verify(3000));
_admin.enableCluster(_clusterName, true);
@@ -231,7 +234,8 @@ public class TestClusterVerifier extends ZkUnitTestBase {
Assert.assertTrue(bestPossibleVerifier.verify(10000));
HelixClusterVerifier strictMatchVerifier =
- new StrictMatchExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient).build();
+ new StrictMatchExternalViewVerifier.Builder(_clusterName).setZkClient(_gZkClient)
+ .setDeactivatedNodeAwareness(true).build();
Assert.assertTrue(strictMatchVerifier.verify(10000));
// Re-start a new participant with sleeping transition(all state model transition cannot finish)
diff --git a/helix-core/src/test/resources/TestStrictMatchExternalViewVerifier.ComputeIdealMapping.json b/helix-core/src/test/resources/TestStrictMatchExternalViewVerifier.ComputeIdealMapping.json
index b237b55..4fde15b 100644
--- a/helix-core/src/test/resources/TestStrictMatchExternalViewVerifier.ComputeIdealMapping.json
+++ b/helix-core/src/test/resources/TestStrictMatchExternalViewVerifier.ComputeIdealMapping.json
@@ -7,7 +7,11 @@
"node_2",
"node_3"
],
- "liveAndEnabledInstances": [],
+ "liveAndEnabledInstances": [
+ "node_1",
+ "node_2",
+ "node_3"
+ ],
"expectedBestPossibleStateMap": {
"node_1": "MASTER",
"node_2": "SLAVE",
@@ -22,11 +26,15 @@
"node_2",
"node_3"
],
- "liveAndEnabledInstances": [],
+ "liveAndEnabledInstances": [
+ "node_1",
+ "node_2",
+ "node_3"
+ ],
"expectedBestPossibleStateMap": {
"node_1": "ONLINE",
"node_2": "ONLINE",
"node_3": "ONLINE"
}
}
-]
\ No newline at end of file
+]