You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by zz...@apache.org on 2013/08/24 00:07:46 UTC
git commit: [HELIX-212] Rebalancer interface should have 1 function
to compute the entire ideal state, rb=13694
Updated Branches:
refs/heads/master a48b89e5a -> d161a4ec3
[HELIX-212] Rebalancer interface should have 1 function to compute the entire ideal state, rb=13694
Project: http://git-wip-us.apache.org/repos/asf/incubator-helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-helix/commit/d161a4ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-helix/tree/d161a4ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-helix/diff/d161a4ec
Branch: refs/heads/master
Commit: d161a4ec30926f1e4338b0ac97581f8a45c932d5
Parents: a48b89e
Author: zzhang <zz...@uci.edu>
Authored: Fri Aug 23 15:07:39 2013 -0700
Committer: zzhang <zz...@uci.edu>
Committed: Fri Aug 23 15:07:39 2013 -0700
----------------------------------------------------------------------
.../java/org/apache/helix/HelixProperty.java | 8 +
.../controller/rebalancer/AutoRebalancer.java | 306 +++----------------
.../controller/rebalancer/CustomRebalancer.java | 22 +-
.../helix/controller/rebalancer/Rebalancer.java | 41 +--
.../rebalancer/SemiAutoRebalancer.java | 23 +-
.../util/ConstraintBasedAssignment.java | 49 +++
.../stages/BestPossibleStateCalcStage.java | 13 +-
.../stages/RebalanceIdealStateStage.java | 15 +-
.../controller/stages/ResourceMapping.java | 58 ----
.../strategy/AutoRebalanceStrategy.java | 117 ++++++-
.../java/org/apache/helix/model/IdealState.java | 30 ++
.../apache/helix/model/ResourceAssignment.java | 68 ++++-
.../TestCustomizedIdealStateRebalancer.java | 31 +-
13 files changed, 379 insertions(+), 402 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/d161a4ec/helix-core/src/main/java/org/apache/helix/HelixProperty.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/HelixProperty.java b/helix-core/src/main/java/org/apache/helix/HelixProperty.java
index 2e19231..9d39400 100644
--- a/helix-core/src/main/java/org/apache/helix/HelixProperty.java
+++ b/helix-core/src/main/java/org/apache/helix/HelixProperty.java
@@ -55,6 +55,14 @@ public class HelixProperty {
}
/**
+ * Initialize the property by copying from another property
+ * @param property
+ */
+ public HelixProperty(HelixProperty property) {
+ _record = new ZNRecord(property.getRecord());
+ }
+
+ /**
* Get the property identifier
* @return the property id
*/
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/d161a4ec/helix-core/src/main/java/org/apache/helix/controller/rebalancer/AutoRebalancer.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/AutoRebalancer.java b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/AutoRebalancer.java
index 4dd5ea6..9564e35 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/AutoRebalancer.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/AutoRebalancer.java
@@ -20,31 +20,27 @@ package org.apache.helix.controller.rebalancer;
*/
import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.TreeMap;
import org.apache.helix.HelixManager;
import org.apache.helix.ZNRecord;
import org.apache.helix.controller.rebalancer.util.ConstraintBasedAssignment;
import org.apache.helix.controller.stages.ClusterDataCache;
import org.apache.helix.controller.stages.CurrentStateOutput;
-import org.apache.helix.controller.stages.ResourceMapping;
import org.apache.helix.controller.strategy.AutoRebalanceStrategy;
import org.apache.helix.controller.strategy.AutoRebalanceStrategy.DefaultPlacementScheme;
import org.apache.helix.controller.strategy.AutoRebalanceStrategy.ReplicaPlacementScheme;
-import org.apache.helix.model.CurrentState;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.IdealState.RebalanceMode;
import org.apache.helix.model.LiveInstance;
import org.apache.helix.model.Partition;
import org.apache.helix.model.Resource;
+import org.apache.helix.model.ResourceAssignment;
import org.apache.helix.model.StateModelDefinition;
import org.apache.log4j.Logger;
@@ -72,8 +68,9 @@ public class AutoRebalancer implements Rebalancer {
}
@Override
- public IdealState computeNewIdealState(String resourceName, IdealState currentIdealState,
+ public ResourceAssignment computeResourceMapping(Resource resource, IdealState currentIdealState,
CurrentStateOutput currentStateOutput, ClusterDataCache clusterData) {
+ // Compute a preference list based on the current ideal state
List<String> partitions = new ArrayList<String>(currentIdealState.getPartitionSet());
String stateModelName = currentIdealState.getStateModelDefRef();
StateModelDefinition stateModelDef = clusterData.getStateModelDef(stateModelName);
@@ -81,10 +78,30 @@ public class AutoRebalancer implements Rebalancer {
String replicas = currentIdealState.getReplicas();
LinkedHashMap<String, Integer> stateCountMap = new LinkedHashMap<String, Integer>();
- stateCountMap = stateCount(stateModelDef, liveInstance.size(), Integer.parseInt(replicas));
+ stateCountMap =
+ ConstraintBasedAssignment.stateCount(stateModelDef, liveInstance.size(),
+ Integer.parseInt(replicas));
List<String> liveNodes = new ArrayList<String>(liveInstance.keySet());
Map<String, Map<String, String>> currentMapping =
- currentMapping(currentStateOutput, resourceName, partitions, stateCountMap);
+ currentMapping(currentStateOutput, resource.getResourceName(), partitions, stateCountMap);
+
+ // If there are nodes tagged with resource name, use only those nodes
+ Set<String> taggedNodes = new HashSet<String>();
+ if (currentIdealState.getInstanceGroupTag() != null) {
+ for (String instanceName : liveNodes) {
+ if (clusterData.getInstanceConfigMap().get(instanceName)
+ .containsTag(currentIdealState.getInstanceGroupTag())) {
+ taggedNodes.add(instanceName);
+ }
+ }
+ }
+ if (taggedNodes.size() > 0) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("found the following instances with tag " + currentIdealState.getResourceName()
+ + " " + taggedNodes);
+ }
+ liveNodes = new ArrayList<String>(taggedNodes);
+ }
List<String> allNodes = new ArrayList<String>(clusterData.getInstanceConfigMap().keySet());
int maxPartition = currentIdealState.getMaxPartitionsPerInstance();
@@ -99,8 +116,8 @@ public class AutoRebalancer implements Rebalancer {
ReplicaPlacementScheme placementScheme = new DefaultPlacementScheme();
placementScheme.init(_manager);
_algorithm =
- new AutoRebalanceStrategy(resourceName, partitions, stateCountMap, maxPartition,
- placementScheme);
+ new AutoRebalanceStrategy(resource.getResourceName(), partitions, stateCountMap,
+ maxPartition, placementScheme);
ZNRecord newMapping =
_algorithm.computePartitionAssignment(liveNodes, currentMapping, allNodes);
@@ -108,55 +125,31 @@ public class AutoRebalancer implements Rebalancer {
LOG.info("newMapping: " + newMapping);
}
- IdealState newIdealState = new IdealState(resourceName);
+ IdealState newIdealState = new IdealState(resource.getResourceName());
newIdealState.getRecord().setSimpleFields(currentIdealState.getRecord().getSimpleFields());
newIdealState.setRebalanceMode(RebalanceMode.FULL_AUTO);
newIdealState.getRecord().setListFields(newMapping.getListFields());
- return newIdealState;
- }
-
- /**
- * @return state count map: state->count
- */
- private LinkedHashMap<String, Integer> stateCount(StateModelDefinition stateModelDef,
- int liveNodesNb, int totalReplicas) {
- LinkedHashMap<String, Integer> stateCountMap = new LinkedHashMap<String, Integer>();
- List<String> statesPriorityList = stateModelDef.getStatesPriorityList();
-
- int replicas = totalReplicas;
- for (String state : statesPriorityList) {
- String num = stateModelDef.getNumInstancesPerState(state);
- if ("N".equals(num)) {
- stateCountMap.put(state, liveNodesNb);
- } else if ("R".equals(num)) {
- // wait until we get the counts for all other states
- continue;
- } else {
- int stateCount = -1;
- try {
- stateCount = Integer.parseInt(num);
- } catch (Exception e) {
- // LOG.error("Invalid count for state: " + state + ", count: " + num +
- // ", use -1 instead");
- }
- if (stateCount > 0) {
- stateCountMap.put(state, stateCount);
- replicas -= stateCount;
- }
- }
+ // compute a full partition mapping for the resource
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Processing resource:" + resource.getResourceName());
}
-
- // get state count for R
- for (String state : statesPriorityList) {
- String num = stateModelDef.getNumInstancesPerState(state);
- if ("R".equals(num)) {
- stateCountMap.put(state, replicas);
- // should have at most one state using R
- break;
- }
+ ResourceAssignment partitionMapping = new ResourceAssignment(resource.getResourceName());
+ for (String partitionName : partitions) {
+ Partition partition = new Partition(partitionName);
+ Map<String, String> currentStateMap =
+ currentStateOutput.getCurrentStateMap(resource.getResourceName(), partition);
+ Set<String> disabledInstancesForPartition =
+ clusterData.getDisabledInstancesForPartition(partition.toString());
+ List<String> preferenceList =
+ ConstraintBasedAssignment.getPreferenceList(clusterData, partition, newIdealState,
+ stateModelDef);
+ Map<String, String> bestStateForPartition =
+ ConstraintBasedAssignment.computeAutoBestStateForPartition(clusterData, stateModelDef,
+ preferenceList, currentStateMap, disabledInstancesForPartition);
+ partitionMapping.addReplicaMap(partition, bestStateForPartition);
}
- return stateCountMap;
+ return partitionMapping;
}
private Map<String, Map<String, String>> currentMapping(CurrentStateOutput currentStateOutput,
@@ -186,211 +179,4 @@ public class AutoRebalancer implements Rebalancer {
}
return map;
}
-
- @Override
- public ResourceMapping computeBestPossiblePartitionState(ClusterDataCache cache,
- IdealState idealState, Resource resource, CurrentStateOutput currentStateOutput) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Processing resource:" + resource.getResourceName());
- }
- String stateModelDefName = idealState.getStateModelDefRef();
- StateModelDefinition stateModelDef = cache.getStateModelDef(stateModelDefName);
- calculateAutoBalancedIdealState(cache, idealState, stateModelDef);
- ResourceMapping partitionMapping = new ResourceMapping();
- for (Partition partition : resource.getPartitions()) {
- Map<String, String> currentStateMap =
- currentStateOutput.getCurrentStateMap(resource.getResourceName(), partition);
- Set<String> disabledInstancesForPartition =
- cache.getDisabledInstancesForPartition(partition.toString());
- List<String> preferenceList =
- ConstraintBasedAssignment.getPreferenceList(cache, partition, idealState, stateModelDef);
- Map<String, String> bestStateForPartition =
- ConstraintBasedAssignment.computeAutoBestStateForPartition(cache, stateModelDef,
- preferenceList, currentStateMap, disabledInstancesForPartition);
- partitionMapping.addReplicaMap(partition, bestStateForPartition);
- }
- return partitionMapping;
- }
-
- /**
- * Compute best state for resource in AUTO_REBALANCE ideal state mode. the algorithm
- * will make sure that the master partition are evenly distributed; Also when instances
- * are added / removed, the amount of diff in master partitions are minimized
- * @param cache
- * @param idealState
- * @param instancePreferenceList
- * @param stateModelDef
- * @param currentStateOutput
- * @return
- */
- private void calculateAutoBalancedIdealState(ClusterDataCache cache, IdealState idealState,
- StateModelDefinition stateModelDef) {
- String topStateValue = stateModelDef.getStatesPriorityList().get(0);
- Set<String> liveInstances = cache.getLiveInstances().keySet();
- Set<String> taggedInstances = new HashSet<String>();
-
- // If there are instances tagged with resource name, use only those instances
- if (idealState.getInstanceGroupTag() != null) {
- for (String instanceName : liveInstances) {
- if (cache.getInstanceConfigMap().get(instanceName)
- .containsTag(idealState.getInstanceGroupTag())) {
- taggedInstances.add(instanceName);
- }
- }
- }
- if (taggedInstances.size() > 0) {
- if (LOG.isInfoEnabled()) {
- LOG.info("found the following instances with tag " + idealState.getResourceName() + " "
- + taggedInstances);
- }
- liveInstances = taggedInstances;
- }
- // Obtain replica number
- int replicas = 1;
- try {
- replicas = Integer.parseInt(idealState.getReplicas());
- } catch (Exception e) {
- LOG.error("", e);
- }
- // Init for all partitions with empty list
- Map<String, List<String>> defaultListFields = new TreeMap<String, List<String>>();
- List<String> emptyList = new ArrayList<String>(0);
- for (String partition : idealState.getPartitionSet()) {
- defaultListFields.put(partition, emptyList);
- }
- idealState.getRecord().setListFields(defaultListFields);
- // Return if no live instance
- if (liveInstances.size() == 0) {
- if (LOG.isInfoEnabled()) {
- LOG.info("No live instances, return. Idealstate : " + idealState.getResourceName());
- }
- return;
- }
- Map<String, List<String>> masterAssignmentMap = new HashMap<String, List<String>>();
- for (String instanceName : liveInstances) {
- masterAssignmentMap.put(instanceName, new ArrayList<String>());
- }
- Set<String> orphanedPartitions = new HashSet<String>();
- orphanedPartitions.addAll(idealState.getPartitionSet());
- // Go through all current states and fill the assignments
- for (String liveInstanceName : liveInstances) {
- CurrentState currentState =
- cache.getCurrentState(liveInstanceName,
- cache.getLiveInstances().get(liveInstanceName).getSessionId())
- .get(idealState.getId());
- if (currentState != null) {
- Map<String, String> partitionStates = currentState.getPartitionStateMap();
- for (String partitionName : partitionStates.keySet()) {
- String state = partitionStates.get(partitionName);
- if (state.equals(topStateValue)) {
- masterAssignmentMap.get(liveInstanceName).add(partitionName);
- orphanedPartitions.remove(partitionName);
- }
- }
- }
- }
- List<String> orphanedPartitionsList = new ArrayList<String>();
- orphanedPartitionsList.addAll(orphanedPartitions);
- int maxPartitionsPerInstance = idealState.getMaxPartitionsPerInstance();
- normalizeAssignmentMap(masterAssignmentMap, orphanedPartitionsList, maxPartitionsPerInstance);
- idealState.getRecord().setListFields(
- generateListFieldFromMasterAssignment(masterAssignmentMap, replicas));
- }
-
- /**
- * Given the current master assignment map and the partitions not hosted, generate an
- * evenly distributed partition assignment map
- * @param masterAssignmentMap
- * current master assignment map
- * @param orphanPartitions
- * partitions not hosted by any instance
- * @return
- */
- private void normalizeAssignmentMap(Map<String, List<String>> masterAssignmentMap,
- List<String> orphanPartitions, int maxPartitionsPerInstance) {
- int totalPartitions = 0;
- String[] instanceNames = new String[masterAssignmentMap.size()];
- masterAssignmentMap.keySet().toArray(instanceNames);
- Arrays.sort(instanceNames);
- // Find out total partition number
- for (String key : masterAssignmentMap.keySet()) {
- totalPartitions += masterAssignmentMap.get(key).size();
- Collections.sort(masterAssignmentMap.get(key));
- }
- totalPartitions += orphanPartitions.size();
-
- // Find out how many partitions an instance should host
- int partitionNumber = totalPartitions / masterAssignmentMap.size();
- int leave = totalPartitions % masterAssignmentMap.size();
-
- for (int i = 0; i < instanceNames.length; i++) {
- int targetPartitionNo = leave > 0 ? (partitionNumber + 1) : partitionNumber;
- leave--;
- // For hosts that has more partitions, move those partitions to "orphaned"
- while (masterAssignmentMap.get(instanceNames[i]).size() > targetPartitionNo) {
- int lastElementIndex = masterAssignmentMap.get(instanceNames[i]).size() - 1;
- orphanPartitions.add(masterAssignmentMap.get(instanceNames[i]).get(lastElementIndex));
- masterAssignmentMap.get(instanceNames[i]).remove(lastElementIndex);
- }
- }
- leave = totalPartitions % masterAssignmentMap.size();
- Collections.sort(orphanPartitions);
- // Assign "orphaned" partitions to hosts that do not have enough partitions
- for (int i = 0; i < instanceNames.length; i++) {
- int targetPartitionNo = leave > 0 ? (partitionNumber + 1) : partitionNumber;
- leave--;
- if (targetPartitionNo > maxPartitionsPerInstance) {
- targetPartitionNo = maxPartitionsPerInstance;
- }
- while (masterAssignmentMap.get(instanceNames[i]).size() < targetPartitionNo) {
- int lastElementIndex = orphanPartitions.size() - 1;
- masterAssignmentMap.get(instanceNames[i]).add(orphanPartitions.get(lastElementIndex));
- orphanPartitions.remove(lastElementIndex);
- }
- }
- if (orphanPartitions.size() > 0) {
- LOG.warn("orphanPartitions still contains elements");
- }
- }
-
- /**
- * Generate full preference list from the master assignment map evenly distribute the
- * slave partitions mastered on a host to other hosts
- * @param masterAssignmentMap
- * current master assignment map
- * @param orphanPartitions
- * partitions not hosted by any instance
- * @return
- */
- private Map<String, List<String>> generateListFieldFromMasterAssignment(
- Map<String, List<String>> masterAssignmentMap, int replicas) {
- Map<String, List<String>> listFields = new HashMap<String, List<String>>();
- int slaves = replicas - 1;
- String[] instanceNames = new String[masterAssignmentMap.size()];
- masterAssignmentMap.keySet().toArray(instanceNames);
- Arrays.sort(instanceNames);
-
- for (int i = 0; i < instanceNames.length; i++) {
- String instanceName = instanceNames[i];
- List<String> otherInstances = new ArrayList<String>(masterAssignmentMap.size() - 1);
- for (int x = 0; x < instanceNames.length - 1; x++) {
- int index = (x + i + 1) % instanceNames.length;
- otherInstances.add(instanceNames[index]);
- }
-
- List<String> partitionList = masterAssignmentMap.get(instanceName);
- for (int j = 0; j < partitionList.size(); j++) {
- String partitionName = partitionList.get(j);
- listFields.put(partitionName, new ArrayList<String>());
- listFields.get(partitionName).add(instanceName);
-
- int slavesCanAssign = Math.min(slaves, otherInstances.size());
- for (int k = 0; k < slavesCanAssign; k++) {
- int index = (j + k + 1) % otherInstances.size();
- listFields.get(partitionName).add(otherInstances.get(index));
- }
- }
- }
- return listFields;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/d161a4ec/helix-core/src/main/java/org/apache/helix/controller/rebalancer/CustomRebalancer.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/CustomRebalancer.java b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/CustomRebalancer.java
index 17dc5c8..8557fa0 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/CustomRebalancer.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/CustomRebalancer.java
@@ -27,11 +27,11 @@ import org.apache.helix.HelixDefinedState;
import org.apache.helix.HelixManager;
import org.apache.helix.controller.stages.ClusterDataCache;
import org.apache.helix.controller.stages.CurrentStateOutput;
-import org.apache.helix.controller.stages.ResourceMapping;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.LiveInstance;
import org.apache.helix.model.Partition;
import org.apache.helix.model.Resource;
+import org.apache.helix.model.ResourceAssignment;
import org.apache.helix.model.StateModelDefinition;
import org.apache.log4j.Logger;
@@ -53,29 +53,23 @@ public class CustomRebalancer implements Rebalancer {
}
@Override
- public IdealState computeNewIdealState(String resourceName, IdealState currentIdealState,
+ public ResourceAssignment computeResourceMapping(Resource resource, IdealState currentIdealState,
CurrentStateOutput currentStateOutput, ClusterDataCache clusterData) {
- return currentIdealState;
- }
-
- @Override
- public ResourceMapping computeBestPossiblePartitionState(ClusterDataCache cache,
- IdealState idealState, Resource resource, CurrentStateOutput currentStateOutput) {
- String stateModelDefName = idealState.getStateModelDefRef();
- StateModelDefinition stateModelDef = cache.getStateModelDef(stateModelDefName);
+ String stateModelDefName = currentIdealState.getStateModelDefRef();
+ StateModelDefinition stateModelDef = clusterData.getStateModelDef(stateModelDefName);
if (LOG.isDebugEnabled()) {
LOG.debug("Processing resource:" + resource.getResourceName());
}
- ResourceMapping partitionMapping = new ResourceMapping();
+ ResourceAssignment partitionMapping = new ResourceAssignment(resource.getResourceName());
for (Partition partition : resource.getPartitions()) {
Map<String, String> currentStateMap =
currentStateOutput.getCurrentStateMap(resource.getResourceName(), partition);
Set<String> disabledInstancesForPartition =
- cache.getDisabledInstancesForPartition(partition.toString());
+ clusterData.getDisabledInstancesForPartition(partition.toString());
Map<String, String> idealStateMap =
- idealState.getInstanceStateMap(partition.getPartitionName());
+ currentIdealState.getInstanceStateMap(partition.getPartitionName());
Map<String, String> bestStateForPartition =
- computeCustomizedBestStateForPartition(cache, stateModelDef, idealStateMap,
+ computeCustomizedBestStateForPartition(clusterData, stateModelDef, idealStateMap,
currentStateMap, disabledInstancesForPartition);
partitionMapping.addReplicaMap(partition, bestStateForPartition);
}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/d161a4ec/helix-core/src/main/java/org/apache/helix/controller/rebalancer/Rebalancer.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/Rebalancer.java b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/Rebalancer.java
index a0cfbb7..e4f165b 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/Rebalancer.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/Rebalancer.java
@@ -22,9 +22,9 @@ package org.apache.helix.controller.rebalancer;
import org.apache.helix.HelixManager;
import org.apache.helix.controller.stages.ClusterDataCache;
import org.apache.helix.controller.stages.CurrentStateOutput;
-import org.apache.helix.controller.stages.ResourceMapping;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.Resource;
+import org.apache.helix.model.ResourceAssignment;
/**
* Allows one to come up with custom implementation of a rebalancer.<br/>
@@ -32,33 +32,24 @@ import org.apache.helix.model.Resource;
* Simply return the newIdealState for a resource in this method.<br/>
*/
public interface Rebalancer {
- void init(HelixManager manager);
-
/**
- * This method provides all the relevant information needed to rebalance a resource.
- * If you need additional information use manager.getAccessor to read the cluster data.
- * This allows one to compute the newIdealState according to app specific requirement.
- * @param resourceName Name of the resource to be rebalanced
- * @param currentIdealState
- * @param currentStateOutput
- * Provides the current state and pending state transition for all
- * partitions
- * @param clusterData Provides additional methods to retrieve cluster data.
- * @return
+ * Initialize the rebalancer with a HelixManager if necessary
+ * @param manager
*/
- IdealState computeNewIdealState(String resourceName, IdealState currentIdealState,
- final CurrentStateOutput currentStateOutput, final ClusterDataCache clusterData);
+ void init(HelixManager manager);
/**
- * Given an ideal state for a resource and the liveness of instances, compute the best possible
- * state assignment for each partition's replicas.
- * @param cache
- * @param idealState
- * @param resource
- * @param currentStateOutput
- * Provides the current state and pending state transitions for all partitions
- * @return
+ * Given an ideal state for a resource and liveness of instances, compute a assignment of
+ * instances and states to each partition of a resource. This method provides all the relevant
+ * information needed to rebalance a resource. If you need additional information use
+ * manager.getAccessor to read the cluster data. This allows one to compute the newIdealState
+ * according to app specific requirements.
+ * @param resourceName the resource for which a mapping will be computed
+ * @param currentIdealState the IdealState that corresponds to this resource
+ * @param currentStateOutput the current states of all partitions
+ * @param clusterData cache of the cluster state
*/
- ResourceMapping computeBestPossiblePartitionState(ClusterDataCache cache, IdealState idealState,
- Resource resource, CurrentStateOutput currentStateOutput);
+ ResourceAssignment computeResourceMapping(final Resource resource,
+ final IdealState currentIdealState, final CurrentStateOutput currentStateOutput,
+ final ClusterDataCache clusterData);
}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/d161a4ec/helix-core/src/main/java/org/apache/helix/controller/rebalancer/SemiAutoRebalancer.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/SemiAutoRebalancer.java b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/SemiAutoRebalancer.java
index bc682ff..b096817 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/SemiAutoRebalancer.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/SemiAutoRebalancer.java
@@ -27,10 +27,10 @@ import org.apache.helix.HelixManager;
import org.apache.helix.controller.rebalancer.util.ConstraintBasedAssignment;
import org.apache.helix.controller.stages.ClusterDataCache;
import org.apache.helix.controller.stages.CurrentStateOutput;
-import org.apache.helix.controller.stages.ResourceMapping;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.Partition;
import org.apache.helix.model.Resource;
+import org.apache.helix.model.ResourceAssignment;
import org.apache.helix.model.StateModelDefinition;
import org.apache.log4j.Logger;
@@ -52,29 +52,24 @@ public class SemiAutoRebalancer implements Rebalancer {
}
@Override
- public IdealState computeNewIdealState(String resourceName, IdealState currentIdealState,
+ public ResourceAssignment computeResourceMapping(Resource resource, IdealState currentIdealState,
CurrentStateOutput currentStateOutput, ClusterDataCache clusterData) {
- return currentIdealState;
- }
-
- @Override
- public ResourceMapping computeBestPossiblePartitionState(ClusterDataCache cache,
- IdealState idealState, Resource resource, CurrentStateOutput currentStateOutput) {
- String stateModelDefName = idealState.getStateModelDefRef();
- StateModelDefinition stateModelDef = cache.getStateModelDef(stateModelDefName);
+ String stateModelDefName = currentIdealState.getStateModelDefRef();
+ StateModelDefinition stateModelDef = clusterData.getStateModelDef(stateModelDefName);
if (LOG.isDebugEnabled()) {
LOG.debug("Processing resource:" + resource.getResourceName());
}
- ResourceMapping partitionMapping = new ResourceMapping();
+ ResourceAssignment partitionMapping = new ResourceAssignment(resource.getResourceName());
for (Partition partition : resource.getPartitions()) {
Map<String, String> currentStateMap =
currentStateOutput.getCurrentStateMap(resource.getResourceName(), partition);
Set<String> disabledInstancesForPartition =
- cache.getDisabledInstancesForPartition(partition.toString());
+ clusterData.getDisabledInstancesForPartition(partition.toString());
List<String> preferenceList =
- ConstraintBasedAssignment.getPreferenceList(cache, partition, idealState, stateModelDef);
+ ConstraintBasedAssignment.getPreferenceList(clusterData, partition, currentIdealState,
+ stateModelDef);
Map<String, String> bestStateForPartition =
- ConstraintBasedAssignment.computeAutoBestStateForPartition(cache, stateModelDef,
+ ConstraintBasedAssignment.computeAutoBestStateForPartition(clusterData, stateModelDef,
preferenceList, currentStateMap, disabledInstancesForPartition);
partitionMapping.addReplicaMap(partition, bestStateForPartition);
}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/d161a4ec/helix-core/src/main/java/org/apache/helix/controller/rebalancer/util/ConstraintBasedAssignment.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/util/ConstraintBasedAssignment.java b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/util/ConstraintBasedAssignment.java
index 3fd52f4..d2dbdef 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/util/ConstraintBasedAssignment.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/util/ConstraintBasedAssignment.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -139,4 +140,52 @@ public class ConstraintBasedAssignment {
}
return instanceStateMap;
}
+
+ /**
+ * Get the number of replicas that should be in each state for a partition
+ * @param stateModelDef StateModelDefinition object
+ * @param liveNodesNb number of live nodes
+ * @param total number of replicas
+ * @return state count map: state->count
+ */
+ public static LinkedHashMap<String, Integer> stateCount(StateModelDefinition stateModelDef,
+ int liveNodesNb, int totalReplicas) {
+ LinkedHashMap<String, Integer> stateCountMap = new LinkedHashMap<String, Integer>();
+ List<String> statesPriorityList = stateModelDef.getStatesPriorityList();
+
+ int replicas = totalReplicas;
+ for (String state : statesPriorityList) {
+ String num = stateModelDef.getNumInstancesPerState(state);
+ if ("N".equals(num)) {
+ stateCountMap.put(state, liveNodesNb);
+ } else if ("R".equals(num)) {
+ // wait until we get the counts for all other states
+ continue;
+ } else {
+ int stateCount = -1;
+ try {
+ stateCount = Integer.parseInt(num);
+ } catch (Exception e) {
+ // LOG.error("Invalid count for state: " + state + ", count: " + num +
+ // ", use -1 instead");
+ }
+
+ if (stateCount > 0) {
+ stateCountMap.put(state, stateCount);
+ replicas -= stateCount;
+ }
+ }
+ }
+
+ // get state count for R
+ for (String state : statesPriorityList) {
+ String num = stateModelDef.getNumInstancesPerState(state);
+ if ("R".equals(num)) {
+ stateCountMap.put(state, replicas);
+ // should have at most one state using R
+ break;
+ }
+ }
+ return stateCountMap;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/d161a4ec/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java b/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java
index 598c318..e812e16 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/BestPossibleStateCalcStage.java
@@ -21,6 +21,7 @@ package org.apache.helix.controller.stages;
import java.util.Map;
+import org.apache.helix.HelixManager;
import org.apache.helix.controller.pipeline.AbstractBaseStage;
import org.apache.helix.controller.pipeline.StageException;
import org.apache.helix.controller.rebalancer.AutoRebalancer;
@@ -28,9 +29,10 @@ import org.apache.helix.controller.rebalancer.CustomRebalancer;
import org.apache.helix.controller.rebalancer.Rebalancer;
import org.apache.helix.controller.rebalancer.SemiAutoRebalancer;
import org.apache.helix.model.IdealState;
+import org.apache.helix.model.IdealState.RebalanceMode;
import org.apache.helix.model.Partition;
import org.apache.helix.model.Resource;
-import org.apache.helix.model.IdealState.RebalanceMode;
+import org.apache.helix.model.ResourceAssignment;
import org.apache.helix.util.HelixUtil;
import org.apache.log4j.Logger;
@@ -112,11 +114,12 @@ public class BestPossibleStateCalcStage extends AbstractBaseStage {
}
}
- ResourceMapping partitionStateAssignment =
- rebalancer.computeBestPossiblePartitionState(cache, idealState, resource,
- currentStateOutput);
+ HelixManager manager = event.getAttribute("helixmanager");
+ rebalancer.init(manager);
+ ResourceAssignment partitionStateAssignment =
+ rebalancer.computeResourceMapping(resource, idealState, currentStateOutput, cache);
for (Partition partition : resource.getPartitions()) {
- Map<String, String> newStateMap = partitionStateAssignment.getInstanceStateMap(partition);
+ Map<String, String> newStateMap = partitionStateAssignment.getReplicaMap(partition);
output.setState(resourceName, partition, newStateMap);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/d161a4ec/helix-core/src/main/java/org/apache/helix/controller/stages/RebalanceIdealStateStage.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/RebalanceIdealStateStage.java b/helix-core/src/main/java/org/apache/helix/controller/stages/RebalanceIdealStateStage.java
index cf1633c..d82ee2f 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/RebalanceIdealStateStage.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/stages/RebalanceIdealStateStage.java
@@ -27,6 +27,8 @@ import org.apache.helix.controller.pipeline.AbstractBaseStage;
import org.apache.helix.controller.rebalancer.Rebalancer;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.IdealState.RebalanceMode;
+import org.apache.helix.model.Resource;
+import org.apache.helix.model.ResourceAssignment;
import org.apache.helix.util.HelixUtil;
import org.apache.log4j.Logger;
@@ -57,10 +59,15 @@ public class RebalanceIdealStateStage extends AbstractBaseStage {
Rebalancer balancer =
(Rebalancer) (HelixUtil.loadClass(getClass(), rebalancerClassName).newInstance());
balancer.init(manager);
- IdealState newIdealState =
- balancer.computeNewIdealState(resourceName, idealStateMap.get(resourceName),
- currentStateOutput, cache);
- updatedIdealStates.put(resourceName, newIdealState);
+ Resource resource = new Resource(resourceName);
+ for (String partitionName : currentIdealState.getPartitionSet()) {
+ resource.addPartition(partitionName);
+ }
+ ResourceAssignment resourceAssignment =
+ balancer.computeResourceMapping(resource, currentIdealState, currentStateOutput,
+ cache);
+ currentIdealState.updateFromAssignment(resourceAssignment);
+ updatedIdealStates.put(resourceName, currentIdealState);
} catch (Exception e) {
LOG.error("Exception while invoking custom rebalancer class:" + rebalancerClassName, e);
}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/d161a4ec/helix-core/src/main/java/org/apache/helix/controller/stages/ResourceMapping.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/stages/ResourceMapping.java b/helix-core/src/main/java/org/apache/helix/controller/stages/ResourceMapping.java
deleted file mode 100644
index 2609791..0000000
--- a/helix-core/src/main/java/org/apache/helix/controller/stages/ResourceMapping.java
+++ /dev/null
@@ -1,58 +0,0 @@
-package org.apache.helix.controller.stages;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.helix.model.Partition;
-
-/**
- * Represents the assignments of replicas for an entire resource, keyed on partitions of the
- * resource. Each partition has its replicas assigned to a node, and each replica is in a state.
- */
-public class ResourceMapping {
-
- private final Map<Partition, Map<String, String>> _resourceMap;
-
- public ResourceMapping() {
- this(new HashMap<Partition, Map<String, String>>());
- }
-
- public ResourceMapping(Map<Partition, Map<String, String>> resourceMap) {
- _resourceMap = resourceMap;
- }
-
- public Map<Partition, Map<String, String>> getResourceMap() {
- return _resourceMap;
- }
-
- public Map<String, String> getInstanceStateMap(Partition partition) {
- if (_resourceMap.containsKey(partition)) {
- return _resourceMap.get(partition);
- }
- return Collections.emptyMap();
- }
-
- public void addReplicaMap(Partition partition, Map<String, String> replicaMap) {
- _resourceMap.put(partition, replicaMap);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/d161a4ec/helix-core/src/main/java/org/apache/helix/controller/strategy/AutoRebalanceStrategy.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/strategy/AutoRebalanceStrategy.java b/helix-core/src/main/java/org/apache/helix/controller/strategy/AutoRebalanceStrategy.java
index 76560a4..72046bf 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/strategy/AutoRebalanceStrategy.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/strategy/AutoRebalanceStrategy.java
@@ -24,6 +24,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -54,6 +55,15 @@ public class AutoRebalanceStrategy {
private Map<Replica, Node> _existingNonPreferredAssignment;
private Set<Replica> _orphaned;
+ /**
+ * Initialize this strategy for a resource
+ * @param resourceName the resource for which an assignment will be computed
+ * @param partitions the partition names for the resource
+ * @param states the states and the number of replicas that should be in each state
+ * @param maximumPerNode the maximum number of replicas any note can hold
+ * @param placementScheme the scheme to use for preferred replica locations. If null, this is
+ * {@link DefaultPlacementScheme}
+ */
public AutoRebalanceStrategy(String resourceName, final List<String> partitions,
final LinkedHashMap<String, Integer> states, int maximumPerNode,
ReplicaPlacementScheme placementScheme) {
@@ -68,11 +78,22 @@ public class AutoRebalanceStrategy {
}
}
+ /**
+ * Initialize the strategy with a default placement scheme and no
+ * @see #AutoRebalanceStrategy(String, List, LinkedHashMap, int, ReplicaPlacementScheme)
+ */
public AutoRebalanceStrategy(String resourceName, final List<String> partitions,
final LinkedHashMap<String, Integer> states) {
this(resourceName, partitions, states, Integer.MAX_VALUE, new DefaultPlacementScheme());
}
+ /**
+ * Determine a preference list and mapping of partitions to nodes for all replicas
+ * @param liveNodes the current list of live participants
+ * @param currentMapping the current assignment of replicas to nodes
+ * @param allNodes the full list of known nodes in the system
+ * @return the preference list and replica mapping
+ */
public ZNRecord computePartitionAssignment(final List<String> liveNodes,
final Map<String, Map<String, String>> currentMapping, final List<String> allNodes) {
int numReplicas = countStateReplicas();
@@ -276,15 +297,6 @@ public class AutoRebalanceStrategy {
znRecord.setMapField(partition, new TreeMap<String, String>());
znRecord.setListField(partition, new ArrayList<String>());
}
- for (Node node : _liveNodesList) {
- for (Replica replica : node.preferred) {
- znRecord.getMapField(replica.partition).put(node.id, _stateMap.get(replica.replicaId));
- }
- for (Replica replica : node.nonPreferred) {
- znRecord.getMapField(replica.partition).put(node.id, _stateMap.get(replica.replicaId));
- }
- }
-
int count = countStateReplicas();
for (int replicaId = 0; replicaId < count; replicaId++) {
for (Node node : _liveNodesList) {
@@ -300,6 +312,93 @@ public class AutoRebalanceStrategy {
}
}
}
+ normalizePreferenceLists(znRecord.getListFields());
+
+ for (Node node : _liveNodesList) {
+ for (Replica replica : node.preferred) {
+ znRecord.getMapField(replica.partition).put(node.id, _stateMap.get(replica.replicaId));
+ }
+ for (Replica replica : node.nonPreferred) {
+ znRecord.getMapField(replica.partition).put(node.id, _stateMap.get(replica.replicaId));
+ }
+ }
+ }
+
+ /**
+ * Adjust preference lists to reduce the number of same replicas on an instance
+ * @param preferenceLists map of (partition --> list of nodes)
+ */
+ private void normalizePreferenceLists(Map<String, List<String>> preferenceLists) {
+ Map<String, Map<Integer, Integer>> nodeReplicaCounts =
+ new HashMap<String, Map<Integer, Integer>>();
+ for (String partition : preferenceLists.keySet()) {
+ normalizePreferenceList(preferenceLists.get(partition), nodeReplicaCounts);
+ }
+ }
+
+ /**
+ * Adjust a single preference list for replica assignment imbalance
+ * @param preferenceList list of node names
+ * @param nodeReplicaCounts map of (node --> replica id --> count)
+ */
+ private void normalizePreferenceList(List<String> preferenceList,
+ Map<String, Map<Integer, Integer>> nodeReplicaCounts) {
+ Set<String> notAssigned = new LinkedHashSet<String>(preferenceList);
+ List<String> newPreferenceList = new ArrayList<String>();
+ int replicas = Math.min(countStateReplicas(), preferenceList.size());
+ for (int i = 0; i < replicas; i++) {
+ String node = getMinimumNodeForReplica(i, notAssigned, nodeReplicaCounts);
+ newPreferenceList.add(node);
+ notAssigned.remove(node);
+ Map<Integer, Integer> counts = nodeReplicaCounts.get(node);
+ counts.put(i, counts.get(i) + 1);
+ }
+ preferenceList.clear();
+ preferenceList.addAll(newPreferenceList);
+ }
+
+ /**
+ * Get the node which hosts the fewest of a given replica
+ * @param replicaId the replica
+ * @param nodes nodes to check
+ * @param nodeReplicaCounts current assignment of replicas
+ * @return the node most willing to accept the replica
+ */
+ private String getMinimumNodeForReplica(int replicaId, Set<String> nodes,
+ Map<String, Map<Integer, Integer>> nodeReplicaCounts) {
+ String minimalNode = null;
+ int minimalCount = Integer.MAX_VALUE;
+ for (String node : nodes) {
+ int count = getReplicaCountForNode(replicaId, node, nodeReplicaCounts);
+ if (count < minimalCount) {
+ minimalCount = count;
+ minimalNode = node;
+ }
+ }
+ return minimalNode;
+ }
+
+ /**
+ * Safe check for the number of replicas of a given id assiged to a node
+ * @param replicaId the replica to assign
+ * @param node the node to check
+ * @param nodeReplicaCounts a map of node to replica id and counts
+ * @return the number of currently assigned replicas of the given id
+ */
+ private int getReplicaCountForNode(int replicaId, String node,
+ Map<String, Map<Integer, Integer>> nodeReplicaCounts) {
+ if (!nodeReplicaCounts.containsKey(node)) {
+ Map<Integer, Integer> replicaCounts = new HashMap<Integer, Integer>();
+ replicaCounts.put(replicaId, 0);
+ nodeReplicaCounts.put(node, replicaCounts);
+ return 0;
+ }
+ Map<Integer, Integer> replicaCounts = nodeReplicaCounts.get(node);
+ if (!replicaCounts.containsKey(replicaId)) {
+ replicaCounts.put(replicaId, 0);
+ return 0;
+ }
+ return replicaCounts.get(replicaId);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/d161a4ec/helix-core/src/main/java/org/apache/helix/model/IdealState.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/model/IdealState.java b/helix-core/src/main/java/org/apache/helix/model/IdealState.java
index 463369a..59bc59c 100644
--- a/helix-core/src/main/java/org/apache/helix/model/IdealState.java
+++ b/helix-core/src/main/java/org/apache/helix/model/IdealState.java
@@ -32,6 +32,8 @@ import org.apache.helix.HelixProperty;
import org.apache.helix.ZNRecord;
import org.apache.log4j.Logger;
+import org.apache.helix.controller.rebalancer.Rebalancer;
+
/**
* The ideal states of all partitions in a resource
*/
@@ -217,6 +219,15 @@ public class IdealState extends HelixProperty {
}
/**
+ * Set the current mapping of a partition
+ * @param partition the partition to set
+ * @param instanceStateMap (instance name, state) pairs
+ */
+ public void setInstanceStateMap(String partition, Map<String, String> instanceStateMap) {
+ _record.setMapField(partition, instanceStateMap);
+ }
+
+ /**
* Get the current mapping of a partition
* @param partitionName the name of the partition
* @return the instances where the replicas live and the state of each
@@ -257,6 +268,15 @@ public class IdealState extends HelixProperty {
}
/**
+ * Set the preference list of a partition
+ * @param partitionName the name of the partition to set
+ * @param preferenceList a list of instances that can serve replicas of the partition
+ */
+ public void setPreferenceList(String partitionName, List<String> preferenceList) {
+ _record.setListField(partitionName, preferenceList);
+ }
+
+ /**
* Get the preference list of a partition
* @param partitionName the name of the partition
* @return a list of instances that can serve replicas of the partition
@@ -443,6 +463,16 @@ public class IdealState extends HelixProperty {
return _record.getSimpleField(IdealStateProperty.INSTANCE_GROUP_TAG.toString());
}
+ public void updateFromAssignment(ResourceAssignment assignment) {
+ _record.getMapFields().clear();
+ _record.getListFields().clear();
+ for (Partition partition : assignment.getMappedPartitions()) {
+ Map<String, String> replicaMap = assignment.getReplicaMap(partition);
+ setInstanceStateMap(partition.getPartitionName(), replicaMap);
+ setPreferenceList(partition.getPartitionName(), new ArrayList<String>(replicaMap.keySet()));
+ }
+ }
+
private RebalanceMode normalizeRebalanceMode(IdealStateModeProperty mode) {
RebalanceMode property;
switch (mode) {
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/d161a4ec/helix-core/src/main/java/org/apache/helix/model/ResourceAssignment.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/model/ResourceAssignment.java b/helix-core/src/main/java/org/apache/helix/model/ResourceAssignment.java
index b0d7f1f..2b3d14d 100644
--- a/helix-core/src/main/java/org/apache/helix/model/ResourceAssignment.java
+++ b/helix-core/src/main/java/org/apache/helix/model/ResourceAssignment.java
@@ -19,6 +19,72 @@ package org.apache.helix.model;
* under the License.
*/
-public class ResourceAssignment {
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import org.apache.helix.HelixProperty;
+
+/**
+ * Represents the assignments of replicas for an entire resource, keyed on partitions of the
+ * resource. Each partition has its replicas assigned to a node, and each replica is in a state.
+ * For example, if there is a partition p with 2 replicas, a valid assignment is:<br />
+ * <br />
+ * p: {(n1, s1), (n2, s2)}<br />
+ * <br />
+ * This means one replica of p is located at node n1 and is in state s1, and another is in node n2
+ * and is in state s2. n1 cannot be equal to n2, but s1 can be equal to s2 if at least two replicas
+ * can be in s1.
+ */
+public class ResourceAssignment extends HelixProperty {
+
+ /**
+ * Initialize an empty mapping
+ * @param resourceName the resource being mapped
+ */
+ public ResourceAssignment(String resourceName) {
+ super(resourceName);
+ }
+
+ /**
+ * Initialize a mapping from an existing ResourceMapping
+ * @param existingMapping pre-populated ResourceMapping
+ */
+ public ResourceAssignment(ResourceAssignment existingMapping) {
+ super(existingMapping);
+ }
+
+ /**
+ * Get the currently mapped partitions
+ * @return list of Partition objects
+ */
+ public List<Partition> getMappedPartitions() {
+ List<Partition> partitions = new ArrayList<Partition>();
+ for (String partitionName : _record.getMapFields().keySet()) {
+ partitions.add(new Partition(partitionName));
+ }
+ return partitions;
+ }
+
+ /**
+ * Get the instance, state pairs for a partition
+ * @param partition the Partition to look up
+ * @return map of (instance name, state)
+ */
+ public Map<String, String> getReplicaMap(Partition partition) {
+ if (_record.getMapFields().containsKey(partition.getPartitionName())) {
+ return _record.getMapField(partition.getPartitionName());
+ }
+ return Collections.emptyMap();
+ }
+
+ /**
+ * Add instance, state pairs for a partition
+ * @param partition the partition to set
+ * @param replicaMap map of (instance name, state)
+ */
+ public void addReplicaMap(Partition partition, Map<String, String> replicaMap) {
+ _record.setMapField(partition.getPartitionName(), replicaMap);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/d161a4ec/helix-core/src/test/java/org/apache/helix/integration/TestCustomizedIdealStateRebalancer.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestCustomizedIdealStateRebalancer.java b/helix-core/src/test/java/org/apache/helix/integration/TestCustomizedIdealStateRebalancer.java
index 7811c0d..70ff6bd 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/TestCustomizedIdealStateRebalancer.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/TestCustomizedIdealStateRebalancer.java
@@ -24,9 +24,9 @@ import java.util.Map;
import org.apache.helix.HelixDataAccessor;
import org.apache.helix.HelixManager;
-import org.apache.helix.ZNRecord;
import org.apache.helix.PropertyKey.Builder;
-import org.apache.helix.controller.rebalancer.SemiAutoRebalancer;
+import org.apache.helix.ZNRecord;
+import org.apache.helix.controller.rebalancer.Rebalancer;
import org.apache.helix.controller.stages.ClusterDataCache;
import org.apache.helix.controller.stages.CurrentStateOutput;
import org.apache.helix.manager.zk.ZKHelixDataAccessor;
@@ -36,6 +36,9 @@ import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.IdealState.IdealStateProperty;
import org.apache.helix.model.IdealState.RebalanceMode;
+import org.apache.helix.model.Partition;
+import org.apache.helix.model.Resource;
+import org.apache.helix.model.ResourceAssignment;
import org.apache.helix.tools.ClusterStateVerifier;
import org.apache.helix.tools.ClusterStateVerifier.ZkVerifier;
import org.testng.Assert;
@@ -46,7 +49,7 @@ public class TestCustomizedIdealStateRebalancer extends
String db2 = TEST_DB + "2";
static boolean testRebalancerCreated = false;
- public static class TestRebalancer extends SemiAutoRebalancer {
+ public static class TestRebalancer implements Rebalancer {
@Override
public void init(HelixManager manager) {
@@ -54,18 +57,22 @@ public class TestCustomizedIdealStateRebalancer extends
}
@Override
- public IdealState computeNewIdealState(String resourceName, IdealState currentIdealState,
+ public ResourceAssignment computeResourceMapping(Resource resource, IdealState currentIdealState,
CurrentStateOutput currentStateOutput, ClusterDataCache clusterData) {
- for (String partition : currentIdealState.getPartitionSet()) {
- String instance = currentIdealState.getPreferenceList(partition).get(0);
- currentIdealState.getPreferenceList(partition).clear();
- currentIdealState.getPreferenceList(partition).add(instance);
-
- currentIdealState.getInstanceStateMap(partition).clear();
- currentIdealState.getInstanceStateMap(partition).put(instance, "MASTER");
+ ResourceAssignment resourceMapping = new ResourceAssignment(resource.getResourceName());
+ for (Partition partition : resource.getPartitions()) {
+ String partitionName = partition.getPartitionName();
+ String instance = currentIdealState.getPreferenceList(partitionName).get(0);
+ currentIdealState.getPreferenceList(partitionName).clear();
+ currentIdealState.getPreferenceList(partitionName).add(instance);
+
+ currentIdealState.getInstanceStateMap(partitionName).clear();
+ currentIdealState.getInstanceStateMap(partitionName).put(instance, "MASTER");
+ resourceMapping.addReplicaMap(partition,
+ currentIdealState.getInstanceStateMap(partitionName));
}
currentIdealState.setReplicas("1");
- return currentIdealState;
+ return resourceMapping;
}
}