You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by zz...@apache.org on 2013/08/21 22:43:39 UTC
[26/51] [partial] [HELIX-198] Unify helix code style, rb=13710
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/f414aad4/helix-core/src/main/java/org/apache/helix/tools/ClusterStateVerifier.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/tools/ClusterStateVerifier.java b/helix-core/src/main/java/org/apache/helix/tools/ClusterStateVerifier.java
index eaada16..7ceee85 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/ClusterStateVerifier.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/ClusterStateVerifier.java
@@ -67,74 +67,59 @@ import org.apache.helix.store.PropertyJsonSerializer;
import org.apache.helix.util.ZKClientPool;
import org.apache.log4j.Logger;
+public class ClusterStateVerifier {
+ public static String cluster = "cluster";
+ public static String zkServerAddress = "zkSvr";
+ public static String help = "help";
+ public static String timeout = "timeout";
+ public static String period = "period";
-public class ClusterStateVerifier
-{
- public static String cluster = "cluster";
- public static String zkServerAddress = "zkSvr";
- public static String help = "help";
- public static String timeout = "timeout";
- public static String period = "period";
+ private static Logger LOG = Logger.getLogger(ClusterStateVerifier.class);
- private static Logger LOG = Logger.getLogger(ClusterStateVerifier.class);
-
- public interface Verifier
- {
+ public interface Verifier {
boolean verify();
}
- public interface ZkVerifier extends Verifier
- {
+ public interface ZkVerifier extends Verifier {
ZkClient getZkClient();
String getClusterName();
}
- static class ExtViewVeriferZkListener implements IZkChildListener, IZkDataListener
- {
+ static class ExtViewVeriferZkListener implements IZkChildListener, IZkDataListener {
final CountDownLatch _countDown;
- final ZkClient _zkClient;
- final Verifier _verifier;
+ final ZkClient _zkClient;
+ final Verifier _verifier;
- public ExtViewVeriferZkListener(CountDownLatch countDown,
- ZkClient zkClient,
- ZkVerifier verifier)
- {
+ public ExtViewVeriferZkListener(CountDownLatch countDown, ZkClient zkClient, ZkVerifier verifier) {
_countDown = countDown;
_zkClient = zkClient;
_verifier = verifier;
}
@Override
- public void handleDataChange(String dataPath, Object data) throws Exception
- {
+ public void handleDataChange(String dataPath, Object data) throws Exception {
boolean result = _verifier.verify();
- if (result == true)
- {
+ if (result == true) {
_countDown.countDown();
}
}
@Override
- public void handleDataDeleted(String dataPath) throws Exception
- {
+ public void handleDataDeleted(String dataPath) throws Exception {
// TODO Auto-generated method stub
}
@Override
- public void handleChildChange(String parentPath, List<String> currentChilds) throws Exception
- {
- for (String child : currentChilds)
- {
- String childPath =
- parentPath.equals("/") ? parentPath + child : parentPath + "/" + child;
+ public void handleChildChange(String parentPath, List<String> currentChilds) throws Exception {
+ for (String child : currentChilds) {
+ String childPath = parentPath.equals("/") ? parentPath + child : parentPath + "/" + child;
_zkClient.subscribeDataChanges(childPath, this);
}
boolean result = _verifier.verify();
- if (result == true)
- {
+ if (result == true) {
_countDown.countDown();
}
}
@@ -144,24 +129,19 @@ public class ClusterStateVerifier
/**
* verifier that verifies best possible state and external view
*/
- public static class BestPossAndExtViewZkVerifier implements ZkVerifier
- {
- private final String zkAddr;
- private final String clusterName;
+ public static class BestPossAndExtViewZkVerifier implements ZkVerifier {
+ private final String zkAddr;
+ private final String clusterName;
private final Map<String, Map<String, String>> errStates;
- private final ZkClient zkClient;
+ private final ZkClient zkClient;
- public BestPossAndExtViewZkVerifier(String zkAddr, String clusterName)
- {
+ public BestPossAndExtViewZkVerifier(String zkAddr, String clusterName) {
this(zkAddr, clusterName, null);
}
- public BestPossAndExtViewZkVerifier(String zkAddr,
- String clusterName,
- Map<String, Map<String, String>> errStates)
- {
- if (zkAddr == null || clusterName == null)
- {
+ public BestPossAndExtViewZkVerifier(String zkAddr, String clusterName,
+ Map<String, Map<String, String>> errStates) {
+ if (zkAddr == null || clusterName == null) {
throw new IllegalArgumentException("requires zkAddr|clusterName");
}
this.zkAddr = zkAddr;
@@ -171,38 +151,30 @@ public class ClusterStateVerifier
}
@Override
- public boolean verify()
- {
- try
- {
+ public boolean verify() {
+ try {
HelixDataAccessor accessor =
- new ZKHelixDataAccessor(clusterName,
- new ZkBaseDataAccessor<ZNRecord>(zkClient));
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(zkClient));
return ClusterStateVerifier.verifyBestPossAndExtView(accessor, errStates);
- }
- catch (Exception e)
- {
+ } catch (Exception e) {
LOG.error("exception in verification", e);
}
return false;
}
@Override
- public ZkClient getZkClient()
- {
+ public ZkClient getZkClient() {
return zkClient;
}
@Override
- public String getClusterName()
- {
+ public String getClusterName() {
return clusterName;
}
@Override
- public String toString()
- {
+ public String toString() {
String verifierName = getClass().getName();
verifierName =
verifierName.substring(verifierName.lastIndexOf('.') + 1, verifierName.length());
@@ -210,16 +182,13 @@ public class ClusterStateVerifier
}
}
- public static class MasterNbInExtViewVerifier implements ZkVerifier
- {
- private final String zkAddr;
- private final String clusterName;
+ public static class MasterNbInExtViewVerifier implements ZkVerifier {
+ private final String zkAddr;
+ private final String clusterName;
private final ZkClient zkClient;
- public MasterNbInExtViewVerifier(String zkAddr, String clusterName)
- {
- if (zkAddr == null || clusterName == null)
- {
+ public MasterNbInExtViewVerifier(String zkAddr, String clusterName) {
+ if (zkAddr == null || clusterName == null) {
throw new IllegalArgumentException("requires zkAddr|clusterName");
}
this.zkAddr = zkAddr;
@@ -228,86 +197,71 @@ public class ClusterStateVerifier
}
@Override
- public boolean verify()
- {
- try
- {
+ public boolean verify() {
+ try {
ZKHelixDataAccessor accessor =
- new ZKHelixDataAccessor(clusterName,
- new ZkBaseDataAccessor<ZNRecord>(zkClient));
+ new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(zkClient));
return ClusterStateVerifier.verifyMasterNbInExtView(accessor);
- }
- catch (Exception e)
- {
+ } catch (Exception e) {
LOG.error("exception in verification", e);
}
return false;
}
@Override
- public ZkClient getZkClient()
- {
+ public ZkClient getZkClient() {
return zkClient;
}
@Override
- public String getClusterName()
- {
+ public String getClusterName() {
return clusterName;
}
}
static boolean verifyBestPossAndExtView(HelixDataAccessor accessor,
- Map<String, Map<String, String>> errStates)
- {
- try
- {
+ Map<String, Map<String, String>> errStates) {
+ try {
Builder keyBuilder = accessor.keyBuilder();
// read cluster once and do verification
ClusterDataCache cache = new ClusterDataCache();
cache.refresh(accessor);
Map<String, IdealState> idealStates = cache.getIdealStates();
- if (idealStates == null) // || idealStates.isEmpty())
+ if (idealStates == null) // || idealStates.isEmpty())
{
// ideal state is null because ideal state is dropped
idealStates = Collections.emptyMap();
}
- Map<String, ExternalView> extViews =
- accessor.getChildValuesMap(keyBuilder.externalViews());
+ Map<String, ExternalView> extViews = accessor.getChildValuesMap(keyBuilder.externalViews());
if (extViews == null) // || extViews.isEmpty())
{
extViews = Collections.emptyMap();
}
// if externalView is not empty and idealState doesn't exist
- // add empty idealState for the resource
- for (String resource : extViews.keySet())
- {
- if (!idealStates.containsKey(resource))
- {
+ // add empty idealState for the resource
+ for (String resource : extViews.keySet()) {
+ if (!idealStates.containsKey(resource)) {
idealStates.put(resource, new IdealState(resource));
}
}
-
+
// calculate best possible state
- BestPossibleStateOutput bestPossOutput =
- ClusterStateVerifier.calcBestPossState(cache);
- Map<String, Map<Partition, Map<String, String>>> bestPossStateMap = bestPossOutput.getStateMap();
+ BestPossibleStateOutput bestPossOutput = ClusterStateVerifier.calcBestPossState(cache);
+ Map<String, Map<Partition, Map<String, String>>> bestPossStateMap =
+ bestPossOutput.getStateMap();
// set error states
- if (errStates != null)
- {
- for (String resourceName : errStates.keySet())
- {
+ if (errStates != null) {
+ for (String resourceName : errStates.keySet()) {
Map<String, String> partErrStates = errStates.get(resourceName);
- for (String partitionName : partErrStates.keySet())
- {
+ for (String partitionName : partErrStates.keySet()) {
String instanceName = partErrStates.get(partitionName);
-
+
if (!bestPossStateMap.containsKey(resourceName)) {
bestPossStateMap.put(resourceName, new HashMap<Partition, Map<String, String>>());
}
@@ -315,19 +269,17 @@ public class ClusterStateVerifier
if (!bestPossStateMap.get(resourceName).containsKey(partition)) {
bestPossStateMap.get(resourceName).put(partition, new HashMap<String, String>());
}
- bestPossStateMap.get(resourceName).get(partition).put(instanceName, HelixDefinedState.ERROR.toString());
+ bestPossStateMap.get(resourceName).get(partition)
+ .put(instanceName, HelixDefinedState.ERROR.toString());
}
}
}
-
+
// System.out.println("stateMap: " + bestPossStateMap);
-
- for (String resourceName : idealStates.keySet())
- {
+ for (String resourceName : idealStates.keySet()) {
ExternalView extView = extViews.get(resourceName);
- if (extView == null)
- {
+ if (extView == null) {
LOG.info("externalView for " + resourceName + " is not available");
return false;
}
@@ -335,27 +287,21 @@ public class ClusterStateVerifier
// step 0: remove empty map and DROPPED state from best possible state
Map<Partition, Map<String, String>> bpStateMap =
bestPossOutput.getResourceMap(resourceName);
- Iterator<Entry<Partition, Map<String, String>>> iter =
- bpStateMap.entrySet().iterator();
- while (iter.hasNext())
- {
+ Iterator<Entry<Partition, Map<String, String>>> iter = bpStateMap.entrySet().iterator();
+ while (iter.hasNext()) {
Map.Entry<Partition, Map<String, String>> entry = iter.next();
Map<String, String> instanceStateMap = entry.getValue();
- if (instanceStateMap.isEmpty())
- {
+ if (instanceStateMap.isEmpty()) {
iter.remove();
- } else
- {
+ } else {
// remove instances with DROPPED state
Iterator<Map.Entry<String, String>> insIter = instanceStateMap.entrySet().iterator();
- while (insIter.hasNext())
- {
+ while (insIter.hasNext()) {
Map.Entry<String, String> insEntry = insIter.next();
String state = insEntry.getValue();
- if (state.equalsIgnoreCase(HelixDefinedState.DROPPED.toString()))
- {
+ if (state.equalsIgnoreCase(HelixDefinedState.DROPPED.toString())) {
insIter.remove();
- }
+ }
}
}
}
@@ -365,15 +311,13 @@ public class ClusterStateVerifier
// step 1: externalView and bestPossibleState has equal size
int extViewSize = extView.getRecord().getMapFields().size();
int bestPossStateSize = bestPossOutput.getResourceMap(resourceName).size();
- if (extViewSize != bestPossStateSize)
- {
- LOG.info("exterView size (" + extViewSize
- + ") is different from bestPossState size (" + bestPossStateSize
- + ") for resource: " + resourceName);
-
-// System.err.println("exterView size (" + extViewSize
-// + ") is different from bestPossState size (" + bestPossStateSize
-// + ") for resource: " + resourceName);
+ if (extViewSize != bestPossStateSize) {
+ LOG.info("exterView size (" + extViewSize + ") is different from bestPossState size ("
+ + bestPossStateSize + ") for resource: " + resourceName);
+
+ // System.err.println("exterView size (" + extViewSize
+ // + ") is different from bestPossState size (" + bestPossStateSize
+ // + ") for resource: " + resourceName);
// System.out.println("extView: " + extView.getRecord().getMapFields());
// System.out.println("bestPossState: " +
// bestPossOutput.getResourceMap(resourceName));
@@ -381,82 +325,66 @@ public class ClusterStateVerifier
}
// step 2: every entry in external view is contained in best possible state
- for (String partition : extView.getRecord().getMapFields().keySet())
- {
- Map<String, String> evInstanceStateMap =
- extView.getRecord().getMapField(partition);
+ for (String partition : extView.getRecord().getMapFields().keySet()) {
+ Map<String, String> evInstanceStateMap = extView.getRecord().getMapField(partition);
Map<String, String> bpInstanceStateMap =
bestPossOutput.getInstanceStateMap(resourceName, new Partition(partition));
boolean result =
ClusterStateVerifier.<String, String> compareMap(evInstanceStateMap,
- bpInstanceStateMap);
- if (result == false)
- {
- LOG.info("externalView is different from bestPossibleState for partition:"
- + partition);
-
-// System.err.println("externalView is different from bestPossibleState for partition: "
-// + partition + ", actual: " + evInstanceStateMap + ", bestPoss: " + bpInstanceStateMap);
+ bpInstanceStateMap);
+ if (result == false) {
+ LOG.info("externalView is different from bestPossibleState for partition:" + partition);
+
+ // System.err.println("externalView is different from bestPossibleState for partition: "
+ // + partition + ", actual: " + evInstanceStateMap + ", bestPoss: " +
+ // bpInstanceStateMap);
return false;
}
}
}
return true;
- }
- catch (Exception e)
- {
+ } catch (Exception e) {
LOG.error("exception in verification", e);
return false;
}
}
- static boolean verifyMasterNbInExtView(HelixDataAccessor accessor)
- {
+ static boolean verifyMasterNbInExtView(HelixDataAccessor accessor) {
Builder keyBuilder = accessor.keyBuilder();
- Map<String, IdealState> idealStates =
- accessor.getChildValuesMap(keyBuilder.idealStates());
- if (idealStates == null || idealStates.size() == 0)
- {
+ Map<String, IdealState> idealStates = accessor.getChildValuesMap(keyBuilder.idealStates());
+ if (idealStates == null || idealStates.size() == 0) {
LOG.info("No resource idealState");
return true;
}
- Map<String, ExternalView> extViews =
- accessor.getChildValuesMap(keyBuilder.externalViews());
- if (extViews == null || extViews.size() < idealStates.size())
- {
+ Map<String, ExternalView> extViews = accessor.getChildValuesMap(keyBuilder.externalViews());
+ if (extViews == null || extViews.size() < idealStates.size()) {
LOG.info("No externalViews | externalView.size() < idealState.size()");
return false;
}
- for (String resource : extViews.keySet())
- {
+ for (String resource : extViews.keySet()) {
int partitions = idealStates.get(resource).getNumPartitions();
Map<String, Map<String, String>> instanceStateMap =
extViews.get(resource).getRecord().getMapFields();
- if (instanceStateMap.size() < partitions)
- {
- LOG.info("Number of externalViews (" + instanceStateMap.size()
- + ") < partitions (" + partitions + ")");
+ if (instanceStateMap.size() < partitions) {
+ LOG.info("Number of externalViews (" + instanceStateMap.size() + ") < partitions ("
+ + partitions + ")");
return false;
}
- for (String partition : instanceStateMap.keySet())
- {
+ for (String partition : instanceStateMap.keySet()) {
boolean foundMaster = false;
- for (String instance : instanceStateMap.get(partition).keySet())
- {
- if (instanceStateMap.get(partition).get(instance).equalsIgnoreCase("MASTER"))
- {
+ for (String instance : instanceStateMap.get(partition).keySet()) {
+ if (instanceStateMap.get(partition).get(instance).equalsIgnoreCase("MASTER")) {
foundMaster = true;
break;
}
}
- if (!foundMaster)
- {
+ if (!foundMaster) {
LOG.info("No MASTER for partition: " + partition);
return false;
}
@@ -465,8 +393,7 @@ public class ClusterStateVerifier
return true;
}
- static void runStage(ClusterEvent event, Stage stage) throws Exception
- {
+ static void runStage(ClusterEvent event, Stage stage) throws Exception {
StageContext context = new StageContext();
stage.init(context);
stage.preProcess();
@@ -477,14 +404,12 @@ public class ClusterStateVerifier
/**
* calculate the best possible state note that DROPPED states are not checked since when
* kick off the BestPossibleStateCalcStage we are providing an empty current state map
- *
* @param cache
* @return
* @throws Exception
*/
- static BestPossibleStateOutput calcBestPossState(ClusterDataCache cache) throws Exception
- {
+ static BestPossibleStateOutput calcBestPossState(ClusterDataCache cache) throws Exception {
ClusterEvent event = new ClusterEvent("sampleEvent");
event.addAttribute("ClusterDataCache", cache);
@@ -503,41 +428,28 @@ public class ClusterStateVerifier
return output;
}
- public static <K, V> boolean compareMap(Map<K, V> map1, Map<K, V> map2)
- {
+ public static <K, V> boolean compareMap(Map<K, V> map1, Map<K, V> map2) {
boolean isEqual = true;
- if (map1 == null && map2 == null)
- {
+ if (map1 == null && map2 == null) {
// OK
- }
- else if (map1 == null && map2 != null)
- {
- if (!map2.isEmpty())
- {
+ } else if (map1 == null && map2 != null) {
+ if (!map2.isEmpty()) {
isEqual = false;
}
- }
- else if (map1 != null && map2 == null)
- {
- if (!map1.isEmpty())
- {
+ } else if (map1 != null && map2 == null) {
+ if (!map1.isEmpty()) {
isEqual = false;
}
- }
- else
- {
+ } else {
// verify size
- if (map1.size() != map2.size())
- {
+ if (map1.size() != map2.size()) {
isEqual = false;
}
// verify each <key, value> in map1 is contained in map2
- for (K key : map1.keySet())
- {
- if (!map1.get(key).equals(map2.get(key)))
- {
- LOG.debug("different value for key: " + key + "(map1: " + map1.get(key)
- + ", map2: " + map2.get(key) + ")");
+ for (K key : map1.keySet()) {
+ if (!map1.get(key).equals(map2.get(key))) {
+ LOG.debug("different value for key: " + key + "(map1: " + map1.get(key) + ", map2: "
+ + map2.get(key) + ")");
isEqual = false;
break;
}
@@ -546,43 +458,32 @@ public class ClusterStateVerifier
return isEqual;
}
- public static boolean verifyByPolling(Verifier verifier)
- {
+ public static boolean verifyByPolling(Verifier verifier) {
return verifyByPolling(verifier, 30 * 1000);
}
- public static boolean verifyByPolling(Verifier verifier, long timeout)
- {
+ public static boolean verifyByPolling(Verifier verifier, long timeout) {
return verifyByPolling(verifier, timeout, 1000);
}
- public static boolean verifyByPolling(Verifier verifier, long timeout, long period)
- {
+ public static boolean verifyByPolling(Verifier verifier, long timeout, long period) {
long startTime = System.currentTimeMillis();
boolean result = false;
- try
- {
+ try {
long curTime;
- do
- {
+ do {
Thread.sleep(period);
result = verifier.verify();
- if (result == true)
- {
+ if (result == true) {
break;
}
curTime = System.currentTimeMillis();
- }
- while (curTime <= startTime + timeout);
+ } while (curTime <= startTime + timeout);
return result;
- }
- catch (Exception e)
- {
+ } catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
- }
- finally
- {
+ } finally {
long endTime = System.currentTimeMillis();
// debug
@@ -593,13 +494,11 @@ public class ClusterStateVerifier
return false;
}
- public static boolean verifyByZkCallback(ZkVerifier verifier)
- {
+ public static boolean verifyByZkCallback(ZkVerifier verifier) {
return verifyByZkCallback(verifier, 30000);
}
- public static boolean verifyByZkCallback(ZkVerifier verifier, long timeout)
- {
+ public static boolean verifyByZkCallback(ZkVerifier verifier, long timeout) {
long startTime = System.currentTimeMillis();
CountDownLatch countDown = new CountDownLatch(1);
ZkClient zkClient = verifier.getZkClient();
@@ -609,34 +508,25 @@ public class ClusterStateVerifier
// so when analyze zk log, we know when a test ends
zkClient.createEphemeral("/" + clusterName + "/CONFIGS/CLUSTER/verify");
- ExtViewVeriferZkListener listener =
- new ExtViewVeriferZkListener(countDown, zkClient, verifier);
+ ExtViewVeriferZkListener listener = new ExtViewVeriferZkListener(countDown, zkClient, verifier);
- String extViewPath =
- PropertyPathConfig.getPath(PropertyType.EXTERNALVIEW, clusterName);
+ String extViewPath = PropertyPathConfig.getPath(PropertyType.EXTERNALVIEW, clusterName);
zkClient.subscribeChildChanges(extViewPath, listener);
- for (String child : zkClient.getChildren(extViewPath))
- {
- String childPath =
- extViewPath.equals("/") ? extViewPath + child : extViewPath + "/" + child;
+ for (String child : zkClient.getChildren(extViewPath)) {
+ String childPath = extViewPath.equals("/") ? extViewPath + child : extViewPath + "/" + child;
zkClient.subscribeDataChanges(childPath, listener);
}
// do initial verify
boolean result = verifier.verify();
- if (result == false)
- {
- try
- {
+ if (result == false) {
+ try {
result = countDown.await(timeout, TimeUnit.MILLISECONDS);
- if (result == false)
- {
+ if (result == false) {
// make a final try if timeout
result = verifier.verify();
}
- }
- catch (Exception e)
- {
+ } catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
@@ -644,10 +534,8 @@ public class ClusterStateVerifier
// clean up
zkClient.unsubscribeChildChanges(extViewPath, listener);
- for (String child : zkClient.getChildren(extViewPath))
- {
- String childPath =
- extViewPath.equals("/") ? extViewPath + child : extViewPath + "/" + child;
+ for (String child : zkClient.getChildren(extViewPath)) {
+ String childPath = extViewPath.equals("/") ? extViewPath + child : extViewPath + "/" + child;
zkClient.unsubscribeDataChanges(childPath, listener);
}
@@ -661,40 +549,33 @@ public class ClusterStateVerifier
}
@SuppressWarnings("static-access")
- private static Options constructCommandLineOptions()
- {
+ private static Options constructCommandLineOptions() {
Option helpOption =
- OptionBuilder.withLongOpt(help)
- .withDescription("Prints command-line options info")
- .create();
+ OptionBuilder.withLongOpt(help).withDescription("Prints command-line options info")
+ .create();
Option zkServerOption =
- OptionBuilder.withLongOpt(zkServerAddress)
- .withDescription("Provide zookeeper address")
- .create();
+ OptionBuilder.withLongOpt(zkServerAddress).withDescription("Provide zookeeper address")
+ .create();
zkServerOption.setArgs(1);
zkServerOption.setRequired(true);
zkServerOption.setArgName("ZookeeperServerAddress(Required)");
Option clusterOption =
- OptionBuilder.withLongOpt(cluster)
- .withDescription("Provide cluster name")
- .create();
+ OptionBuilder.withLongOpt(cluster).withDescription("Provide cluster name").create();
clusterOption.setArgs(1);
clusterOption.setRequired(true);
clusterOption.setArgName("Cluster name (Required)");
Option timeoutOption =
- OptionBuilder.withLongOpt(timeout)
- .withDescription("Timeout value for verification")
- .create();
+ OptionBuilder.withLongOpt(timeout).withDescription("Timeout value for verification")
+ .create();
timeoutOption.setArgs(1);
timeoutOption.setArgName("Timeout value (Optional), default=30s");
Option sleepIntervalOption =
- OptionBuilder.withLongOpt(period)
- .withDescription("Polling period for verification")
- .create();
+ OptionBuilder.withLongOpt(period).withDescription("Polling period for verification")
+ .create();
sleepIntervalOption.setArgs(1);
sleepIntervalOption.setArgName("Polling period value (Optional), default=1s");
@@ -708,25 +589,20 @@ public class ClusterStateVerifier
return options;
}
- public static void printUsage(Options cliOptions)
- {
+ public static void printUsage(Options cliOptions) {
HelpFormatter helpFormatter = new HelpFormatter();
helpFormatter.setWidth(1000);
helpFormatter.printHelp("java " + ClusterSetup.class.getName(), cliOptions);
}
- public static CommandLine processCommandLineArgs(String[] cliArgs)
- {
+ public static CommandLine processCommandLineArgs(String[] cliArgs) {
CommandLineParser cliParser = new GnuParser();
Options cliOptions = constructCommandLineOptions();
// CommandLine cmd = null;
- try
- {
+ try {
return cliParser.parse(cliOptions, cliArgs);
- }
- catch (ParseException pe)
- {
+ } catch (ParseException pe) {
System.err.println("CommandLineClient: failed to parse command-line options: "
+ pe.toString());
printUsage(cliOptions);
@@ -735,42 +611,31 @@ public class ClusterStateVerifier
return null;
}
- public static boolean verifyState(String[] args)
- {
+ public static boolean verifyState(String[] args) {
// TODO Auto-generated method stub
String clusterName = "storage-cluster";
String zkServer = "localhost:2181";
long timeoutValue = 0;
long periodValue = 1000;
- if (args.length > 0)
- {
+ if (args.length > 0) {
CommandLine cmd = processCommandLineArgs(args);
zkServer = cmd.getOptionValue(zkServerAddress);
clusterName = cmd.getOptionValue(cluster);
String timeoutStr = cmd.getOptionValue(timeout);
String periodStr = cmd.getOptionValue(period);
- if (timeoutStr != null)
- {
- try
- {
+ if (timeoutStr != null) {
+ try {
timeoutValue = Long.parseLong(timeoutStr);
- }
- catch (Exception e)
- {
- System.err.println("Exception in converting " + timeoutStr
- + " to long. Use default (0)");
+ } catch (Exception e) {
+ System.err.println("Exception in converting " + timeoutStr + " to long. Use default (0)");
}
}
- if (periodStr != null)
- {
- try
- {
+ if (periodStr != null) {
+ try {
periodValue = Long.parseLong(periodStr);
- }
- catch (Exception e)
- {
+ } catch (Exception e) {
System.err.println("Exception in converting " + periodStr
+ " to long. Use default (1000)");
}
@@ -781,12 +646,10 @@ public class ClusterStateVerifier
// timeoutValue,
// periodValue);
- return verifyByZkCallback(new BestPossAndExtViewZkVerifier(zkServer, clusterName),
- timeoutValue);
+ return verifyByZkCallback(new BestPossAndExtViewZkVerifier(zkServer, clusterName), timeoutValue);
}
- public static void main(String[] args)
- {
+ public static void main(String[] args) {
boolean result = verifyState(args);
System.out.println(result ? "Successful" : "failed");
System.exit(1);
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/f414aad4/helix-core/src/main/java/org/apache/helix/tools/DefaultIdealStateCalculator.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/tools/DefaultIdealStateCalculator.java b/helix-core/src/main/java/org/apache/helix/tools/DefaultIdealStateCalculator.java
index 3c63269..dea8e0a 100644
--- a/helix-core/src/main/java/org/apache/helix/tools/DefaultIdealStateCalculator.java
+++ b/helix-core/src/main/java/org/apache/helix/tools/DefaultIdealStateCalculator.java
@@ -30,32 +30,30 @@ import org.apache.helix.HelixException;
import org.apache.helix.ZNRecord;
import org.apache.helix.model.IdealState.IdealStateProperty;
-
/**
* DefaultIdealStateCalculator tries to optimally allocate master/slave partitions among
* espresso storage nodes.
- *
- * Given a batch of storage nodes, the partition and replication factor, the algorithm first given a initial state
- * When new batches of storage nodes are added, the algorithm will calculate the new ideal state such that the total
+ * Given a batch of storage nodes, the partition and replication factor, the algorithm first given a
+ * initial state
+ * When new batches of storage nodes are added, the algorithm will calculate the new ideal state
+ * such that the total
* partition movements are minimized.
- *
*/
-public class DefaultIdealStateCalculator
-{
+public class DefaultIdealStateCalculator {
static final String _MasterAssignmentMap = "MasterAssignmentMap";
static final String _SlaveAssignmentMap = "SlaveAssignmentMap";
static final String _partitions = "partitions";
static final String _replicas = "replicas";
/**
- * Calculate the initial ideal state given a batch of storage instances, the replication factor and
+ * Calculate the initial ideal state given a batch of storage instances, the replication factor
+ * and
* number of partitions
- *
* 1. Calculate the master assignment by random shuffling
- * 2. for each storage instance, calculate the 1st slave assignment map, by another random shuffling
+ * 2. for each storage instance, calculate the 1st slave assignment map, by another random
+ * shuffling
* 3. for each storage instance, calculate the i-th slave assignment map
* 4. Combine the i-th slave assignment maps together
- *
* @param instanceNames
* list of storage node instances
* @param partitions
@@ -69,34 +67,26 @@ public class DefaultIdealStateCalculator
* @param resourceName
* @return a ZNRecord that contain the idealstate info
*/
- public static ZNRecord calculateIdealState(List<String> instanceNames, int partitions, int replicas, String resourceName,
- String masterStateValue, String slaveStateValue)
- {
+ public static ZNRecord calculateIdealState(List<String> instanceNames, int partitions,
+ int replicas, String resourceName, String masterStateValue, String slaveStateValue) {
Collections.sort(instanceNames);
- if(instanceNames.size() < replicas + 1)
- {
+ if (instanceNames.size() < replicas + 1) {
throw new HelixException("Number of instances must not be less than replicas + 1. "
- + "instanceNr:" + instanceNames.size()
- + ", replicas:" + replicas);
- }
- else if(partitions < instanceNames.size())
- {
- ZNRecord idealState = IdealStateCalculatorByShuffling.calculateIdealState(instanceNames, partitions, replicas, resourceName, 12345, masterStateValue, slaveStateValue);
+ + "instanceNr:" + instanceNames.size() + ", replicas:" + replicas);
+ } else if (partitions < instanceNames.size()) {
+ ZNRecord idealState =
+ IdealStateCalculatorByShuffling.calculateIdealState(instanceNames, partitions, replicas,
+ resourceName, 12345, masterStateValue, slaveStateValue);
int i = 0;
- for(String partitionId : idealState.getMapFields().keySet())
- {
+ for (String partitionId : idealState.getMapFields().keySet()) {
Map<String, String> partitionAssignmentMap = idealState.getMapField(partitionId);
List<String> partitionAssignmentPriorityList = new ArrayList<String>();
String masterInstance = "";
- for(String instanceName : partitionAssignmentMap.keySet())
- {
- if(partitionAssignmentMap.get(instanceName).equalsIgnoreCase(masterStateValue)
- && masterInstance.equals(""))
- {
+ for (String instanceName : partitionAssignmentMap.keySet()) {
+ if (partitionAssignmentMap.get(instanceName).equalsIgnoreCase(masterStateValue)
+ && masterInstance.equals("")) {
masterInstance = instanceName;
- }
- else
- {
+ } else {
partitionAssignmentPriorityList.add(instanceName);
}
}
@@ -112,13 +102,13 @@ public class DefaultIdealStateCalculator
return convertToZNRecord(result, resourceName, masterStateValue, slaveStateValue);
}
- public static ZNRecord calculateIdealStateBatch(List<List<String>> instanceBatches, int partitions, int replicas, String resourceName,
- String masterStateValue, String slaveStateValue)
- {
- Map<String, Object> result = calculateInitialIdealState(instanceBatches.get(0), partitions, replicas);
+ public static ZNRecord calculateIdealStateBatch(List<List<String>> instanceBatches,
+ int partitions, int replicas, String resourceName, String masterStateValue,
+ String slaveStateValue) {
+ Map<String, Object> result =
+ calculateInitialIdealState(instanceBatches.get(0), partitions, replicas);
- for(int i = 1; i < instanceBatches.size(); i++)
- {
+ for (int i = 1; i < instanceBatches.size(); i++) {
result = calculateNextIdealState(instanceBatches.get(i), result);
}
@@ -129,62 +119,51 @@ public class DefaultIdealStateCalculator
* Convert the internal result (stored as a Map<String, Object>) into ZNRecord.
*/
public static ZNRecord convertToZNRecord(Map<String, Object> result, String resourceName,
- String masterStateValue, String slaveStateValue)
- {
- Map<String, List<Integer>> nodeMasterAssignmentMap
- = (Map<String, List<Integer>>) (result.get(_MasterAssignmentMap));
- Map<String, Map<String, List<Integer>>> nodeSlaveAssignmentMap
- = (Map<String, Map<String, List<Integer>>>)(result.get(_SlaveAssignmentMap));
+ String masterStateValue, String slaveStateValue) {
+ Map<String, List<Integer>> nodeMasterAssignmentMap =
+ (Map<String, List<Integer>>) (result.get(_MasterAssignmentMap));
+ Map<String, Map<String, List<Integer>>> nodeSlaveAssignmentMap =
+ (Map<String, Map<String, List<Integer>>>) (result.get(_SlaveAssignmentMap));
- int partitions = (Integer)(result.get("partitions"));
+ int partitions = (Integer) (result.get("partitions"));
ZNRecord idealState = new ZNRecord(resourceName);
- idealState.setSimpleField(IdealStateProperty.NUM_PARTITIONS.toString(), String.valueOf(partitions));
+ idealState.setSimpleField(IdealStateProperty.NUM_PARTITIONS.toString(),
+ String.valueOf(partitions));
-
- for(String instanceName : nodeMasterAssignmentMap.keySet())
- {
- for(Integer partitionId : nodeMasterAssignmentMap.get(instanceName))
- {
- String partitionName = resourceName+"_"+partitionId;
- if(!idealState.getMapFields().containsKey(partitionName))
- {
+ for (String instanceName : nodeMasterAssignmentMap.keySet()) {
+ for (Integer partitionId : nodeMasterAssignmentMap.get(instanceName)) {
+ String partitionName = resourceName + "_" + partitionId;
+ if (!idealState.getMapFields().containsKey(partitionName)) {
idealState.setMapField(partitionName, new TreeMap<String, String>());
}
idealState.getMapField(partitionName).put(instanceName, masterStateValue);
}
}
- for(String instanceName : nodeSlaveAssignmentMap.keySet())
- {
+ for (String instanceName : nodeSlaveAssignmentMap.keySet()) {
Map<String, List<Integer>> slaveAssignmentMap = nodeSlaveAssignmentMap.get(instanceName);
- for(String slaveNode: slaveAssignmentMap.keySet())
- {
+ for (String slaveNode : slaveAssignmentMap.keySet()) {
List<Integer> slaveAssignment = slaveAssignmentMap.get(slaveNode);
- for(Integer partitionId: slaveAssignment)
- {
- String partitionName = resourceName+"_"+partitionId;
+ for (Integer partitionId : slaveAssignment) {
+ String partitionName = resourceName + "_" + partitionId;
idealState.getMapField(partitionName).put(slaveNode, slaveStateValue);
}
}
}
- // generate the priority list of instances per partition. Master should be at front and slave follows.
+ // generate the priority list of instances per partition. Master should be at front and slave
+ // follows.
- for(String partitionId : idealState.getMapFields().keySet())
- {
+ for (String partitionId : idealState.getMapFields().keySet()) {
Map<String, String> partitionAssignmentMap = idealState.getMapField(partitionId);
List<String> partitionAssignmentPriorityList = new ArrayList<String>();
String masterInstance = "";
- for(String instanceName : partitionAssignmentMap.keySet())
- {
- if(partitionAssignmentMap.get(instanceName).equalsIgnoreCase(masterStateValue)
- && masterInstance.equals(""))
- {
+ for (String instanceName : partitionAssignmentMap.keySet()) {
+ if (partitionAssignmentMap.get(instanceName).equalsIgnoreCase(masterStateValue)
+ && masterInstance.equals("")) {
masterInstance = instanceName;
- }
- else
- {
+ } else {
partitionAssignmentPriorityList.add(instanceName);
}
}
@@ -192,19 +171,21 @@ public class DefaultIdealStateCalculator
partitionAssignmentPriorityList.add(0, masterInstance);
idealState.setListField(partitionId, partitionAssignmentPriorityList);
}
- assert(result.containsKey("replicas"));
- idealState.setSimpleField(IdealStateProperty.REPLICAS.toString(), result.get("replicas").toString());
+ assert (result.containsKey("replicas"));
+ idealState.setSimpleField(IdealStateProperty.REPLICAS.toString(), result.get("replicas")
+ .toString());
return idealState;
}
+
/**
- * Calculate the initial ideal state given a batch of storage instances, the replication factor and
+ * Calculate the initial ideal state given a batch of storage instances, the replication factor
+ * and
* number of partitions
- *
* 1. Calculate the master assignment by random shuffling
- * 2. for each storage instance, calculate the 1st slave assignment map, by another random shuffling
+ * 2. for each storage instance, calculate the 1st slave assignment map, by another random
+ * shuffling
* 3. for each storage instance, calculate the i-th slave assignment map
* 4. Combine the i-th slave assignment maps together
- *
* @param instanceNames
* list of storage node instances
* @param weight
@@ -215,27 +196,24 @@ public class DefaultIdealStateCalculator
* The number of replicas (slave partitions) per master partition
* @return a map that contain the idealstate info
*/
- public static Map<String, Object> calculateInitialIdealState(List<String> instanceNames, int partitions, int replicas)
- {
+ public static Map<String, Object> calculateInitialIdealState(List<String> instanceNames,
+ int partitions, int replicas) {
Random r = new Random(54321);
- assert(replicas <= instanceNames.size() - 1);
+ assert (replicas <= instanceNames.size() - 1);
ArrayList<Integer> masterPartitionAssignment = new ArrayList<Integer>();
- for(int i = 0;i< partitions; i++)
- {
+ for (int i = 0; i < partitions; i++) {
masterPartitionAssignment.add(i);
}
// shuffle the partition id array
Collections.shuffle(masterPartitionAssignment, new Random(r.nextInt()));
// 1. Generate the random master partition assignment
- // instanceName -> List of master partitions on that instance
+ // instanceName -> List of master partitions on that instance
Map<String, List<Integer>> nodeMasterAssignmentMap = new TreeMap<String, List<Integer>>();
- for(int i = 0; i < masterPartitionAssignment.size(); i++)
- {
+ for (int i = 0; i < masterPartitionAssignment.size(); i++) {
String instanceName = instanceNames.get(i % instanceNames.size());
- if(!nodeMasterAssignmentMap.containsKey(instanceName))
- {
+ if (!nodeMasterAssignmentMap.containsKey(instanceName)) {
nodeMasterAssignmentMap.put(instanceName, new ArrayList<Integer>());
}
nodeMasterAssignmentMap.get(instanceName).add(masterPartitionAssignment.get(i));
@@ -243,36 +221,35 @@ public class DefaultIdealStateCalculator
// instanceName -> slave assignment for its master partitions
// slave assignment: instanceName -> list of slave partitions on it
- List<Map<String, Map<String, List<Integer>>>> nodeSlaveAssignmentMapsList = new ArrayList<Map<String, Map<String, List<Integer>>>>(replicas);
+ List<Map<String, Map<String, List<Integer>>>> nodeSlaveAssignmentMapsList =
+ new ArrayList<Map<String, Map<String, List<Integer>>>>(replicas);
- Map<String, Map<String, List<Integer>>> firstNodeSlaveAssignmentMap = new TreeMap<String, Map<String, List<Integer>>>();
- Map<String, Map<String, List<Integer>>> combinedNodeSlaveAssignmentMap = new TreeMap<String, Map<String, List<Integer>>>();
+ Map<String, Map<String, List<Integer>>> firstNodeSlaveAssignmentMap =
+ new TreeMap<String, Map<String, List<Integer>>>();
+ Map<String, Map<String, List<Integer>>> combinedNodeSlaveAssignmentMap =
+ new TreeMap<String, Map<String, List<Integer>>>();
- if(replicas > 0)
- {
+ if (replicas > 0) {
// 2. For each node, calculate the evenly distributed slave as the first slave assignment
- // We will figure out the 2nd ...replicas-th slave assignment based on the first level slave assignment
- for(int i = 0; i < instanceNames.size(); i++)
- {
+ // We will figure out the 2nd ...replicas-th slave assignment based on the first level slave
+ // assignment
+ for (int i = 0; i < instanceNames.size(); i++) {
List<String> slaveInstances = new ArrayList<String>();
ArrayList<Integer> slaveAssignment = new ArrayList<Integer>();
TreeMap<String, List<Integer>> slaveAssignmentMap = new TreeMap<String, List<Integer>>();
- for(int j = 0;j < instanceNames.size(); j++)
- {
- if(j != i)
- {
+ for (int j = 0; j < instanceNames.size(); j++) {
+ if (j != i) {
slaveInstances.add(instanceNames.get(j));
slaveAssignmentMap.put(instanceNames.get(j), new ArrayList<Integer>());
}
}
// Get the number of master partitions on instanceName
- List<Integer> masterAssignment = nodeMasterAssignmentMap.get(instanceNames.get(i));
- // do a random shuffling as in step 1, so that the first-level slave are distributed among rest instances
-
+ List<Integer> masterAssignment = nodeMasterAssignmentMap.get(instanceNames.get(i));
+ // do a random shuffling as in step 1, so that the first-level slave are distributed among
+ // rest instances
- for(int j = 0;j < masterAssignment.size(); j++)
- {
+ for (int j = 0; j < masterAssignment.size(); j++) {
slaveAssignment.add(j);
}
Collections.shuffle(slaveAssignment, new Random(r.nextInt()));
@@ -280,11 +257,10 @@ public class DefaultIdealStateCalculator
Collections.shuffle(slaveInstances, new Random(instanceNames.get(i).hashCode()));
// Get the slave assignment map of node instanceName
- for(int j = 0;j < masterAssignment.size(); j++)
- {
- String slaveInstanceName = slaveInstances.get(slaveAssignment.get(j) % slaveInstances.size());
- if(!slaveAssignmentMap.containsKey(slaveInstanceName))
- {
+ for (int j = 0; j < masterAssignment.size(); j++) {
+ String slaveInstanceName =
+ slaveInstances.get(slaveAssignment.get(j) % slaveInstances.size());
+ if (!slaveAssignmentMap.containsKey(slaveInstanceName)) {
slaveAssignmentMap.put(slaveInstanceName, new ArrayList<Integer>());
}
slaveAssignmentMap.get(slaveInstanceName).add(masterAssignment.get(j));
@@ -293,31 +269,28 @@ public class DefaultIdealStateCalculator
}
nodeSlaveAssignmentMapsList.add(firstNodeSlaveAssignmentMap);
// From the first slave assignment map, calculate the rest slave assignment maps
- for(int replicaOrder = 1; replicaOrder < replicas; replicaOrder++)
- {
+ for (int replicaOrder = 1; replicaOrder < replicas; replicaOrder++) {
// calculate the next slave partition assignment map
- Map<String, Map<String, List<Integer>>> nextNodeSlaveAssignmentMap
- = calculateNextSlaveAssignemntMap(firstNodeSlaveAssignmentMap, replicaOrder);
+ Map<String, Map<String, List<Integer>>> nextNodeSlaveAssignmentMap =
+ calculateNextSlaveAssignemntMap(firstNodeSlaveAssignmentMap, replicaOrder);
nodeSlaveAssignmentMapsList.add(nextNodeSlaveAssignmentMap);
}
// Combine the calculated 1...replicas-th slave assignment map together
- for(String instanceName : nodeMasterAssignmentMap.keySet())
- {
- Map<String, List<Integer>> combinedSlaveAssignmentMap = new TreeMap<String, List<Integer>>();
+ for (String instanceName : nodeMasterAssignmentMap.keySet()) {
+ Map<String, List<Integer>> combinedSlaveAssignmentMap =
+ new TreeMap<String, List<Integer>>();
- for(Map<String, Map<String, List<Integer>>> slaveNodeAssignmentMap : nodeSlaveAssignmentMapsList)
- {
+ for (Map<String, Map<String, List<Integer>>> slaveNodeAssignmentMap : nodeSlaveAssignmentMapsList) {
Map<String, List<Integer>> slaveAssignmentMap = slaveNodeAssignmentMap.get(instanceName);
- for(String slaveInstance : slaveAssignmentMap.keySet())
- {
- if(!combinedSlaveAssignmentMap.containsKey(slaveInstance))
- {
+ for (String slaveInstance : slaveAssignmentMap.keySet()) {
+ if (!combinedSlaveAssignmentMap.containsKey(slaveInstance)) {
combinedSlaveAssignmentMap.put(slaveInstance, new ArrayList<Integer>());
}
- combinedSlaveAssignmentMap.get(slaveInstance).addAll(slaveAssignmentMap.get(slaveInstance));
+ combinedSlaveAssignmentMap.get(slaveInstance).addAll(
+ slaveAssignmentMap.get(slaveInstance));
}
}
migrateSlaveAssignMapToNewInstances(combinedSlaveAssignmentMap, new ArrayList<String>());
@@ -325,45 +298,47 @@ public class DefaultIdealStateCalculator
}
}
/*
- // Print the result master and slave assignment maps
- System.out.println("Master assignment:");
- for(String instanceName : nodeMasterAssignmentMap.keySet())
- {
- System.out.println(instanceName+":");
- for(Integer x : nodeMasterAssignmentMap.get(instanceName))
- {
- System.out.print(x+" ");
- }
- System.out.println();
- System.out.println("Slave assignment:");
-
- int slaveOrder = 1;
- for(Map<String, Map<String, List<Integer>>> slaveNodeAssignmentMap : nodeSlaveAssignmentMapsList)
- {
- System.out.println("Slave assignment order :" + (slaveOrder++));
- Map<String, List<Integer>> slaveAssignmentMap = slaveNodeAssignmentMap.get(instanceName);
- for(String slaveName : slaveAssignmentMap.keySet())
- {
- System.out.print("\t" + slaveName +":\n\t" );
- for(Integer x : slaveAssignmentMap.get(slaveName))
- {
- System.out.print(x + " ");
- }
- System.out.println("\n");
- }
- }
- System.out.println("\nCombined slave assignment map");
- Map<String, List<Integer>> slaveAssignmentMap = combinedNodeSlaveAssignmentMap.get(instanceName);
- for(String slaveName : slaveAssignmentMap.keySet())
- {
- System.out.print("\t" + slaveName +":\n\t" );
- for(Integer x : slaveAssignmentMap.get(slaveName))
- {
- System.out.print(x + " ");
- }
- System.out.println("\n");
- }
- }*/
+ * // Print the result master and slave assignment maps
+ * System.out.println("Master assignment:");
+ * for(String instanceName : nodeMasterAssignmentMap.keySet())
+ * {
+ * System.out.println(instanceName+":");
+ * for(Integer x : nodeMasterAssignmentMap.get(instanceName))
+ * {
+ * System.out.print(x+" ");
+ * }
+ * System.out.println();
+ * System.out.println("Slave assignment:");
+ * int slaveOrder = 1;
+ * for(Map<String, Map<String, List<Integer>>> slaveNodeAssignmentMap :
+ * nodeSlaveAssignmentMapsList)
+ * {
+ * System.out.println("Slave assignment order :" + (slaveOrder++));
+ * Map<String, List<Integer>> slaveAssignmentMap = slaveNodeAssignmentMap.get(instanceName);
+ * for(String slaveName : slaveAssignmentMap.keySet())
+ * {
+ * System.out.print("\t" + slaveName +":\n\t" );
+ * for(Integer x : slaveAssignmentMap.get(slaveName))
+ * {
+ * System.out.print(x + " ");
+ * }
+ * System.out.println("\n");
+ * }
+ * }
+ * System.out.println("\nCombined slave assignment map");
+ * Map<String, List<Integer>> slaveAssignmentMap =
+ * combinedNodeSlaveAssignmentMap.get(instanceName);
+ * for(String slaveName : slaveAssignmentMap.keySet())
+ * {
+ * System.out.print("\t" + slaveName +":\n\t" );
+ * for(Integer x : slaveAssignmentMap.get(slaveName))
+ * {
+ * System.out.print(x + " ");
+ * }
+ * System.out.println("\n");
+ * }
+ * }
+ */
Map<String, Object> result = new TreeMap<String, Object>();
result.put("MasterAssignmentMap", nodeMasterAssignmentMap);
result.put("SlaveAssignmentMap", combinedNodeSlaveAssignmentMap);
@@ -371,50 +346,46 @@ public class DefaultIdealStateCalculator
result.put("partitions", new Integer(partitions));
return result;
}
+
/**
- * In the case there are more than 1 slave, we use the following algorithm to calculate the n-th slave
+ * In the case there are more than 1 slave, we use the following algorithm to calculate the n-th
+ * slave
* assignment map based on the first level slave assignment map.
- *
- * @param firstInstanceSlaveAssignmentMap the first slave assignment map for all instances
+ * @param firstInstanceSlaveAssignmentMap the first slave assignment map for all instances
* @param order of the slave
* @return the n-th slave assignment map for all the instances
- * */
- static Map<String, Map<String, List<Integer>>> calculateNextSlaveAssignemntMap(Map<String, Map<String, List<Integer>>> firstInstanceSlaveAssignmentMap, int replicaOrder)
- {
- Map<String, Map<String, List<Integer>>> result = new TreeMap<String, Map<String, List<Integer>>>();
+ */
+ static Map<String, Map<String, List<Integer>>> calculateNextSlaveAssignemntMap(
+ Map<String, Map<String, List<Integer>>> firstInstanceSlaveAssignmentMap, int replicaOrder) {
+ Map<String, Map<String, List<Integer>>> result =
+ new TreeMap<String, Map<String, List<Integer>>>();
- for(String currentInstance : firstInstanceSlaveAssignmentMap.keySet())
- {
+ for (String currentInstance : firstInstanceSlaveAssignmentMap.keySet()) {
Map<String, List<Integer>> resultAssignmentMap = new TreeMap<String, List<Integer>>();
result.put(currentInstance, resultAssignmentMap);
}
- for(String currentInstance : firstInstanceSlaveAssignmentMap.keySet())
- {
- Map<String, List<Integer>> previousSlaveAssignmentMap = firstInstanceSlaveAssignmentMap.get(currentInstance);
+ for (String currentInstance : firstInstanceSlaveAssignmentMap.keySet()) {
+ Map<String, List<Integer>> previousSlaveAssignmentMap =
+ firstInstanceSlaveAssignmentMap.get(currentInstance);
Map<String, List<Integer>> resultAssignmentMap = result.get(currentInstance);
int offset = replicaOrder - 1;
- for(String instance : previousSlaveAssignmentMap.keySet())
- {
+ for (String instance : previousSlaveAssignmentMap.keySet()) {
List<String> otherInstances = new ArrayList<String>(previousSlaveAssignmentMap.size() - 1);
// Obtain an array of other instances
- for(String otherInstance : previousSlaveAssignmentMap.keySet())
- {
+ for (String otherInstance : previousSlaveAssignmentMap.keySet()) {
otherInstances.add(otherInstance);
}
Collections.sort(otherInstances);
int instanceIndex = -1;
- for(int index = 0;index < otherInstances.size(); index++)
- {
- if(otherInstances.get(index).equalsIgnoreCase(instance))
- {
+ for (int index = 0; index < otherInstances.size(); index++) {
+ if (otherInstances.get(index).equalsIgnoreCase(instance)) {
instanceIndex = index;
}
}
- assert(instanceIndex >= 0);
- if(instanceIndex == otherInstances.size() - 1)
- {
- instanceIndex --;
+ assert (instanceIndex >= 0);
+ if (instanceIndex == otherInstances.size() - 1) {
+ instanceIndex--;
}
// Since we need to evenly distribute the slaves on "instance" to other partitions, we
// need to remove "instance" from the array.
@@ -422,14 +393,12 @@ public class DefaultIdealStateCalculator
// distribute previous slave assignment to other instances.
List<Integer> previousAssignmentList = previousSlaveAssignmentMap.get(instance);
- for(int i = 0; i < previousAssignmentList.size(); i++)
- {
+ for (int i = 0; i < previousAssignmentList.size(); i++) {
// Evenly distribute the previousAssignmentList to the remaining other instances
int newInstanceIndex = (i + offset + instanceIndex) % otherInstances.size();
String newInstance = otherInstances.get(newInstanceIndex);
- if(!resultAssignmentMap.containsKey(newInstance))
- {
+ if (!resultAssignmentMap.containsKey(newInstance)) {
resultAssignmentMap.put(newInstance, new ArrayList<Integer>());
}
resultAssignmentMap.get(newInstance).add(previousAssignmentList.get(i));
@@ -442,24 +411,20 @@ public class DefaultIdealStateCalculator
/**
* Given the current idealState, and the list of new Instances needed to be added, calculate the
* new Ideal state.
- *
* 1. Calculate how many master partitions should be moved to the new cluster of instances
* 2. assign the number of master partitions px to be moved to each previous node
* 3. for each previous node,
- * 3.1 randomly choose px nodes, move them to temp list
- * 3.2 for each px nodes, remove them from the slave assignment map; record the map position of
- * the partition;
- * 3.3 calculate # of new nodes that should be put in the slave assignment map
- * 3.4 even-fy the slave assignment map;
- * 3.5 randomly place # of new nodes that should be placed in
- *
+ * 3.1 randomly choose px nodes, move them to temp list
+ * 3.2 for each px nodes, remove them from the slave assignment map; record the map position of
+ * the partition;
+ * 3.3 calculate # of new nodes that should be put in the slave assignment map
+ * 3.4 even-fy the slave assignment map;
+ * 3.5 randomly place # of new nodes that should be placed in
* 4. from all the temp master node list get from 3.1,
- * 4.1 randomly assign them to nodes in the new cluster
- *
+ * 4.1 randomly assign them to nodes in the new cluster
* 5. for each node in the new cluster,
- * 5.1 assemble the slave assignment map
- * 5.2 even-fy the slave assignment map
- *
+ * 5.1 assemble the slave assignment map
+ * 5.2 even-fy the slave assignment map
* @param newInstances
* list of new added storage node instances
* @param weight
@@ -467,67 +432,64 @@ public class DefaultIdealStateCalculator
* @param previousIdealState
* The previous ideal state
* @return a map that contain the updated idealstate info
- * */
- public static Map<String, Object> calculateNextIdealState(List<String> newInstances, Map<String, Object> previousIdealState)
- {
+ */
+ public static Map<String, Object> calculateNextIdealState(List<String> newInstances,
+ Map<String, Object> previousIdealState) {
// Obtain the master / slave assignment info maps
Collections.sort(newInstances);
- Map<String, List<Integer>> previousMasterAssignmentMap
- = (Map<String, List<Integer>>) (previousIdealState.get("MasterAssignmentMap"));
- Map<String, Map<String, List<Integer>>> nodeSlaveAssignmentMap
- = (Map<String, Map<String, List<Integer>>>)(previousIdealState.get("SlaveAssignmentMap"));
+ Map<String, List<Integer>> previousMasterAssignmentMap =
+ (Map<String, List<Integer>>) (previousIdealState.get("MasterAssignmentMap"));
+ Map<String, Map<String, List<Integer>>> nodeSlaveAssignmentMap =
+ (Map<String, Map<String, List<Integer>>>) (previousIdealState.get("SlaveAssignmentMap"));
List<String> oldInstances = new ArrayList<String>();
- for(String oldInstance : previousMasterAssignmentMap.keySet())
- {
+ for (String oldInstance : previousMasterAssignmentMap.keySet()) {
oldInstances.add(oldInstance);
}
int previousInstanceNum = previousMasterAssignmentMap.size();
- int partitions = (Integer)(previousIdealState.get("partitions"));
+ int partitions = (Integer) (previousIdealState.get("partitions"));
// TODO: take weight into account when calculate this
- int totalMasterParitionsToMove
- = partitions * (newInstances.size()) / (previousInstanceNum + newInstances.size());
+ int totalMasterParitionsToMove =
+ partitions * (newInstances.size()) / (previousInstanceNum + newInstances.size());
int numMastersFromEachNode = totalMasterParitionsToMove / previousInstanceNum;
int remain = totalMasterParitionsToMove % previousInstanceNum;
- // Note that when remain > 0, we should make [remain] moves with (numMastersFromEachNode + 1) partitions.
- // And we should first choose those (numMastersFromEachNode + 1) moves from the instances that has more
+ // Note that when remain > 0, we should make [remain] moves with (numMastersFromEachNode + 1)
+ // partitions.
+ // And we should first choose those (numMastersFromEachNode + 1) moves from the instances that
+ // has more
// master partitions
List<Integer> masterPartitionListToMove = new ArrayList<Integer>();
- // For corresponding moved slave partitions, keep track of their original location; the new node does not
+ // For corresponding moved slave partitions, keep track of their original location; the new node
+ // does not
// need to migrate all of them.
Map<String, List<Integer>> slavePartitionsToMoveMap = new TreeMap<String, List<Integer>>();
// Make sure that the instances that holds more master partitions are put in front
List<String> bigList = new ArrayList<String>(), smallList = new ArrayList<String>();
- for(String oldInstance : previousMasterAssignmentMap.keySet())
- {
+ for (String oldInstance : previousMasterAssignmentMap.keySet()) {
List<Integer> masterAssignmentList = previousMasterAssignmentMap.get(oldInstance);
- if(masterAssignmentList.size() > numMastersFromEachNode)
- {
+ if (masterAssignmentList.size() > numMastersFromEachNode) {
bigList.add(oldInstance);
- }
- else
- {
+ } else {
smallList.add(oldInstance);
}
}
- // "sort" the list, such that the nodes that has more master partitions moves more partitions to the
+ // "sort" the list, such that the nodes that has more master partitions moves more partitions to
+ // the
// new added batch of instances.
bigList.addAll(smallList);
int totalSlaveMoves = 0;
- for(String oldInstance : bigList)
- {
+ for (String oldInstance : bigList) {
List<Integer> masterAssignmentList = previousMasterAssignmentMap.get(oldInstance);
int numToChoose = numMastersFromEachNode;
- if(remain > 0)
- {
+ if (remain > 0) {
numToChoose = numMastersFromEachNode + 1;
- remain --;
+ remain--;
}
// randomly remove numToChoose of master partitions to the new added nodes
ArrayList<Integer> masterPartionsMoved = new ArrayList<Integer>();
@@ -535,11 +497,14 @@ public class DefaultIdealStateCalculator
masterPartitionListToMove.addAll(masterPartionsMoved);
Map<String, List<Integer>> slaveAssignmentMap = nodeSlaveAssignmentMap.get(oldInstance);
- removeFromSlaveAssignmentMap(slaveAssignmentMap, masterPartionsMoved, slavePartitionsToMoveMap);
+ removeFromSlaveAssignmentMap(slaveAssignmentMap, masterPartionsMoved,
+ slavePartitionsToMoveMap);
// Make sure that for old instances, the slave placement map is evenly distributed
- // Trace the "local slave moves", which should together contribute to most of the slave migrations
- int movesWithinInstance = migrateSlaveAssignMapToNewInstances(slaveAssignmentMap, newInstances);
+ // Trace the "local slave moves", which should together contribute to most of the slave
+ // migrations
+ int movesWithinInstance =
+ migrateSlaveAssignMapToNewInstances(slaveAssignmentMap, newInstances);
// System.out.println("local moves: "+ movesWithinInstance);
totalSlaveMoves += movesWithinInstance;
}
@@ -549,42 +514,33 @@ public class DefaultIdealStateCalculator
// We already have the list of master partitions that will migrate to new batch of instances,
// shuffle the partitions and assign them to new instances
Collections.shuffle(masterPartitionListToMove, new Random(12345));
- for(int i = 0;i < newInstances.size(); i++)
- {
+ for (int i = 0; i < newInstances.size(); i++) {
String newInstance = newInstances.get(i);
List<Integer> masterPartitionList = new ArrayList<Integer>();
- for(int j = 0;j < masterPartitionListToMove.size(); j++)
- {
- if(j % newInstances.size() == i)
- {
+ for (int j = 0; j < masterPartitionListToMove.size(); j++) {
+ if (j % newInstances.size() == i) {
masterPartitionList.add(masterPartitionListToMove.get(j));
}
}
Map<String, List<Integer>> slavePartitionMap = new TreeMap<String, List<Integer>>();
- for(String oldInstance : oldInstances)
- {
+ for (String oldInstance : oldInstances) {
slavePartitionMap.put(oldInstance, new ArrayList<Integer>());
}
// Build the slave assignment map for the new instance, based on the saved information
// about those slave partition locations in slavePartitionsToMoveMap
- for(Integer x : masterPartitionList)
- {
- for(String oldInstance : slavePartitionsToMoveMap.keySet())
- {
+ for (Integer x : masterPartitionList) {
+ for (String oldInstance : slavePartitionsToMoveMap.keySet()) {
List<Integer> slaves = slavePartitionsToMoveMap.get(oldInstance);
- if(slaves.contains(x))
- {
+ if (slaves.contains(x)) {
slavePartitionMap.get(oldInstance).add(x);
}
}
}
// add entry for other new instances into the slavePartitionMap
List<String> otherNewInstances = new ArrayList<String>();
- for(String instance : newInstances)
- {
- if(!instance.equalsIgnoreCase(newInstance))
- {
+ for (String instance : newInstances) {
+ if (!instance.equalsIgnoreCase(newInstance)) {
otherNewInstances.add(instance);
}
}
@@ -598,78 +554,71 @@ public class DefaultIdealStateCalculator
}
/*
- // Print content of the master/ slave assignment maps
- for(String instanceName : previousMasterAssignmentMap.keySet())
- {
- System.out.println(instanceName+":");
- for(Integer x : previousMasterAssignmentMap.get(instanceName))
- {
- System.out.print(x+" ");
- }
- System.out.println("\nmaster partition moved:");
-
- System.out.println();
- System.out.println("Slave assignment:");
-
- Map<String, List<Integer>> slaveAssignmentMap = nodeSlaveAssignmentMap.get(instanceName);
- for(String slaveName : slaveAssignmentMap.keySet())
- {
- System.out.print("\t" + slaveName +":\n\t" );
- for(Integer x : slaveAssignmentMap.get(slaveName))
- {
- System.out.print(x + " ");
- }
- System.out.println("\n");
- }
- }
-
- System.out.println("Master partitions migrated to new instances");
- for(Integer x : masterPartitionListToMove)
- {
- System.out.print(x+" ");
- }
- System.out.println();
-
- System.out.println("Slave partitions migrated to new instances");
- for(String oldInstance : slavePartitionsToMoveMap.keySet())
- {
- System.out.print(oldInstance + ": ");
- for(Integer x : slavePartitionsToMoveMap.get(oldInstance))
- {
- System.out.print(x+" ");
- }
- System.out.println();
- }
- */
+ * // Print content of the master/ slave assignment maps
+ * for(String instanceName : previousMasterAssignmentMap.keySet())
+ * {
+ * System.out.println(instanceName+":");
+ * for(Integer x : previousMasterAssignmentMap.get(instanceName))
+ * {
+ * System.out.print(x+" ");
+ * }
+ * System.out.println("\nmaster partition moved:");
+ * System.out.println();
+ * System.out.println("Slave assignment:");
+ * Map<String, List<Integer>> slaveAssignmentMap = nodeSlaveAssignmentMap.get(instanceName);
+ * for(String slaveName : slaveAssignmentMap.keySet())
+ * {
+ * System.out.print("\t" + slaveName +":\n\t" );
+ * for(Integer x : slaveAssignmentMap.get(slaveName))
+ * {
+ * System.out.print(x + " ");
+ * }
+ * System.out.println("\n");
+ * }
+ * }
+ * System.out.println("Master partitions migrated to new instances");
+ * for(Integer x : masterPartitionListToMove)
+ * {
+ * System.out.print(x+" ");
+ * }
+ * System.out.println();
+ * System.out.println("Slave partitions migrated to new instances");
+ * for(String oldInstance : slavePartitionsToMoveMap.keySet())
+ * {
+ * System.out.print(oldInstance + ": ");
+ * for(Integer x : slavePartitionsToMoveMap.get(oldInstance))
+ * {
+ * System.out.print(x+" ");
+ * }
+ * System.out.println();
+ * }
+ */
return previousIdealState;
}
-
- public ZNRecord calculateNextIdealState(List<String> newInstances, Map<String, Object> previousIdealState,
- String resourceName, String masterStateValue, String slaveStateValue)
- {
- return convertToZNRecord(calculateNextIdealState(newInstances, previousIdealState), resourceName, masterStateValue, slaveStateValue);
+
+ public ZNRecord calculateNextIdealState(List<String> newInstances,
+ Map<String, Object> previousIdealState, String resourceName, String masterStateValue,
+ String slaveStateValue) {
+ return convertToZNRecord(calculateNextIdealState(newInstances, previousIdealState),
+ resourceName, masterStateValue, slaveStateValue);
}
+
/**
* Given the list of master partition that will be migrated away from the storage instance,
* Remove their entries from the local instance slave assignment map.
- *
- * @param slaveAssignmentMap the local instance slave assignment map
+ * @param slaveAssignmentMap the local instance slave assignment map
* @param masterPartionsMoved the list of master partition ids that will be migrated away
* @param removedAssignmentMap keep track of the removed slave assignment info. The info can be
- * used by new added storage nodes.
- * */
- static void removeFromSlaveAssignmentMap( Map<String, List<Integer>>slaveAssignmentMap, List<Integer> masterPartionsMoved, Map<String, List<Integer>> removedAssignmentMap)
- {
- for(String instanceName : slaveAssignmentMap.keySet())
- {
+ * used by new added storage nodes.
+ */
+ static void removeFromSlaveAssignmentMap(Map<String, List<Integer>> slaveAssignmentMap,
+ List<Integer> masterPartionsMoved, Map<String, List<Integer>> removedAssignmentMap) {
+ for (String instanceName : slaveAssignmentMap.keySet()) {
List<Integer> slaveAssignment = slaveAssignmentMap.get(instanceName);
- for(Integer partitionId: masterPartionsMoved)
- {
- if(slaveAssignment.contains(partitionId))
- {
+ for (Integer partitionId : masterPartionsMoved) {
+ if (slaveAssignment.contains(partitionId)) {
slaveAssignment.remove(partitionId);
- if(!removedAssignmentMap.containsKey(instanceName))
- {
+ if (!removedAssignmentMap.containsKey(instanceName)) {
removedAssignmentMap.put(instanceName, new ArrayList<Integer>());
}
removedAssignmentMap.get(instanceName).add(partitionId);
@@ -681,58 +630,46 @@ public class DefaultIdealStateCalculator
/**
* Since some new storage instances are added, each existing storage instance should migrate some
* slave partitions to the new added instances.
- *
* The algorithm keeps moving one partition to from the instance that hosts most slave partitions
* to the instance that hosts least number of partitions, until max-min <= 1.
- *
- * In this way we can guarantee that all instances hosts almost same number of slave partitions, also
+ * In this way we can guarantee that all instances hosts almost same number of slave partitions,
+ * also
* slave partitions are evenly distributed.
- *
- * @param slaveAssignmentMap the local instance slave assignment map
+ * @param slaveAssignmentMap the local instance slave assignment map
* @param masterPartionsMoved the list of master partition ids that will be migrated away
* @param removedAssignmentMap keep track of the removed slave assignment info. The info can be
- * used by new added storage nodes.
- * */
- static int migrateSlaveAssignMapToNewInstances(Map<String, List<Integer>> nodeSlaveAssignmentMap, List<String> newInstances)
- {
+ * used by new added storage nodes.
+ */
+ static int migrateSlaveAssignMapToNewInstances(Map<String, List<Integer>> nodeSlaveAssignmentMap,
+ List<String> newInstances) {
int moves = 0;
boolean done = false;
- for(String newInstance : newInstances)
- {
+ for (String newInstance : newInstances) {
nodeSlaveAssignmentMap.put(newInstance, new ArrayList<Integer>());
}
- while(!done)
- {
+ while (!done) {
List<Integer> maxAssignment = null, minAssignment = null;
int minCount = Integer.MAX_VALUE, maxCount = Integer.MIN_VALUE;
String minInstance = "";
- for(String instanceName : nodeSlaveAssignmentMap.keySet())
- {
+ for (String instanceName : nodeSlaveAssignmentMap.keySet()) {
List<Integer> slaveAssignment = nodeSlaveAssignmentMap.get(instanceName);
- if(minCount > slaveAssignment.size())
- {
+ if (minCount > slaveAssignment.size()) {
minCount = slaveAssignment.size();
minAssignment = slaveAssignment;
minInstance = instanceName;
}
- if(maxCount < slaveAssignment.size())
- {
+ if (maxCount < slaveAssignment.size()) {
maxCount = slaveAssignment.size();
maxAssignment = slaveAssignment;
}
}
- if(maxCount - minCount <= 1 )
- {
+ if (maxCount - minCount <= 1) {
done = true;
- }
- else
- {
+ } else {
int indexToMove = -1;
// find a partition that is not contained in the minAssignment list
- for(int i = 0; i < maxAssignment.size(); i++ )
- {
- if(!minAssignment.contains(maxAssignment.get(i)))
- {
+ for (int i = 0; i < maxAssignment.size(); i++) {
+ if (!minAssignment.contains(maxAssignment.get(i))) {
indexToMove = i;
break;
}
@@ -741,8 +678,7 @@ public class DefaultIdealStateCalculator
minAssignment.add(maxAssignment.get(indexToMove));
maxAssignment.remove(indexToMove);
- if(newInstances.contains(minInstance))
- {
+ if (newInstances.contains(minInstance)) {
moves++;
}
}
@@ -753,40 +689,34 @@ public class DefaultIdealStateCalculator
/**
* Randomly select a number of elements from original list and put them in the selectedList
* The algorithm is used to select master partitions to be migrated when new instances are added.
- *
- *
- * @param originalList the original list
- * @param selectedList the list that contain selected elements
+ * @param originalList the original list
+ * @param selectedList the list that contain selected elements
* @param num number of elements to be selected
- * */
- static void randomSelect(List<Integer> originalList, List<Integer> selectedList, int num)
- {
- assert(originalList.size() >= num);
+ */
+ static void randomSelect(List<Integer> originalList, List<Integer> selectedList, int num) {
+ assert (originalList.size() >= num);
int[] indexArray = new int[originalList.size()];
- for(int i = 0;i < indexArray.length; i++)
- {
+ for (int i = 0; i < indexArray.length; i++) {
indexArray[i] = i;
}
int numRemains = originalList.size();
Random r = new Random(numRemains);
- for(int j = 0;j < num; j++)
- {
+ for (int j = 0; j < num; j++) {
int randIndex = r.nextInt(numRemains);
selectedList.add(originalList.get(randIndex));
originalList.remove(randIndex);
- numRemains --;
+ numRemains--;
}
}
- public static void main(String args[])
- {
+ public static void main(String args[]) {
List<String> instanceNames = new ArrayList<String>();
- for(int i = 0;i < 10; i++)
- {
+ for (int i = 0; i < 10; i++) {
instanceNames.add("localhost:123" + i);
}
- int partitions = 48*3, replicas = 3;
- Map<String, Object> resultOriginal = DefaultIdealStateCalculator.calculateInitialIdealState(instanceNames, partitions, replicas);
+ int partitions = 48 * 3, replicas = 3;
+ Map<String, Object> resultOriginal =
+ DefaultIdealStateCalculator.calculateInitialIdealState(instanceNames, partitions, replicas);
}
}