You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by lx...@apache.org on 2018/03/23 18:31:52 UTC
helix git commit: Retrieve cached idealMappings for all Rebalancers
(AutoRebalancer,
DelayedRebalancer and CustomRebalancer) for any rebalance strategies. This
will avoid recompute idealmapping (preference list) if there is no
idealstate/instanceconfig/L
Repository: helix
Updated Branches:
refs/heads/master df012d5b8 -> a5de29c2a
Retrieve cached idealMappings for all Rebalancers (AutoRebalancer, DelayedRebalancer and CustomRebalancer) for any rebalance strategies. This will avoid recompute idealmapping (preference list) if there is no idealstate/instanceconfig/LiveInstances changes.
Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/a5de29c2
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/a5de29c2
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/a5de29c2
Branch: refs/heads/master
Commit: a5de29c2ac5b1a5567e50f2febbfec492a47c799
Parents: df012d5
Author: Lei Xia <lx...@linkedin.com>
Authored: Fri Feb 9 10:28:25 2018 -0800
Committer: Lei Xia <lx...@linkedin.com>
Committed: Fri Mar 23 11:21:16 2018 -0700
----------------------------------------------------------------------
.../rebalancer/AbstractRebalancer.java | 19 ++++++++++++++++++-
.../controller/rebalancer/AutoRebalancer.java | 11 +++++++++--
.../controller/rebalancer/CustomRebalancer.java | 2 ++
.../rebalancer/DelayedAutoRebalancer.java | 19 +++++++------------
.../rebalancer/SemiAutoRebalancer.java | 8 --------
5 files changed, 36 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/helix/blob/a5de29c2/helix-core/src/main/java/org/apache/helix/controller/rebalancer/AbstractRebalancer.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/AbstractRebalancer.java b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/AbstractRebalancer.java
index 5d77eb3..32057ff 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/AbstractRebalancer.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/AbstractRebalancer.java
@@ -32,6 +32,7 @@ import java.util.Set;
import org.apache.helix.HelixDefinedState;
import org.apache.helix.HelixException;
import org.apache.helix.HelixManager;
+import org.apache.helix.ZNRecord;
import org.apache.helix.controller.rebalancer.internal.MappingCalculator;
import org.apache.helix.controller.rebalancer.strategy.AutoRebalanceStrategy;
import org.apache.helix.controller.rebalancer.strategy.RebalanceStrategy;
@@ -103,10 +104,26 @@ public abstract class AbstractRebalancer implements Rebalancer, MappingCalculato
return partitionMapping;
}
+ /**
+ * Looking for cached ideal mapping for this resource, if it is already there, do not recompute it
+ * again. The cached mapping will be cleared in ClusterDataCache if there is anything changed in
+ * cluster state that can cause the potential changes in ideal state. This will avoid flip-flop
+ * issue we saw in AutoRebalanceStrategy, and also improve the performance by avoiding recompute
+ * IS everytime.
+ */
+ protected IdealState getCachedIdealState(String resourceName, ClusterDataCache clusterData) {
+ ZNRecord cachedIdealMapping = clusterData.getCachedIdealMapping(resourceName);
+ if (cachedIdealMapping != null) {
+ return new IdealState(cachedIdealMapping);
+ }
+
+ return null;
+ }
+
protected Map<String, Map<String, String>> currentMapping(CurrentStateOutput currentStateOutput,
String resourceName, List<String> partitions, Map<String, Integer> stateCountMap) {
- Map<String, Map<String, String>> map = new HashMap<String, Map<String, String>>();
+ Map<String, Map<String, String>> map = new HashMap<>();
for (String partition : partitions) {
Map<String, String> curStateMap =
http://git-wip-us.apache.org/repos/asf/helix/blob/a5de29c2/helix-core/src/main/java/org/apache/helix/controller/rebalancer/AutoRebalancer.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/AutoRebalancer.java b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/AutoRebalancer.java
index a82865b..1c385dd 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/AutoRebalancer.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/AutoRebalancer.java
@@ -21,14 +21,12 @@ package org.apache.helix.controller.rebalancer;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.helix.HelixDefinedState;
import org.apache.helix.HelixException;
import org.apache.helix.ZNRecord;
import org.apache.helix.controller.stages.ClusterDataCache;
@@ -57,6 +55,15 @@ public class AutoRebalancer extends AbstractRebalancer {
public IdealState computeNewIdealState(String resourceName,
IdealState currentIdealState, CurrentStateOutput currentStateOutput,
ClusterDataCache clusterData) {
+
+ IdealState cachedIdealState = getCachedIdealState(resourceName, clusterData);
+ if (cachedIdealState != null) {
+ LOG.debug("Use cached IdealState for " + resourceName);
+ return cachedIdealState;
+ }
+
+ LOG.info("Computing IdealState for " + resourceName);
+
List<String> partitions = new ArrayList<String>(currentIdealState.getPartitionSet());
String stateModelName = currentIdealState.getStateModelDefRef();
StateModelDefinition stateModelDef = clusterData.getStateModelDef(stateModelName);
http://git-wip-us.apache.org/repos/asf/helix/blob/a5de29c2/helix-core/src/main/java/org/apache/helix/controller/rebalancer/CustomRebalancer.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/CustomRebalancer.java b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/CustomRebalancer.java
index e769886..dae9f57 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/CustomRebalancer.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/CustomRebalancer.java
@@ -65,6 +65,8 @@ public class CustomRebalancer extends AbstractRebalancer {
return partitionMapping;
}
+ LOG.info("Computing BestPossibleMapping for " + resource);
+
String stateModelDefName = idealState.getStateModelDefRef();
StateModelDefinition stateModelDef = cache.getStateModelDef(stateModelDefName);
if (LOG.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/helix/blob/a5de29c2/helix-core/src/main/java/org/apache/helix/controller/rebalancer/DelayedAutoRebalancer.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/DelayedAutoRebalancer.java b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/DelayedAutoRebalancer.java
index f943abf..dc1517a 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/DelayedAutoRebalancer.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/DelayedAutoRebalancer.java
@@ -33,6 +33,7 @@ import java.util.Set;
import org.apache.helix.HelixDefinedState;
import org.apache.helix.ZNRecord;
import org.apache.helix.controller.rebalancer.strategy.AutoRebalanceStrategy;
+import org.apache.helix.controller.rebalancer.strategy.RebalanceStrategy;
import org.apache.helix.controller.rebalancer.util.RebalanceScheduler;
import org.apache.helix.controller.stages.ClusterDataCache;
import org.apache.helix.controller.stages.CurrentStateOutput;
@@ -59,20 +60,14 @@ public class DelayedAutoRebalancer extends AbstractRebalancer {
IdealState currentIdealState, CurrentStateOutput currentStateOutput,
ClusterDataCache clusterData) {
- // Looking for cached ideal mapping for this resource, if it is already there, do not recompute it again.
- // The cached mapping will be cleared in ClusterDataCache if there is anything changed in cluster state that can
- // cause the potential changes in ideal state.
- // this will avoid flip-flop issue we saw in AutoRebalanceStrategy.
- ZNRecord znRecord = clusterData.getCachedIdealMapping(resourceName);
- if (znRecord != null) {
- // TODO: only apply to legacy Auto-RebalanceStrategy at this time, need to apply to any strategy in future.
- if (AutoRebalanceStrategy.class.getName().equals(currentIdealState.getRebalanceStrategy())) {
- LOG.info("Use cached idealstate for " + resourceName);
- IdealState idealState = new IdealState(znRecord);
- return idealState;
- }
+ IdealState cachedIdealState = getCachedIdealState(resourceName, clusterData);
+ if (cachedIdealState != null) {
+ LOG.debug("Use cached IdealState for " + resourceName);
+ return cachedIdealState;
}
+ LOG.info("Computing IdealState for " + resourceName);
+
List<String> allPartitions = new ArrayList<>(currentIdealState.getPartitionSet());
if (allPartitions.size() == 0) {
LOG.info("Partition count is 0 for resource " + resourceName
http://git-wip-us.apache.org/repos/asf/helix/blob/a5de29c2/helix-core/src/main/java/org/apache/helix/controller/rebalancer/SemiAutoRebalancer.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/SemiAutoRebalancer.java b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/SemiAutoRebalancer.java
index fd422d5..940d45a 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/SemiAutoRebalancer.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/SemiAutoRebalancer.java
@@ -18,17 +18,9 @@ package org.apache.helix.controller.rebalancer;
* specific language governing permissions and limitations
* under the License.
*/
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.helix.HelixDefinedState;
import org.apache.helix.controller.stages.ClusterDataCache;
import org.apache.helix.controller.stages.CurrentStateOutput;
import org.apache.helix.model.IdealState;
-import org.apache.helix.model.StateModelDefinition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;