You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by ka...@apache.org on 2013/11/11 23:12:11 UTC
[1/2] git commit: [HELIX-290] Prefer ideal state for 0.7.0 rebalancer
if not user defined, rb=15381
Updated Branches:
refs/heads/master ebd37fed4 -> 00ff8c06c
[HELIX-290] Prefer ideal state for 0.7.0 rebalancer if not user defined, rb=15381
Project: http://git-wip-us.apache.org/repos/asf/incubator-helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-helix/commit/0cb37755
Tree: http://git-wip-us.apache.org/repos/asf/incubator-helix/tree/0cb37755
Diff: http://git-wip-us.apache.org/repos/asf/incubator-helix/diff/0cb37755
Branch: refs/heads/master
Commit: 0cb3775564cb53c4129b9afcaecc0876a33a120f
Parents: ea8c61d
Author: Kanak Biscuitwala <ka...@apache.org>
Authored: Fri Nov 8 16:03:24 2013 -0800
Committer: Kanak Biscuitwala <ka...@apache.org>
Committed: Mon Nov 11 14:09:17 2013 -0800
----------------------------------------------------------------------
.../helix/api/accessor/ResourceAccessor.java | 8 ++++--
.../context/CustomRebalancerContext.java | 3 +-
.../context/FullAutoRebalancerContext.java | 3 +-
.../context/PartitionedRebalancerContext.java | 30 ++++++++++++++++----
.../context/SemiAutoRebalancerContext.java | 8 +++---
5 files changed, 38 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/0cb37755/helix-core/src/main/java/org/apache/helix/api/accessor/ResourceAccessor.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/api/accessor/ResourceAccessor.java b/helix-core/src/main/java/org/apache/helix/api/accessor/ResourceAccessor.java
index 0dfceca..e5ac57c 100644
--- a/helix-core/src/main/java/org/apache/helix/api/accessor/ResourceAccessor.java
+++ b/helix-core/src/main/java/org/apache/helix/api/accessor/ResourceAccessor.java
@@ -436,12 +436,14 @@ public class ResourceAccessor {
int bucketSize = 0;
boolean batchMessageMode = false;
if (idealState != null) {
- if (resourceConfiguration != null) {
+ if (resourceConfiguration != null
+ && idealState.getRebalanceMode() != RebalanceMode.USER_DEFINED) {
+ // prefer rebalancer context for non-user_defined data rebalancing
rebalancerContext =
resourceConfiguration.getRebalancerContext(PartitionedRebalancerContext.class);
}
if (rebalancerContext == null) {
- // fallback: get rebalancer context from ideal state
+ // prefer ideal state for non-user_defined data rebalancing
rebalancerContext = PartitionedRebalancerContext.from(idealState);
}
bucketSize = idealState.getBucketSize();
@@ -454,7 +456,7 @@ public class ResourceAccessor {
rebalancerContext = rebalancerConfig.getRebalancerContext(RebalancerContext.class);
}
if (rebalancerContext == null) {
- rebalancerContext = new PartitionedRebalancerContext(RebalanceMode.NONE);
+ rebalancerContext = new PartitionedRebalancerContext();
}
return new Resource(resourceId, type, idealState, resourceAssignment, externalView,
rebalancerContext, userConfig, bucketSize, batchMessageMode);
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/0cb37755/helix-core/src/main/java/org/apache/helix/controller/rebalancer/context/CustomRebalancerContext.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/context/CustomRebalancerContext.java b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/context/CustomRebalancerContext.java
index 1fc1cda..0d2c1f2 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/context/CustomRebalancerContext.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/context/CustomRebalancerContext.java
@@ -53,7 +53,7 @@ public class CustomRebalancerContext extends PartitionedRebalancerContext {
* Instantiate a CustomRebalancerContext
*/
public CustomRebalancerContext() {
- super(RebalanceMode.CUSTOMIZED);
+ setRebalanceMode(RebalanceMode.CUSTOMIZED);
setRebalancerRef(RebalancerRef.from(CustomRebalancer.class));
_preferenceMaps = Maps.newHashMap();
}
@@ -133,6 +133,7 @@ public class CustomRebalancerContext extends PartitionedRebalancerContext {
public Builder(ResourceId resourceId) {
super(resourceId);
super.rebalancerRef(RebalancerRef.from(CustomRebalancer.class));
+ super.rebalanceMode(RebalanceMode.CUSTOMIZED);
_preferenceMaps = Maps.newHashMap();
}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/0cb37755/helix-core/src/main/java/org/apache/helix/controller/rebalancer/context/FullAutoRebalancerContext.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/context/FullAutoRebalancerContext.java b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/context/FullAutoRebalancerContext.java
index 2db9ac6..2400707 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/context/FullAutoRebalancerContext.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/context/FullAutoRebalancerContext.java
@@ -30,7 +30,7 @@ import org.apache.helix.model.IdealState.RebalanceMode;
*/
public class FullAutoRebalancerContext extends PartitionedRebalancerContext {
public FullAutoRebalancerContext() {
- super(RebalanceMode.FULL_AUTO);
+ setRebalanceMode(RebalanceMode.FULL_AUTO);
setRebalancerRef(RebalancerRef.from(FullAutoRebalancer.class));
}
@@ -46,6 +46,7 @@ public class FullAutoRebalancerContext extends PartitionedRebalancerContext {
public Builder(ResourceId resourceId) {
super(resourceId);
super.rebalancerRef(RebalancerRef.from(FullAutoRebalancer.class));
+ super.rebalanceMode(RebalanceMode.FULL_AUTO);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/0cb37755/helix-core/src/main/java/org/apache/helix/controller/rebalancer/context/PartitionedRebalancerContext.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/context/PartitionedRebalancerContext.java b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/context/PartitionedRebalancerContext.java
index 9d24e68..15fcf9c 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/context/PartitionedRebalancerContext.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/context/PartitionedRebalancerContext.java
@@ -49,17 +49,17 @@ public class PartitionedRebalancerContext extends BasicRebalancerContext impleme
private boolean _anyLiveParticipant;
private int _replicaCount;
private int _maxPartitionsPerParticipant;
- private final RebalanceMode _rebalanceMode;
+ private RebalanceMode _rebalanceMode;
/**
* Instantiate a DataRebalancerContext
*/
- public PartitionedRebalancerContext(RebalanceMode rebalanceMode) {
+ public PartitionedRebalancerContext() {
_partitionMap = Collections.emptyMap();
_replicaCount = 1;
_anyLiveParticipant = false;
_maxPartitionsPerParticipant = Integer.MAX_VALUE;
- _rebalanceMode = rebalanceMode;
+ _rebalanceMode = RebalanceMode.USER_DEFINED;
}
/**
@@ -140,6 +140,14 @@ public class PartitionedRebalancerContext extends BasicRebalancerContext impleme
}
/**
+ * Set the rebalancer mode of the partitioned resource
+ * @param rebalanceMode {@link RebalanceMode} enum value
+ */
+ public void setRebalanceMode(RebalanceMode rebalanceMode) {
+ _rebalanceMode = rebalanceMode;
+ }
+
+ /**
* Get the rebalancer mode of the resource
* @return RebalanceMode
*/
@@ -258,8 +266,7 @@ public class PartitionedRebalancerContext extends BasicRebalancerContext impleme
@Override
public PartitionedRebalancerContext build() {
- PartitionedRebalancerContext context =
- new PartitionedRebalancerContext(RebalanceMode.USER_DEFINED);
+ PartitionedRebalancerContext context = new PartitionedRebalancerContext();
super.update(context);
return context;
}
@@ -272,6 +279,7 @@ public class PartitionedRebalancerContext extends BasicRebalancerContext impleme
extends BasicRebalancerContext.AbstractBuilder<T> {
private final ResourceId _resourceId;
private final Map<PartitionId, Partition> _partitionMap;
+ private RebalanceMode _rebalanceMode;
private boolean _anyLiveParticipant;
private int _replicaCount;
private int _maxPartitionsPerParticipant;
@@ -284,12 +292,23 @@ public class PartitionedRebalancerContext extends BasicRebalancerContext impleme
super(resourceId);
_resourceId = resourceId;
_partitionMap = Maps.newHashMap();
+ _rebalanceMode = RebalanceMode.USER_DEFINED;
_anyLiveParticipant = false;
_replicaCount = 1;
_maxPartitionsPerParticipant = Integer.MAX_VALUE;
}
/**
+ * Set the rebalance mode for a partitioned rebalancer context
+ * @param rebalanceMode {@link RebalanceMode} enum value
+ * @return Builder
+ */
+ public T rebalanceMode(RebalanceMode rebalanceMode) {
+ _rebalanceMode = rebalanceMode;
+ return self();
+ }
+
+ /**
* Add a partition that the resource serves
* @param partition fully-qualified partition
* @return Builder
@@ -364,6 +383,7 @@ public class PartitionedRebalancerContext extends BasicRebalancerContext impleme
if (_partitionMap.isEmpty()) {
addPartitions(1);
}
+ context.setRebalanceMode(_rebalanceMode);
context.setPartitionMap(_partitionMap);
context.setAnyLiveParticipant(_anyLiveParticipant);
context.setMaxPartitionsPerParticipant(_maxPartitionsPerParticipant);
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/0cb37755/helix-core/src/main/java/org/apache/helix/controller/rebalancer/context/SemiAutoRebalancerContext.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/context/SemiAutoRebalancerContext.java b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/context/SemiAutoRebalancerContext.java
index f574a62..afa81e2 100644
--- a/helix-core/src/main/java/org/apache/helix/controller/rebalancer/context/SemiAutoRebalancerContext.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/rebalancer/context/SemiAutoRebalancerContext.java
@@ -56,7 +56,7 @@ public final class SemiAutoRebalancerContext extends PartitionedRebalancerContex
* Instantiate a SemiAutoRebalancerContext
*/
public SemiAutoRebalancerContext() {
- super(RebalanceMode.SEMI_AUTO);
+ setRebalanceMode(RebalanceMode.SEMI_AUTO);
setRebalancerRef(RebalancerRef.from(SemiAutoRebalancer.class));
_preferenceLists = Maps.newHashMap();
}
@@ -110,9 +110,8 @@ public final class SemiAutoRebalancerContext extends PartitionedRebalancerContex
Set<ParticipantId> disabledParticipants = Collections.emptySet();
Map<ParticipantId, State> emptyCurrentState = Collections.emptyMap();
Map<ParticipantId, State> initialMap =
- ConstraintBasedAssignment.computeAutoBestStateForPartition(upperBounds,
- participantSet, stateModelDef, preferenceList, emptyCurrentState,
- disabledParticipants);
+ ConstraintBasedAssignment.computeAutoBestStateForPartition(upperBounds, participantSet,
+ stateModelDef, preferenceList, emptyCurrentState, disabledParticipants);
currentMapping.put(partitionId, initialMap);
}
}
@@ -148,6 +147,7 @@ public final class SemiAutoRebalancerContext extends PartitionedRebalancerContex
public Builder(ResourceId resourceId) {
super(resourceId);
super.rebalancerRef(RebalancerRef.from(SemiAutoRebalancer.class));
+ super.rebalanceMode(RebalanceMode.SEMI_AUTO);
_preferenceLists = Maps.newHashMap();
}
[2/2] git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-helix
Posted by ka...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-helix
Project: http://git-wip-us.apache.org/repos/asf/incubator-helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-helix/commit/00ff8c06
Tree: http://git-wip-us.apache.org/repos/asf/incubator-helix/tree/00ff8c06
Diff: http://git-wip-us.apache.org/repos/asf/incubator-helix/diff/00ff8c06
Branch: refs/heads/master
Commit: 00ff8c06c3402c205a56945552e6a6cc57bfec3b
Parents: 0cb3775 ebd37fe
Author: Kanak Biscuitwala <ka...@apache.org>
Authored: Mon Nov 11 14:09:44 2013 -0800
Committer: Kanak Biscuitwala <ka...@apache.org>
Committed: Mon Nov 11 14:09:44 2013 -0800
----------------------------------------------------------------------
.../tools/TestHelixAdminScenariosRest.java | 889 +++++++++------
.../apache/helix/tools/TestResetInstance.java | 21 +-
.../helix/tools/TestResetPartitionState.java | 24 +-
.../apache/helix/tools/TestResetResource.java | 22 +-
.../org/apache/helix/agent/TestHelixAgent.java | 27 +-
.../java/org/apache/helix/HelixManager.java | 7 +
.../java/org/apache/helix/HelixProperty.java | 6 +-
.../rebalancer/SemiAutoRebalancer.java | 3 +
.../util/ConstraintBasedAssignment.java | 55 +-
.../stages/BestPossibleStateCalcStage.java | 64 +-
.../stages/CurrentStateComputationStage.java | 13 +-
.../stages/MessageGenerationStage.java | 2 +-
.../stages/MessageSelectionStage.java | 9 +-
.../controller/stages/MessageThrottleStage.java | 11 +-
.../controller/stages/ResourceCurrentState.java | 13 +-
.../controller/stages/TaskAssignmentStage.java | 12 +-
.../helix/manager/zk/AbstractManager.java | 691 ------------
.../helix/manager/zk/ControllerManager.java | 174 ---
.../manager/zk/ControllerManagerHelper.java | 7 +-
.../DefaultSchedulerMessageHandlerFactory.java | 1 +
.../zk/DistributedControllerManager.java | 190 ----
.../manager/zk/DistributedLeaderElection.java | 16 +-
.../manager/zk/HelixConnectionAdaptor.java | 6 +
.../helix/manager/zk/ParticipantManager.java | 153 ---
.../manager/zk/ParticipantManagerHelper.java | 27 +-
.../apache/helix/manager/zk/ZKHelixManager.java | 1041 +++++++++---------
.../helix/manager/zk/ZkAsyncCallbacks.java | 3 +-
.../helix/manager/zk/ZkStateChangeListener.java | 127 ---
.../messaging/DefaultMessagingService.java | 4 +-
.../apache/helix/model/ResourceAssignment.java | 19 +-
.../src/test/java/org/apache/helix/Mocks.java | 6 +
.../test/java/org/apache/helix/TestHelper.java | 97 +-
.../org/apache/helix/TestZkClientWrapper.java | 45 +-
.../java/org/apache/helix/TestZnodeModify.java | 3 +-
.../org/apache/helix/ZkHelixTestManager.java | 44 -
.../org/apache/helix/api/TestNewStages.java | 12 +-
.../controller/stages/DummyClusterManager.java | 6 +
.../stages/TestMessageThrottleStage.java | 30 +-
.../stages/TestParseInfoFromAlert.java | 5 +-
.../stages/TestRebalancePipeline.java | 19 +-
.../strategy/TestShufflingTwoStateStrategy.java | 68 +-
.../helix/healthcheck/TestAddDropAlert.java | 35 +-
.../healthcheck/TestAlertActionTriggering.java | 21 +-
.../helix/healthcheck/TestAlertFireHistory.java | 38 +-
.../helix/healthcheck/TestDummyAlerts.java | 14 +-
.../helix/healthcheck/TestExpandAlert.java | 40 +-
.../helix/healthcheck/TestSimpleAlert.java | 40 +-
.../healthcheck/TestSimpleWildcardAlert.java | 42 +-
.../helix/healthcheck/TestStalenessAlert.java | 36 +-
.../helix/healthcheck/TestWildcardAlert.java | 36 +-
.../helix/integration/TestAddClusterV2.java | 78 +-
.../TestAddNodeAfterControllerStart.java | 56 +-
.../TestAddStateModelFactoryAfterConnect.java | 14 +-
.../integration/TestAutoIsWithEmptyMap.java | 14 +-
.../helix/integration/TestAutoRebalance.java | 64 +-
.../TestAutoRebalancePartitionLimit.java | 102 +-
.../helix/integration/TestBatchMessage.java | 50 +-
.../integration/TestBatchMessageWrapper.java | 18 +-
.../integration/TestBucketizedResource.java | 15 +-
.../integration/TestCarryOverBadCurState.java | 19 +-
.../integration/TestCleanupExternalView.java | 20 +-
.../helix/integration/TestClusterStartsup.java | 15 +-
.../helix/integration/TestCustomIdealState.java | 12 -
.../TestCustomizedIdealStateRebalancer.java | 4 +-
.../apache/helix/integration/TestDisable.java | 36 +-
.../helix/integration/TestDisableNode.java | 2 +-
.../helix/integration/TestDisablePartition.java | 2 +-
.../integration/TestDistributedCMMain.java | 18 +-
.../TestDistributedClusterController.java | 18 +-
.../apache/helix/integration/TestDriver.java | 80 +-
.../org/apache/helix/integration/TestDrop.java | 150 ++-
.../helix/integration/TestDropResource.java | 9 +-
.../TestEnablePartitionDuringDisable.java | 22 +-
.../helix/integration/TestErrorPartition.java | 28 +-
.../integration/TestExternalViewUpdates.java | 21 +-
.../integration/TestHelixCustomCodeRunner.java | 40 +-
.../helix/integration/TestHelixInstanceTag.java | 3 +-
.../helix/integration/TestInstanceAutoJoin.java | 25 +-
.../integration/TestInvalidAutoIdealState.java | 14 +-
.../TestMessagePartitionStateMismatch.java | 4 +-
.../helix/integration/TestMessageThrottle.java | 16 +-
.../helix/integration/TestMessageThrottle2.java | 4 +-
.../helix/integration/TestMessagingService.java | 81 +-
.../integration/TestNonOfflineInitState.java | 28 +-
.../helix/integration/TestNullReplica.java | 15 +-
.../TestParticipantErrorMessage.java | 14 +-
.../TestParticipantNameCollision.java | 9 +-
.../helix/integration/TestPauseSignal.java | 15 +-
.../integration/TestRedefineStateModelDef.java | 13 +-
.../helix/integration/TestRenamePartition.java | 45 +-
.../helix/integration/TestResetInstance.java | 18 +-
.../integration/TestResetPartitionState.java | 17 +-
.../helix/integration/TestResetResource.java | 18 +-
.../integration/TestRestartParticipant.java | 41 +-
.../helix/integration/TestSchedulerMessage.java | 586 +++++++---
.../integration/TestSchedulerMsgContraints.java | 254 -----
.../integration/TestSchedulerMsgUsingQueue.java | 181 ---
.../helix/integration/TestSchemataSM.java | 14 +-
.../TestSessionExpiryInTransition.java | 29 +-
.../helix/integration/TestStandAloneCMMain.java | 40 +-
.../TestStandAloneCMSessionExpiry.java | 22 +-
...estStartMultipleControllersWithSameName.java | 9 +-
.../integration/TestStateTransitionTimeout.java | 38 +-
.../helix/integration/TestSwapInstance.java | 16 +-
.../integration/TestZkCallbackHandlerLeak.java | 89 +-
.../integration/ZkIntegrationTestBase.java | 49 +-
.../integration/ZkStandAloneCMTestBase.java | 63 +-
...dAloneCMTestBaseWithPropertyServerCheck.java | 25 +-
.../manager/ClusterControllerManager.java | 17 +-
.../manager/ClusterDistributedController.java | 20 +-
.../manager/MockParticipantManager.java | 8 +-
.../TestDistributedControllerManager.java | 9 +-
.../manager/TestParticipantManager.java | 26 +-
.../apache/helix/josql/TestJosqlProcessor.java | 8 +-
.../TestDefaultControllerMsgHandlerFactory.java | 9 +-
.../helix/manager/zk/TestHandleNewSession.java | 16 +-
.../manager/zk/TestLiveInstanceBounce.java | 20 +-
.../zk/TestZKPropertyTransferServer.java | 21 +-
.../helix/manager/zk/TestZkClusterManager.java | 16 +-
.../apache/helix/manager/zk/TestZkFlapping.java | 14 +-
.../zk/TestZkManagerFlappingDetection.java | 117 +-
.../manager/zk/TestZkStateChangeListener.java | 54 +-
.../handling/TestConfigThreadpoolSize.java | 10 +-
.../handling/TestResourceThreadpoolSize.java | 8 +-
.../mock/controller/ClusterController.java | 127 ---
.../MockHealthReportParticipant.java | 16 +-
.../helix/mock/participant/MockParticipant.java | 181 ---
.../helix/participant/MockZKHelixManager.java | 6 +
.../TestDistControllerStateModel.java | 13 +-
.../apache/helix/tools/TestHelixAdminCli.java | 162 +--
.../helix/userdefinedrebalancer/Lock.java | 5 +
131 files changed, 3175 insertions(+), 4684 deletions(-)
----------------------------------------------------------------------