You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by zz...@apache.org on 2014/08/21 23:03:59 UTC
[2/2] git commit: [HELIX-329] HelixAutoController isn't a very clear
name, consider renaming, rb=24807
[HELIX-329] HelixAutoController isn't a very clear name, consider renaming, rb=24807
Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/3365bac2
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/3365bac2
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/3365bac2
Branch: refs/heads/master
Commit: 3365bac22c3d7f39efcac9b4e74d50515f6ead9f
Parents: 0746d07 363c049
Author: zzhang <zz...@apache.org>
Authored: Thu Aug 21 14:03:43 2014 -0700
Committer: zzhang <zz...@apache.org>
Committed: Thu Aug 21 14:03:43 2014 -0700
----------------------------------------------------------------------
helix-core/helix-core-0.7.1-SNAPSHOT.ivy | 3 +-
.../main/java/org/apache/helix/api/Cluster.java | 16 +-
.../helix/api/accessor/ClusterAccessor.java | 3 +-
.../controller/GenericHelixController.java | 54 ++-
.../helix/controller/HelixControllerMain.java | 9 +-
.../controller/stages/ClusterDataCache.java | 24 +-
.../manager/zk/ControllerManagerHelper.java | 5 +
.../helix/manager/zk/ZkCallbackHandler.java | 4 +
.../helix/manager/zk/ZkHelixConnection.java | 9 +-
.../helix/manager/zk/ZkHelixController.java | 6 +
.../org/apache/helix/model/InstanceConfig.java | 2 +-
.../MultiClusterControllerStateModel.java | 135 ------
...MultiClusterControllerStateModelFactory.java | 39 --
...MultiClusterControllerTransitionHandler.java | 135 ++++++
...usterControllerTransitionHandlerFactory.java | 38 ++
.../java/org/apache/helix/task/JobContext.java | 11 +
.../java/org/apache/helix/task/JobQueue.java | 99 ++++
.../java/org/apache/helix/task/TaskConfig.java | 37 +-
.../java/org/apache/helix/task/TaskDriver.java | 173 ++++++-
.../org/apache/helix/task/TaskRebalancer.java | 78 ++-
.../org/apache/helix/task/TaskStateModel.java | 6 +
.../java/org/apache/helix/task/TaskUtil.java | 37 +-
.../java/org/apache/helix/task/Workflow.java | 28 +-
.../org/apache/helix/task/WorkflowConfig.java | 50 +-
.../org/apache/helix/task/WorkflowContext.java | 8 +-
.../integration/TestZkCallbackHandlerLeak.java | 35 +-
.../manager/TestZkCallbackHandlerLeak.java | 474 -------------------
.../integration/task/TestTaskRebalancer.java | 59 ++-
.../apache/helix/integration/task/TestUtil.java | 6 +-
.../manager/zk/MockMultiClusterController.java | 6 +-
.../TestMultiClusterControllerStateModel.java | 6 +-
...MultiClusterControllerStateModelFactory.java | 6 +-
.../helix/provisioning/yarn/AppLauncher.java | 6 +-
.../yarn/AppStatusReportGenerator.java | 14 +-
.../provisioning/yarn/YarnProvisioner.java | 6 +-
pom.xml | 15 +-
36 files changed, 868 insertions(+), 774 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/helix/blob/3365bac2/helix-core/src/main/java/org/apache/helix/controller/HelixControllerMain.java
----------------------------------------------------------------------
diff --cc helix-core/src/main/java/org/apache/helix/controller/HelixControllerMain.java
index 2f5686c,b652f35..50f5ca0
--- a/helix-core/src/main/java/org/apache/helix/controller/HelixControllerMain.java
+++ b/helix-core/src/main/java/org/apache/helix/controller/HelixControllerMain.java
@@@ -49,7 -49,8 +49,8 @@@ import org.apache.helix.HelixManagerFac
import org.apache.helix.InstanceType;
import org.apache.helix.api.id.StateModelDefId;
import org.apache.helix.manager.zk.HelixManagerShutdownHook;
- import org.apache.helix.participant.MultiClusterControllerStateModelFactory;
++import org.apache.helix.participant.MultiClusterControllerTransitionHandlerFactory;
+ import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
-import org.apache.helix.participant.DistClusterControllerStateModelFactory;
import org.apache.helix.participant.StateMachineEngine;
import org.apache.log4j.Logger;
@@@ -162,8 -165,8 +165,8 @@@ public class HelixControllerMain
HelixManagerFactory.getZKHelixManager(clusterName, controllerName,
InstanceType.CONTROLLER_PARTICIPANT, zkConnectString);
- MultiClusterControllerStateModelFactory stateModelFactory =
- new MultiClusterControllerStateModelFactory(zkConnectString);
- DistClusterControllerStateModelFactory stateModelFactory =
- new DistClusterControllerStateModelFactory(zkConnectString);
++ MultiClusterControllerTransitionHandlerFactory stateModelFactory =
++ new MultiClusterControllerTransitionHandlerFactory(zkConnectString);
StateMachineEngine stateMach = manager.getStateMachineEngine();
stateMach.registerStateModelFactory(StateModelDefId.LeaderStandby, stateModelFactory);
http://git-wip-us.apache.org/repos/asf/helix/blob/3365bac2/helix-core/src/main/java/org/apache/helix/manager/zk/ZkHelixConnection.java
----------------------------------------------------------------------
diff --cc helix-core/src/main/java/org/apache/helix/manager/zk/ZkHelixConnection.java
index 32191ec,38332c5..222f15c
--- a/helix-core/src/main/java/org/apache/helix/manager/zk/ZkHelixConnection.java
+++ b/helix-core/src/main/java/org/apache/helix/manager/zk/ZkHelixConnection.java
@@@ -210,7 -210,8 +210,8 @@@ public class ZkHelixConnection implemen
}
@Override
- public HelixMultiClusterController createMultiClusterController(ClusterId clusterId, ControllerId controllerId) {
- public HelixMultiClusterController createAutoController(ClusterId clusterId,
++ public HelixMultiClusterController createMultiClusterController(ClusterId clusterId,
+ ControllerId controllerId) {
return new ZkHelixMultiClusterController(this, clusterId, controllerId);
}
http://git-wip-us.apache.org/repos/asf/helix/blob/3365bac2/helix-core/src/main/java/org/apache/helix/participant/MultiClusterControllerTransitionHandler.java
----------------------------------------------------------------------
diff --cc helix-core/src/main/java/org/apache/helix/participant/MultiClusterControllerTransitionHandler.java
index 0000000,0000000..46f3c0f
new file mode 100644
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/participant/MultiClusterControllerTransitionHandler.java
@@@ -1,0 -1,0 +1,135 @@@
++package org.apache.helix.participant;
++
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing,
++ * software distributed under the License is distributed on an
++ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++ * KIND, either express or implied. See the License for the
++ * specific language governing permissions and limitations
++ * under the License.
++ */
++
++import org.apache.helix.HelixManager;
++import org.apache.helix.HelixManagerFactory;
++import org.apache.helix.InstanceType;
++import org.apache.helix.NotificationContext;
++import org.apache.helix.api.TransitionHandler;
++import org.apache.helix.model.Message;
++import org.apache.helix.participant.statemachine.StateModelInfo;
++import org.apache.helix.participant.statemachine.StateModelParser;
++import org.apache.helix.participant.statemachine.StateTransitionError;
++import org.apache.helix.participant.statemachine.Transition;
++import org.apache.log4j.Logger;
++
++@StateModelInfo(initialState = "OFFLINE", states = {
++ "LEADER", "STANDBY"
++})
++public class MultiClusterControllerTransitionHandler extends TransitionHandler {
++ private static Logger logger = Logger.getLogger(MultiClusterControllerTransitionHandler.class);
++ private HelixManager _controller = null;
++ private final String _zkAddr;
++
++ public MultiClusterControllerTransitionHandler(String zkAddr) {
++ StateModelParser parser = new StateModelParser();
++ _currentState = parser.getInitialState(MultiClusterControllerTransitionHandler.class);
++ _zkAddr = zkAddr;
++ }
++
++ @Transition(to = "STANDBY", from = "OFFLINE")
++ public void onBecomeStandbyFromOffline(Message message, NotificationContext context) {
++ logger.info("Becoming standby from offline");
++ }
++
++ @Transition(to = "LEADER", from = "STANDBY")
++ public void onBecomeLeaderFromStandby(Message message, NotificationContext context)
++ throws Exception {
++ String clusterName = message.getPartitionId().stringify();
++ String controllerName = message.getTgtName();
++
++ logger.info(controllerName + " becomes leader from standby for " + clusterName);
++ // System.out.println(controllerName + " becomes leader from standby for " + clusterName);
++
++ if (_controller == null) {
++ _controller =
++ HelixManagerFactory.getZKHelixManager(clusterName, controllerName,
++ InstanceType.CONTROLLER, _zkAddr);
++ _controller.connect();
++ _controller.startTimerTasks();
++ } else {
++ logger.error("controller already exists:" + _controller.getInstanceName() + " for "
++ + clusterName);
++ }
++
++ }
++
++ @Transition(to = "STANDBY", from = "LEADER")
++ public void onBecomeStandbyFromLeader(Message message, NotificationContext context) {
++ String clusterName = message.getPartitionId().stringify();
++ String controllerName = message.getTgtName();
++
++ logger.info(controllerName + " becoming standby from leader for " + clusterName);
++
++ if (_controller != null) {
++ _controller.disconnect();
++ _controller = null;
++ } else {
++ logger.error("No controller exists for " + clusterName);
++ }
++ }
++
++ @Transition(to = "OFFLINE", from = "STANDBY")
++ public void onBecomeOfflineFromStandby(Message message, NotificationContext context) {
++ String clusterName = message.getPartitionId().stringify();
++ String controllerName = message.getTgtName();
++
++ logger.info(controllerName + " becoming offline from standby for cluster:" + clusterName);
++
++ }
++
++ @Transition(to = "DROPPED", from = "OFFLINE")
++ public void onBecomeDroppedFromOffline(Message message, NotificationContext context) {
++ logger.info("Becoming dropped from offline");
++ }
++
++ @Transition(to = "OFFLINE", from = "DROPPED")
++ public void onBecomeOfflineFromDropped(Message message, NotificationContext context) {
++ logger.info("Becoming offline from dropped");
++ }
++
++ @Override
++ public void rollbackOnError(Message message, NotificationContext context,
++ StateTransitionError error) {
++ String clusterName = message.getPartitionId().stringify();
++ String controllerName = message.getTgtName();
++
++ logger.error(controllerName + " rollbacks on error for " + clusterName);
++
++ if (_controller != null) {
++ _controller.disconnect();
++ _controller = null;
++ }
++
++ }
++
++ @Override
++ public void reset() {
++ if (_controller != null) {
++ // System.out.println("disconnect " + _controller.getInstanceName()
++ // + "(" + _controller.getInstanceType()
++ // + ") from " + _controller.getClusterName());
++ _controller.disconnect();
++ _controller = null;
++ }
++
++ }
++}
http://git-wip-us.apache.org/repos/asf/helix/blob/3365bac2/helix-core/src/main/java/org/apache/helix/participant/MultiClusterControllerTransitionHandlerFactory.java
----------------------------------------------------------------------
diff --cc helix-core/src/main/java/org/apache/helix/participant/MultiClusterControllerTransitionHandlerFactory.java
index 0000000,0000000..f368c7a
new file mode 100644
--- /dev/null
+++ b/helix-core/src/main/java/org/apache/helix/participant/MultiClusterControllerTransitionHandlerFactory.java
@@@ -1,0 -1,0 +1,38 @@@
++package org.apache.helix.participant;
++
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one
++ * or more contributor license agreements. See the NOTICE file
++ * distributed with this work for additional information
++ * regarding copyright ownership. The ASF licenses this file
++ * to you under the Apache License, Version 2.0 (the
++ * "License"); you may not use this file except in compliance
++ * with the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing,
++ * software distributed under the License is distributed on an
++ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++ * KIND, either express or implied. See the License for the
++ * specific language governing permissions and limitations
++ * under the License.
++ */
++
++import org.apache.helix.api.StateTransitionHandlerFactory;
++import org.apache.helix.api.id.PartitionId;
++
++public class MultiClusterControllerTransitionHandlerFactory extends
++ StateTransitionHandlerFactory<MultiClusterControllerTransitionHandler> {
++ private final String _zkAddr;
++
++ public MultiClusterControllerTransitionHandlerFactory(String zkAddr) {
++ _zkAddr = zkAddr;
++ }
++
++ @Override
++ public MultiClusterControllerTransitionHandler createStateTransitionHandler(PartitionId partition) {
++ return new MultiClusterControllerTransitionHandler(_zkAddr);
++ }
++
++}
http://git-wip-us.apache.org/repos/asf/helix/blob/3365bac2/helix-core/src/test/java/org/apache/helix/manager/zk/MockMultiClusterController.java
----------------------------------------------------------------------
diff --cc helix-core/src/test/java/org/apache/helix/manager/zk/MockMultiClusterController.java
index 63c3e84,f68444c..79e64d0
--- a/helix-core/src/test/java/org/apache/helix/manager/zk/MockMultiClusterController.java
+++ b/helix-core/src/test/java/org/apache/helix/manager/zk/MockMultiClusterController.java
@@@ -25,7 -25,7 +25,7 @@@ import java.util.concurrent.CountDownLa
import org.apache.helix.InstanceType;
import org.apache.helix.api.id.StateModelDefId;
- import org.apache.helix.participant.MultiClusterControllerStateModelFactory;
-import org.apache.helix.participant.DistClusterControllerStateModelFactory;
++import org.apache.helix.participant.MultiClusterControllerTransitionHandlerFactory;
import org.apache.helix.participant.StateMachineEngine;
import org.apache.log4j.Logger;
@@@ -63,8 -63,8 +63,8 @@@ public class MockMultiClusterControlle
public void run() {
try {
StateMachineEngine stateMach = getStateMachineEngine();
- MultiClusterControllerStateModelFactory lsModelFactory =
- new MultiClusterControllerStateModelFactory(_zkAddress);
- DistClusterControllerStateModelFactory lsModelFactory =
- new DistClusterControllerStateModelFactory(_zkAddress);
++ MultiClusterControllerTransitionHandlerFactory lsModelFactory =
++ new MultiClusterControllerTransitionHandlerFactory(_zkAddress);
stateMach.registerStateModelFactory(StateModelDefId.LeaderStandby, lsModelFactory);
connect();
http://git-wip-us.apache.org/repos/asf/helix/blob/3365bac2/helix-core/src/test/java/org/apache/helix/participant/TestMultiClusterControllerStateModel.java
----------------------------------------------------------------------
diff --cc helix-core/src/test/java/org/apache/helix/participant/TestMultiClusterControllerStateModel.java
index c991976,0000000..6714ed8
mode 100644,000000..100644
--- a/helix-core/src/test/java/org/apache/helix/participant/TestMultiClusterControllerStateModel.java
+++ b/helix-core/src/test/java/org/apache/helix/participant/TestMultiClusterControllerStateModel.java
@@@ -1,120 -1,0 +1,120 @@@
+package org.apache.helix.participant;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.helix.NotificationContext;
+import org.apache.helix.TestHelper;
+import org.apache.helix.api.id.MessageId;
+import org.apache.helix.api.id.PartitionId;
+import org.apache.helix.model.Message;
+import org.apache.helix.model.Message.MessageType;
- import org.apache.helix.participant.MultiClusterControllerStateModel;
++import org.apache.helix.participant.MultiClusterControllerTransitionHandler;
+import org.apache.helix.testutil.ZkTestBase;
+import org.apache.log4j.Logger;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class TestMultiClusterControllerStateModel extends ZkTestBase {
+ private static Logger LOG = Logger.getLogger(TestMultiClusterControllerStateModel.class);
+
+ final String clusterName = "TestMultiClusterControllerStateModel";
- MultiClusterControllerStateModel stateModel = null;
++ MultiClusterControllerTransitionHandler stateModel = null;
+
+ @BeforeMethod()
+ public void beforeMethod() {
- stateModel = new MultiClusterControllerStateModel(_zkaddr);
++ stateModel = new MultiClusterControllerTransitionHandler(_zkaddr);
+ if (_zkclient.exists("/" + clusterName)) {
+ _zkclient.deleteRecursive("/" + clusterName);
+ }
+ TestHelper.setupEmptyCluster(_zkclient, clusterName);
+ }
+
+ @Test()
+ public void testOnBecomeStandbyFromOffline() {
+ stateModel.onBecomeStandbyFromOffline(null, null);
+ }
+
+ @Test()
+ public void testOnBecomeLeaderFromStandby() {
+ Message message = new Message(MessageType.STATE_TRANSITION, MessageId.from("0"));
+ message.setPartitionId(PartitionId.from(clusterName));
+ message.setTgtName("controller_0");
+ try {
+ stateModel.onBecomeLeaderFromStandby(message, new NotificationContext(null));
+ } catch (Exception e) {
+ LOG.error("Exception becoming leader from standby", e);
+ }
+ stateModel.onBecomeStandbyFromLeader(message, new NotificationContext(null));
+ }
+
+ @Test()
+ public void testOnBecomeStandbyFromLeader() {
+ Message message = new Message(MessageType.STATE_TRANSITION, MessageId.from("0"));
+ message.setPartitionId(PartitionId.from(clusterName));
+ message.setTgtName("controller_0");
+ stateModel.onBecomeStandbyFromLeader(message, new NotificationContext(null));
+ }
+
+ @Test()
+ public void testOnBecomeOfflineFromStandby() {
+ Message message = new Message(MessageType.STATE_TRANSITION, MessageId.from("0"));
+ message.setPartitionId(PartitionId.from(clusterName));
+ message.setTgtName("controller_0");
+
+ stateModel.onBecomeOfflineFromStandby(message, null);
+ }
+
+ @Test()
+ public void testOnBecomeDroppedFromOffline() {
+ stateModel.onBecomeDroppedFromOffline(null, null);
+ }
+
+ @Test()
+ public void testOnBecomeOfflineFromDropped() {
+ stateModel.onBecomeOfflineFromDropped(null, null);
+ }
+
+ @Test()
+ public void testRollbackOnError() {
+ Message message = new Message(MessageType.STATE_TRANSITION, MessageId.from("0"));
+ message.setPartitionId(PartitionId.from(clusterName));
+ message.setTgtName("controller_0");
+ try {
+ stateModel.onBecomeLeaderFromStandby(message, new NotificationContext(null));
+ } catch (Exception e) {
+ LOG.error("Exception becoming leader from standby", e);
+ }
+ stateModel.rollbackOnError(message, new NotificationContext(null), null);
+ }
+
+ @Test()
+ public void testReset() {
+ Message message = new Message(MessageType.STATE_TRANSITION, MessageId.from("0"));
+ message.setPartitionId(PartitionId.from(clusterName));
+ message.setTgtName("controller_0");
+ try {
+ stateModel.onBecomeLeaderFromStandby(message, new NotificationContext(null));
+ } catch (Exception e) {
+ LOG.error("Exception becoming leader from standby", e);
+ }
+ stateModel.reset();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/helix/blob/3365bac2/helix-core/src/test/java/org/apache/helix/participant/TestMultiClusterControllerStateModelFactory.java
----------------------------------------------------------------------
diff --cc helix-core/src/test/java/org/apache/helix/participant/TestMultiClusterControllerStateModelFactory.java
index 43eded0,0000000..922a60d
mode 100644,000000..100644
--- a/helix-core/src/test/java/org/apache/helix/participant/TestMultiClusterControllerStateModelFactory.java
+++ b/helix-core/src/test/java/org/apache/helix/participant/TestMultiClusterControllerStateModelFactory.java
@@@ -1,35 -1,0 +1,35 @@@
+package org.apache.helix.participant;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.helix.api.id.PartitionId;
+import org.testng.annotations.Test;
+
+public class TestMultiClusterControllerStateModelFactory {
+
+ @Test()
+ public void test() {
- MultiClusterControllerStateModelFactory factory =
- new MultiClusterControllerStateModelFactory("localhost:2181");
- MultiClusterControllerStateModel stateModel =
++ MultiClusterControllerTransitionHandlerFactory factory =
++ new MultiClusterControllerTransitionHandlerFactory("localhost:2181");
++ MultiClusterControllerTransitionHandler stateModel =
+ factory.createStateTransitionHandler(PartitionId.from("key"));
+ stateModel.onBecomeStandbyFromOffline(null, null);
+ }
+}