You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by sw...@apache.org on 2013/09/22 23:49:30 UTC
git commit: added first cut of helix yarn test
Updated Branches:
refs/heads/helix-yarn e38aa54b0 -> b9ada66a8
added first cut of helix yarn test
Project: http://git-wip-us.apache.org/repos/asf/incubator-helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-helix/commit/b9ada66a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-helix/tree/b9ada66a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-helix/diff/b9ada66a
Branch: refs/heads/helix-yarn
Commit: b9ada66a8bac9e022f72f7f59264361cb4d3ac13
Parents: e38aa54
Author: Swaroop Jagadish <sj...@sjagadis-ld.linkedin.biz>
Authored: Sun Sep 22 14:48:44 2013 -0700
Committer: Swaroop Jagadish <sj...@sjagadis-ld.linkedin.biz>
Committed: Sun Sep 22 14:48:44 2013 -0700
----------------------------------------------------------------------
.../apache/helix/autoscale/AbstractTask.java | 12 +++
.../helix/autoscale/HelixYarnProcess.java | 57 +++++++++++++
.../apache/helix/autoscale/HelixYarnTest.java | 86 ++++++++++++++++++++
.../helix/autoscale/MyAppTargetProvider.java | 38 +++++++++
.../org/apache/helix/autoscale/MyAppTask.java | 27 ++++++
.../src/test/resources/myapp.properties | 10 +++
6 files changed, 230 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/b9ada66a/recipes/auto-scale/src/test/java/org/apache/helix/autoscale/AbstractTask.java
----------------------------------------------------------------------
diff --git a/recipes/auto-scale/src/test/java/org/apache/helix/autoscale/AbstractTask.java b/recipes/auto-scale/src/test/java/org/apache/helix/autoscale/AbstractTask.java
new file mode 100644
index 0000000..8af29af
--- /dev/null
+++ b/recipes/auto-scale/src/test/java/org/apache/helix/autoscale/AbstractTask.java
@@ -0,0 +1,12 @@
+package org.apache.helix.autoscale;
+import java.util.Properties;
+
+import org.apache.helix.task.Task;
+
+public abstract class AbstractTask implements Task {
+ protected Properties _taskConfig;
+
+ public AbstractTask(Properties config) {
+ _taskConfig = config;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/b9ada66a/recipes/auto-scale/src/test/java/org/apache/helix/autoscale/HelixYarnProcess.java
----------------------------------------------------------------------
diff --git a/recipes/auto-scale/src/test/java/org/apache/helix/autoscale/HelixYarnProcess.java b/recipes/auto-scale/src/test/java/org/apache/helix/autoscale/HelixYarnProcess.java
new file mode 100644
index 0000000..f4abd5f
--- /dev/null
+++ b/recipes/auto-scale/src/test/java/org/apache/helix/autoscale/HelixYarnProcess.java
@@ -0,0 +1,57 @@
+package org.apache.helix.autoscale;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerFactory;
+import org.apache.helix.InstanceType;
+import org.apache.helix.autoscale.container.ContainerProcess;
+import org.apache.helix.autoscale.container.ContainerProcessProperties;
+import org.apache.helix.autoscale.impl.container.RedisServerProcess.RedisServerModelFactory;
+import org.apache.helix.integration.task.WorkflowGenerator;
+import org.apache.helix.task.Task;
+import org.apache.helix.task.TaskConfig;
+import org.apache.helix.task.TaskDriver;
+import org.apache.helix.task.TaskFactory;
+import org.apache.helix.task.Workflow;
+
+import com.google.common.base.Preconditions;
+
+public abstract class HelixYarnProcess extends ContainerProcess {
+ protected final String _appName;
+ private String _taskClassName;
+
+ public HelixYarnProcess(ContainerProcessProperties properties) throws Exception {
+ configure(properties);
+ setModelName("Task");
+
+ _appName = properties.getProperty(ContainerProcessProperties.NAME);
+ _taskClassName = properties.getProperty("taskClass");
+
+ }
+
+ protected void startContainer() throws Exception {
+ }
+
+ protected void startParticipant() throws Exception {
+ participantManager = HelixManagerFactory.getZKHelixManager(clusterName, instanceName, InstanceType.PARTICIPANT, zookeeperAddress);
+
+ }
+
+
+ private Map<String, TaskFactory> getTaskFactoryReg(String appName, String taskClassName, Properties taskConfig) {
+ Map<String, TaskFactory> taskFactoryReg = new HashMap<String, TaskFactory>();
+ taskFactoryReg.put(_appName, new TaskFactory()
+ {
+ @Override
+ public Task createNewTask(String config)
+ {
+ return null;
+ }
+ });
+
+ return taskFactoryReg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/b9ada66a/recipes/auto-scale/src/test/java/org/apache/helix/autoscale/HelixYarnTest.java
----------------------------------------------------------------------
diff --git a/recipes/auto-scale/src/test/java/org/apache/helix/autoscale/HelixYarnTest.java b/recipes/auto-scale/src/test/java/org/apache/helix/autoscale/HelixYarnTest.java
new file mode 100644
index 0000000..50ca9ab
--- /dev/null
+++ b/recipes/auto-scale/src/test/java/org/apache/helix/autoscale/HelixYarnTest.java
@@ -0,0 +1,86 @@
+package org.apache.helix.autoscale;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerFactory;
+import org.apache.helix.InstanceType;
+import org.apache.helix.autoscale.impl.yarn.YarnContainerProviderProcess;
+import org.apache.helix.autoscale.impl.yarn.YarnStatusProvider;
+import org.apache.helix.integration.task.WorkflowGenerator;
+import org.apache.helix.task.TaskConfig;
+import org.apache.helix.task.TaskDriver;
+import org.apache.helix.task.Workflow;
+import org.apache.log4j.Logger;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class HelixYarnTest {
+
+ static final Logger log = Logger.getLogger(HelixYarnTest.class);
+
+ static final long TEST_TIMEOUT = 20000;
+ static final long REBALANCE_TIMEOUT = 10000;
+
+
+ YarnContainerProviderProcess _containerProvider;
+ YarnStatusProvider _containerStatusProvider;
+
+ @BeforeMethod(alwaysRun = true)
+ public void setupTest() throws Exception {
+ teardownTest();
+ TestUtils.configure();
+ TestUtils.startZookeeper();
+ _containerProvider = TestUtils.makeYarnProvider("provider_0");
+ _containerStatusProvider = new YarnStatusProvider(TestUtils.zkAddress);
+ }
+
+ @AfterMethod(alwaysRun = true)
+ public void teardownTest() throws Exception {
+ TestUtils.stopTestCluster();
+ TestUtils.stopZookeeper();
+ }
+
+ @Test(timeOut = TestUtils.TEST_TIMEOUT)
+ public void testBasic() throws Exception {
+ Properties applicationSpec = getProperties("myapp.properties");
+ TargetProviderService targetProvider = getTargetProviderService(applicationSpec);
+ TestUtils.startTestCluster(targetProvider, _containerStatusProvider, _containerProvider);
+
+ HelixManager taskClusterManager = HelixManagerFactory.getZKHelixManager(TestUtils.managedClusterName,
+ "Admin", InstanceType.ADMINISTRATOR, TestUtils.zkAddress);
+ taskClusterManager.connect();
+ TaskDriver driver = new TaskDriver(taskClusterManager);
+ /*Workflow flow = WorkflowGenerator.generateDefaultSingleTaskWorkflowBuilderWithExtraConfigs(taskName, TaskConfig.COMMAND_CONFIG,
+ String.valueOf(100)).setExpiry(expiry).build();
+
+ driver.start(flow);*/
+ //TODO: Submit Workflow using TaskDriver
+ }
+
+ private TargetProviderService getTargetProviderService(Properties applicationSpec) throws Exception {
+ String className = applicationSpec.getProperty("targetProviderClass");
+ Class<?> clazz = Class.forName(className);
+ return createInstance(clazz);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static <T> T createInstance(Class<?> clazz) throws Exception {
+ try {
+ log.debug(String.format("checking for default constructor in class '%s'", clazz.getSimpleName()));
+ return (T) clazz.getConstructor().newInstance();
+ } catch (Exception e) {
+ log.debug("no default constructor found");
+ }
+
+ throw new Exception(String.format("no suitable constructor for class '%s'", clazz.getSimpleName()));
+ }
+
+ static Properties getProperties(String resourcePath) throws IOException {
+ Properties properties = new Properties();
+ properties.load(ClassLoader.getSystemResourceAsStream(resourcePath));
+ return properties;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/b9ada66a/recipes/auto-scale/src/test/java/org/apache/helix/autoscale/MyAppTargetProvider.java
----------------------------------------------------------------------
diff --git a/recipes/auto-scale/src/test/java/org/apache/helix/autoscale/MyAppTargetProvider.java b/recipes/auto-scale/src/test/java/org/apache/helix/autoscale/MyAppTargetProvider.java
new file mode 100644
index 0000000..09584e2
--- /dev/null
+++ b/recipes/auto-scale/src/test/java/org/apache/helix/autoscale/MyAppTargetProvider.java
@@ -0,0 +1,38 @@
+package org.apache.helix.autoscale;
+
+import java.util.Properties;
+
+public class MyAppTargetProvider implements TargetProviderService {
+ private int _minCount = 0;
+ private int _maxCount = 0;
+
+ @Override
+ public int getTargetContainerCount(String containerType) throws Exception {
+ return _minCount; //TODO: Change this
+ }
+
+ @Override
+ public void configure(Properties properties) throws Exception {
+ String minContainersStr = (String) properties.get("minContainers");
+ String maxContainersStr = (String) properties.get("maxContainers");
+
+ if(minContainersStr != null) {
+ _minCount = Integer.parseInt(minContainersStr);
+ }
+
+ if(maxContainersStr != null) {
+ _maxCount = Integer.parseInt(maxContainersStr);
+ }
+ }
+
+ @Override
+ public void start() throws Exception {
+
+ }
+
+ @Override
+ public void stop() throws Exception {
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/b9ada66a/recipes/auto-scale/src/test/java/org/apache/helix/autoscale/MyAppTask.java
----------------------------------------------------------------------
diff --git a/recipes/auto-scale/src/test/java/org/apache/helix/autoscale/MyAppTask.java b/recipes/auto-scale/src/test/java/org/apache/helix/autoscale/MyAppTask.java
new file mode 100644
index 0000000..96f87df
--- /dev/null
+++ b/recipes/auto-scale/src/test/java/org/apache/helix/autoscale/MyAppTask.java
@@ -0,0 +1,27 @@
+package org.apache.helix.autoscale;
+
+import java.util.Properties;
+
+import org.apache.helix.task.TaskResult;
+
+public class MyAppTask extends AbstractTask {
+
+ public MyAppTask(Properties config) {
+ super(config);
+ }
+
+ @Override
+ public TaskResult run() {
+ System.out.println("Task running");
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ return new TaskResult(TaskResult.Status.COMPLETED, "Done");
+ }
+
+ @Override
+ public void cancel() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-helix/blob/b9ada66a/recipes/auto-scale/src/test/resources/myapp.properties
----------------------------------------------------------------------
diff --git a/recipes/auto-scale/src/test/resources/myapp.properties b/recipes/auto-scale/src/test/resources/myapp.properties
new file mode 100644
index 0000000..38a23f8
--- /dev/null
+++ b/recipes/auto-scale/src/test/resources/myapp.properties
@@ -0,0 +1,10 @@
+name=myapp
+minContainers=5
+maxContainers=10
+taskClass=MyAppTask
+targetProviderClass=MyAppTargetProvider
+serviceClass=MyAppService
+numTasks=50
+maxTasksPerContainer=10
+taskConfig.key1=foo
+taskConfig.key2=bar