[GitHub] [helix] rabashizade commented on a change in pull request #1169: Dynamically Load Tasks in Task Framework

[GitBox <gi...@apache.org>]

rabashizade commented on a change in pull request #1169:
URL: https://github.com/apache/helix/pull/1169#discussion_r470269287



##########
File path: helix-core/src/test/java/org/apache/helix/task/TestDynamicTaskLoading.java
##########
@@ -0,0 +1,251 @@
+package org.apache.helix.task;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.helix.AccessOption;
+import org.apache.helix.HelixException;
+import org.apache.helix.HelixManager;
+import org.apache.helix.HelixManagerFactory;
+import org.apache.helix.InstanceType;
+import org.apache.helix.TestHelper;
+import org.apache.helix.common.ZkTestBase;
+import org.apache.helix.examples.MasterSlaveStateModelFactory;
+import org.apache.helix.integration.manager.ClusterControllerManager;
+import org.apache.helix.integration.manager.MockParticipantManager;
+import org.apache.helix.integration.task.MockTask;
+import org.apache.helix.participant.StateMachineEngine;
+import org.apache.helix.participant.statemachine.StateModel;
+import org.apache.helix.participant.statemachine.StateModelFactory;
+import org.testng.AssertJUnit;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+public class TestDynamicTaskLoading extends ZkTestBase {
+  private final String _clusterName = CLUSTER_PREFIX + "_" + getShortClassName();
+  private static final String _instanceName = "localhost_12913";
+  private HelixManager _manager;
+  private MockParticipantManager _participant;
+  private ClusterControllerManager _controller;
+
+  @BeforeMethod
+  public void beforeMethod() throws Exception {
+    _gSetupTool.addCluster(_clusterName, true);
+
+    _manager = HelixManagerFactory
+        .getZKHelixManager(_clusterName, "Admin", InstanceType.ADMINISTRATOR, ZK_ADDR);
+    _manager.connect();
+
+    _gSetupTool.addInstanceToCluster(_clusterName, _instanceName);
+    _participant = new MockParticipantManager(ZK_ADDR, _clusterName, _instanceName);
+    StateModelFactory<StateModel> stateModelFactory =
+        new MasterSlaveStateModelFactory(_instanceName, 0);
+    StateMachineEngine stateMach = _participant.getStateMachineEngine();
+    stateMach.registerStateModelFactory("MasterSlave", stateModelFactory);
+    Map<String, TaskFactory> taskFactoryReg = new HashMap<>();
+    _participant.getStateMachineEngine().registerStateModelFactory(TaskConstants.STATE_MODEL_NAME,
+        new TaskStateModelFactory(_participant, taskFactoryReg));
+    _participant.syncStart();
+
+    _controller = new ClusterControllerManager(ZK_ADDR, _clusterName, null);
+    _controller.syncStart();
+  }
+
+  private void removePathIfExists(String path) {
+    if (_manager.getHelixDataAccessor().getBaseDataAccessor().exists(path, 0)) {
+      _manager.getHelixDataAccessor().getBaseDataAccessor().remove(path, 0);
+    }
+  }
+
+  /**
+   * Submit a workflow consisting of a job with a MockTask task.
+   * @param workflowName name of the workflow
+   * @param driver {@link TaskDriver} to submit workflowName to
+   */
+  private void submitWorkflow(String workflowName, TaskDriver driver) {
+    JobConfig.Builder job = new JobConfig.Builder();
+    job.setJobCommandConfigMap(Collections.singletonMap(MockTask.JOB_DELAY, "100"));
+    Workflow.Builder workflow = new Workflow.Builder(workflowName);
+    job.setWorkflow(workflowName);
+    TaskConfig taskConfig =
+        new TaskConfig(MockTask.TASK_COMMAND, new HashMap<String, String>(), null, null);
+    job.addTaskConfigMap(Collections.singletonMap(taskConfig.getId(), taskConfig));
+    job.setJobId(TaskUtil.getNamespacedJobName(workflowName, "JOB"));
+    workflow.addJob("JOB", job);
+    driver.start(workflow.build());
+  }
+
+  @Test
+  public void testDynamicTaskLoading() throws Exception {
+    // Add task definition information as a DynamicTaskConfig.
+    List<String> taskClasses = new ArrayList<String>();
+    taskClasses.add("com.mycompany.mocktask.MockTask");
+    DynamicTaskConfig taskConfig =
+        new DynamicTaskConfig("Reindex", "src/test/resources/Reindex.jar", "1.0.0", taskClasses,
+            "com.mycompany.mocktask.MockTaskFactory");
+    String path = TaskConstants.DYNAMICALLY_LOADED_TASK_PATH + "/Reindex";
+    removePathIfExists(path);
+    _manager.getHelixDataAccessor().getBaseDataAccessor()
+        .create(path, taskConfig.getTaskConfigZNRecord(), AccessOption.PERSISTENT);
+
+    // Submit workflow
+    TaskDriver driver = new TaskDriver(_manager);
+    String workflowName = TestHelper.getTestMethodName();
+    submitWorkflow(workflowName, driver);
+
+    try {
+      // Wait for the workflow to either complete or fail.
+      TaskState finalState =
+          driver.pollForWorkflowState(workflowName, TaskState.COMPLETED, TaskState.FAILED);
+      AssertJUnit.assertEquals(finalState, TaskState.COMPLETED);
+    } catch (HelixException e) {
+      AssertJUnit.fail(e.getMessage());
+    }
+  }
+
+  @Test
+  public void testDynamicTaskLoadingNonexistingJar() throws Exception {
+    // Add task definition information as a DynamicTaskConfig.
+    List<String> taskClasses = new ArrayList<String>();
+    taskClasses.add("com.mycompany.mocktask.MockTask");
+    DynamicTaskConfig taskConfig =
+        new DynamicTaskConfig("Reindex", "src/test/resources/Random.jar", "1.0.0", taskClasses,
+            "com.mycompany.mocktask.MockTaskFactory");
+    String path = TaskConstants.DYNAMICALLY_LOADED_TASK_PATH + "/Reindex";
+    removePathIfExists(path);
+    _manager.getHelixDataAccessor().getBaseDataAccessor()
+        .create(path, taskConfig.getTaskConfigZNRecord(), AccessOption.PERSISTENT);
+
+    // Submit workflow
+    TaskDriver driver = new TaskDriver(_manager);
+    String workflowName = TestHelper.getTestMethodName();
+    submitWorkflow(workflowName, driver);
+
+    try {
+      // Wait for the workflow to either complete or fail.
+      TaskState finalState =
+          driver.pollForWorkflowState(workflowName, TaskState.COMPLETED, TaskState.FAILED);
+      AssertJUnit.assertEquals(finalState, TaskState.FAILED);
+    } catch (HelixException e) {
+      AssertJUnit.fail(e.getMessage());
+    }
+  }
+
+  @Test
+  public void testDynamicTaskLoadingNonexistingTaskConfig() throws Exception {
+    // Remove task config ZNRecord if it exists.
+    String path = TaskConstants.DYNAMICALLY_LOADED_TASK_PATH + "/Reindex";
+    removePathIfExists(path);
+
+    // Submit workflow
+    TaskDriver driver = new TaskDriver(_manager);
+    String workflowName = TestHelper.getTestMethodName();
+    submitWorkflow(workflowName, driver);
+
+    try {
+      // Wait for the workflow to either complete or fail.
+      TaskState finalState =
+          driver.pollForWorkflowState(workflowName, TaskState.COMPLETED, TaskState.FAILED);
+      AssertJUnit.assertEquals(finalState, TaskState.FAILED);
+    } catch (HelixException e) {
+      AssertJUnit.fail(e.getMessage());
+    }
+  }
+
+  @Test
+  public void testDynamicTaskLoadingNonexistingTaskClass() throws Exception {
+    // Add task definition information as a DynamicTaskConfig.
+    List<String> taskClasses = new ArrayList<String>();
+    taskClasses.add("com.mycompany.mocktask.RandomTask");
+    DynamicTaskConfig taskConfig =
+        new DynamicTaskConfig("Reindex", "src/test/resources/Reindex.jar", "1.0.0", taskClasses,
+            "com.mycompany.mocktask.MockTaskFactory");
+    String path = TaskConstants.DYNAMICALLY_LOADED_TASK_PATH + "/Reindex";
+    removePathIfExists(path);
+    _manager.getHelixDataAccessor().getBaseDataAccessor()
+        .create(path, taskConfig.getTaskConfigZNRecord(), AccessOption.PERSISTENT);
+
+    // Submit workflow
+    TaskDriver driver = new TaskDriver(_manager);
+    String workflowName = TestHelper.getTestMethodName();
+    submitWorkflow(workflowName, driver);
+
+    try {
+      // Wait for the workflow to either complete or fail.
+      TaskState finalState =
+          driver.pollForWorkflowState(workflowName, TaskState.COMPLETED, TaskState.FAILED);
+      AssertJUnit.assertEquals(finalState, TaskState.FAILED);
+    } catch (HelixException e) {
+      AssertJUnit.fail(e.getMessage());
+    }
+  }
+
+  @Test
+  public void testDynamicTaskLoadingNonexistingTaskFactory() throws Exception {
+    // Add task definition information as a DynamicTaskConfig.
+    List<String> taskClasses = new ArrayList<String>();
+    taskClasses.add("com.mycompany.mocktask.MockTask");
+    DynamicTaskConfig taskConfig =
+        new DynamicTaskConfig("Reindex", "src/test/resources/Reindex.jar", "1.0.0", taskClasses,
+            "com.mycompany.mocktask.RandomTaskFactory");
+    String path = TaskConstants.DYNAMICALLY_LOADED_TASK_PATH + "/Reindex";
+    removePathIfExists(path);
+    _manager.getHelixDataAccessor().getBaseDataAccessor()
+        .create(path, taskConfig.getTaskConfigZNRecord(), AccessOption.PERSISTENT);
+
+    // Submit workflow
+    TaskDriver driver = new TaskDriver(_manager);
+    String workflowName = TestHelper.getTestMethodName();
+    submitWorkflow(workflowName, driver);
+
+    try {
+      // Wait for the workflow to either complete or fail.
+      TaskState finalState =
+          driver.pollForWorkflowState(workflowName, TaskState.COMPLETED, TaskState.FAILED);
+      AssertJUnit.assertEquals(finalState, TaskState.FAILED);
+    } catch (HelixException e) {
+      AssertJUnit.fail(e.getMessage());
+    }
+  }
+
+  @AfterMethod
+  public void afterMethod() {

Review comment:
       Regarding this and BeforeMethod vs. BeforeClass, I wanted to make sure no dynamically loaded class remains loaded between the tests. I tried with BeforeClass and AfterClass, and all tests would result in TaskState.COMPLETED whereas only one should complete and the others should fail.




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@helix.apache.org
For additional commands, e-mail: reviews-help@helix.apache.org