You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2014/05/21 19:56:57 UTC
[43/51] [abbrv] git commit: fixing
https://issues.apache.org/jira/browse/AIRAVATA-1200
fixing https://issues.apache.org/jira/browse/AIRAVATA-1200
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/7ec932b5
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/7ec932b5
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/7ec932b5
Branch: refs/heads/temp
Commit: 7ec932b5e48b9b1c42355ffc42abada59bed1eb3
Parents: a9e022b
Author: lahiru <la...@apache.org>
Authored: Tue May 6 13:42:40 2014 -0400
Committer: lahiru <la...@apache.org>
Committed: Tue May 6 13:42:40 2014 -0400
----------------------------------------------------------------------
.../server/OrchestratorServerHandler.java | 37 +++++++++++++++
.../core/OrchestratorConfiguration.java | 11 +++--
.../core/context/OrchestratorContext.java | 2 +-
.../core/utils/OrchestratorUtils.java | 3 +-
.../core/validator/JobMetadataValidator.java | 9 ++--
.../impl/AbstractJobMetadataValidator.java | 30 ------------
.../validator/impl/SimpleAppDataValidator.java | 40 ++++------------
.../airavata/orchestrator/cpi/Orchestrator.java | 11 +++++
.../cpi/impl/SimpleOrchestratorImpl.java | 50 ++++++++++++--------
.../orchestrator.cpi.service.thrift | 4 +-
10 files changed, 102 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java b/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
index 073f65d..f456c65 100644
--- a/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
+++ b/modules/orchestrator/airavata-orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
@@ -107,6 +107,43 @@ public class OrchestratorServerHandler implements OrchestratorService.Iface {
}
return true;
}
+
+
+ /**
+ * This method will validate the experiment before launching, if is failed we do not run the launch in airavata
+ * thrift service (only if validation is enabled
+ * @param experimentId
+ * @return
+ * @throws TException
+ */
+ public boolean validateExperiment(String experimentId) throws TException {
+ //TODO: Write the Orchestrator implementaion
+ try {
+ List<TaskDetails> tasks = orchestrator.createTasks(experimentId);
+ if (tasks.size() > 1) {
+ log.info("There are multiple tasks for this experiment, So Orchestrator will launch multiple Jobs");
+ }
+ List<String> ids = registry.getIds(RegistryModelType.WORKFLOW_NODE_DETAIL,WorkflowNodeConstants.EXPERIMENT_ID,experimentId);
+ for (String workflowNodeId : ids) {
+ WorkflowNodeDetails workflowNodeDetail = (WorkflowNodeDetails)registry.get(RegistryModelType.WORKFLOW_NODE_DETAIL, workflowNodeId);
+ List<Object> taskDetailList = registry.get(RegistryModelType.TASK_DETAIL, TaskDetailConstants.NODE_ID, workflowNodeId);
+ for (Object o : taskDetailList) {
+ TaskDetails taskID = (TaskDetails) o;
+ //iterate through all the generated tasks and performs the job submisssion+monitoring
+ Experiment experiment = (Experiment) registry.get(RegistryModelType.EXPERIMENT, experimentId);
+ if (experiment == null) {
+ log.error("Error retrieving the Experiment by the given experimentID: " + experimentId);
+ return false;
+ }
+ return orchestrator.validateExperiment(experiment, workflowNodeDetail, taskID);
+ }
+ }
+
+ } catch (Exception e) {
+ throw new TException(e);
+ }
+ return false;
+ }
public boolean terminateExperiment(String experimentId) throws TException {
try {
orchestrator.cancelExperiment(experimentId);
http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java
index fcb3552..b0cbe07 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/OrchestratorConfiguration.java
@@ -23,6 +23,7 @@ package org.apache.airavata.orchestrator.core;
import org.apache.airavata.client.api.AiravataAPI;
import java.net.URL;
+import java.util.List;
/**
* This keeps configuration of orchestrator, mostly this keep static
@@ -46,17 +47,17 @@ public class OrchestratorConfiguration {
private boolean embeddedMode;
- private String validatorClass;
+ private List<String> validatorClasses;
private boolean enableValidation;
- public String getValidatorClass() {
- return validatorClass;
+ public List<String> getValidatorClasses() {
+ return validatorClasses;
}
- public void setValidatorClass(String validatorClass) {
- this.validatorClass = validatorClass;
+ public void setValidatorClasses(List<String> validatorClasses) {
+ this.validatorClasses = validatorClasses;
}
public boolean isEmbeddedMode() {
http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/context/OrchestratorContext.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/context/OrchestratorContext.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/context/OrchestratorContext.java
index eb3f463..8de0482 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/context/OrchestratorContext.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/context/OrchestratorContext.java
@@ -29,7 +29,7 @@ import org.apache.airavata.registry.api.AiravataRegistry2;
import org.apache.airavata.registry.cpi.Registry;
/**
- * This is the context object used in orchestrator whic h
+ * This is the context object used in orchestrator which
*/
public class OrchestratorContext {
private List<GFACInstance> gfacInstanceList;
http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java
index 5abbe03..0af1ed8 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/utils/OrchestratorUtils.java
@@ -21,6 +21,7 @@
package org.apache.airavata.orchestrator.core.utils;
import java.io.IOException;
+import java.util.Arrays;
import org.apache.airavata.common.exception.ApplicationSettingsException;
import org.apache.airavata.common.utils.ServerSettings;
@@ -53,7 +54,7 @@ public class OrchestratorUtils {
orchestratorConfiguration.setEmbeddedMode(Boolean.valueOf(ServerSettings.getSetting(OrchestratorConstants.EMBEDDED_MODE)));
orchestratorConfiguration.setEnableValidation(Boolean.valueOf(ServerSettings.getSetting(OrchestratorConstants.ENABLE_VALIDATION)));
if (orchestratorConfiguration.isEnableValidation()) {
- orchestratorConfiguration.setValidatorClass((String) ServerSettings.getSetting(OrchestratorConstants.JOB_VALIDATOR));
+ orchestratorConfiguration.setValidatorClasses(Arrays.asList(ServerSettings.getSetting(OrchestratorConstants.JOB_VALIDATOR).split(",")));
}
return orchestratorConfiguration;
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/JobMetadataValidator.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/JobMetadataValidator.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/JobMetadataValidator.java
index d365c87..1c4eaba 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/JobMetadataValidator.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/JobMetadataValidator.java
@@ -20,13 +20,12 @@
*/
package org.apache.airavata.orchestrator.core.validator;
+import org.apache.airavata.model.workspace.experiment.Experiment;
+import org.apache.airavata.model.workspace.experiment.TaskDetails;
+import org.apache.airavata.model.workspace.experiment.WorkflowNodeDetails;
import org.apache.airavata.orchestrator.core.exception.OrchestratorException;
public interface JobMetadataValidator {
- boolean runBasicValidation(String experimentID)throws OrchestratorException;
-
- boolean runAppSpecificValidation(String experimentID)throws OrchestratorException;
-
- boolean validate(String experimentID) throws OrchestratorException;
+ boolean validate(Experiment experiment, WorkflowNodeDetails workflowNodeDetail, TaskDetails taskID) throws OrchestratorException;
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/AbstractJobMetadataValidator.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/AbstractJobMetadataValidator.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/AbstractJobMetadataValidator.java
deleted file mode 100644
index e83d5c9..0000000
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/AbstractJobMetadataValidator.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.orchestrator.core.validator.impl;
-
-import org.apache.airavata.orchestrator.core.validator.JobMetadataValidator;
-
-public abstract class AbstractJobMetadataValidator implements JobMetadataValidator {
- public boolean runBasicValidation(String experimentID) {
- //todo implement basic validation which is not an application specific
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java
index 53ecfe8..a756a0a 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/SimpleAppDataValidator.java
@@ -21,13 +21,16 @@
package org.apache.airavata.orchestrator.core.validator.impl;
import org.apache.airavata.model.workspace.experiment.Experiment;
+import org.apache.airavata.model.workspace.experiment.TaskDetails;
+import org.apache.airavata.model.workspace.experiment.WorkflowNodeDetails;
import org.apache.airavata.orchestrator.core.exception.OrchestratorException;
+import org.apache.airavata.orchestrator.core.validator.JobMetadataValidator;
import org.apache.airavata.persistance.registry.jpa.impl.RegistryFactory;
import org.apache.airavata.registry.cpi.Registry;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class SimpleAppDataValidator extends AbstractJobMetadataValidator {
+public class SimpleAppDataValidator implements JobMetadataValidator {
private final static Logger logger = LoggerFactory.getLogger(SimpleAppDataValidator.class);
private Registry registry;
@@ -36,38 +39,11 @@ public class SimpleAppDataValidator extends AbstractJobMetadataValidator {
this.registry = RegistryFactory.getDefaultRegistry();
}
- public boolean runAppSpecificValidation(String experimentID) throws OrchestratorException{
- // implement simple application specific validator to be used for
- // all the applications.
- return true; //To change body of implemented methods use File | Settings | File Templates.
- }
-
- public boolean validate(String experimentID) throws OrchestratorException {
+ public boolean validate(Experiment experiment, WorkflowNodeDetails workflowNodeDetail, TaskDetails taskID) throws OrchestratorException {
boolean result = false;
- if (super.runBasicValidation(experimentID)) {
-
- Experiment experiment = null;
- try {
- experiment = (Experiment) registry.get(org.apache.airavata.registry.cpi.RegistryModelType.EXPERIMENT, experimentID);
- } catch (Exception e) {
- throw new OrchestratorException(e);
- }
- if (experiment.getUserConfigurationData().isAiravataAutoSchedule()) {
- logger.error("We dont' support auto scheduling at this point, We will simply use user data as it is");
- }
-
- /* todo like this do more validation and if they are suppose to fail return false otherwise give some
- log messages in server side logs
- */
- if (runAppSpecificValidation(experimentID)) {
- return true;
- }
- String error = "Application data validation steps failed";
- logger.error(error);
- return false;
+ if (experiment.getUserConfigurationData().isAiravataAutoSchedule()) {
+ logger.error("We dont' support auto scheduling at this point, We will simply use user data as it is");
}
- String error = "Basic validation steps failed";
- logger.error(error);
- return false;
+ return true;
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java
index d02155e..3a47585 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/Orchestrator.java
@@ -34,6 +34,17 @@ import java.util.List;
public interface Orchestrator {
/**
+ * This method can be used to run all custom validators plugged in to the orchestrator and make
+ * sure the experiment is ready to launch and if its not this will return false
+ * @param experiment
+ * @param workflowNodeDetail
+ * @param taskID
+ * @return boolean if the experiments are valids after executing all the validators return true otherwise it iwll return false
+ * @throws OrchestratorException
+ */
+ boolean validateExperiment(Experiment experiment, WorkflowNodeDetails workflowNodeDetail,
+ TaskDetails taskID) throws OrchestratorException;
+ /**
* After creating the experiment Data user have the
* experimentID as the handler to the experiment, during the launchExperiment
* We just have to give the experimentID
http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
index afd7425..08df639 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
@@ -49,9 +49,6 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{
// this is going to be null unless the thread count is 0
private JobSubmitter jobSubmitter = null;
- private JobMetadataValidator jobMetadataValidator = null;
-
-
public SimpleOrchestratorImpl() throws OrchestratorException {
try {
@@ -60,14 +57,7 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{
Class<? extends JobSubmitter> aClass = Class.forName(submitterClass.trim()).asSubclass(JobSubmitter.class);
jobSubmitter = aClass.newInstance();
jobSubmitter.initialize(this.orchestratorContext);
- String validatorClzz = this.orchestratorContext.getOrchestratorConfiguration().getValidatorClass();
- if (this.orchestratorConfiguration.isEnableValidation()) {
- if (validatorClzz == null) {
- logger.error("Job validation class is not properly set, so Validation will be turned off !");
- }
- Class<? extends JobMetadataValidator> vClass = Class.forName(validatorClzz.trim()).asSubclass(JobMetadataValidator.class);
- jobMetadataValidator = vClass.newInstance();
- }
+
} catch (Exception e) {
String error = "Error creating JobSubmitter in non threaded mode ";
logger.error(error);
@@ -123,7 +113,35 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{
return tasks;
}
- public void cancelExperiment(String experimentID)
+ public boolean validateExperiment(Experiment experiment, WorkflowNodeDetails workflowNodeDetail, TaskDetails taskID) throws OrchestratorException {
+ if (this.orchestratorConfiguration.isEnableValidation()) {
+ List<String> validatorClzzez = this.orchestratorContext.getOrchestratorConfiguration().getValidatorClasses();
+ for (String validator : validatorClzzez) {
+ try {
+ Class<? extends JobMetadataValidator> vClass = Class.forName(validator.trim()).asSubclass(JobMetadataValidator.class);
+ JobMetadataValidator jobMetadataValidator = vClass.newInstance();
+ if(jobMetadataValidator.validate(experiment, workflowNodeDetail, taskID)){
+ logger.info("Validation of " + validator + " is SUCCESSFUL");
+ }else {
+ logger.error("Validation of " + validator + " is FAILED");
+ return false;
+ }
+ } catch (ClassNotFoundException e) {
+ logger.error("Error loading the validation class: ", validator, e);
+ return false;
+ } catch (InstantiationException e) {
+ logger.error("Error loading the validation class: ", validator, e);
+ return false;
+ } catch (IllegalAccessException e) {
+ logger.error("Error loading the validation class: ", validator, e);
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ public void cancelExperiment(String experimentID)
throws OrchestratorException {
throw new OrchestratorException(new OperationNotSupportedException());
}
@@ -137,14 +155,6 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{
this.executor = executor;
}
- public JobMetadataValidator getJobMetadataValidator() {
- return jobMetadataValidator;
- }
-
- public void setJobMetadataValidator(JobMetadataValidator jobMetadataValidator) {
- this.jobMetadataValidator = jobMetadataValidator;
- }
-
public JobSubmitter getJobSubmitter() {
return jobSubmitter;
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/7ec932b5/modules/orchestrator/orchestrator-thrift-descriptions/orchestrator.cpi.service.thrift
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-thrift-descriptions/orchestrator.cpi.service.thrift b/modules/orchestrator/orchestrator-thrift-descriptions/orchestrator.cpi.service.thrift
index c03e81f..6dc452e 100644
--- a/modules/orchestrator/orchestrator-thrift-descriptions/orchestrator.cpi.service.thrift
+++ b/modules/orchestrator/orchestrator-thrift-descriptions/orchestrator.cpi.service.thrift
@@ -43,5 +43,7 @@ service OrchestratorService {
**/
bool launchExperiment (1: required string experimentId),
- bool terminateExperiment (1: required string experimentId)
+ bool terminateExperiment (1: required string experimentId),
+
+ bool validateExperiment(1: required string experimentId)
}
\ No newline at end of file