You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sh...@apache.org on 2015/11/06 23:26:04 UTC
[1/2] airavata git commit: check monitor mode before create
monitoring task
Repository: airavata
Updated Branches:
refs/heads/master b4e3c33fd -> 85fb6b694
check monitor mode before create monitoring task
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/687d8126
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/687d8126
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/687d8126
Branch: refs/heads/master
Commit: 687d812617a0fb236a7c44a2a894d71c641e51d6
Parents: 5eb3c26
Author: Shameera Rathnayaka <sh...@gmail.com>
Authored: Fri Nov 6 17:25:22 2015 -0500
Committer: Shameera Rathnayaka <sh...@gmail.com>
Committed: Fri Nov 6 17:25:22 2015 -0500
----------------------------------------------------------------------
.../cpi/impl/SimpleOrchestratorImpl.java | 61 ++++++++++++--------
.../server/OrchestratorServerHandler.java | 4 +-
2 files changed, 39 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/687d8126/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
index 1e2ad58..ee4d2c6 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
@@ -20,14 +20,11 @@
*/
package org.apache.airavata.orchestrator.cpi.impl;
-import org.apache.airavata.common.exception.ApplicationSettingsException;
import org.apache.airavata.common.utils.AiravataUtils;
-import org.apache.airavata.common.utils.ServerSettings;
import org.apache.airavata.common.utils.ThriftUtils;
import org.apache.airavata.gfac.core.task.TaskException;
import org.apache.airavata.model.appcatalog.computeresource.*;
import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference;
-import org.apache.airavata.model.appcatalog.gatewayprofile.DataStoragePreference;
import org.apache.airavata.model.application.io.DataType;
import org.apache.airavata.model.application.io.InputDataObjectType;
import org.apache.airavata.model.application.io.OutputDataObjectType;
@@ -288,6 +285,7 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{
List<String> taskIdList = createAndSaveEnvSetupTask(gatewayId, processModel, experimentCatalog);
taskIdList.addAll(createAndSaveInputDataStagingTasks(processModel, gatewayId));
+ JobSubmissionInterface preferredJobSubmissionInterface = OrchestratorUtils.getPreferredJobSubmissionInterface(orchestratorContext, processModel, gatewayId);
if (autoSchedule) {
List<BatchQueue> definedBatchQueues = computeResource.getBatchQueues();
for (BatchQueue batchQueue : definedBatchQueues) {
@@ -298,19 +296,19 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{
// need to create more job submissions
int numOfMaxWallTimeJobs = ((int) Math.floor(userGivenWallTime / maxRunTime));
for (int i = 1; i <= numOfMaxWallTimeJobs; i++) {
- taskIdList.addAll(createAndSaveSubmissionTasks(gatewayId, processModel, maxRunTime));
+ taskIdList.addAll(createAndSaveSubmissionTasks(gatewayId,preferredJobSubmissionInterface, processModel, maxRunTime));
}
int leftWallTime = userGivenWallTime % maxRunTime;
if (leftWallTime != 0) {
- taskIdList.addAll(createAndSaveSubmissionTasks(gatewayId, processModel, leftWallTime));
+ taskIdList.addAll(createAndSaveSubmissionTasks(gatewayId,preferredJobSubmissionInterface, processModel, leftWallTime));
}
} else {
- taskIdList.addAll(createAndSaveSubmissionTasks(gatewayId, processModel, userGivenWallTime));
+ taskIdList.addAll(createAndSaveSubmissionTasks(gatewayId,preferredJobSubmissionInterface, processModel, userGivenWallTime));
}
}
}
} else {
- taskIdList.addAll(createAndSaveSubmissionTasks(gatewayId, processModel, userGivenWallTime));
+ taskIdList.addAll(createAndSaveSubmissionTasks(gatewayId,preferredJobSubmissionInterface, processModel, userGivenWallTime));
}
taskIdList.addAll(createAndSaveOutputDataStagingTasks(processModel, gatewayId));
// update process scheduling
@@ -442,8 +440,19 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{
return dataStagingTaskIds;
}
- private List<String> createAndSaveSubmissionTasks(String gatewayId, ProcessModel processModel, int wallTime)
- throws TException, RegistryException {
+ private List<String> createAndSaveSubmissionTasks(String gatewayId, JobSubmissionInterface jobSubmissionInterface, ProcessModel processModel, int wallTime)
+ throws TException, RegistryException, OrchestratorException {
+
+ JobSubmissionProtocol jobSubmissionProtocol = jobSubmissionInterface.getJobSubmissionProtocol();
+ MonitorMode monitorMode = null;
+ if (jobSubmissionProtocol == JobSubmissionProtocol.SSH || jobSubmissionProtocol == JobSubmissionProtocol.SSH_FORK) {
+ SSHJobSubmission sshJobSubmission = OrchestratorUtils.getSSHJobSubmission(orchestratorContext, jobSubmissionInterface.getJobSubmissionInterfaceId());
+ monitorMode = sshJobSubmission.getMonitorMode();
+ }else {
+ logger.error("expId : {}, processId : {} :- Unsupported Job submission protocol {}.",
+ processModel.getExperimentId(), processModel.getProcessId(), jobSubmissionProtocol.name());
+ throw new OrchestratorException("Unsupported Job Submission Protocol " + jobSubmissionProtocol.name());
+ }
List<String> submissionTaskIds = new ArrayList<>();
TaskModel taskModel = new TaskModel();
taskModel.setParentProcessId(processModel.getProcessId());
@@ -454,7 +463,7 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{
taskModel.setTaskStatus(taskStatus);
taskModel.setTaskType(TaskTypes.JOB_SUBMISSION);
JobSubmissionTaskModel submissionSubTask = new JobSubmissionTaskModel();
- submissionSubTask.setMonitorMode(MonitorMode.JOB_EMAIL_NOTIFICATION_MONITOR);
+ submissionSubTask.setMonitorMode(monitorMode);
submissionSubTask.setJobSubmissionProtocol(
OrchestratorUtils.getPreferredJobSubmissionProtocol(orchestratorContext, processModel, gatewayId));
submissionSubTask.setWallTime(wallTime);
@@ -465,21 +474,23 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{
taskModel.setTaskId(taskId);
submissionTaskIds.add(taskModel.getTaskId());
- // create monitor task for this job
- TaskModel monitorTaskModel = new TaskModel();
- monitorTaskModel.setParentProcessId(processModel.getProcessId());
- monitorTaskModel.setCreationTime(new Date().getTime());
- monitorTaskModel.setLastUpdateTime(monitorTaskModel.getCreationTime());
- TaskStatus monitorTaskStatus = new TaskStatus(TaskState.CREATED);
- monitorTaskStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
- monitorTaskModel.setTaskStatus(monitorTaskStatus);
- monitorTaskModel.setTaskType(TaskTypes.MONITORING);
- MonitorTaskModel monitorSubTaskModel = new MonitorTaskModel();
- monitorSubTaskModel.setMonitorMode(MonitorMode.JOB_EMAIL_NOTIFICATION_MONITOR);
- monitorTaskModel.setSubTaskModel(ThriftUtils.serializeThriftObject(monitorSubTaskModel));
- String mTaskId = (String) orchestratorContext.getRegistry().getExperimentCatalog().add(ExpCatChildDataType.TASK, monitorTaskModel, processModel.getProcessId());
- monitorTaskModel.setTaskId(mTaskId);
- submissionTaskIds.add(monitorTaskModel.getTaskId());
+ // create monitor task for this Email based monitor mode job
+ if (monitorMode == MonitorMode.JOB_EMAIL_NOTIFICATION_MONITOR) {
+ TaskModel monitorTaskModel = new TaskModel();
+ monitorTaskModel.setParentProcessId(processModel.getProcessId());
+ monitorTaskModel.setCreationTime(new Date().getTime());
+ monitorTaskModel.setLastUpdateTime(monitorTaskModel.getCreationTime());
+ TaskStatus monitorTaskStatus = new TaskStatus(TaskState.CREATED);
+ monitorTaskStatus.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime());
+ monitorTaskModel.setTaskStatus(monitorTaskStatus);
+ monitorTaskModel.setTaskType(TaskTypes.MONITORING);
+ MonitorTaskModel monitorSubTaskModel = new MonitorTaskModel();
+ monitorSubTaskModel.setMonitorMode(monitorMode);
+ monitorTaskModel.setSubTaskModel(ThriftUtils.serializeThriftObject(monitorSubTaskModel));
+ String mTaskId = (String) orchestratorContext.getRegistry().getExperimentCatalog().add(ExpCatChildDataType.TASK, monitorTaskModel, processModel.getProcessId());
+ monitorTaskModel.setTaskId(mTaskId);
+ submissionTaskIds.add(monitorTaskModel.getTaskId());
+ }
return submissionTaskIds;
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/687d8126/modules/orchestrator/orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java b/modules/orchestrator/orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
index f75c91e..c1e9d65 100644
--- a/modules/orchestrator/orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
+++ b/modules/orchestrator/orchestrator-service/src/main/java/org/apache/airavata/orchestrator/server/OrchestratorServerHandler.java
@@ -154,7 +154,9 @@ public class OrchestratorServerHandler implements OrchestratorService.Iface {
log.error("Validating process fails for given experiment Id : {}", experimentId);
return false;
}
- ComputeResourcePreference computeResourcePreference = appCatalog.getGatewayProfile().getComputeResourcePreference(gatewayId, experiment.getUserConfigurationData().getComputationalResourceScheduling().getResourceHostId());
+ ComputeResourcePreference computeResourcePreference = appCatalog.getGatewayProfile().
+ getComputeResourcePreference(gatewayId,
+ experiment.getUserConfigurationData().getComputationalResourceScheduling().getResourceHostId());
String token = computeResourcePreference.getResourceSpecificCredentialStoreToken();
if (token == null || token.isEmpty()){
// try with gateway profile level token
[2/2] airavata git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/airavata
Posted by sh...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/airavata
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/85fb6b69
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/85fb6b69
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/85fb6b69
Branch: refs/heads/master
Commit: 85fb6b69443c4be9736c9a3af641341c5f957db8
Parents: 687d812 b4e3c33
Author: Shameera Rathnayaka <sh...@gmail.com>
Authored: Fri Nov 6 17:25:44 2015 -0500
Committer: Shameera Rathnayaka <sh...@gmail.com>
Committed: Fri Nov 6 17:25:44 2015 -0500
----------------------------------------------------------------------
.../api/server/util/AppCatalogInitUtil.java | 1 +
.../java/org/apache/airavata/api/Airavata.java | 2 +-
.../src/main/resources/lib/airavata/Airavata.h | 152 +++++++++----------
.../application_interface_model_types.cpp | 4 +-
.../application_interface_model_types.h | 4 +-
.../airavata/application_io_models_types.cpp | 26 +++-
.../lib/airavata/application_io_models_types.h | 16 +-
.../lib/airavata/experiment_model_types.cpp | 4 +-
.../lib/airavata/experiment_model_types.h | 4 +-
.../gateway_resource_profile_model_types.cpp | 37 ++++-
.../gateway_resource_profile_model_types.h | 11 +-
.../lib/airavata/messaging_events_types.cpp | 4 +-
.../lib/airavata/messaging_events_types.h | 4 +-
.../lib/airavata/process_model_types.cpp | 4 +-
.../lib/airavata/process_model_types.h | 4 +-
.../resources/lib/airavata/task_model_types.cpp | 4 +-
.../resources/lib/airavata/task_model_types.h | 4 +-
.../lib/airavata/workflow_data_model_types.cpp | 4 +-
.../lib/airavata/workflow_data_model_types.h | 4 +-
.../Model/AppCatalog/GatewayProfile/Types.php | 37 ++++-
.../lib/Airavata/Model/Application/Io/Types.php | 23 +++
.../model/appcatalog/gatewayprofile/ttypes.py | 33 ++--
.../airavata/model/application/io/ttypes.py | 15 +-
.../client/samples/CreateLaunchExperiment.java | 4 +-
.../org/apache/airavata/model/Workflow.java | 2 +-
.../ApplicationDeploymentDescription.java | 2 +-
.../appdeployment/ApplicationModule.java | 2 +-
.../appcatalog/appdeployment/CommandObject.java | 2 +-
.../appcatalog/appdeployment/SetEnvPaths.java | 2 +-
.../ApplicationInterfaceDescription.java | 2 +-
.../appcatalog/computeresource/BatchQueue.java | 2 +-
.../computeresource/CloudJobSubmission.java | 2 +-
.../ComputeResourceDescription.java | 2 +-
.../computeresource/DataMovementInterface.java | 2 +-
.../computeresource/GlobusJobSubmission.java | 2 +-
.../computeresource/GridFTPDataMovement.java | 2 +-
.../computeresource/JobSubmissionInterface.java | 2 +-
.../computeresource/LOCALDataMovement.java | 2 +-
.../computeresource/LOCALSubmission.java | 2 +-
.../computeresource/ResourceJobManager.java | 2 +-
.../computeresource/SCPDataMovement.java | 2 +-
.../computeresource/SSHJobSubmission.java | 2 +-
.../computeresource/UnicoreDataMovement.java | 2 +-
.../computeresource/UnicoreJobSubmission.java | 2 +-
.../ComputeResourcePreference.java | 2 +-
.../gatewayprofile/DataStoragePreference.java | 2 +-
.../gatewayprofile/GatewayResourceProfile.java | 129 ++++++++++++++--
.../application/io/InputDataObjectType.java | 2 +-
.../application/io/OutputDataObjectType.java | 109 ++++++++++++-
.../airavata/model/commons/ErrorModel.java | 2 +-
.../model/commons/ValidationResults.java | 2 +-
.../airavata/model/commons/ValidatorResult.java | 2 +-
.../model/error/AiravataClientException.java | 2 +-
.../model/error/AiravataSystemException.java | 2 +-
.../model/error/AuthenticationException.java | 2 +-
.../model/error/AuthorizationException.java | 2 +-
.../error/ExperimentNotFoundException.java | 2 +-
.../model/error/InvalidRequestException.java | 2 +-
.../model/error/LaunchValidationException.java | 2 +-
.../model/error/ProjectNotFoundException.java | 2 +-
.../airavata/model/error/TimedOutException.java | 2 +-
.../airavata/model/error/ValidationResults.java | 2 +-
.../airavata/model/error/ValidatorResult.java | 2 +-
.../model/experiment/ExperimentModel.java | 2 +-
.../model/experiment/ExperimentStatistics.java | 2 +-
.../experiment/ExperimentSummaryModel.java | 2 +-
.../experiment/UserConfigurationDataModel.java | 2 +-
.../org/apache/airavata/model/job/JobModel.java | 2 +-
.../event/ExperimentStatusChangeEvent.java | 2 +-
.../model/messaging/event/JobIdentifier.java | 2 +-
.../messaging/event/JobStatusChangeEvent.java | 2 +-
.../event/JobStatusChangeRequestEvent.java | 2 +-
.../airavata/model/messaging/event/Message.java | 2 +-
.../messaging/event/ProcessIdentifier.java | 2 +-
.../event/ProcessStatusChangeEvent.java | 2 +-
.../event/ProcessStatusChangeRequestEvent.java | 2 +-
.../messaging/event/ProcessSubmitEvent.java | 2 +-
.../messaging/event/ProcessTerminateEvent.java | 2 +-
.../model/messaging/event/TaskIdentifier.java | 2 +-
.../messaging/event/TaskOutputChangeEvent.java | 2 +-
.../messaging/event/TaskStatusChangeEvent.java | 2 +-
.../event/TaskStatusChangeRequestEvent.java | 2 +-
.../airavata/model/process/ProcessModel.java | 2 +-
.../ComputationalResourceSchedulingModel.java | 2 +-
.../airavata/model/security/AuthzToken.java | 2 +-
.../airavata/model/status/ExperimentStatus.java | 2 +-
.../apache/airavata/model/status/JobStatus.java | 2 +-
.../airavata/model/status/ProcessStatus.java | 2 +-
.../airavata/model/status/TaskStatus.java | 2 +-
.../model/task/DataStagingTaskModel.java | 2 +-
.../model/task/EnvironmentSetupTaskModel.java | 2 +-
.../model/task/JobSubmissionTaskModel.java | 2 +-
.../airavata/model/task/MonitorTaskModel.java | 2 +-
.../apache/airavata/model/task/TaskModel.java | 2 +-
.../airavata/model/workspace/Gateway.java | 2 +-
.../apache/airavata/model/workspace/Group.java | 2 +-
.../airavata/model/workspace/Project.java | 2 +-
.../apache/airavata/model/workspace/User.java | 2 +-
.../impl/task/AdvancedSCPDataStageTask.java | 1 +
.../catalog/impl/ApplicationInterfaceImpl.java | 2 +
.../catalog/impl/GwyResourceProfileImpl.java | 2 +
.../app/catalog/model/ApplicationIntOutput.java | 10 ++
.../core/app/catalog/model/GatewayProfile.java | 11 ++
.../resources/ApplicationOutputResource.java | 10 ++
.../resources/GatewayProfileResource.java | 11 ++
.../app/catalog/util/AppCatalogJPAUtils.java | 2 +
.../util/AppCatalogThriftConversion.java | 2 +
.../catalog/impl/ExperimentRegistry.java | 4 +
.../catalog/model/ExperimentOutput.java | 11 ++
.../experiment/catalog/model/ProcessOutput.java | 12 +-
.../resources/ExperimentOutputResource.java | 10 ++
.../resources/ProcessOutputResource.java | 10 ++
.../experiment/catalog/resources/Utils.java | 2 +
.../utils/ThriftDataModelConversion.java | 2 +
.../src/main/resources/appcatalog-derby.sql | 2 +
.../src/main/resources/appcatalog-mysql.sql | 2 +
.../src/main/resources/expcatalog-derby.sql | 2 +
.../src/main/resources/expcatalog-mysql.sql | 2 +
.../airavata-api/application_io_models.thrift | 3 +-
.../gateway_resource_profile_model.thrift | 7 +-
120 files changed, 675 insertions(+), 232 deletions(-)
----------------------------------------------------------------------