You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sh...@apache.org on 2016/03/23 23:06:07 UTC
[1/3] airavata git commit: Fixed experiment lauch error with unicore
Repository: airavata
Updated Branches:
refs/heads/develop 199c30107 -> 6c483bc4c
Fixed experiment lauch error with unicore
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/24421596
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/24421596
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/24421596
Branch: refs/heads/develop
Commit: 244215968409100ebccef0425c505ac7f6e52fc1
Parents: 6c5645d
Author: Shameera Rathnayaka <sh...@gmail.com>
Authored: Wed Mar 23 11:26:28 2016 -0400
Committer: Shameera Rathnayaka <sh...@gmail.com>
Committed: Wed Mar 23 11:26:28 2016 -0400
----------------------------------------------------------------------
.../orchestrator/cpi/impl/SimpleOrchestratorImpl.java | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/24421596/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
index 427be5b..15c8556 100644
--- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
+++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/cpi/impl/SimpleOrchestratorImpl.java
@@ -284,14 +284,15 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{
}
ComputeResourceDescription computeResource = appCatalog.getComputeResource().getComputeResource(resourceHostId);
JobSubmissionInterface preferredJobSubmissionInterface = OrchestratorUtils.getPreferredJobSubmissionInterface(orchestratorContext, processModel, gatewayId);
- List<String> taskIdList = createAndSaveEnvSetupTask(gatewayId, processModel, experimentCatalog);
-
ComputeResourcePreference resourcePreference = OrchestratorUtils.getComputeResourcePreference(orchestratorContext, processModel, gatewayId);
+ List<String> taskIdList = new ArrayList<>();
+
if (resourcePreference.getPreferredJobSubmissionProtocol() == JobSubmissionProtocol.UNICORE) {
+ // TODO - breakdown unicore all in one task to multiple tasks, then we don't need to handle UNICORE here.
taskIdList.addAll(createAndSaveSubmissionTasks(gatewayId, preferredJobSubmissionInterface, processModel, userGivenWallTime));
} else {
+ taskIdList.addAll(createAndSaveEnvSetupTask(gatewayId, processModel, experimentCatalog));
taskIdList.addAll(createAndSaveInputDataStagingTasks(processModel, gatewayId));
-
if (autoSchedule) {
List<BatchQueue> definedBatchQueues = computeResource.getBatchQueues();
for (BatchQueue batchQueue : definedBatchQueues) {
@@ -316,10 +317,8 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator{
} else {
taskIdList.addAll(createAndSaveSubmissionTasks(gatewayId,preferredJobSubmissionInterface, processModel, userGivenWallTime));
}
-
taskIdList.addAll(createAndSaveOutputDataStagingTasks(processModel, gatewayId));
}
-
// update process scheduling
experimentCatalog.update(ExperimentCatalogModelType.PROCESS, processModel, processModel.getProcessId());
return getTaskDag(taskIdList);
[2/3] airavata git commit: Fixed issues with BES security context
Posted by sh...@apache.org.
Fixed issues with BES security context
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/f30e73fd
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/f30e73fd
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/f30e73fd
Branch: refs/heads/develop
Commit: f30e73fd10668d2ae4735702eda0021f0490abf0
Parents: 2442159
Author: Shameera Rathnayaka <sh...@gmail.com>
Authored: Wed Mar 23 18:05:46 2016 -0400
Committer: Shameera Rathnayaka <sh...@gmail.com>
Committed: Wed Mar 23 18:05:46 2016 -0400
----------------------------------------------------------------------
.../airavata/gfac/impl/GFacEngineImpl.java | 12 ++++++++---
.../gfac/impl/task/BESJobSubmissionTask.java | 22 +++++++++++++-------
.../impl/task/utils/bes/DataTransferrer.java | 13 +++++++-----
.../task/utils/bes/UNICORESecurityContext.java | 7 ++++---
4 files changed, 35 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/f30e73fd/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/GFacEngineImpl.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/GFacEngineImpl.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/GFacEngineImpl.java
index 0622e58..a4f101e 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/GFacEngineImpl.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/GFacEngineImpl.java
@@ -178,9 +178,15 @@ public class GFacEngineImpl implements GFacEngine {
processModel.setProcessOutputs(applicationOutputs);
processContext.setSshKeyAuthentication(Factory.getComputerResourceSSHKeyAuthentication(processContext));
- processContext.setResourceJobManager(getResourceJobManager(processContext));
- processContext.setJobSubmissionRemoteCluster(Factory.getJobSubmissionRemoteCluster(processContext));
- processContext.setDataMovementRemoteCluster(Factory.getDataMovementRemoteCluster(processContext));
+ if (processContext.getJobSubmissionProtocol() == JobSubmissionProtocol.UNICORE) {
+ // process monitor mode set in getResourceJobManager method, but unicore doesn't have resource job manager.
+ // hence we set process monitor mode here.
+ processContext.setMonitorMode(MonitorMode.FORK);
+ } else {
+ processContext.setResourceJobManager(getResourceJobManager(processContext));
+ processContext.setJobSubmissionRemoteCluster(Factory.getJobSubmissionRemoteCluster(processContext));
+ processContext.setDataMovementRemoteCluster(Factory.getDataMovementRemoteCluster(processContext));
+ }
String inputPath = ServerSettings.getLocalDataLocation();
if (inputPath != null) {
http://git-wip-us.apache.org/repos/asf/airavata/blob/f30e73fd/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java
index cf8e4a0..145cde9 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java
@@ -35,8 +35,10 @@ import org.apache.airavata.gfac.core.context.TaskContext;
import org.apache.airavata.gfac.core.task.JobSubmissionTask;
import org.apache.airavata.gfac.core.task.TaskException;
import org.apache.airavata.gfac.impl.task.utils.bes.*;
+import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionInterface;
import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
import org.apache.airavata.model.appcatalog.computeresource.UnicoreJobSubmission;
+import org.apache.airavata.model.experiment.UserConfigurationDataModel;
import org.apache.airavata.model.job.JobModel;
import org.apache.airavata.model.status.JobState;
import org.apache.airavata.model.status.JobStatus;
@@ -92,10 +94,11 @@ public class BESJobSubmissionTask implements JobSubmissionTask {
try {
ProcessContext processContext = taskContext.getParentProcessContext();
JobSubmissionProtocol protocol = processContext.getJobSubmissionProtocol();
- String interfaceId = processContext.getApplicationInterfaceDescription().getApplicationInterfaceId();
+ JobSubmissionInterface jobSubmissionInterface = GFacUtils.getPreferredJobSubmissionInterface(processContext);
String factoryUrl = null;
if (protocol.equals(JobSubmissionProtocol.UNICORE)) {
- UnicoreJobSubmission unicoreJobSubmission = GFacUtils.getUnicoreJobSubmission(interfaceId);
+ UnicoreJobSubmission unicoreJobSubmission = GFacUtils.getUnicoreJobSubmission(
+ jobSubmissionInterface.getJobSubmissionInterfaceId());
factoryUrl = unicoreJobSubmission.getUnicoreEndPointURL();
}
EndpointReferenceType eprt = EndpointReferenceType.Factory.newInstance();
@@ -200,7 +203,7 @@ public class BESJobSubmissionTask implements JobSubmissionTask {
log.error("Error while retrieving UNICORE job submission..");
taskStatus.setState(TaskState.FAILED);
} catch (Exception e) {
- log.error("Cannot create storage..");
+ log.error("Cannot create storage..", e);
taskStatus.setState(TaskState.FAILED);
}
@@ -211,10 +214,13 @@ public class BESJobSubmissionTask implements JobSubmissionTask {
DefaultClientConfiguration clientConfig = null;
try {
UNICORESecurityContext unicoreSecurityContext = SecurityUtils.getSecurityContext(pc);
- UserConfigurationData userConfigData = (UserConfigurationData) pc.getExperimentCatalog().
+ UserConfigurationDataModel userConfigDataModel = (UserConfigurationDataModel) pc.getExperimentCatalog().
get(ExperimentCatalogModelType.USER_CONFIGURATION_DATA, pc.getExperimentId());
- if (userConfigData.getGenerateCert()) {
- clientConfig = unicoreSecurityContext.getDefaultConfiguration(false, userConfigData);
+ // FIXME - remove following setter lines, and use original value comes with user configuration data model.
+ userConfigDataModel.setGenerateCert(true);
+ userConfigDataModel.setUserDN("CN=swus3, O=Ultrascan Gateway, C=DE");
+ if (userConfigDataModel.isGenerateCert()) {
+ clientConfig = unicoreSecurityContext.getDefaultConfiguration(false, userConfigDataModel);
} else {
clientConfig = unicoreSecurityContext.getDefaultConfiguration(false);
}
@@ -266,12 +272,12 @@ public class BESJobSubmissionTask implements JobSubmissionTask {
@Override
public TaskStatus recover(TaskContext taskContext) {
- return null;
+ return execute(taskContext);
}
@Override
public TaskTypes getType() {
- return null;
+ return TaskTypes.JOB_SUBMISSION;
}
protected ActivityStatusType getStatus(FactoryClient fc, EndpointReferenceType activityEpr)
http://git-wip-us.apache.org/repos/asf/airavata/blob/f30e73fd/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/DataTransferrer.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/DataTransferrer.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/DataTransferrer.java
index e480f92..4a0cbbf 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/DataTransferrer.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/DataTransferrer.java
@@ -57,8 +57,8 @@ public class DataTransferrer {
protected String gatewayDownloadLocation, stdoutLocation, stderrLocation;
- public DataTransferrer(ProcessContext jobContext, StorageClient storageClient) {
- this.processContext = jobContext;
+ public DataTransferrer(ProcessContext processContext, StorageClient storageClient) {
+ this.processContext = processContext;
this.storageClient = storageClient;
resultantOutputsLst = new ArrayList<OutputDataObjectType>();
initStdoutsLocation();
@@ -103,13 +103,16 @@ public class DataTransferrer {
}
public void uploadLocalFiles() throws GFacException {
- List<String> inFilePrms = extractInFileParams();
+ List<String> inFilePrms = new ArrayList<>();
+ // FIXME - remove hard coded file path.
+// inFilePrms.addAll(extractInFileParams());
+ inFilePrms.add("file://home/airavata/test/hpcinput-localhost-uslims3_cauma3d-00950.tar");
for (String uri : inFilePrms) {
String fileName = new File(uri).getName();
if (uri.startsWith("file")) {
try {
- String uriWithoutProtocol = uri.substring(uri.lastIndexOf("://") + 1, uri.length());
- FileUploader fileUploader = new FileUploader(uriWithoutProtocol,fileName,Mode.overwrite);
+ String uriWithoutProtocol = uri.substring(uri.lastIndexOf("://") + 2, uri.length());
+ FileUploader fileUploader = new FileUploader(uriWithoutProtocol, fileName, Mode.overwrite, false);
fileUploader.perform(storageClient);
} catch (FileNotFoundException e3) {
throw new GFacException(
http://git-wip-us.apache.org/repos/asf/airavata/blob/f30e73fd/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/UNICORESecurityContext.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/UNICORESecurityContext.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/UNICORESecurityContext.java
index a383b40..609e1fa 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/UNICORESecurityContext.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/UNICORESecurityContext.java
@@ -31,6 +31,7 @@ import org.apache.airavata.common.utils.ServerSettings;
import org.apache.airavata.credential.store.store.CredentialReader;
import org.apache.airavata.gfac.core.GFacException;
import org.apache.airavata.gfac.core.RequestData;
+import org.apache.airavata.model.experiment.UserConfigurationDataModel;
import org.apache.airavata.registry.core.experiment.catalog.model.UserConfigurationData;
import org.bouncycastle.asn1.x500.style.BCStyle;
import org.slf4j.Logger;
@@ -68,13 +69,13 @@ public class UNICORESecurityContext extends X509SecurityContext {
return secProperties;
}
- public DefaultClientConfiguration getDefaultConfiguration(Boolean enableMessageLogging, UserConfigurationData userData) throws GFacException, ApplicationSettingsException {
+ public DefaultClientConfiguration getDefaultConfiguration(Boolean enableMessageLogging, UserConfigurationDataModel userDataModel) throws GFacException, ApplicationSettingsException {
X509Credential cred = null;
try{
- boolean genCert = userData.getGenerateCert();
+ boolean genCert = userDataModel.isGenerateCert();
if(genCert) {
- String userDN = userData.getUserDn();
+ String userDN = userDataModel.getUserDN();
if (userDN == null || "".equals(userDN)){
log.warn("Cannot generate cert, falling back to GFAC configured MyProxy credentials");
return getDefaultConfiguration(enableMessageLogging);
[3/3] airavata git commit: Merge branch 'develop' of
https://git-wip-us.apache.org/repos/asf/airavata into develop
Posted by sh...@apache.org.
Merge branch 'develop' of https://git-wip-us.apache.org/repos/asf/airavata into develop
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/6c483bc4
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/6c483bc4
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/6c483bc4
Branch: refs/heads/develop
Commit: 6c483bc4cf9f7a7c7301449a02dd50e499baccd5
Parents: f30e73f 199c301
Author: Shameera Rathnayaka <sh...@gmail.com>
Authored: Wed Mar 23 18:05:55 2016 -0400
Committer: Shameera Rathnayaka <sh...@gmail.com>
Committed: Wed Mar 23 18:05:55 2016 -0400
----------------------------------------------------------------------
.../client/samples/DataCatalogSample.java | 2 +-
.../test/resources/airavata-server.properties | 13 ----
.../server/OrchestratorServerHandler.java | 37 +++++++----
.../core/data/catalog/impl/DataCatalogImpl.java | 64 +++++++++++++++++++-
.../airavata/data/catalog/DataCatalogTest.java | 6 +-
.../airavata/registry/cpi/DataCatalog.java | 12 ++--
6 files changed, 96 insertions(+), 38 deletions(-)
----------------------------------------------------------------------