You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ms...@apache.org on 2014/12/10 13:38:12 UTC
[1/5] airavata git commit: modified provider sources to use
appcatalog types
Repository: airavata
Updated Branches:
refs/heads/master d35226d88 -> 5b118f8c4
modified provider sources to use appcatalog types
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/28a235e8
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/28a235e8
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/28a235e8
Branch: refs/heads/master
Commit: 28a235e87c5820ab2d5e7e9849be5f041823b35d
Parents: 7f4faeb
Author: msmemon <sh...@gmail.com>
Authored: Wed Dec 3 13:24:55 2014 +0100
Committer: msmemon <sh...@gmail.com>
Committed: Wed Dec 3 13:24:55 2014 +0100
----------------------------------------------------------------------
.../gfac/bes/handlers/AbstractSMSHandler.java | 2 -
.../gfac/bes/provider/impl/BESProvider.java | 81 +++++----
.../gfac/bes/utils/ApplicationProcessor.java | 60 ++++---
.../gfac/bes/utils/DataServiceInfo.java | 62 -------
.../gfac/bes/utils/DataTransferrer.java | 158 +++++-----------
.../airavata/gfac/bes/utils/JSDLGenerator.java | 84 ++-------
.../gfac/bes/utils/ResourceProcessor.java | 149 +++++----------
.../gfac/bes/utils/UASDataStagingProcessor.java | 180 ++++++-------------
8 files changed, 230 insertions(+), 546 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/28a235e8/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/handlers/AbstractSMSHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/handlers/AbstractSMSHandler.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/handlers/AbstractSMSHandler.java
index 71ca0db..a23a096 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/handlers/AbstractSMSHandler.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/handlers/AbstractSMSHandler.java
@@ -17,8 +17,6 @@ import org.apache.airavata.gfac.core.utils.GFacUtils;
import org.apache.airavata.model.appcatalog.computeresource.*;
import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
import org.apache.airavata.model.workspace.experiment.ErrorCategory;
-import org.apache.airavata.schemas.gfac.JobDirectoryModeDocument.JobDirectoryMode;
-import org.apache.airavata.schemas.gfac.UnicoreHostType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3.x2005.x08.addressing.EndpointReferenceType;
http://git-wip-us.apache.org/repos/asf/airavata/blob/28a235e8/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java
index 7cf2d7c..6fdadfb 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java
@@ -115,8 +115,7 @@ public class BESProvider extends AbstractProvider implements GFacProvider,
UnicoreJobSubmission unicoreJobSubmission = GFacUtils.getUnicoreJobSubmission(interfaceId);
factoryUrl = unicoreJobSubmission.getUnicoreEndPointURL();
}
- EndpointReferenceType eprt = EndpointReferenceType.Factory
- .newInstance();
+ EndpointReferenceType eprt = EndpointReferenceType.Factory.newInstance();
eprt.addNewAddress().setStringValue(factoryUrl);
String userDN = getUserName(jobExecutionContext);
@@ -124,20 +123,15 @@ public class BESProvider extends AbstractProvider implements GFacProvider,
if (userDN == null || userDN.equalsIgnoreCase("admin")) {
userDN = "CN=zdv575, O=Ultrascan Gateway, C=DE";
}
- CreateActivityDocument cad = CreateActivityDocument.Factory
- .newInstance();
- JobDefinitionDocument jobDefDoc = JobDefinitionDocument.Factory
- .newInstance();
+ CreateActivityDocument cad = CreateActivityDocument.Factory.newInstance();
+ JobDefinitionDocument jobDefDoc = JobDefinitionDocument.Factory.newInstance();
// create storage
- StorageCreator storageCreator = new StorageCreator(secProperties,
- factoryUrl, 5, null);
+ StorageCreator storageCreator = new StorageCreator(secProperties, factoryUrl, 5, null);
sc = storageCreator.createStorage();
- JobDefinitionType jobDefinition = JSDLGenerator.buildJSDLInstance(
- jobExecutionContext, sc.getUrl()).getJobDefinition();
- cad.addNewCreateActivity().addNewActivityDocument()
- .setJobDefinition(jobDefinition);
+ JobDefinitionType jobDefinition = JSDLGenerator.buildJSDLInstance(jobExecutionContext, sc.getUrl()).getJobDefinition();
+ cad.addNewCreateActivity().addNewActivityDocument().setJobDefinition(jobDefinition);
log.info("JSDL" + jobDefDoc.toString());
// upload files if any
@@ -180,28 +174,7 @@ public class BESProvider extends AbstractProvider implements GFacProvider,
.getStringValue(), factory.getActivityStatus(activityEpr)
.toString()));
- // TODO publish the status messages to the message bus
- while ((factory.getActivityStatus(activityEpr) != ActivityStateEnumeration.FINISHED)
- && (factory.getActivityStatus(activityEpr) != ActivityStateEnumeration.FAILED)
- && (factory.getActivityStatus(activityEpr) != ActivityStateEnumeration.CANCELLED)) {
-
- ActivityStatusType activityStatus = getStatus(factory, activityEpr);
- JobState applicationJobStatus = getApplicationJobStatus(activityStatus);
- String jobStatusMessage = "Status of job " + jobId + "is "
- + applicationJobStatus;
- GFacUtils.updateJobStatus(jobExecutionContext, jobDetails, applicationJobStatus);
-
- jobExecutionContext.getNotifier().publish(
- new StatusChangeEvent(jobStatusMessage));
-
- // GFacUtils.updateApplicationJobStatus(jobExecutionContext,jobId,
- // applicationJobStatus);
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- }
- continue;
- }
+ waitUntilDone(factory, activityEpr, jobDetails);
ActivityStatusType activityStatus = null;
activityStatus = getStatus(factory, activityEpr);
@@ -217,11 +190,12 @@ public class BESProvider extends AbstractProvider implements GFacProvider,
+ activityStatus.getFault().getFaultstring()
+ "\n EXITCODE: " + activityStatus.getExitCode();
log.info(error);
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- }
+
+ try {Thread.sleep(5000);} catch (InterruptedException e) {}
+
+ //What if job is failed before execution and there are not stdouts generated yet?
dt.downloadStdOuts();
+
} else if (activityStatus.getState() == ActivityStateEnumeration.CANCELLED) {
JobState applicationJobStatus = JobState.CANCELED;
String jobStatusMessage = "Status of job " + jobId + "is "
@@ -357,8 +331,7 @@ public class BESProvider extends AbstractProvider implements GFacProvider,
// }
}
- protected ActivityStatusType getStatus(FactoryClient fc,
- EndpointReferenceType activityEpr)
+ protected ActivityStatusType getStatus(FactoryClient fc, EndpointReferenceType activityEpr)
throws UnknownActivityIdentifierFault {
GetActivityStatusesDocument stats = GetActivityStatusesDocument.Factory
@@ -427,6 +400,32 @@ public class BESProvider extends AbstractProvider implements GFacProvider,
public void cancelJob(JobExecutionContext jobExecutionContext)
throws GFacProviderException, GFacException {
// TODO Auto-generated method stub
-
+ }
+
+ protected void waitUntilDone(FactoryClient factory, EndpointReferenceType activityEpr, JobDetails jobDetails) throws Exception {
+
+ try {
+ while ((factory.getActivityStatus(activityEpr) != ActivityStateEnumeration.FINISHED)
+ && (factory.getActivityStatus(activityEpr) != ActivityStateEnumeration.FAILED)
+ && (factory.getActivityStatus(activityEpr) != ActivityStateEnumeration.CANCELLED)) {
+
+ ActivityStatusType activityStatus = getStatus(factory, activityEpr);
+ JobState applicationJobStatus = getApplicationJobStatus(activityStatus);
+ String jobStatusMessage = "Status of job " + jobId + "is " + applicationJobStatus;
+ GFacUtils.updateJobStatus(jobExecutionContext, jobDetails, applicationJobStatus);
+
+ jobExecutionContext.getNotifier().publish(new StatusChangeEvent(jobStatusMessage));
+
+ // GFacUtils.updateApplicationJobStatus(jobExecutionContext,jobId,
+ // applicationJobStatus);
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {}
+ continue;
+ }
+ } catch(Exception e) {
+ log.error("Error monitoring job status..");
+ throw e;
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/28a235e8/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java
index ee58565..91c27f9 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java
@@ -24,8 +24,6 @@ package org.apache.airavata.gfac.bes.utils;
import org.apache.airavata.gfac.core.context.JobExecutionContext;
import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
import org.apache.airavata.model.appcatalog.appdeployment.ApplicationParallelismType;
-import org.apache.airavata.schemas.gfac.ExtendedKeyValueType;
-import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
import org.ggf.schemas.jsdl.x2005.x11.jsdl.ApplicationType;
import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.FileNameType;
@@ -47,10 +45,9 @@ public class ApplicationProcessor {
ApplicationDeploymentDescription appDep= context.getApplicationContext().getApplicationDeploymentDescription();
String appname = context.getApplicationContext().getApplicationInterfaceDescription().getApplicationName();
ApplicationParallelismType parallelism = appDep.getParallelism();
-
ApplicationType appType = JSDLUtils.getOrCreateApplication(value);
appType.setApplicationName(appname);
- JSDLUtils.getOrCreateJobIdentification(value).setJobName(appname);
+
// if (appDep.getSetEnvironment().size() > 0) {
// createApplicationEnvironment(value, appDep.getSetEnvironment(), parallelism);
@@ -58,10 +55,11 @@ public class ApplicationProcessor {
//
String stdout = context.getStandardOutput();
String stderr = context.getStandardError();
+
if (appDep.getExecutablePath() != null) {
FileNameType fNameType = FileNameType.Factory.newInstance();
fNameType.setStringValue(appDep.getExecutablePath());
- if(parallelism.equals(ApplicationParallelismType.MPI) || parallelism.equals(ApplicationParallelismType.OPENMP_MPI)) {
+ if(isParallelJob(context)) {
JSDLUtils.getOrCreateSPMDApplication(value).setExecutable(fNameType);
if (parallelism.equals(ApplicationParallelismType.OPENMP_MPI)){
JSDLUtils.getSPMDApplication(value).setSPMDVariation(SPMDVariations.OpenMPI.value());
@@ -85,11 +83,11 @@ public class ApplicationProcessor {
}
int totalThreadCount = context.getTaskData().getTaskScheduling().getNumberOfThreads();
+
if(totalThreadCount > 0){
ThreadsPerProcessType tpp = ThreadsPerProcessType.Factory.newInstance();
tpp.setStringValue(String.valueOf(totalThreadCount));
JSDLUtils.getSPMDApplication(value).setThreadsPerProcess(tpp);
-
}
if(userName != null) {
@@ -134,45 +132,49 @@ public class ApplicationProcessor {
public static String getUserNameFromContext(JobExecutionContext jobContext) {
if(jobContext.getTaskData() == null)
return null;
- //FIXME: Discuss to get user and change this
+ //TODO: Extend unicore model to specify optional unix user id (allocation account)
return "admin";
}
- public static void addApplicationArgument(JobDefinitionType value, HpcApplicationDeploymentType appDepType, String stringPrm) {
- if(isParallelJob(appDepType))
- JSDLUtils.getOrCreateSPMDApplication(value)
- .addNewArgument().setStringValue(stringPrm);
- else
- JSDLUtils.getOrCreatePOSIXApplication(value)
- .addNewArgument().setStringValue(stringPrm);
-
+ public static void addApplicationArgument(JobDefinitionType value, JobExecutionContext context, String stringPrm) {
+ if(isParallelJob(context)){
+ JSDLUtils.getOrCreateSPMDApplication(value).addNewArgument().setStringValue(stringPrm);
+ }
+ else {
+ JSDLUtils.getOrCreatePOSIXApplication(value).addNewArgument().setStringValue(stringPrm);
+ }
}
- public static String getApplicationStdOut(JobDefinitionType value, HpcApplicationDeploymentType appDepType) throws RuntimeException {
- if (isParallelJob(appDepType)) return JSDLUtils.getOrCreateSPMDApplication(value).getOutput().getStringValue();
+ public static String getApplicationStdOut(JobDefinitionType value, JobExecutionContext context) throws RuntimeException {
+ if (isParallelJob(context)) return JSDLUtils.getOrCreateSPMDApplication(value).getOutput().getStringValue();
else return JSDLUtils.getOrCreatePOSIXApplication(value).getOutput().getStringValue();
}
- public static String getApplicationStdErr(JobDefinitionType value, HpcApplicationDeploymentType appDepType) throws RuntimeException {
- if (isParallelJob(appDepType)) return JSDLUtils.getOrCreateSPMDApplication(value).getError().getStringValue();
+ public static String getApplicationStdErr(JobDefinitionType value, JobExecutionContext context) throws RuntimeException {
+ if (isParallelJob(context)) return JSDLUtils.getOrCreateSPMDApplication(value).getError().getStringValue();
else return JSDLUtils.getOrCreatePOSIXApplication(value).getError().getStringValue();
}
public static void createGenericApplication(JobDefinitionType value, String appName) {
ApplicationType appType = JSDLUtils.getOrCreateApplication(value);
appType.setApplicationName(appName);
- JSDLUtils.getOrCreateJobIdentification(value).setJobName(appName);
}
-
-
- public static String getValueFromMap(HpcApplicationDeploymentType appDepType, String name) {
- ExtendedKeyValueType[] extended = appDepType.getKeyValuePairsArray();
- for(ExtendedKeyValueType e: extended) {
- if(e.getName().equalsIgnoreCase(name)) {
- return e.getStringValue();
- }
+
+ public static boolean isParallelJob(JobExecutionContext context) {
+
+ ApplicationDeploymentDescription appDep = context.getApplicationContext().getApplicationDeploymentDescription();
+ ApplicationParallelismType parallelism = appDep.getParallelism();
+
+ boolean isParallel = false;
+
+ if(parallelism.equals(ApplicationParallelismType.MPI) ||
+ parallelism.equals(ApplicationParallelismType.OPENMP_MPI) ||
+ parallelism.equals(ApplicationParallelismType.OPENMP )) {
+ isParallel = true;
}
- return null;
+
+ return isParallel;
}
+
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/28a235e8/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataServiceInfo.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataServiceInfo.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataServiceInfo.java
deleted file mode 100644
index b63dcb2..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataServiceInfo.java
+++ /dev/null
@@ -1,62 +0,0 @@
-package org.apache.airavata.gfac.bes.utils;
-
-import java.io.Serializable;
-
-import org.apache.airavata.gfac.core.context.JobExecutionContext;
-import org.apache.airavata.schemas.gfac.JobDirectoryModeDocument.JobDirectoryMode;
-import org.apache.airavata.schemas.gfac.UnicoreHostType;
-import org.w3.x2005.x08.addressing.EndpointReferenceType;
-
-
-/**
- * A value object carrying information about data service access mode.
- * */
-public class DataServiceInfo implements BESConstants, Serializable {
-
- private static final long serialVersionUID = 1L;
-
- public enum DirectoryAccessMode {
- GridFTP, SMSBYTEIO, RNSBYTEIO
- }
-
- /*
- * basically only uses information to hold gridftp address or an optional
- * pointer to a remote StorageManagementService instance.
- */
- private String dataServiceUrl;
-
- private DirectoryAccessMode directoryAccesMode = DirectoryAccessMode.SMSBYTEIO;
-
- public DataServiceInfo(JobExecutionContext c) {
- JobDirectoryMode.Enum directoryAccess = ((UnicoreHostType)c.getApplicationContext().getHostDescription().getType()).getJobDirectoryMode();
-
- switch(directoryAccess.intValue()) {
- case JobDirectoryMode.INT_SMS_BYTE_IO:
- directoryAccesMode = DirectoryAccessMode.SMSBYTEIO;
- EndpointReferenceType s = (EndpointReferenceType) c
- .getProperty(PROP_SMS_EPR);
- dataServiceUrl = s.getAddress().getStringValue();
- break;
- case JobDirectoryMode.INT_GRID_FTP:
- case JobDirectoryMode.INT_RNS_BYTE_IO:
- default:
- directoryAccesMode = DirectoryAccessMode.GridFTP;
- break;
- }
-
- }
-
- public String getDataServiceUrl() {
- return dataServiceUrl;
- }
-
- public void setDataServiceUrl(String dataServiceUrl) {
- this.dataServiceUrl = dataServiceUrl;
- }
-
- public DirectoryAccessMode getDirectoryAccesMode() {
- return directoryAccesMode;
- }
-
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/28a235e8/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java
index fa9eb83..4aa6cc1 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java
@@ -27,22 +27,16 @@ import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
-import org.apache.airavata.commons.gfac.type.ActualParameter;
-import org.apache.airavata.commons.gfac.type.ApplicationDescription;
import org.apache.airavata.gfac.Constants;
import org.apache.airavata.gfac.core.context.JobExecutionContext;
import org.apache.airavata.gfac.core.provider.GFacProviderException;
+import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
+import org.apache.airavata.model.appcatalog.appinterface.DataType;
+import org.apache.airavata.model.appcatalog.appinterface.InputDataObjectType;
+import org.apache.airavata.model.appcatalog.appinterface.OutputDataObjectType;
import org.apache.airavata.model.workspace.experiment.TaskDetails;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
-import org.apache.airavata.schemas.gfac.StringArrayType;
-import org.apache.airavata.schemas.gfac.StringParameterType;
-import org.apache.airavata.schemas.gfac.URIArrayType;
-import org.apache.airavata.schemas.gfac.URIParameterType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,15 +57,9 @@ public class DataTransferrer {
public void uploadLocalFiles() throws GFacProviderException {
- Map<String, Object> inputParams = jobContext.getInMessageContext()
- .getParameters();
- for (String paramKey : inputParams.keySet()) {
- ActualParameter inParam = (ActualParameter) inputParams
- .get(paramKey);
- String paramDataType = inParam.getType().getType().toString();
- if("URI".equals(paramDataType)) {
- String uri = ((URIParameterType) inParam.getType()).getValue();
- String fileName = new File(uri).getName();
+ List<String> inFilePrms = extractInFileParams();
+ for (String uri : inFilePrms) {
+ String fileName = new File(uri).getName();
if (uri.startsWith("file")) {
try {
String uriWithoutProtocol = uri.substring(uri.lastIndexOf("://") + 1, uri.length());
@@ -86,7 +74,6 @@ public class DataTransferrer {
}
}
- }
}
}
@@ -103,63 +90,16 @@ public class DataTransferrer {
if(!file.exists()){
file.mkdirs();
}
-
- Map<String, ActualParameter> stringMap = new HashMap<String, ActualParameter>();
-
- Map<String, Object> outputParams = jobContext.getOutMessageContext()
- .getParameters();
-
- for (String paramKey : outputParams.keySet()) {
-
- ActualParameter outParam = (ActualParameter) outputParams
- .get(paramKey);
-
- String paramDataType = outParam.getType().getType().toString();
-
- if ("String".equals(paramDataType)) {
- String stringPrm = ((StringParameterType) outParam
- .getType()).getValue();
- String localFileName = null;
- //TODO: why analysis.tar? it wont scale to other gateways..
- if(stringPrm == null || stringPrm.isEmpty()){
- continue;
-// localFileName = "analysis-results.tar";
- }else{
- localFileName = stringPrm.substring(stringPrm.lastIndexOf("/")+1);
- }
- String outputLocation = downloadLocation+File.separator+localFileName;
- FileDownloader fileDownloader = new FileDownloader(stringPrm,outputLocation, Mode.overwrite);
+ List<String> outPrms = extractOutParams(jobContext);
+ for (String outPrm : outPrms) {
+ String outputLocation = downloadLocation+File.separator+outPrm;
+ FileDownloader fileDownloader = new FileDownloader(outPrm,outputLocation, Mode.overwrite);
try {
fileDownloader.perform(storageClient);
- ((StringParameterType) outParam.getType()).setValue(outputLocation);
- stringMap.put(paramKey, outParam);
} catch (Exception e) {
throw new GFacProviderException(e.getLocalizedMessage(),e);
}
- }
-
- else if ("StringArray".equals(paramDataType)) {
- String[] valueArray = ((StringArrayType) outParam.getType())
- .getValueArray();
- for (String v : valueArray) {
- String localFileName = v.substring(v.lastIndexOf("/")+1);;
- String outputLocation = downloadLocation+File.separator+localFileName;
- FileDownloader fileDownloader = new FileDownloader(v,outputLocation, Mode.overwrite);
- try {
- fileDownloader.perform(storageClient);
- ((StringParameterType) outParam.getType()).setValue(outputLocation);
- stringMap.put(paramKey, outParam);
- } catch (Exception e) {
- throw new GFacProviderException(e.getLocalizedMessage(),e);
- }
- }
- }
}
- if (stringMap == null || stringMap.isEmpty()) {
- log.warn("Empty Output returned from the Application, Double check the application" +
- "and ApplicationDescriptor output Parameter Names");
- }
-
downloadStdOuts();
}
@@ -171,12 +111,8 @@ public class DataTransferrer {
file.mkdirs();
}
- HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) jobContext
- .getApplicationContext().getApplicationDeploymentDescription()
- .getType();
-
- String stdout = appDepType.getStandardOutput();
- String stderr = appDepType.getStandardError();
+ String stdout = jobContext.getStandardOutput();
+ String stderr = jobContext.getStandardError();
if(stdout != null) {
stdout = stdout.substring(stdout.lastIndexOf('/')+1);
}
@@ -190,18 +126,15 @@ public class DataTransferrer {
String stderrFileName = (stdout == null || stderr.equals("")) ? "stderr"
: stderr;
- ApplicationDescription application = jobContext.getApplicationContext().getApplicationDeploymentDescription();
- ApplicationDeploymentDescriptionType appDesc = application.getType();
-
+ ApplicationDeploymentDescription application = jobContext.getApplicationContext().getApplicationDeploymentDescription();
+
String stdoutLocation = downloadLocation+File.separator+stdoutFileName;
FileDownloader f1 = new FileDownloader(stdoutFileName,stdoutLocation, Mode.overwrite);
try {
f1.perform(storageClient);
log.info("Downloading stdout and stderr..");
String stdoutput = readFile(stdoutLocation);
- log.info("Stdout downloaded to "+stdoutLocation);
- appDesc.setStandardOutput(stdoutput);
-
+ log.info("Stdout downloaded to -> "+stdoutLocation);
if(UASDataStagingProcessor.isUnicoreEndpoint(jobContext)) {
String scriptExitCodeFName = "UNICORE_SCRIPT_EXIT_CODE";
String scriptCodeLocation = downloadLocation+File.separator+scriptExitCodeFName;
@@ -210,52 +143,47 @@ public class DataTransferrer {
f1.perform(storageClient);
log.info("UNICORE_SCRIPT_EXIT_CODE downloaded to "+scriptCodeLocation);
}
-
String stderrLocation = downloadLocation+File.separator+stderrFileName;
f1.setFrom(stderrFileName);
f1.setTo(stderrLocation);
f1.perform(storageClient);
String stderror = readFile(stderrLocation);
- log.info("Stderr downloaded to "+stderrLocation);
- appDesc.setStandardError(stderror);
+ log.info("Stderr downloaded to -> "+stderrLocation);
} catch (Exception e) {
throw new GFacProviderException(e.getLocalizedMessage(),e);
}
}
- public List<String> extractOutStringParams(JobExecutionContext context) {
-
- Map<String, Object> outputParams = context.getOutMessageContext()
- .getParameters();
-
+ public List<String> extractOutParams(JobExecutionContext context) {
List<String> outPrmsList = new ArrayList<String>();
-
- for (String paramKey : outputParams.keySet()) {
-
- ActualParameter outParam = (ActualParameter) outputParams
- .get(paramKey);
-
- String paramDataType = outParam.getType().getType().toString();
-
- if ("String".equals(paramDataType)) {
- String strPrm = ((StringParameterType) outParam.getType())
- .getValue();
- outPrmsList.add(strPrm);
- }
-
- else if (("StringArray").equals(paramDataType)) {
- String[] uriArray = ((URIArrayType) outParam.getType())
- .getValueArray();
- for (String u : uriArray) {
- outPrmsList.add(u);
- }
-
- }
- }
-
+ List<OutputDataObjectType> applicationOutputs = jobContext.getTaskData().getApplicationOutputs();
+ if (applicationOutputs != null && !applicationOutputs.isEmpty()){
+ for (OutputDataObjectType output : applicationOutputs){
+ if(output.getType().equals(DataType.STRING)) {
+ outPrmsList.add(output.getValue());
+ }
+ else if(output.getType().equals(DataType.FLOAT) || output.getType().equals(DataType.INTEGER)) {
+ outPrmsList.add(String.valueOf(output.getValue()));
+
+ }
+ }
+ }
return outPrmsList;
}
+
+ public List<String> extractInFileParams() {
+ List<String> filePrmsList = new ArrayList<String>();
+ List<InputDataObjectType> applicationInputs = jobContext.getTaskData().getApplicationInputs();
+ if (applicationInputs != null && !applicationInputs.isEmpty()){
+ for (InputDataObjectType output : applicationInputs){
+ if(output.getType().equals(DataType.URI)) {
+ filePrmsList.add(output.getValue());
+ }
+ }
+ }
+ return filePrmsList;
+ }
private String readFile(String localFile) throws IOException {
http://git-wip-us.apache.org/repos/asf/airavata/blob/28a235e8/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/JSDLGenerator.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/JSDLGenerator.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/JSDLGenerator.java
index c29e12d..9755bc7 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/JSDLGenerator.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/JSDLGenerator.java
@@ -21,8 +21,8 @@
package org.apache.airavata.gfac.bes.utils;
+import org.apache.airavata.gfac.core.context.ApplicationContext;
import org.apache.airavata.gfac.core.context.JobExecutionContext;
-import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionDocument;
import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
import org.slf4j.Logger;
@@ -41,19 +41,15 @@ public class JSDLGenerator implements BESConstants {
protected final Logger log = LoggerFactory.getLogger(this.getClass());
- public synchronized static JobDefinitionDocument buildJSDLInstance(
- JobExecutionContext context) throws Exception {
+ public synchronized static JobDefinitionDocument buildJSDLInstance(JobExecutionContext context) throws Exception {
JobDefinitionDocument jobDefDoc = JobDefinitionDocument.Factory
.newInstance();
JobDefinitionType value = jobDefDoc.addNewJobDefinition();
- HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
- .getApplicationContext().getApplicationDeploymentDescription()
- .getType();
-
+
// build Identification
- createJobIdentification(value, appDepType);
+ createJobIdentification(value, context);
ResourceProcessor.generateResourceElements(value, context);
@@ -70,12 +66,9 @@ public class JSDLGenerator implements BESConstants {
.newInstance();
JobDefinitionType value = jobDefDoc.addNewJobDefinition();
- HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
- .getApplicationContext().getApplicationDeploymentDescription()
- .getType();
-
+
// build Identification
- createJobIdentification(value, appDepType);
+ createJobIdentification(value, context);
ResourceProcessor.generateResourceElements(value, context);
@@ -87,41 +80,6 @@ public class JSDLGenerator implements BESConstants {
}
public synchronized static JobDefinitionDocument buildJSDLInstance(
- JobExecutionContext context, DataServiceInfo dataInfo)
- throws Exception {
-
- JobDefinitionDocument jobDefDoc = JobDefinitionDocument.Factory
- .newInstance();
- JobDefinitionType value = jobDefDoc.addNewJobDefinition();
-
- HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
- .getApplicationContext().getApplicationDeploymentDescription()
- .getType();
-
- createJobIdentification(value, appDepType);
-
- ResourceProcessor.generateResourceElements(value, context);
-
- ApplicationProcessor.generateJobSpecificAppElements(value, context);
-
- switch (dataInfo.getDirectoryAccesMode()) {
- case SMSBYTEIO:
- if(null == dataInfo.getDataServiceUrl() || "".equals(dataInfo.getDataServiceUrl()))
- throw new Exception("No SMS address found");
- UASDataStagingProcessor.generateDataStagingElements(value, context,
- dataInfo.getDataServiceUrl());
- break;
- case RNSBYTEIO:
- case GridFTP:
- default:
- DataStagingProcessor.generateDataStagingElements(value, context);
- break;
-
- }
- return jobDefDoc;
- }
-
- public synchronized static JobDefinitionDocument buildJSDLInstance(
JobExecutionContext context, String smsUrl, Object jobDirectoryMode)
throws Exception {
@@ -129,12 +87,8 @@ public class JSDLGenerator implements BESConstants {
.newInstance();
JobDefinitionType value = jobDefDoc.addNewJobDefinition();
- HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
- .getApplicationContext().getApplicationDeploymentDescription()
- .getType();
-
// build Identification
- createJobIdentification(value, appDepType);
+ createJobIdentification(value, context);
ResourceProcessor.generateResourceElements(value, context);
@@ -146,18 +100,18 @@ public class JSDLGenerator implements BESConstants {
return jobDefDoc;
}
- private static void createJobIdentification(JobDefinitionType value,
- HpcApplicationDeploymentType appDepType) {
- if (appDepType.getProjectAccount() != null) {
-
- if (appDepType.getProjectAccount().getProjectAccountNumber() != null)
- JSDLUtils.addProjectName(value, appDepType.getProjectAccount()
- .getProjectAccountNumber());
-
- if (appDepType.getProjectAccount().getProjectAccountDescription() != null)
- JSDLUtils.getOrCreateJobIdentification(value).setDescription(
- appDepType.getProjectAccount()
- .getProjectAccountDescription());
+ private static void createJobIdentification(JobDefinitionType value, JobExecutionContext context) {
+ ApplicationContext appCtxt = context.getApplicationContext();
+
+ if (appCtxt != null) {
+ if (appCtxt.getComputeResourcePreference().getAllocationProjectNumber() != null)
+ JSDLUtils.addProjectName(value, appCtxt.getComputeResourcePreference().getAllocationProjectNumber());
+
+ if (appCtxt.getApplicationInterfaceDescription().getApplicationDescription() != null)
+ JSDLUtils.getOrCreateJobIdentification(value).setDescription(appCtxt.getApplicationInterfaceDescription().getApplicationDescription());
+
+ if (appCtxt.getApplicationInterfaceDescription().getApplicationName() != null)
+ JSDLUtils.getOrCreateJobIdentification(value).setJobName(appCtxt.getApplicationInterfaceDescription().getApplicationName());
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/28a235e8/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java
index 5df9a0f..fce0c31 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java
@@ -25,125 +25,64 @@ import org.apache.airavata.gfac.core.context.JobExecutionContext;
import org.apache.airavata.gfac.core.provider.GFacProviderException;
import org.apache.airavata.model.workspace.experiment.ComputationalResourceScheduling;
import org.apache.airavata.model.workspace.experiment.TaskDetails;
-import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
-import org.apache.airavata.schemas.gfac.QueueType;
import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
-import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.NumberOfProcessesType;
public class ResourceProcessor {
public static void generateResourceElements(JobDefinitionType value, JobExecutionContext context) throws Exception{
- HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
- .getApplicationContext().getApplicationDeploymentDescription()
- .getType();
-
- createMemory(value, appDepType);
TaskDetails taskData = context.getTaskData();
- if(taskData != null && taskData.isSetTaskScheduling()){
- ComputationalResourceScheduling computionResource= taskData.getTaskScheduling();
- try {
- int cpuCount = computionResource.getTotalCPUCount();
- if(cpuCount>0){
-// appDepType.setCpuCount(cpuCount);
- NumberOfProcessesType num = NumberOfProcessesType.Factory.newInstance();
- String processers = Integer.toString(cpuCount);
- num.setStringValue(processers);
- JSDLUtils.getOrCreateSPMDApplication(value).setNumberOfProcesses(num);
- }
- } catch (NullPointerException e) {
- new GFacProviderException("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used",e);
- }
- try {
- int nodeCount = computionResource.getNodeCount();
- if(nodeCount>0){
- appDepType.setNodeCount(nodeCount);
- }
- } catch (NullPointerException e) {
- new GFacProviderException("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used",e);
- }
- try {
- String queueName = computionResource.getQueueName();
- if (queueName != null) {
- if(appDepType.getQueue() == null){
- QueueType queueType = appDepType.addNewQueue();
- queueType.setQueueName(queueName);
- }else{
- appDepType.getQueue().setQueueName(queueName);
- }
- }
- } catch (NullPointerException e) {
- new GFacProviderException("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used",e);
- }
- try {
- int maxwallTime = computionResource.getWallTimeLimit();
- if(maxwallTime>0){
- appDepType.setMaxWallTime(maxwallTime);
- }
- } catch (NullPointerException e) {
- new GFacProviderException("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used",e);
- }
- }
- if (appDepType.getCpuCount() > 0) {
- RangeValueType rangeType = new RangeValueType();
- rangeType.setLowerBound(Double.NaN);
- rangeType.setUpperBound(Double.NaN);
- rangeType.setExact(appDepType.getCpuCount());
- JSDLUtils.setTotalCPUCountRequirements(value, rangeType);
- }
+ if(taskData != null && taskData.isSetTaskScheduling()){
+ try {
+ ComputationalResourceScheduling crs = taskData.getTaskScheduling();
+
+ if (crs.getTotalPhysicalMemory() > 0) {
+ RangeValueType rangeType = new RangeValueType();
+ rangeType.setLowerBound(Double.NaN);
+ rangeType.setUpperBound(Double.NaN);
+ rangeType.setExact(crs.getTotalPhysicalMemory());
+ JSDLUtils.setIndividualPhysicalMemoryRequirements(value, rangeType);
+ }
+
+ if (crs.getNodeCount() > 0) {
+ RangeValueType rangeType = new RangeValueType();
+ rangeType.setLowerBound(Double.NaN);
+ rangeType.setUpperBound(Double.NaN);
+ rangeType.setExact(crs.getNodeCount());
+ JSDLUtils.setTotalResourceCountRequirements(value, rangeType);
+ }
+
+ if(crs.getWallTimeLimit() > 0) {
+ RangeValueType cpuTime = new RangeValueType();
+ cpuTime.setLowerBound(Double.NaN);
+ cpuTime.setUpperBound(Double.NaN);
+ long wallTime = crs.getWallTimeLimit() * 60;
+ cpuTime.setExact(wallTime);
+ JSDLUtils.setIndividualCPUTimeRequirements(value, cpuTime);
+ }
+
+ if(crs.getTotalCPUCount() > 0) {
+ RangeValueType rangeType = new RangeValueType();
+ rangeType.setLowerBound(Double.NaN);
+ rangeType.setUpperBound(Double.NaN);
+ rangeType.setExact(crs.getTotalCPUCount());
+ JSDLUtils.setTotalCPUCountRequirements(value, rangeType);
+ }
+ } catch (NullPointerException npe) {
+ new GFacProviderException("No value set for resource requirements.",npe);
+ }
+
+
+ }
- if (appDepType.getProcessorsPerNode() > 0) {
- RangeValueType rangeType = new RangeValueType();
- rangeType.setLowerBound(Double.NaN);
- rangeType.setUpperBound(Double.NaN);
- rangeType.setExact(appDepType.getProcessorsPerNode());
- JSDLUtils.setIndividualCPUCountRequirements(value, rangeType);
- }
- if (appDepType.getNodeCount() > 0) {
- RangeValueType rangeType = new RangeValueType();
- rangeType.setLowerBound(Double.NaN);
- rangeType.setUpperBound(Double.NaN);
- rangeType.setExact(appDepType.getNodeCount());
- JSDLUtils.setTotalResourceCountRequirements(value, rangeType);
- }
-
- if(appDepType.getMaxWallTime() > 0) {
- RangeValueType cpuTime = new RangeValueType();
- cpuTime.setLowerBound(Double.NaN);
- cpuTime.setUpperBound(Double.NaN);
- long wallTime = appDepType.getMaxWallTime() * 60;
- cpuTime.setExact(wallTime);
- JSDLUtils.setIndividualCPUTimeRequirements(value, cpuTime);
- }
+
}
- private static void createMemory(JobDefinitionType value, HpcApplicationDeploymentType appDepType){
- if (appDepType.getMinMemory() > 0 && appDepType.getMaxMemory() > 0) {
- RangeValueType rangeType = new RangeValueType();
- rangeType.setLowerBound(appDepType.getMinMemory());
- rangeType.setUpperBound(appDepType.getMaxMemory());
- JSDLUtils.setIndividualPhysicalMemoryRequirements(value, rangeType);
- }
-
- else if (appDepType.getMinMemory() > 0 && appDepType.getMaxMemory() <= 0) {
- // TODO set Wall time
- RangeValueType rangeType = new RangeValueType();
- rangeType.setLowerBound(appDepType.getMinMemory());
- JSDLUtils.setIndividualPhysicalMemoryRequirements(value, rangeType);
- }
-
- else if (appDepType.getMinMemory() <= 0 && appDepType.getMaxMemory() > 0) {
- // TODO set Wall time
- RangeValueType rangeType = new RangeValueType();
- rangeType.setUpperBound(appDepType.getMinMemory());
- JSDLUtils.setIndividualPhysicalMemoryRequirements(value, rangeType);
- }
-
- }
+
http://git-wip-us.apache.org/repos/asf/airavata/blob/28a235e8/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java
index 76624cc..9c92789 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java
@@ -22,72 +22,45 @@
package org.apache.airavata.gfac.bes.utils;
import java.io.File;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
-
-import org.apache.airavata.commons.gfac.type.ActualParameter;
import org.apache.airavata.gfac.core.context.JobExecutionContext;
-import org.apache.airavata.gfac.core.context.MessageContext;
-import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
-import org.apache.airavata.schemas.gfac.StringArrayType;
-import org.apache.airavata.schemas.gfac.StringParameterType;
-import org.apache.airavata.schemas.gfac.URIArrayType;
-import org.apache.airavata.schemas.gfac.URIParameterType;
-import org.apache.airavata.schemas.gfac.UnicoreHostType;
+import org.apache.airavata.model.appcatalog.appinterface.DataType;
+import org.apache.airavata.model.appcatalog.appinterface.InputDataObjectType;
+import org.apache.airavata.model.appcatalog.appinterface.OutputDataObjectType;
+import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
public class UASDataStagingProcessor {
public static void generateDataStagingElements(JobDefinitionType value, JobExecutionContext context, String smsUrl) throws Exception{
-
- HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
- .getApplicationContext().getApplicationDeploymentDescription()
- .getType();
-
smsUrl = "BFT:"+smsUrl;
-
- if (context.getInMessageContext().getParameters().size() > 0) {
- buildDataStagingFromInputContext(context, value, smsUrl, appDepType);
+
+ if (context.getTaskData().getApplicationInputs().size() > 0) {
+ buildDataStagingFromInputContext(context, value, smsUrl);
}
-// MessageContext outMessage = new MessageContext();
-// ActualParameter a1 = new ActualParameter();
-// a1.getType().changeType(StringParameterType.type);
-// ((StringParameterType)a1.getType()).setValue("analysis-results.tar");
-// outMessage.addParameter("o1", a1);
-// context.setOutMessageContext(outMessage);
- // now download for string typed outputs are to be done
- if (context.getOutMessageContext().getParameters().size() > 0) {
- buildFromOutputContext(context, value, smsUrl, appDepType);
+ if (context.getTaskData().getApplicationOutputs().size() > 0) {
+ buildFromOutputContext(context, value, smsUrl);
}
- //TODO need a review for us3 gateway..
-// createStdOutURIs(value, appDepType, smsUrl, isUnicoreEndpoint(context));
}
- private static void createInURISMSElement(JobDefinitionType value,
- String smsUrl, String inputDir, ActualParameter inParam)
+ private static void createInURISMSElement(JobDefinitionType value, String smsUrl, String uri)
throws Exception {
-
- String uri = ((URIParameterType) inParam.getType()).getValue();
- //TODO: To add this input file name setting part of Airavata API
String fileName = "input/" + new File(uri).getName();
if (uri.startsWith("file")) {
- String fileUri = smsUrl+"#/"+fileName;
-
- JSDLUtils.addDataStagingSourceElement(value, fileUri, null, fileName);
- } else if (uri.startsWith("gsiftp") || uri.startsWith("http")
- || uri.startsWith("rns")) {
- // no need to stage-in those files to the input
- // directory because unicore site will fetch them for the user
- JSDLUtils.addDataStagingSourceElement(value, uri, null, fileName);
- }
+ uri = smsUrl+"#/"+fileName;
+
+ }
+ // no need to stage-in those files to the input
+ // directory because unicore site will fetch them for the user
+ // supported third party transfers include
+ // gsiftp, http, rns, ftp
+ JSDLUtils.addDataStagingSourceElement(value, uri, null, fileName);
}
-
- private static void createStdOutURIs(JobDefinitionType value,
- HpcApplicationDeploymentType appDepType, String smsUrl,
- boolean isUnicore) throws Exception {
+
+ //TODO: will be deprecated
+ private static void createStdOutURIs(JobDefinitionType value, JobExecutionContext context, String smsUrl, boolean isUnicore) throws Exception {
// no need to use smsUrl for output location, because output location is activity's working directory
@@ -99,9 +72,9 @@ public class UASDataStagingProcessor {
}
if(!isUnicore) {
- String stdout = ApplicationProcessor.getApplicationStdOut(value, appDepType);
+ String stdout = ApplicationProcessor.getApplicationStdOut(value, context);
- String stderr = ApplicationProcessor.getApplicationStdErr(value, appDepType);
+ String stderr = ApplicationProcessor.getApplicationStdErr(value, context);
String stdoutFileName = (stdout == null || stdout.equals("")) ? "stdout"
: stdout;
@@ -120,14 +93,10 @@ public class UASDataStagingProcessor {
}
-
- private static void createOutStringElements(JobDefinitionType value,
- HpcApplicationDeploymentType appDeptype, String smsUrl, String prmValue) throws Exception {
-
+ // TODO: this should be deprecated, because the outputs are fetched using activity working dir from data transferrer
+ private static void createOutStringElements(JobDefinitionType value, String smsUrl, String prmValue) throws Exception {
if(prmValue == null || "".equals(prmValue)) return;
-
String finalSMSPath = smsUrl + "#/output/"+prmValue;
-
JSDLUtils.addDataStagingTargetElement(value, null, prmValue, null);
}
@@ -140,88 +109,45 @@ public class UASDataStagingProcessor {
private static JobDefinitionType buildFromOutputContext(JobExecutionContext context,
- JobDefinitionType value, String smsUrl,
- HpcApplicationDeploymentType appDepType) throws Exception {
-
- Map<String, Object> outputParams = context.getOutMessageContext()
- .getParameters();
-
- for (String paramKey : outputParams.keySet()) {
-
- ActualParameter outParam = (ActualParameter) outputParams
- .get(paramKey);
-
- String paramDataType = outParam.getType().getType().toString();
-
- if ("URI".equals(paramDataType)) {
- String uriPrm = ((URIParameterType) outParam.getType())
- .getValue();
- createOutURIElement(value, uriPrm);
- }
-
- else if (("URIArray").equals(paramDataType)) {
- String[] uriArray = ((URIArrayType) outParam.getType())
- .getValueArray();
- for (String u : uriArray) {
-
- createOutURIElement(value, u);
- }
-
- }
-// else if ("String".equals(paramDataType)) {
-// String stringPrm = ((StringParameterType) outParam
-// .getType()).getValue();
-// createOutStringElements(value, appDepType, smsUrl, stringPrm);
-// }
-//
-// else if ("StringArray".equals(paramDataType)) {
-// String[] valueArray = ((StringArrayType) outParam.getType())
-// .getValueArray();
-// for (String v : valueArray) {
-// createOutStringElements(value, appDepType, smsUrl, v);
-// }
-// }
- }
-
+ JobDefinitionType value, String smsUrl) throws Exception {
+ List<OutputDataObjectType> applicationOutputs = context.getTaskData().getApplicationOutputs();
+ if (applicationOutputs != null && !applicationOutputs.isEmpty()){
+ for (OutputDataObjectType output : applicationOutputs){
+ if(output.getType().equals(DataType.URI)) {
+ createOutURIElement(value, output.getValue());
+ }
+ else if(output.getType().equals(DataType.STRING)) {
+ // TODO: remove this check, as out string
+ createOutStringElements(value, smsUrl, output.getValue());
+ }
+ }
+ }
return value;
}
- private static void buildDataStagingFromInputContext(JobExecutionContext context, JobDefinitionType value, String smsUrl, HpcApplicationDeploymentType appDepType)
+ private static void buildDataStagingFromInputContext(JobExecutionContext context, JobDefinitionType value, String smsUrl)
throws Exception {
+ List<InputDataObjectType> applicationInputs = context.getApplicationContext().getApplicationInterfaceDescription().getApplicationInputs();
- // TODO set data directory
- Map<String, Object> inputParams = context.getInMessageContext()
- .getParameters();
-
- for (String paramKey : inputParams.keySet()) {
-
- ActualParameter inParam = (ActualParameter) inputParams
- .get(paramKey);
-
- // if single urls then convert each url into jsdl source
- // elements,
- // that are formed by concat of gridftpurl+inputdir+filename
-
- String paramDataType = inParam.getType().getType().toString();
-
- if ("URI".equals(paramDataType)) {
- createInURISMSElement(value, smsUrl,
- appDepType.getInputDataDirectory(), inParam);
- }
-
- // string params are converted into the job arguments
-
- else if ("String".equals(paramDataType)) {
- String stringPrm = ((StringParameterType) inParam.getType())
- .getValue();
- ApplicationProcessor.addApplicationArgument(value, appDepType, stringPrm);
+ if (applicationInputs != null && !applicationInputs.isEmpty()){
+ for (InputDataObjectType input : applicationInputs){
+ if(input.getType().equals(DataType.URI)){
+ //TODO: set the in sms url
+ createInURISMSElement(value, smsUrl, input.getValue());
+ }
+ else if(input.getType().equals(DataType.STRING)){
+ ApplicationProcessor.addApplicationArgument(value, context, input.getValue());
+ }
+ else if (input.getType().equals(DataType.FLOAT) || input.getType().equals(DataType.INTEGER)){
+ ApplicationProcessor.addApplicationArgument(value, context, String.valueOf(input.getValue()));
+ }
}
}
}
public static boolean isUnicoreEndpoint(JobExecutionContext context) {
- return ( (context.getApplicationContext().getHostDescription().getType() instanceof UnicoreHostType)?true:false );
+ return context.getPreferredJobSubmissionProtocol().equals(JobSubmissionProtocol.UNICORE);
}
}
[3/5] airavata git commit: fixed data staging issues,
supporting local stageins to remote job working directory,
use of delegation tokens while job runs
Posted by ms...@apache.org.
fixed data staging issues, supporting local stageins to remote job
working directory, use of delegation tokens while job runs
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/f4a32fd0
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/f4a32fd0
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/f4a32fd0
Branch: refs/heads/master
Commit: f4a32fd04e1cf2dbc24b59303cba9fcdb6855a9d
Parents: 576c87d
Author: msmemon <sh...@gmail.com>
Authored: Fri Dec 5 17:23:45 2014 +0100
Committer: msmemon <sh...@gmail.com>
Committed: Fri Dec 5 17:23:45 2014 +0100
----------------------------------------------------------------------
.../client/samples/CreateLaunchExperiment.java | 38 ++-
.../tools/RegisterSampleApplications.java | 22 +-
.../gfac/bes/provider/impl/BESProvider.java | 9 +-
.../bes/security/UNICORESecurityContext.java | 2 +-
.../gfac/bes/utils/ApplicationProcessor.java | 21 +-
.../gfac/bes/utils/DataStagingProcessor.java | 236 -------------------
.../gfac/bes/utils/DataTransferrer.java | 4 +-
.../airavata/gfac/bes/utils/JSDLGenerator.java | 3 +-
.../airavata/gfac/bes/utils/SecurityUtils.java | 7 +-
.../gfac/bes/utils/UASDataStagingProcessor.java | 10 +-
.../org/apache/airavata/gfac/Scheduler.java | 7 +
.../airavata/gfac/core/cpi/BetterGfacImpl.java | 16 +-
modules/gfac/pom.xml | 2 +-
13 files changed, 85 insertions(+), 292 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/f4a32fd0/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
index 34d419c..5c3ef19 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
@@ -57,7 +57,7 @@ public class CreateLaunchExperiment {
private static final String DEFAULT_GATEWAY = "default.registry.gateway";
private static Airavata.Client airavataClient;
- private static String echoAppId = "Echo_36fbb479-5b41-4f48-a9c5-382ee910ac6b";
+ private static String echoAppId = "Echo_753d7cf6-f79a-4f7f-8ada-5d707e90c383";
private static String wrfAppId = "WRF_7ad5da38-c08b-417c-a9ea-da9298839762";
private static String amberAppId = "Amber_49b16f6f-93ab-4885-9971-6ab2ab5eb3d3";
private static String gromacsAppId = "GROMACS_05622038-9edd-4cb1-824e-0b7cb993364b";
@@ -93,9 +93,9 @@ public class CreateLaunchExperiment {
try {
for (int i = 0; i < 1; i++) {
// final String expId = createExperimentForSSHHost(airavata);
-// final String expId = createEchoExperimentForFSD(airavataClient);
+ final String expId = createEchoExperimentForFSD(airavataClient);
// final String expId = createEchoExperimentForStampede(airavataClient);
- final String expId = createEchoExperimentForTrestles(airavataClient);
+// final String expId = createEchoExperimentForTrestles(airavataClient);
// final String expId = createExperimentEchoForLocalHost(airavataClient);
// final String expId = createExperimentWRFTrestles(airavataClient);
// final String expId = createExperimentForBR2(airavataClient);
@@ -161,7 +161,6 @@ public class CreateLaunchExperiment {
ucrJobSubmission.setSecurityProtocol(securityProtocol);
ucrJobSubmission.setUnicoreEndPointURL(unicoreEndPointURL);
-
return jobSubmission.getJobSubmissionInterfaceId();
}
@@ -180,7 +179,13 @@ public class CreateLaunchExperiment {
output.setType(DataType.STRING);
output.setValue("");
exOut.add(output);
-
+
+ OutputDataObjectType output2 = new OutputDataObjectType();
+ output2.setName("Echoed_Output2");
+ output2.setType(DataType.URI);
+ output2.setValue("file:///tmp/test.txt");
+ exOut.add(output2);
+
Experiment simpleExperiment =
ExperimentModelUtil.createSimpleExperiment("default", "admin", "echoExperiment", "SimpleEcho2", echoAppId, exInputs);
simpleExperiment.setExperimentOutputs(exOut);
@@ -223,23 +228,32 @@ public class CreateLaunchExperiment {
InputDataObjectType input = new InputDataObjectType();
input.setName("Input_to_Echo");
input.setType(DataType.STRING);
- input.setValue("Echoed_Output=Hello World");
- exInputs.add(input);
+ input.setValue("Hello World");
+
+
InputDataObjectType i2 = new InputDataObjectType();
- i2.setName("Input_to_Echo1");
+ i2.setName("Input_to_Echo2");
i2.setType(DataType.URI);
- i2.setValue("http://shrib.com/22QmrrX4");
+ i2.setValue("http://www.textfiles.com/100/ad.txt");
+
+ InputDataObjectType i3 = new InputDataObjectType();
+ i3.setName("Input_to_Echo3");
+ i3.setType(DataType.URI);
+ i3.setValue("file:///tmp/test.txt");
+
+ exInputs.add(input);
exInputs.add(i2);
+ exInputs.add(i3);
List<OutputDataObjectType> exOut = new ArrayList<OutputDataObjectType>();
+
OutputDataObjectType output = new OutputDataObjectType();
output.setName("Echoed_Output");
output.setType(DataType.STRING);
- output.setValue("22QmrrX4");
+ output.setValue("test.txt");
exOut.add(output);
-
Experiment simpleExperiment =
ExperimentModelUtil.createSimpleExperiment("default", "admin", "echoExperiment", "SimpleEcho2", echoAppId, exInputs);
simpleExperiment.setExperimentOutputs(exOut);
@@ -251,7 +265,7 @@ public class CreateLaunchExperiment {
for (String id : computeResources.keySet()) {
String resourceName = computeResources.get(id);
if (resourceName.equals(unicoreHostName)) {
- ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling(id, 1, 1, 1, "normal", 30, 0, 1, "sds128");
+ ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling(id, 1, 1, 1, "normal", 30, 0, 1048576, "sds128");
UserConfigurationData userConfigurationData = new UserConfigurationData();
userConfigurationData.setAiravataAutoSchedule(false);
userConfigurationData.setOverrideManualScheduledParams(false);
http://git-wip-us.apache.org/repos/asf/airavata/blob/f4a32fd0/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
index 41020a0..ce26814 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
@@ -210,7 +210,6 @@ public class RegisterSampleApplications {
throw new AiravataClientException();
System.out.println("FSD Compute ResourceID: "+fsdResourceId);
-
JobSubmissionInterface jobSubmission = RegisterSampleApplicationsUtils.createJobSubmissionInterface(fsdResourceId, protocol, 2);
UnicoreJobSubmission ucrJobSubmission = new UnicoreJobSubmission();
ucrJobSubmission.setSecurityProtocol(securityProtocol);
@@ -222,11 +221,6 @@ public class RegisterSampleApplications {
return jobSubmission.getJobSubmissionInterfaceId();
}
- public void registerfsd(){
- System.out.println("\n #### Registering XSEDE Computational Resources #### \n");
-
- }
-
public void registerAppModules() {
try {
System.out.println("\n #### Registering Application Modules #### \n");
@@ -529,17 +523,25 @@ public class RegisterSampleApplications {
List<String> appModules = new ArrayList<String>();
appModules.add(echoModuleId);
- InputDataObjectType input1 = RegisterSampleApplicationsUtils.createAppInput("Input_to_Echo", "Hello World",
+ InputDataObjectType input1 = RegisterSampleApplicationsUtils.createAppInput("Input_to_Echo", "",
DataType.STRING, null, false, "A test string to Echo", null);
+ InputDataObjectType input2 = RegisterSampleApplicationsUtils.createAppInput("Input_to_Echo2", "",
+ DataType.URI, null, false, "A sample input remote file", null);
+
+ InputDataObjectType input3 = RegisterSampleApplicationsUtils.createAppInput("Input_to_Echo3", "file:///tmp/test.txt",
+ DataType.URI, null, false, "A sample input local file", null);
+
+
List<InputDataObjectType> applicationInputs = new ArrayList<InputDataObjectType>();
applicationInputs.add(input1);
OutputDataObjectType output1 = RegisterSampleApplicationsUtils.createAppOutput("Echoed_Output",
- "22QmrrX4", DataType.STRING);
-
+ "", DataType.STRING);
+
List<OutputDataObjectType> applicationOutputs = new ArrayList<OutputDataObjectType>();
applicationOutputs.add(output1);
+
echoInterfaceId = airavataClient.registerApplicationInterface(
RegisterSampleApplicationsUtils.createApplicationInterfaceDescription(echoName, echoDescription,
@@ -1104,7 +1106,7 @@ public class RegisterSampleApplications {
"/N/dc2/scratch/cgateway/gta-work-dirs");
ComputeResourcePreference fsdResourcePreferences = RegisterSampleApplicationsUtils.
- createComputeResourcePreference(fsdResourceId, null, false, null, null, null,null);
+ createComputeResourcePreference(fsdResourceId, null, false, null, JobSubmissionProtocol.UNICORE, DataMovementProtocol.UNICORE_STORAGE_SERVICE,null);
GatewayResourceProfile gatewayResourceProfile = new GatewayResourceProfile();
gatewayResourceProfile.setGatewayID(DEFAULT_GATEWAY);
http://git-wip-us.apache.org/repos/asf/airavata/blob/f4a32fd0/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java
index 6fdadfb..d490269 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java
@@ -167,9 +167,9 @@ public class BESProvider extends AbstractProvider implements GFacProvider,
.toString()));
jobExecutionContext.getNotifier().publish(new UnicoreJobIDEvent(jobId));
- GFacUtils.saveJobStatus(jobExecutionContext, details, JobState.SUBMITTED);
+// GFacUtils.saveJobStatus(jobExecutionContext, details, JobState.SUBMITTED);
- factory.getActivityStatus(activityEpr);
+
log.info(formatStatusMessage(activityEpr.getAddress()
.getStringValue(), factory.getActivityStatus(activityEpr)
.toString()));
@@ -202,8 +202,7 @@ public class BESProvider extends AbstractProvider implements GFacProvider,
+ applicationJobStatus;
jobExecutionContext.getNotifier().publish(
new StatusChangeEvent(jobStatusMessage));
- GFacUtils.updateJobStatus(jobExecutionContext, jobDetails,
- applicationJobStatus);
+ GFacUtils.updateJobStatus(jobExecutionContext, jobDetails, applicationJobStatus);
throw new GFacProviderException(
jobExecutionContext.getExperimentID() + "Job Canceled");
} else if (activityStatus.getState() == ActivityStateEnumeration.FINISHED) {
@@ -412,7 +411,7 @@ public class BESProvider extends AbstractProvider implements GFacProvider,
ActivityStatusType activityStatus = getStatus(factory, activityEpr);
JobState applicationJobStatus = getApplicationJobStatus(activityStatus);
String jobStatusMessage = "Status of job " + jobId + "is " + applicationJobStatus;
- GFacUtils.updateJobStatus(jobExecutionContext, jobDetails, applicationJobStatus);
+// GFacUtils.updateJobStatus(jobExecutionContext, jobDetails, applicationJobStatus);
jobExecutionContext.getNotifier().publish(new StatusChangeEvent(jobStatusMessage));
http://git-wip-us.apache.org/repos/asf/airavata/blob/f4a32fd0/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/security/UNICORESecurityContext.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/security/UNICORESecurityContext.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/security/UNICORESecurityContext.java
index 57fc2d5..e2896e8 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/security/UNICORESecurityContext.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/security/UNICORESecurityContext.java
@@ -52,7 +52,7 @@ public class UNICORESecurityContext extends X509SecurityContext {
secProperties.getETDSettings().setExtendTrustDelegation(true);
// secProperties.setMessageLogging(true);
// secProperties.setDoSignMessage(true);
-// secProperties.getETDSettings().setIssuerCertificateChain(secProperties.getCredential().getCertificateChain());
+ secProperties.getETDSettings().setIssuerCertificateChain(secProperties.getCredential().getCertificateChain());
http://git-wip-us.apache.org/repos/asf/airavata/blob/f4a32fd0/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java
index 91c27f9..7c7a59e 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java
@@ -38,9 +38,9 @@ public class ApplicationProcessor {
public static void generateJobSpecificAppElements(JobDefinitionType value, JobExecutionContext context){
String userName = getUserNameFromContext(context);
- if (userName.equalsIgnoreCase("admin")){
- userName = "CN=zdv575, O=Ultrascan Gateway, C=DE";
- }
+// if (userName.equalsIgnoreCase("admin")){
+// userName = "CN=zdv575, O=Ultrascan Gateway, C=DE";
+// }
ApplicationDeploymentDescription appDep= context.getApplicationContext().getApplicationDeploymentDescription();
String appname = context.getApplicationContext().getApplicationInterfaceDescription().getApplicationName();
@@ -53,9 +53,20 @@ public class ApplicationProcessor {
// createApplicationEnvironment(value, appDep.getSetEnvironment(), parallelism);
// }
//
- String stdout = context.getStandardOutput();
- String stderr = context.getStandardError();
+ String stdout = context.getStandardOutput();
+ String stderr = context.getStandardError();
+ if(stdout != null) {
+ stdout = stdout.substring(stdout.lastIndexOf('/')+1);
+ }
+
+ if(stderr != null) {
+ stderr = stderr.substring(stderr.lastIndexOf('/')+1);
+ }
+
+ stdout = (stdout == null || stdout.equals("")) ? "stdout":stdout;
+ stderr = (stdout == null || stderr.equals("")) ? "stderr":stderr;
+
if (appDep.getExecutablePath() != null) {
FileNameType fNameType = FileNameType.Factory.newInstance();
fNameType.setStringValue(appDep.getExecutablePath());
http://git-wip-us.apache.org/repos/asf/airavata/blob/f4a32fd0/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataStagingProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataStagingProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataStagingProcessor.java
deleted file mode 100644
index 0541d47..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataStagingProcessor.java
+++ /dev/null
@@ -1,236 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.bes.utils;
-
-import java.io.File;
-import java.net.URI;
-import java.util.Map;
-
-import org.apache.airavata.commons.gfac.type.ActualParameter;
-import org.apache.airavata.gfac.core.context.JobExecutionContext;
-import org.apache.airavata.gfac.core.utils.GFacUtils;
-import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
-import org.apache.airavata.schemas.gfac.StringArrayType;
-import org.apache.airavata.schemas.gfac.StringParameterType;
-import org.apache.airavata.schemas.gfac.URIArrayType;
-import org.apache.airavata.schemas.gfac.URIParameterType;
-import org.apache.airavata.schemas.gfac.UnicoreHostType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
-
-public class DataStagingProcessor {
-
- public static void generateDataStagingElements(JobDefinitionType value, JobExecutionContext context) throws Exception{
-
- HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
- .getApplicationContext().getApplicationDeploymentDescription()
- .getType();
-
-
- String gridftpEndpoint = ((UnicoreHostType) context.getApplicationContext().getHostDescription().getType())
- .getGridFTPEndPointArray()[0];
-
-
- if (context.getInMessageContext().getParameters().size() > 0) {
- buildDataStagingFromInputContext(context, value, gridftpEndpoint, appDepType);
- }
-
- if (context.getOutMessageContext().getParameters().size() > 0) {
- buildFromOutputContext(context, value, gridftpEndpoint, appDepType);
- }
-
- createStdOutURIs(value, appDepType, gridftpEndpoint, isUnicoreEndpoint(context));
-
- }
-
- private static void createInURIElement(JobDefinitionType value,
- String endpoint, String inputDir, ActualParameter inParam)
- throws Exception {
-
- String uri = ((URIParameterType) inParam.getType()).getValue();
- String fileName = new File(uri).getName();
- if (uri.startsWith("file")) {
- URI gridFTPInputDirURI = URIUtils.createGsiftpURI(endpoint,
- inputDir);
- String filePath = gridFTPInputDirURI.toString() + File.separator
- + fileName;
- JSDLUtils
- .addDataStagingSourceElement(value, filePath, null, fileName);
- } else if (uri.startsWith("gsiftp") || uri.startsWith("http")
- || uri.startsWith("rns")) {
- // no need to stage-in those files to the input
- // directory
- JSDLUtils.addDataStagingSourceElement(value, uri, null, fileName);
- }
-
- }
-
- private static void createStdOutURIs(JobDefinitionType value,
- HpcApplicationDeploymentType appDepType, String endpoint,
- boolean isUnicore) throws Exception {
-
- URI remoteOutputDir = URIUtils.createGsiftpURI(endpoint,
- appDepType.getOutputDataDirectory());
-
- String stdout = ApplicationProcessor.getApplicationStdOut(value, appDepType);
-
- String stderr = ApplicationProcessor.getApplicationStdErr(value, appDepType);
-
- String stdoutFileName = (stdout == null || stdout.equals("")) ? "stdout"
- : stdout;
- String stdoutURI = GFacUtils.createGsiftpURIAsString(
- remoteOutputDir.toString(), stdoutFileName);
- JSDLUtils.addDataStagingTargetElement(value, null, stdoutFileName,
- stdoutURI);
-
- String stderrFileName = (stdout == null || stderr.equals("")) ? "stderr"
- : stderr;
- String stderrURI = GFacUtils.createGsiftpURIAsString(
- remoteOutputDir.toString(), stderrFileName);
- JSDLUtils.addDataStagingTargetElement(value, null, stderrFileName,
- stderrURI);
-
- if(isUnicore) {
- String scriptExitCodeFName = "UNICORE_SCRIPT_EXIT_CODE";
- String scriptExitCode = GFacUtils.createGsiftpURIAsString(
- remoteOutputDir.toString(), scriptExitCodeFName);
- JSDLUtils.addDataStagingTargetElement(value, null,
- scriptExitCodeFName, scriptExitCode.toString());
- }
-
- }
-
-
- private static void createOutStringElements(JobDefinitionType value,
- HpcApplicationDeploymentType appDeptype, String endpoint, String prmValue) throws Exception {
-
- if(prmValue == null || "".equals(prmValue)) return;
-
-
- String outputUri = GFacUtils.createGsiftpURIAsString(endpoint, appDeptype.getOutputDataDirectory());
-
- URI finalOutputUri = URIUtils.createGsiftpURI(outputUri, prmValue);
- JSDLUtils.addDataStagingTargetElement(value, null, prmValue, finalOutputUri.toString());
- }
-
-
- private static void createOutURIElement(JobDefinitionType value,
- String prmValue) throws Exception {
- String fileName = new File(prmValue.toString()).getName();
- JSDLUtils.addDataStagingTargetElement(value, null, fileName, prmValue);
- }
-
-
- private static JobDefinitionType buildFromOutputContext(JobExecutionContext context,
- JobDefinitionType value, String gridftpEndpoint,
- HpcApplicationDeploymentType appDepType) throws Exception {
-
- Map<String, Object> outputParams = context.getOutMessageContext()
- .getParameters();
-
- for (String paramKey : outputParams.keySet()) {
-
- ActualParameter outParam = (ActualParameter) outputParams
- .get(paramKey);
-
- // if single urls then convert each url into jsdl source
- // elements,
- // that are formed by concat of gridftpurl+inputdir+filename
-
- String paramDataType = outParam.getType().getType().toString();
-
- if ("URI".equals(paramDataType)) {
- String uriPrm = ((URIParameterType) outParam.getType())
- .getValue();
- createOutURIElement(value, uriPrm);
- }
-
- // string params are converted into the job arguments
-
- else if (("URIArray").equals(paramDataType)) {
- String[] uriArray = ((URIArrayType) outParam.getType())
- .getValueArray();
- for (String u : uriArray) {
-
- createOutURIElement(value, u);
- }
-
- }
- else if ("String".equals(paramDataType)) {
- String stringPrm = ((StringParameterType) outParam
- .getType()).getValue();
- createOutStringElements(value, appDepType, gridftpEndpoint, stringPrm);
- }
-
- else if ("StringArray".equals(paramDataType)) {
- String[] valueArray = ((StringArrayType) outParam.getType())
- .getValueArray();
- for (String v : valueArray) {
- createOutStringElements(value, appDepType, gridftpEndpoint, v);
- }
- }
- }
-
- return value;
- }
-
-
- private static void buildDataStagingFromInputContext(JobExecutionContext context, JobDefinitionType value, String gridftpEndpoint, HpcApplicationDeploymentType appDepType)
- throws Exception {
-
- // TODO set data directory
- Map<String, Object> inputParams = context.getInMessageContext()
- .getParameters();
-
- for (String paramKey : inputParams.keySet()) {
-
- ActualParameter inParam = (ActualParameter) inputParams
- .get(paramKey);
-
- // if single urls then convert each url into jsdl source
- // elements,
- // that are formed by concat of gridftpurl+inputdir+filename
-
- String paramDataType = inParam.getType().getType().toString();
-
- if ("URI".equals(paramDataType)) {
- createInURIElement(value, gridftpEndpoint,
- appDepType.getInputDataDirectory(), inParam);
- }
-
- // string params are converted into the job arguments
-
- else if ("String".equals(paramDataType)) {
- String stringPrm = ((StringParameterType) inParam.getType())
- .getValue();
- ApplicationProcessor.addApplicationArgument(value, appDepType, stringPrm);
- }
- }
-
- }
-
-
- public static boolean isUnicoreEndpoint(JobExecutionContext context) {
- return ( (context.getApplicationContext().getHostDescription().getType() instanceof UnicoreHostType)?true:false );
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/f4a32fd0/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java
index 4aa6cc1..cb036fa 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java
@@ -63,7 +63,7 @@ public class DataTransferrer {
if (uri.startsWith("file")) {
try {
String uriWithoutProtocol = uri.substring(uri.lastIndexOf("://") + 1, uri.length());
- FileUploader fileUploader = new FileUploader(uriWithoutProtocol,"input/" + fileName,Mode.overwrite);
+ FileUploader fileUploader = new FileUploader(uriWithoutProtocol,fileName,Mode.overwrite);
fileUploader.perform(storageClient);
} catch (FileNotFoundException e3) {
throw new GFacProviderException(
@@ -139,7 +139,7 @@ public class DataTransferrer {
String scriptExitCodeFName = "UNICORE_SCRIPT_EXIT_CODE";
String scriptCodeLocation = downloadLocation+File.separator+scriptExitCodeFName;
f1.setFrom(scriptExitCodeFName);
- f1.setTo(downloadLocation+File.separator+scriptCodeLocation);
+ f1.setTo(scriptCodeLocation);
f1.perform(storageClient);
log.info("UNICORE_SCRIPT_EXIT_CODE downloaded to "+scriptCodeLocation);
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/f4a32fd0/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/JSDLGenerator.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/JSDLGenerator.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/JSDLGenerator.java
index 9755bc7..4743522 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/JSDLGenerator.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/JSDLGenerator.java
@@ -55,8 +55,7 @@ public class JSDLGenerator implements BESConstants {
ApplicationProcessor.generateJobSpecificAppElements(value, context);
- DataStagingProcessor.generateDataStagingElements(value, context);
-
+
return jobDefDoc;
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/f4a32fd0/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/SecurityUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/SecurityUtils.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/SecurityUtils.java
index ac79f33..1da0bac 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/SecurityUtils.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/SecurityUtils.java
@@ -16,7 +16,6 @@ import javax.security.auth.x500.X500Principal;
import org.apache.airavata.common.exception.ApplicationSettingsException;
import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.commons.gfac.type.HostDescription;
import org.apache.airavata.credential.store.store.CredentialReader;
import org.apache.airavata.gfac.GFacException;
import org.apache.airavata.gfac.RequestData;
@@ -24,14 +23,13 @@ import org.apache.airavata.gfac.bes.security.UNICORESecurityContext;
import org.apache.airavata.gfac.bes.security.X509SecurityContext;
import org.apache.airavata.gfac.core.context.JobExecutionContext;
import org.apache.airavata.gfac.core.utils.GFacUtils;
-import org.apache.airavata.schemas.gfac.UnicoreHostType;
+import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
import org.bouncycastle.asn1.ASN1InputStream;
import org.bouncycastle.asn1.x500.X500Name;
import org.bouncycastle.asn1.x509.AlgorithmIdentifier;
import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import eu.emi.security.authn.x509.helpers.CertificateHelpers;
import eu.emi.security.authn.x509.helpers.proxy.X509v3CertificateBuilder;
import eu.emi.security.authn.x509.impl.CertificateUtils;
@@ -45,8 +43,7 @@ public class SecurityUtils {
public static void addSecurityContext(JobExecutionContext jobExecutionContext) throws GFacException {
- HostDescription registeredHost = jobExecutionContext.getApplicationContext().getHostDescription();
- if (! (registeredHost.getType() instanceof UnicoreHostType)) {
+ if (!jobExecutionContext.getPreferredJobSubmissionProtocol().equals(JobSubmissionProtocol.UNICORE)) {
logger.error("This is a wrong method to invoke for UNICORE host types,please check your gfac-config.xml");
}
else
http://git-wip-us.apache.org/repos/asf/airavata/blob/f4a32fd0/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java
index 9c92789..e82edef 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java
@@ -46,7 +46,7 @@ public class UASDataStagingProcessor {
private static void createInURISMSElement(JobDefinitionType value, String smsUrl, String uri)
throws Exception {
- String fileName = "input/" + new File(uri).getName();
+ String fileName = new File(uri).getName();
if (uri.startsWith("file")) {
uri = smsUrl+"#/"+fileName;
@@ -113,13 +113,9 @@ public class UASDataStagingProcessor {
List<OutputDataObjectType> applicationOutputs = context.getTaskData().getApplicationOutputs();
if (applicationOutputs != null && !applicationOutputs.isEmpty()){
for (OutputDataObjectType output : applicationOutputs){
- if(output.getType().equals(DataType.URI)) {
+ if(output.getType().equals(DataType.URI) && !output.getValue().startsWith("file:")) {
createOutURIElement(value, output.getValue());
}
- else if(output.getType().equals(DataType.STRING)) {
- // TODO: remove this check, as out string
- createOutStringElements(value, smsUrl, output.getValue());
- }
}
}
return value;
@@ -128,7 +124,7 @@ public class UASDataStagingProcessor {
private static void buildDataStagingFromInputContext(JobExecutionContext context, JobDefinitionType value, String smsUrl)
throws Exception {
- List<InputDataObjectType> applicationInputs = context.getApplicationContext().getApplicationInterfaceDescription().getApplicationInputs();
+ List<InputDataObjectType> applicationInputs = context.getTaskData().getApplicationInputs();
if (applicationInputs != null && !applicationInputs.isEmpty()){
for (InputDataObjectType input : applicationInputs){
http://git-wip-us.apache.org/repos/asf/airavata/blob/f4a32fd0/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/Scheduler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/Scheduler.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/Scheduler.java
index a9940b8..853ffc8 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/Scheduler.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/Scheduler.java
@@ -32,6 +32,7 @@ import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionInterfa
import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
import org.apache.airavata.model.appcatalog.computeresource.LOCALSubmission;
import org.apache.airavata.model.appcatalog.computeresource.SSHJobSubmission;
+import org.apache.airavata.model.appcatalog.computeresource.UnicoreJobSubmission;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
@@ -42,6 +43,7 @@ import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPathExpressionException;
+
import java.io.File;
import java.io.IOException;
import java.net.URL;
@@ -118,6 +120,7 @@ public class Scheduler {
JobSubmissionProtocol jobSubmissionProtocol = jobExecutionContext.getPreferredJobSubmissionProtocol();
SSHJobSubmission sshJobSubmission;
LOCALSubmission localSubmission;
+ UnicoreJobSubmission unicoreSubmission;
String securityProtocol = null;
try {
AppCatalog appCatalog = jobExecutionContext.getAppCatalog();
@@ -130,6 +133,10 @@ public class Scheduler {
}else if (jobSubmissionProtocol == JobSubmissionProtocol.LOCAL) {
localSubmission = appCatalog.getComputeResource().getLocalJobSubmission(jobExecutionContext.getPreferredJobSubmissionInterface().getJobSubmissionInterfaceId());
}
+ else if (jobSubmissionProtocol == JobSubmissionProtocol.UNICORE) {
+ unicoreSubmission = appCatalog.getComputeResource().getUNICOREJobSubmission(jobExecutionContext.getPreferredJobSubmissionInterface().getJobSubmissionInterfaceId());
+ securityProtocol = unicoreSubmission.getSecurityProtocol().toString();
+ }
List<Element> elements = GFacUtils.getElementList(GFacConfiguration.getHandlerDoc(), Constants.XPATH_EXPR_PROVIDER_ON_SUBMISSION + jobSubmissionProtocol + "']");
for (Element element : elements) {
String security = element.getAttribute(Constants.GFAC_CONFIG_SECURITY_ATTRIBUTE);
http://git-wip-us.apache.org/repos/asf/airavata/blob/f4a32fd0/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
index aaa5461..8403f8c 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
@@ -370,12 +370,16 @@ public class BetterGfacImpl implements GFac,Watcher {
jobExecutionContext.setPreferredDataMovementInterface(jobExecutionContext.getHostPrioritizedDataMovementInterfaces().get(0));
jobExecutionContext.setPreferredDataMovementProtocol(jobExecutionContext.getPreferredDataMovementInterface().getDataMovementProtocol());
} else {
- for (DataMovementInterface dataMovementInterface : dataMovementInterfaces) {
- if (gatewayResourcePreferences.getPreferredDataMovementProtocol() == dataMovementInterface.getDataMovementProtocol()) {
- jobExecutionContext.setPreferredDataMovementInterface(dataMovementInterface);
- break;
- }
- }
+ // this check is to avoid NPE when job submission endpoints do
+ // not contain any data movement interfaces.
+ if((dataMovementInterfaces != null) && (!dataMovementInterfaces.isEmpty())) {
+ for (DataMovementInterface dataMovementInterface : dataMovementInterfaces) {
+ if (gatewayResourcePreferences.getPreferredDataMovementProtocol() == dataMovementInterface.getDataMovementProtocol()) {
+ jobExecutionContext.setPreferredDataMovementInterface(dataMovementInterface);
+ break;
+ }
+ }
+ }
}
} else {
setUpWorkingLocation(jobExecutionContext, applicationInterface, "/tmp");
http://git-wip-us.apache.org/repos/asf/airavata/blob/f4a32fd0/modules/gfac/pom.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/pom.xml b/modules/gfac/pom.xml
index 34253b9..2e7df07 100644
--- a/modules/gfac/pom.xml
+++ b/modules/gfac/pom.xml
@@ -38,7 +38,7 @@
<!--<module>gfac-hadoop</module>-->
<!--<module>gfac-gram</module>-->
<module>gfac-gsissh</module>
- <!--<module>gfac-bes</module>-->
+ <module>gfac-bes</module>
<module>gfac-monitor</module>
<module>airavata-gfac-service</module>
<module>airavata-gfac-stubs</module>
[5/5] airavata git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/airavata
Posted by ms...@apache.org.
Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/airavata
Conflicts:
airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/5b118f8c
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/5b118f8c
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/5b118f8c
Branch: refs/heads/master
Commit: 5b118f8c41580d79dfb5707814cac1c628fa2b07
Parents: 6644942 d35226d
Author: msmemon <sh...@gmail.com>
Authored: Wed Dec 10 13:41:18 2014 +0100
Committer: msmemon <sh...@gmail.com>
Committed: Wed Dec 10 13:41:18 2014 +0100
----------------------------------------------------------------------
.../applicationInterfaceModel_types.cpp | 212 ++++++++--
.../airavata/applicationInterfaceModel_types.h | 124 +++++-
.../lib/airavata/experimentModel_types.cpp | 12 +-
.../lib/airavata/experimentModel_types.h | 12 +-
.../lib/airavata/messagingEvents_types.cpp | 4 +-
.../lib/airavata/messagingEvents_types.h | 4 +-
.../lib/airavata/workflowDataModel_types.cpp | 4 +-
.../lib/airavata/workflowDataModel_types.h | 4 +-
.../Model/AppCatalog/AppInterface/Types.php | 167 ++++++++
.../client/samples/RegisterSampleData.java | 14 +-
.../tools/RegisterOGCEUS3Application.java | 2 +-
.../tools/RegisterSampleApplications.java | 85 ++--
.../tools/RegisterSampleApplicationsUtils.java | 3 +-
.../client/tools/RegisterUS3Application.java | 2 +-
.../appinterface/InputDataObjectType.java | 424 ++++++++++++++++++-
.../appinterface/OutputDataObjectType.java | 320 +++++++++++++-
.../airavata/model/util/AppInterfaceUtil.java | 2 +
.../applicationInterfaceModel.thrift | 29 +-
.../data/impl/ApplicationInterfaceImpl.java | 6 +-
.../catalog/data/impl/WorkflowCatalogImpl.java | 4 +-
.../catalog/data/model/ApplicationInput.java | 40 ++
.../catalog/data/model/ApplicationOutput.java | 30 ++
.../catalog/data/model/WorkflowInput.java | 44 +-
.../catalog/data/model/WorkflowOutput.java | 30 ++
.../resources/ApplicationInputResource.java | 52 ++-
.../resources/ApplicationOutputResource.java | 30 ++
.../data/resources/WorkflowInputResource.java | 51 ++-
.../data/resources/WorkflowOutputResource.java | 33 ++
.../catalog/data/util/AppCatalogJPAUtils.java | 18 +-
.../data/util/AppCatalogThriftConversion.java | 28 +-
.../src/main/resources/appcatalog-derby.sql | 7 +
.../src/main/resources/appcatalog-mysql.sql | 7 +
.../app/catalog/test/AppInterfaceTest.java | 2 +
.../src/test/resources/appcatalog-derby.sql | 7 +
.../gfac/local/provider/impl/LocalProvider.java | 41 +-
.../registry/jpa/impl/ExperimentRegistry.java | 38 +-
.../registry/jpa/model/ApplicationInput.java | 86 +++-
.../registry/jpa/model/ApplicationOutput.java | 53 ++-
.../registry/jpa/model/Experiment_Input.java | 87 +++-
.../registry/jpa/model/Experiment_Output.java | 53 ++-
.../registry/jpa/model/NodeInput.java | 85 +++-
.../registry/jpa/model/NodeOutput.java | 53 ++-
.../jpa/resources/ApplicationInputResource.java | 91 +++-
.../resources/ApplicationOutputResource.java | 58 ++-
.../jpa/resources/ExperimentInputResource.java | 92 +++-
.../jpa/resources/ExperimentOutputResource.java | 58 ++-
.../jpa/resources/NodeInputResource.java | 92 +++-
.../jpa/resources/NodeOutputResource.java | 58 ++-
.../registry/jpa/resources/Utils.java | 46 +-
.../jpa/utils/ThriftDataModelConversion.java | 79 +++-
.../src/main/resources/registry-derby.sql | 45 +-
.../src/main/resources/registry-mysql.sql | 45 +-
.../jpa/ExperimentInputResourceTest.java | 2 +-
.../jpa/ExperimentOutputResourceTest.java | 2 +-
.../src/test/resources/registry-derby.sql | 45 +-
.../engine/interpretor/WorkflowInterpreter.java | 14 +-
.../model/component/ws/WSComponent.java | 3 +-
.../component/ws/WSComponentApplication.java | 97 +++--
.../ws/WSComponentApplicationParameter.java | 39 +-
.../model/component/ws/WSComponentPort.java | 23 +-
.../workflow/model/gpel/script/BPELScript.java | 2 +-
.../model/gpel/script/WorkflowWSDL.java | 3 +-
.../workflow/model/graph/GraphSchema.java | 1 +
.../workflow/model/graph/system/InputNode.java | 23 +-
.../workflow/model/utils/WorkflowConstants.java | 16 +-
.../registry/ComponentRegistryLoader.java | 9 +-
.../graph/system/InputConfigurationDialog.java | 91 ++--
.../WorkflowInterpreterLaunchWindow.java | 3 +
68 files changed, 2860 insertions(+), 486 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/5b118f8c/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
----------------------------------------------------------------------
diff --cc airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
index 9a2aaf7,e4e1e32..18408b7
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
@@@ -536,25 -529,17 +536,26 @@@ public class RegisterSampleApplication
List<String> appModules = new ArrayList<String>();
appModules.add(echoModuleId);
- InputDataObjectType input1 = RegisterSampleApplicationsUtils.createAppInput("Input_to_Echo", "Hello World",
++
+ InputDataObjectType input1 = RegisterSampleApplicationsUtils.createAppInput("Input_to_Echo", "",
- DataType.STRING, null, false, "A test string to Echo", null);
+ DataType.STRING, null, 1, false, "A test string to Echo", null);
+ InputDataObjectType input2 = RegisterSampleApplicationsUtils.createAppInput("Input_to_Echo2", "",
- DataType.URI, null, false, "A sample input remote file", null);
++ DataType.URI, null, 2, false, "A sample input remote file", null);
+
+ InputDataObjectType input3 = RegisterSampleApplicationsUtils.createAppInput("Input_to_Echo3", "file:///tmp/test.txt",
- DataType.URI, null, false, "A sample input local file", null);
++ DataType.URI, null, 3, false, "A sample input local file", null);
+
+
List<InputDataObjectType> applicationInputs = new ArrayList<InputDataObjectType>();
-- applicationInputs.add(input1);
++ applicationInputs.add(input1); applicationInputs.add(input2); applicationInputs.add(input3);
OutputDataObjectType output1 = RegisterSampleApplicationsUtils.createAppOutput("Echoed_Output",
- "22QmrrX4", DataType.STRING);
-
+ "", DataType.STRING);
+
List<OutputDataObjectType> applicationOutputs = new ArrayList<OutputDataObjectType>();
applicationOutputs.add(output1);
+
echoInterfaceId = airavataClient.registerApplicationInterface(
RegisterSampleApplicationsUtils.createApplicationInterfaceDescription(echoName, echoDescription,
@@@ -565,39 -550,7 +566,39 @@@
e.printStackTrace();
}
}
+
+
+ public void registerMPIInterface() {
+ try {
+ System.out.println("#### Registering MPI Interface #### \n");
+
+ List<String> appModules = new ArrayList<String>();
+ appModules.add(mpiModuleId);
+ InputDataObjectType input1 = RegisterSampleApplicationsUtils.createAppInput("Sample_Input", "",
- DataType.STRING, null, false, "An optional MPI source file", null);
++ DataType.STRING, null, 1, false, "An optional MPI source file", null);
+
+ List<InputDataObjectType> applicationInputs = new ArrayList<InputDataObjectType>();
+ applicationInputs.add(input1);
+
+ OutputDataObjectType output1 = RegisterSampleApplicationsUtils.createAppOutput("Sample_Output",
+ "", DataType.STRING);
+
+ List<OutputDataObjectType> applicationOutputs = new ArrayList<OutputDataObjectType>();
+ applicationOutputs.add(output1);
+
+
+ mpiInterfaceId = airavataClient.registerApplicationInterface(
+ RegisterSampleApplicationsUtils.createApplicationInterfaceDescription(mpiName, mpiDescription,
+ appModules, applicationInputs, applicationOutputs));
+ System.out.println("MPI Application Interface Id " + mpiInterfaceId);
+
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+ }
+
+
public void registerAmberInterface() {
try {
System.out.println("#### Registering Amber Interface #### \n");
[2/5] airavata git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/airavata
Posted by ms...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/airavata
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/576c87d2
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/576c87d2
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/576c87d2
Branch: refs/heads/master
Commit: 576c87d2241f1036fdd267503381f59de11c43b3
Parents: 28a235e 6f89736
Author: msmemon <sh...@gmail.com>
Authored: Wed Dec 3 13:25:17 2014 +0100
Committer: msmemon <sh...@gmail.com>
Committed: Wed Dec 3 13:25:17 2014 +0100
----------------------------------------------------------------------
modules/gfac/gfac-monitor/pom.xml | 9 +----
.../airavata/gfac/monitor/util/X509Helper.java | 35 +++++++++++---------
tools/gsissh/pom.xml | 12 ++++++-
3 files changed, 31 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
[4/5] airavata git commit: working mpi example
Posted by ms...@apache.org.
working mpi example
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/6644942b
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/6644942b
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/6644942b
Branch: refs/heads/master
Commit: 6644942bfd1a3b2214a0497bc7ddaaf6d839fe1d
Parents: f4a32fd
Author: msmemon <sh...@gmail.com>
Authored: Wed Dec 10 13:31:14 2014 +0100
Committer: msmemon <sh...@gmail.com>
Committed: Wed Dec 10 13:31:14 2014 +0100
----------------------------------------------------------------------
.../client/samples/CreateLaunchExperiment.java | 68 +++++++++++++++++++-
.../tools/RegisterSampleApplications.java | 53 ++++++++++++++-
modules/distribution/server/pom.xml | 10 +--
3 files changed, 123 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/6644942b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
index 5c3ef19..aa3e654 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
@@ -58,6 +58,7 @@ public class CreateLaunchExperiment {
private static Airavata.Client airavataClient;
private static String echoAppId = "Echo_753d7cf6-f79a-4f7f-8ada-5d707e90c383";
+ private static String mpiAppId = "HelloMPI_da45305f-5d90-4a18-8716-8dd54c3b2376";
private static String wrfAppId = "WRF_7ad5da38-c08b-417c-a9ea-da9298839762";
private static String amberAppId = "Amber_49b16f6f-93ab-4885-9971-6ab2ab5eb3d3";
private static String gromacsAppId = "GROMACS_05622038-9edd-4cb1-824e-0b7cb993364b";
@@ -81,7 +82,7 @@ public class CreateLaunchExperiment {
public static void main(String[] args) throws Exception {
airavataClient = AiravataClientFactory.createAiravataClient(THRIFT_SERVER_HOST, THRIFT_SERVER_PORT);
System.out.println("API version is " + airavataClient.getAPIVersion());
-// registerApplications(); // run this only the first time
+// registerApplications(); // run this only the first time
createAndLaunchExp();
}
@@ -93,7 +94,8 @@ public class CreateLaunchExperiment {
try {
for (int i = 0; i < 1; i++) {
// final String expId = createExperimentForSSHHost(airavata);
- final String expId = createEchoExperimentForFSD(airavataClient);
+// final String expId = createEchoExperimentForFSD(airavataClient);
+ final String expId = createMPIExperimentForFSD(airavataClient);
// final String expId = createEchoExperimentForStampede(airavataClient);
// final String expId = createEchoExperimentForTrestles(airavataClient);
// final String expId = createExperimentEchoForLocalHost(airavataClient);
@@ -296,9 +298,71 @@ public class CreateLaunchExperiment {
}
return null;
}
+
+
+ public static String createMPIExperimentForFSD(Airavata.Client client) throws TException {
+ try {
+
+ List<InputDataObjectType> exInputs = new ArrayList<InputDataObjectType>();
+ InputDataObjectType input = new InputDataObjectType();
+ input.setName("Sample_Input");
+ input.setType(DataType.STRING);
+ input.setValue("");
+ exInputs.add(input);
+
+ List<OutputDataObjectType> exOut = new ArrayList<OutputDataObjectType>();
+ OutputDataObjectType output = new OutputDataObjectType();
+ output.setName("Sample_Output");
+ output.setType(DataType.STRING);
+ output.setValue("");
+ exOut.add(output);
+
+ Experiment simpleExperiment =
+ ExperimentModelUtil.createSimpleExperiment("default", "admin", "mpiExperiment", "HelloMPI", mpiAppId, null);
+// simpleExperiment.setExperimentOutputs(exOut);
+
+
+
+ Map<String, String> computeResources = airavataClient.getAvailableAppInterfaceComputeResources(mpiAppId);
+ if (computeResources != null && computeResources.size() != 0) {
+ for (String id : computeResources.keySet()) {
+ String resourceName = computeResources.get(id);
+ if (resourceName.equals(unicoreHostName)) {
+ ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling(id, 2, 1, 2, "normal", 30, 0, 1048576, "sds128");
+ UserConfigurationData userConfigurationData = new UserConfigurationData();
+ userConfigurationData.setAiravataAutoSchedule(false);
+ userConfigurationData.setOverrideManualScheduledParams(false);
+ userConfigurationData.setComputationalResourceScheduling(scheduling);
+
+ // set output directory
+ AdvancedOutputDataHandling dataHandling = new AdvancedOutputDataHandling();
+ dataHandling.setOutputDataDir("/tmp/airavata/output/"+UUID.randomUUID().toString()+"/");
+ userConfigurationData.setAdvanceOutputDataHandling(dataHandling);
+ simpleExperiment.setUserConfigurationData(userConfigurationData);
+
+ return client.createExperiment(simpleExperiment);
+ }
+ }
+ }
+ } catch (AiravataSystemException e) {
+ logger.error("Error occured while creating the experiment...", e.getMessage());
+ throw new AiravataSystemException(e);
+ } catch (InvalidRequestException e) {
+ logger.error("Error occured while creating the experiment...", e.getMessage());
+ throw new InvalidRequestException(e);
+ } catch (AiravataClientException e) {
+ logger.error("Error occured while creating the experiment...", e.getMessage());
+ throw new AiravataClientException(e);
+ } catch (TException e) {
+ logger.error("Error occured while creating the experiment...", e.getMessage());
+ throw new TException(e);
+ }
+ return null;
+ }
+
public static String createExperimentWRFStampede(Airavata.Client client) throws TException {
try {
List<InputDataObjectType> exInputs = new ArrayList<InputDataObjectType>();
http://git-wip-us.apache.org/repos/asf/airavata/blob/6644942b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
index ce26814..9a2aaf7 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
@@ -71,6 +71,7 @@ public class RegisterSampleApplications {
private static final String trinityName = "Trinity";
private static final String wrfName = "WRF";
private static final String phastaName = "PHASTA";
+ private static final String mpiName = "HelloMPI";
//Appplication Descriptions
private static final String echoDescription = "A Simple Echo Application";
@@ -83,6 +84,7 @@ public class RegisterSampleApplications {
private static final String trinityDescription = "de novo reconstruction of transcriptomes from RNA-seq data";
private static final String wrfDescription = "Weather Research and Forecasting";
private static final String phastaDescription = "Computational fluid dynamics solver";
+ private static final String mpiDescription = "A Hello MPI Application";
//App Module Id's
private static String echoModuleId;
@@ -95,9 +97,11 @@ public class RegisterSampleApplications {
private static String trinityModuleId = "Trinity_8af45ca0-b628-4614-9087-c7b73f5f2fb6";
private static String wrfModuleId;
private static String phastaModuleId;
+ private static String mpiModuleId;
//App Interface Id's
private static String echoInterfaceId = "";
+ private static String mpiInterfaceId = "";
private static String echoLocalInterfaceId = "";
private static String amberInterfaceId = "";
private static String autoDockInterfaceId = "";
@@ -230,6 +234,12 @@ public class RegisterSampleApplications {
RegisterSampleApplicationsUtils.createApplicationModule(
echoName, "1.0", echoDescription));
System.out.println("Echo Module Id " + echoModuleId);
+
+ mpiModuleId = airavataClient.registerApplicationModule(
+ RegisterSampleApplicationsUtils.createApplicationModule(
+ mpiName, "1.0", mpiDescription));
+ System.out.println("MPI Module Id " + mpiModuleId);
+
//Register Amber
amberModuleId = airavataClient.registerApplicationModule(
@@ -320,6 +330,9 @@ public class RegisterSampleApplications {
//Registering Echo
registerEchoInterface();
+
+ //Registering MPI
+ registerMPIInterface();
//Registering Amber
registerAmberInterface();
@@ -552,7 +565,39 @@ public class RegisterSampleApplications {
e.printStackTrace();
}
}
+
+
+ public void registerMPIInterface() {
+ try {
+ System.out.println("#### Registering MPI Interface #### \n");
+
+ List<String> appModules = new ArrayList<String>();
+ appModules.add(mpiModuleId);
+ InputDataObjectType input1 = RegisterSampleApplicationsUtils.createAppInput("Sample_Input", "",
+ DataType.STRING, null, false, "An optional MPI source file", null);
+
+ List<InputDataObjectType> applicationInputs = new ArrayList<InputDataObjectType>();
+ applicationInputs.add(input1);
+
+ OutputDataObjectType output1 = RegisterSampleApplicationsUtils.createAppOutput("Sample_Output",
+ "", DataType.STRING);
+
+ List<OutputDataObjectType> applicationOutputs = new ArrayList<OutputDataObjectType>();
+ applicationOutputs.add(output1);
+
+
+ mpiInterfaceId = airavataClient.registerApplicationInterface(
+ RegisterSampleApplicationsUtils.createApplicationInterfaceDescription(mpiName, mpiDescription,
+ appModules, applicationInputs, applicationOutputs));
+ System.out.println("MPI Application Interface Id " + mpiInterfaceId);
+
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+ }
+
+
public void registerAmberInterface() {
try {
System.out.println("#### Registering Amber Interface #### \n");
@@ -1039,7 +1084,13 @@ public class RegisterSampleApplications {
String echoAppDeployId = airavataClient.registerApplicationDeployment(
RegisterSampleApplicationsUtils.createApplicationDeployment(echoModuleId, fsdResourceId,
"/bin/echo", ApplicationParallelismType.SERIAL, echoDescription));
- System.out.println("Echo on FSD deployment Id " + echoAppDeployId);
+ System.out.println("Echo on FSD deployment Id: " + echoAppDeployId);
+
+ //Register MPI
+ String mpiAppDeployId = airavataClient.registerApplicationDeployment(
+ RegisterSampleApplicationsUtils.createApplicationDeployment(mpiModuleId, fsdResourceId,
+ "/home/bes/hellompi", ApplicationParallelismType.OPENMP_MPI, mpiDescription));
+ System.out.println("MPI on FSD deployment Id: " + mpiAppDeployId);
} catch (TException e) {
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/airavata/blob/6644942b/modules/distribution/server/pom.xml
----------------------------------------------------------------------
diff --git a/modules/distribution/server/pom.xml b/modules/distribution/server/pom.xml
index 1f8b659..fb018a0 100644
--- a/modules/distribution/server/pom.xml
+++ b/modules/distribution/server/pom.xml
@@ -319,11 +319,11 @@
<!--<artifactId>airavata-gfac-gram</artifactId>-->
<!--<version>${project.version}</version>-->
<!--</dependency>-->
- <!--<dependency>-->
- <!--<groupId>org.apache.airavata</groupId>-->
- <!--<artifactId>airavata-gfac-bes</artifactId>-->
- <!--<version>${project.version}</version>-->
- <!--</dependency>-->
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-gfac-bes</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>airavata-gfac-gsissh</artifactId>