You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ra...@apache.org on 2013/02/28 01:26:09 UTC
svn commit: r1451054 [1/2] - in /airavata/trunk/modules/gfac-core: ./
src/main/java/org/apache/airavata/gfac/
src/main/java/org/apache/airavata/gfac/provider/impl/
src/main/java/org/apache/airavata/gfac/provider/utils/
src/test/java/org/apache/airavata...
Author: raminder
Date: Thu Feb 28 00:26:09 2013
New Revision: 1451054
URL: http://svn.apache.org/r1451054
Log:
Applying the patch from Shahbaz for unicore provider. AIRAVATA-770
Added:
airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/ApplicationProcessor.java
airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/DataStagingProcessor.java
airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/ResourceProcessor.java
airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/SPMDProcessor.java
airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/SPMDVariations.java
airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/AbstractBESTest.java
airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/BESParallelJobTest.java
airavata/trunk/modules/gfac-core/src/test/resources/logging.properties
Modified:
airavata/trunk/modules/gfac-core/pom.xml
airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/GFacConfiguration.java
airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/impl/BESProvider.java
airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/JSDLGenerator.java
airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/JSDLUtils.java
airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/BESProviderTest.java
airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/GramProviderTest.java
airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTest.java
airavata/trunk/modules/gfac-core/src/test/resources/gfac-config.xml
Modified: airavata/trunk/modules/gfac-core/pom.xml
URL: http://svn.apache.org/viewvc/airavata/trunk/modules/gfac-core/pom.xml?rev=1451054&r1=1451053&r2=1451054&view=diff
==============================================================================
--- airavata/trunk/modules/gfac-core/pom.xml (original)
+++ airavata/trunk/modules/gfac-core/pom.xml Thu Feb 28 00:26:09 2013
@@ -1,14 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
-<!--Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under
- the Apache License, Version 2.0 (theà "License"); you may not use this file except in compliance with the License. You may
- obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to
- in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
- ANY ~ KIND, either express or implied. See the License for the specific language governing permissions and limitations under
+<!--Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under
+ the Apache License, Version 2.0 (theà "License"); you may not use this file except in compliance with the License. You may
+ obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to
+ in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+ ANY ~ KIND, either express or implied. See the License for the specific language governing permissions and limitations under
the License. -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.airavata</groupId>
<artifactId>airavata</artifactId>
@@ -24,27 +25,14 @@
<dependencies>
<dependency>
- <groupId>cog-globus</groupId>
- <artifactId>cog-jglobus</artifactId>
- <version>1.8.0-bc146</version>
- </dependency>
- <!-- Experimental dependencies for XSP integration
- <dependency>
<groupId>cog-jglobus</groupId>
<artifactId>cog-jglobus</artifactId>
- <version>1.8.0-mod</version>
- </dependency>
- <dependency>
- <groupId>edu.iub.soic.damsl</groupId>
- <artifactId>jXSP</artifactId>
- <version>0.0.1-SNAPSHOT</version>
+ <version>1.8.0_bc</version>
</dependency>
- -->
<dependency>
<groupId>cog-globus</groupId>
<artifactId>puretls</artifactId>
<type>jar</type>
- <!-- <version>0.9b4-1</version> -->
<version>1.0</version>
</dependency>
<dependency>
@@ -53,10 +41,21 @@
<version>1.0</version>
</dependency>
<dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk16</artifactId>
+ <version>1.46</version>
+ </dependency>
+ <dependency>
<groupId>cog-globus</groupId>
<artifactId>cryptix-asn1</artifactId>
<version>1.0</version>
</dependency>
+ <!-- dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ <version>1.6</version>
+ </dependency>
+ -->
<!-- Logging -->
<dependency>
<groupId>org.slf4j</groupId>
@@ -64,11 +63,6 @@
</dependency>
<!-- GFAC schemas -->
- <!--dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>airavata-gfac-schema-utils</artifactId>
- <version>${project.version}</version>
- </dependency-->
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>airavata-client-api</artifactId>
@@ -95,11 +89,11 @@
</dependency>
<!-- Amazon -->
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk</artifactId>
- <version>1.3.20</version>
- </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk</artifactId>
+ <version>1.3.20</version>
+ </dependency>
<!-- Test -->
<dependency>
@@ -107,16 +101,16 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <scope>test</scope>
- </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>j2ssh</groupId>
<artifactId>j2ssh-core</artifactId>
@@ -134,24 +128,8 @@
<groupId>eu.unicore</groupId>
<artifactId>ogsabes-client</artifactId>
<version>1.6.0-SNAPSHOT</version>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
</dependency>
-
- <!-- unicore dependencies finished -->
-
- <!-- Guava -->
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>12.0</version>
- </dependency>
-
- <!-- Hadoop provider related dependencies -->
+ <!-- Hadoop provider related dependencies -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
@@ -197,6 +175,30 @@
<artifactId>jopt-simple</artifactId>
<version>3.2</version>
</dependency>
+ <!-- Guava -->
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>12.0</version>
+ </dependency>
</dependencies>
-</project>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <!--skip>true</skip-->
+ <excludes>
+ <exclude>**/services/**</exclude>
+ </excludes>
+ <forkMode>always</forkMode>
+ <useSystemClassloader>true</useSystemClassloader>
+ <failIfNoTests>false</failIfNoTests>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
\ No newline at end of file
Modified: airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/GFacConfiguration.java
URL: http://svn.apache.org/viewvc/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/GFacConfiguration.java?rev=1451054&r1=1451053&r2=1451054&view=diff
==============================================================================
--- airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/GFacConfiguration.java (original)
+++ airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/GFacConfiguration.java Thu Feb 28 00:26:09 2013
@@ -200,34 +200,34 @@ public class GFacConfiguration {
/**
* Parse GFac configuration file and populate GFacConfiguration object. XML configuration
* file for GFac will look like below.
- * <p/>
- * <GFac>
- * <GlobalHandlers>
- * <InHandlers>
- * <Handler class="org.apache.airavata.gfac.GlobalHandler1">
- * </InHandler>
- * <OutHandlers>
- * <Handler class="org.apache.airavata.gfac.GlabalHandler2">
- * </OutHandlers>
- * </GlobalHandlers>
- * <Provider class="org.apache.airavata.gfac.providers.LocalProvider" host="LocalHost">
- * <InHandlers>
- * <Handler class="org.apache.airavata.gfac.handlers.LocalEvenSetupHandler">
- * </InHandlers>
- * <OutHandlers>
- * <Handler>org.apache.airavata.LocalOutHandler1</Handler>
- * </OutHandlers>
- * </Provider>
- * <Application name="UltraScan">
- * <InHandlers>
- * <Handler class="org.apache.airavata.gfac.handlers.LocalEvenSetupHandler">
- * </InHandlers>
- * <OutHandlers>
- * <Handler class="org.apache.airavata.gfac.LocalOutHandler1">
- * </OutHandlers>
- * </Application>
- * </GFac>
- *
+ * <p>
+ * <GFac> <br/>
+ * <GlobalHandlers> <br/>
+ * <InHandlers> <br/>
+ * <Handler class="org.apache.airavata.gfac.GlobalHandler1"> <br/>
+ * </InHandler> <br/>
+ * <OutHandlers> <br/>
+ * <Handler class="org.apache.airavata.gfac.GlabalHandler2"> <br/>
+ * </OutHandlers> <br/>
+ * </GlobalHandlers> <br/>
+ * <Provider class="org.apache.airavata.gfac.providers.LocalProvider" host="LocalHost"> <br/>
+ * <InHandlers> <br/>
+ * <Handler class="org.apache.airavata.gfac.handlers.LocalEvenSetupHandler"> <br/>
+ * </InHandlers> <br/>
+ * <OutHandlers> <br/>
+ * <Handler>org.apache.airavata.LocalOutHandler1</Handler> <br/>
+ * </OutHandlers> <br/>
+ * </Provider> <br/>
+ * <Application name="UltraScan"> <br/>
+ * <InHandlers> <br/>
+ * <Handler class="org.apache.airavata.gfac.handlers.LocalEvenSetupHandler"> <br/>
+ * </InHandlers> <br/>
+ * <OutHandlers> <br/>
+ * <Handler class="org.apache.airavata.gfac.LocalOutHandler1"> <br/>
+ * </OutHandlers> <br/>
+ * </Application> <br/>
+ * </GFac> <br/>
+ * </p>
* @param configFile configuration file
* @return GFacConfiguration object.
*/
Modified: airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/impl/BESProvider.java
URL: http://svn.apache.org/viewvc/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/impl/BESProvider.java?rev=1451054&r1=1451053&r2=1451054&view=diff
==============================================================================
--- airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/impl/BESProvider.java (original)
+++ airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/impl/BESProvider.java Thu Feb 28 00:26:09 2013
@@ -65,13 +65,13 @@ public class BESProvider implements GFac
private DefaultClientConfiguration secProperties;
private String jobId;
-
-
-
+
+
+
@Override
public void initialize(JobExecutionContext jobExecutionContext)
throws GFacProviderException {
-
+
log.info("Initializing GFAC's <<< UNICORE Provider >>>");
initSecurityProperties(jobExecutionContext);
log.debug("initialized security properties");
@@ -82,7 +82,7 @@ public class BESProvider implements GFac
public void execute(JobExecutionContext jobExecutionContext)
throws GFacProviderException {
UnicoreHostType host = (UnicoreHostType) jobExecutionContext.getApplicationContext().getHostDescription().getType();
-
+
String factoryUrl = host.getUnicoreHostAddressArray()[0];
EndpointReferenceType eprt = EndpointReferenceType.Factory.newInstance();
@@ -98,8 +98,8 @@ public class BESProvider implements GFac
}
CreateActivityDocument cad = CreateActivityDocument.Factory
.newInstance();
-
-
+
+
try {
cad.addNewCreateActivity().addNewActivityDocument()
.setJobDefinition(JSDLGenerator.buildJSDLInstance(jobExecutionContext).getJobDefinition());
@@ -107,7 +107,7 @@ public class BESProvider implements GFac
// TODO Auto-generated catch block
e1.printStackTrace();
}
-
+
CreateActivityResponseDocument response = null;
try {
response = factory.createActivity(cad);
@@ -120,27 +120,27 @@ public class BESProvider implements GFac
}
EndpointReferenceType activityEpr = response
.getCreateActivityResponse().getActivityIdentifier();
-
+
log.debug("Job EPR: "+activityEpr);
-
+
log.info("Job: "+activityEpr.getAddress().getStringValue()+ " Submitted.");
-
+
//factory.waitWhileActivityIsDone(activityEpr, 1000);
jobId = WSUtilities.extractResourceID(activityEpr);
if (jobId == null) {
jobId = new Long(Calendar.getInstance().getTimeInMillis())
.toString();
}
-
+
String status = String.format("Job %s is %s.\n", activityEpr.getAddress()
.getStringValue(), factory.getActivityStatus(activityEpr)
.toString()).toString();
-
+
log.info(status);
-
-
+
+
while ((factory.getActivityStatus(activityEpr) != ActivityStateEnumeration.FINISHED) &&
(factory.getActivityStatus(activityEpr) != ActivityStateEnumeration.FAILED)){
status = String.format("Job %s is %s.\n", activityEpr.getAddress()
@@ -166,62 +166,62 @@ public class BESProvider implements GFac
@Override
public void dispose(JobExecutionContext jobExecutionContext)
throws GFacProviderException {
-
+
secProperties = null;
-
+
}
-
-
+
+
protected void initSecurityProperties(JobExecutionContext jobExecutionContext) throws GFacProviderException{
-
+
if (secProperties != null) return;
-
+
GSISecurityContext gssContext = new GSISecurityContext(jobExecutionContext.getGFacConfiguration());
GlobusCredential credentials = gssContext.getGlobusCredential();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
-
+
BufferedOutputStream bufos = new BufferedOutputStream(bos);
-
+
ByteArrayInputStream bis = null;
BufferedInputStream bufis = null;
try{
credentials.save(bufos);
bufos.flush();
-
+
//TODO: to be supported by airavata gsscredential class
List<String> trustedCert = new ArrayList<String>();
trustedCert.add(gssContext.getTrustedCertLoc()+"/*.0");
trustedCert.add(gssContext.getTrustedCertLoc()+"/*.pem");
-
+
char[] c = null;
-
+
DirectoryCertChainValidator dcValidator = new DirectoryCertChainValidator(trustedCert, Encoding.PEM, -1, 60000, null);
bis = new ByteArrayInputStream(bos.toByteArray());
bufis = new BufferedInputStream(bis);
PEMCredential pem = new PEMCredential(bufis, c);
-
+
secProperties = new DefaultClientConfiguration(dcValidator, pem);
secProperties.doSSLAuthn();
-
+
String[] outHandlers = secProperties.getOutHandlerClassNames();
-
- Set<String> outHandlerLst = null;
-
+
+ Set<String> outHandlerLst = null;
+
if(outHandlers == null) {
outHandlerLst = new HashSet<String>();
}
else {
outHandlerLst = new HashSet<String>(Arrays.asList(outHandlers));
}
-
+
outHandlerLst.add("de.fzj.unicore.uas.security.ProxyCertOutHandler");
-
+
secProperties.setOutHandlerClassNames(outHandlerLst.toArray(new String[outHandlerLst.size()]));
-
+
}
catch (Exception e) {
- throw new GFacProviderException(e.getMessage(), e);
- }
+ throw new GFacProviderException(e.getMessage(), e);
+ }
finally{
try {
if(bos!=null)bos.close();
@@ -233,27 +233,10 @@ public class BESProvider implements GFac
}
}
}
-
-
-// public static void main(String[] args) throws Exception {
-//
-// PropertyConfigurator.configure("src/test/resources/logging.properties");
-//
-//// String url = "https://daemon.india.futuregrid.org:8081/INDIA/services/BESFactory?res=default_bes_factory";
-//
-// String url = "https://zam1161v01.zam.kfa-juelich.de:8002/INTEROP1/services/BESFactory?res=default_bes_factory";
-//
-// ClientProperties cp = new ClientProperties(new File("src/test/resources/sec.properties"));
-//
-// EndpointReferenceType epr = WSUtilities.makeServiceEPR(url);
-//
-// FactoryClient bes = new FactoryClient(url, epr, cp);
-//
-// System.out.println(bes.getFactoryAttributesDocument());
-//
-// }
-
-
-
-
+
+
+
+
+
+
}
Added: airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/ApplicationProcessor.java
URL: http://svn.apache.org/viewvc/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/ApplicationProcessor.java?rev=1451054&view=auto
==============================================================================
--- airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/ApplicationProcessor.java (added)
+++ airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/ApplicationProcessor.java Thu Feb 28 00:26:09 2013
@@ -0,0 +1,232 @@
+package org.apache.airavata.gfac.provider.utils;
+
+import java.io.File;
+import java.math.BigInteger;
+
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.schemas.gfac.ExtendedKeyValueType;
+import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
+import org.apache.airavata.schemas.gfac.JobTypeType;
+import org.apache.airavata.schemas.gfac.NameValuePairType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.ApplicationType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.EnvironmentType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.FileNameType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.LimitsType;
+import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.NumberOfProcessesType;
+import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.ProcessesPerHostType;
+import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.ThreadsPerProcessType;
+
+
+public class ApplicationProcessor {
+
+ public static void generateJobSpecificAppElements(JobDefinitionType value, JobExecutionContext context){
+
+ HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
+ .getApplicationContext().getApplicationDeploymentDescription()
+ .getType();
+
+ createGenericApplication(value, appDepType);
+
+ if (appDepType.getMaxWallTime() > 0) {
+
+ if(!isParallelJob(appDepType)) {
+ LimitsType limitType = LimitsType.Factory.newInstance();
+ BigInteger bi = BigInteger.valueOf(appDepType.getMaxWallTime());
+ limitType.setBigIntegerValue(bi);
+ JSDLUtils.getOrCreatePOSIXApplication(value).setWallTimeLimit(limitType);
+ }
+
+ else {
+ LimitsType limitType = LimitsType.Factory.newInstance();
+ Integer i = appDepType.getMaxWallTime();
+ limitType.setStringValue(i.toString());
+ JSDLUtils.getOrCreateSPMDApplication(value).setWallTimeLimit(limitType);
+ }
+ }
+
+
+ if (appDepType.getApplicationEnvironmentArray().length > 0) {
+ createApplicationEnvironment(value,
+ appDepType.getApplicationEnvironmentArray(), appDepType);
+ }
+
+
+ if (appDepType.getExecutableLocation() != null) {
+ FileNameType fNameType = FileNameType.Factory.newInstance();
+ fNameType.setStringValue(appDepType.getExecutableLocation());
+ if(isParallelJob(appDepType)) {
+ JSDLUtils.getOrCreateSPMDApplication(value).setExecutable(fNameType);
+ JSDLUtils.getSPMDApplication(value).setSPMDVariation(getSPMDVariation(appDepType));
+
+ if(getValueFromMap(appDepType, JSDLUtils.NUMBEROFPROCESSES)!=null){
+ NumberOfProcessesType num = NumberOfProcessesType.Factory.newInstance();
+ num.setStringValue(getValueFromMap(appDepType, JSDLUtils.NUMBEROFPROCESSES));
+ JSDLUtils.getSPMDApplication(value).setNumberOfProcesses(num);
+ }
+
+ if(getValueFromMap(appDepType, JSDLUtils.PROCESSESPERHOST)!=null){
+ ProcessesPerHostType pph = ProcessesPerHostType.Factory.newInstance();
+ pph.setStringValue(getValueFromMap(appDepType, JSDLUtils.PROCESSESPERHOST));
+ JSDLUtils.getSPMDApplication(value).setProcessesPerHost(pph);
+ }
+
+ if(getValueFromMap(appDepType, JSDLUtils.THREADSPERHOST)!=null){
+ ThreadsPerProcessType tpp = ThreadsPerProcessType.Factory.newInstance();
+ tpp.setStringValue(getValueFromMap(appDepType, JSDLUtils.THREADSPERHOST));
+ JSDLUtils.getSPMDApplication(value).setThreadsPerProcess(tpp);
+ }
+ }
+ else
+ JSDLUtils.getOrCreatePOSIXApplication(value).setExecutable(fNameType);
+ }
+
+ if(appDepType.getStandardOutput() != null) {
+ String stdout = new File(appDepType.getStandardOutput()).getName();
+ ApplicationProcessor.setApplicationStdOut(value, appDepType, stdout);
+ }
+
+ if(appDepType.getStandardError() != null) {
+ String stderr = new File(appDepType.getStandardError()).getName();
+ ApplicationProcessor.setApplicationStdErr(value, appDepType, stderr);
+ }
+
+
+
+
+ }
+
+
+ public static boolean isParallelJob(HpcApplicationDeploymentType appDepType) {
+
+ boolean isParallel = false;
+
+ if (appDepType.getJobType() != null) {
+ // TODO set data output directory
+ int status = appDepType.getJobType().intValue();
+
+ switch (status) {
+ // TODO: this check should be done outside this class
+ case JobTypeType.INT_MPI:
+ case JobTypeType.INT_OPEN_MP:
+ isParallel = true;
+ break;
+
+ case JobTypeType.INT_SERIAL:
+ case JobTypeType.INT_SINGLE:
+ isParallel = false;
+ break;
+
+ default:
+ isParallel = false;
+ break;
+ }
+ }
+ return isParallel;
+ }
+
+
+ public static void createApplicationEnvironment(JobDefinitionType value, NameValuePairType[] nameValuePairs, HpcApplicationDeploymentType appDepType) {
+
+ if(isParallelJob(appDepType)) {
+ for (NameValuePairType nv : nameValuePairs) {
+ EnvironmentType envType = JSDLUtils.getOrCreateSPMDApplication(value).addNewEnvironment();
+ envType.setName(nv.getName());
+ envType.setStringValue(nv.getValue());
+ }
+ }
+ else {
+ for (NameValuePairType nv : nameValuePairs) {
+ EnvironmentType envType = JSDLUtils.getOrCreatePOSIXApplication(value).addNewEnvironment();
+ envType.setName(nv.getName());
+ envType.setStringValue(nv.getValue());
+ }
+ }
+
+ }
+
+
+ public static String getSPMDVariation (HpcApplicationDeploymentType appDepType) {
+
+ String variation = null;
+
+ if (appDepType.getJobType() != null) {
+ // TODO set data output directory
+ int status = appDepType.getJobType().intValue();
+
+ switch (status) {
+ // TODO: this check should be done outside this class
+ case JobTypeType.INT_MPI:
+ variation = SPMDVariations.MPI.value();
+ break;
+
+ case JobTypeType.INT_OPEN_MP:
+ variation = SPMDVariations.OpenMPI.value();
+ break;
+
+ }
+ }
+ return variation;
+ }
+
+
+ public static void addApplicationArgument(JobDefinitionType value, HpcApplicationDeploymentType appDepType, String stringPrm) {
+ if(isParallelJob(appDepType))
+ JSDLUtils.getOrCreateSPMDApplication(value)
+ .addNewArgument().setStringValue(stringPrm);
+ else
+ JSDLUtils.getOrCreatePOSIXApplication(value)
+ .addNewArgument().setStringValue(stringPrm);
+
+ }
+
+ public static void setApplicationStdErr(JobDefinitionType value, HpcApplicationDeploymentType appDepType, String stderr) {
+ FileNameType fName = FileNameType.Factory.newInstance();
+ fName.setStringValue(stderr);
+ if (isParallelJob(appDepType))
+ JSDLUtils.getOrCreateSPMDApplication(value).setError(fName);
+ else
+ JSDLUtils.getOrCreatePOSIXApplication(value).setError(fName);
+ }
+
+ public static void setApplicationStdOut(JobDefinitionType value, HpcApplicationDeploymentType appDepType, String stderr) {
+ FileNameType fName = FileNameType.Factory.newInstance();
+ fName.setStringValue(stderr);
+ if (isParallelJob(appDepType))
+ JSDLUtils.getOrCreateSPMDApplication(value).setOutput(fName);
+ else
+ JSDLUtils.getOrCreatePOSIXApplication(value).setOutput(fName);
+ }
+
+ public static String getApplicationStdOut(JobDefinitionType value, HpcApplicationDeploymentType appDepType) {
+ if (isParallelJob(appDepType)) return JSDLUtils.getOrCreateSPMDApplication(value).getOutput().getStringValue();
+ else return JSDLUtils.getOrCreatePOSIXApplication(value).getOutput().getStringValue();
+ }
+
+ public static String getApplicationStdErr(JobDefinitionType value, HpcApplicationDeploymentType appDepType) {
+ if (isParallelJob(appDepType)) return JSDLUtils.getOrCreateSPMDApplication(value).getError().getStringValue();
+ else return JSDLUtils.getOrCreatePOSIXApplication(value).getError().getStringValue();
+ }
+
+ public static void createGenericApplication(JobDefinitionType value, HpcApplicationDeploymentType appDepType) {
+ if (appDepType.getApplicationName() != null) {
+ ApplicationType appType = JSDLUtils.getOrCreateApplication(value);
+ String appName = appDepType.getApplicationName()
+ .getStringValue();
+ appType.setApplicationName(appName);
+ JSDLUtils.getOrCreateJobIdentification(value).setJobName(appName);
+ }
+ }
+
+
+ public static String getValueFromMap(HpcApplicationDeploymentType appDepType, String name) {
+ ExtendedKeyValueType[] extended = appDepType.getKeyValuePairsArray();
+ for(ExtendedKeyValueType e: extended) {
+ if(e.getName().equalsIgnoreCase(name)) {
+ return e.getStringValue();
+ }
+ }
+ return null;
+ }
+
+}
Added: airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/DataStagingProcessor.java
URL: http://svn.apache.org/viewvc/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/DataStagingProcessor.java?rev=1451054&view=auto
==============================================================================
--- airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/DataStagingProcessor.java (added)
+++ airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/DataStagingProcessor.java Thu Feb 28 00:26:09 2013
@@ -0,0 +1,211 @@
+package org.apache.airavata.gfac.provider.utils;
+
+import java.io.File;
+import java.net.URI;
+import java.util.Map;
+
+import org.apache.airavata.commons.gfac.type.ActualParameter;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.utils.GFacUtils;
+import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
+import org.apache.airavata.schemas.gfac.StringArrayType;
+import org.apache.airavata.schemas.gfac.StringParameterType;
+import org.apache.airavata.schemas.gfac.URIArrayType;
+import org.apache.airavata.schemas.gfac.URIParameterType;
+import org.apache.airavata.schemas.gfac.UnicoreHostType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
+
+public class DataStagingProcessor {
+
+ public static void generateDataStagingElements(JobDefinitionType value, JobExecutionContext context) throws Exception{
+
+ HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
+ .getApplicationContext().getApplicationDeploymentDescription()
+ .getType();
+
+
+ String gridftpEndpoint = ((UnicoreHostType) context.getApplicationContext().getHostDescription().getType())
+ .getGridFTPEndPointArray()[0];
+
+
+ if (context.getInMessageContext().getParameters().size() > 0) {
+ buildDataStagingFromInputContext(context, value, gridftpEndpoint, appDepType);
+ }
+
+ if (context.getOutMessageContext().getParameters().size() > 0) {
+ buildFromOutputContext(context, value, gridftpEndpoint, appDepType);
+ }
+
+ createStdOutURIs(value, appDepType, gridftpEndpoint, isUnicoreEndpoint(context));
+
+ }
+
+ private static void createInURIElement(JobDefinitionType value,
+ String endpoint, String inputDir, ActualParameter inParam)
+ throws Exception {
+
+ String uri = ((URIParameterType) inParam.getType()).getValue();
+ String fileName = new File(uri).getName();
+ if (uri.startsWith("file")) {
+ URI gridFTPInputDirURI = GFacUtils.createGsiftpURI(endpoint,
+ inputDir);
+ String filePath = gridFTPInputDirURI.toString() + File.separator
+ + fileName;
+ JSDLUtils
+ .addDataStagingSourceElement(value, filePath, null, fileName);
+ } else if (uri.startsWith("gsiftp") || uri.startsWith("http")
+ || uri.startsWith("rns")) {
+ // no need to stage-in those files to the input
+ // directory
+ JSDLUtils.addDataStagingSourceElement(value, uri, null, fileName);
+ }
+
+ }
+
+ private static void createStdOutURIs(JobDefinitionType value,
+ HpcApplicationDeploymentType appDepType, String endpoint,
+ boolean isUnicore) throws Exception {
+
+ URI remoteOutputDir = GFacUtils.createGsiftpURI(endpoint,
+ appDepType.getOutputDataDirectory());
+
+ String stdout = ApplicationProcessor.getApplicationStdOut(value, appDepType);
+
+ String stderr = ApplicationProcessor.getApplicationStdErr(value, appDepType);
+
+ String stdoutFileName = (stdout == null || stdout.equals("")) ? "stdout"
+ : stdout;
+ String stdoutURI = GFacUtils.createGsiftpURIAsString(
+ remoteOutputDir.toString(), stdoutFileName);
+ JSDLUtils.addDataStagingTargetElement(value, null, stdoutFileName,
+ stdoutURI);
+
+ String stderrFileName = (stdout == null || stderr.equals("")) ? "stderr"
+ : stderr;
+ String stderrURI = GFacUtils.createGsiftpURIAsString(
+ remoteOutputDir.toString(), stderrFileName);
+ JSDLUtils.addDataStagingTargetElement(value, null, stderrFileName,
+ stderrURI);
+
+ if(isUnicore) {
+ String scriptExitCodeFName = "UNICORE_SCRIPT_EXIT_CODE";
+ String scriptExitCode = GFacUtils.createGsiftpURIAsString(
+ remoteOutputDir.toString(), scriptExitCodeFName);
+ JSDLUtils.addDataStagingTargetElement(value, null,
+ scriptExitCodeFName, scriptExitCode.toString());
+ }
+
+ }
+
+
+ private static void createOutStringElements(JobDefinitionType value,
+ HpcApplicationDeploymentType appDeptype, String endpoint, String prmValue) throws Exception {
+
+ String outputUri = GFacUtils.createGsiftpURIAsString(endpoint, appDeptype.getOutputDataDirectory());
+
+ URI finalOutputUri = GFacUtils.createGsiftpURI(outputUri, prmValue);
+ JSDLUtils.addDataStagingTargetElement(value, null, prmValue, finalOutputUri.toString());
+ }
+
+
+ private static void createOutURIElement(JobDefinitionType value,
+ String prmValue) throws Exception {
+ String fileName = new File(prmValue.toString()).getName();
+ JSDLUtils.addDataStagingTargetElement(value, null, fileName, prmValue);
+ }
+
+
+ private static JobDefinitionType buildFromOutputContext(JobExecutionContext context,
+ JobDefinitionType value, String gridftpEndpoint,
+ HpcApplicationDeploymentType appDepType) throws Exception {
+
+ Map<String, Object> outputParams = context.getOutMessageContext()
+ .getParameters();
+
+ for (String paramKey : outputParams.keySet()) {
+
+ ActualParameter outParam = (ActualParameter) outputParams
+ .get(paramKey);
+
+ // if single urls then convert each url into jsdl source
+ // elements,
+ // that are formed by concat of gridftpurl+inputdir+filename
+
+ String paramDataType = outParam.getType().getType().toString();
+
+ if ("URI".equals(paramDataType)) {
+ String uriPrm = ((URIParameterType) outParam.getType())
+ .getValue();
+ createOutURIElement(value, uriPrm);
+ }
+
+ // string params are converted into the job arguments
+
+ else if (("URIArray").equals(paramDataType)) {
+ String[] uriArray = ((URIArrayType) outParam.getType())
+ .getValueArray();
+ for (String u : uriArray) {
+ createOutURIElement(value, u);
+ }
+
+ }
+ else if ("String".equals(paramDataType)) {
+ String stringPrm = ((StringParameterType) outParam
+ .getType()).getValue();
+ createOutStringElements(value, appDepType, gridftpEndpoint, stringPrm);
+ }
+
+ else if ("StringArray".equals(paramDataType)) {
+ String[] valueArray = ((StringArrayType) outParam.getType())
+ .getValueArray();
+ for (String v : valueArray) {
+ createOutStringElements(value, appDepType, gridftpEndpoint, v);
+ }
+ }
+ }
+
+ return value;
+ }
+
+
+ private static void buildDataStagingFromInputContext(JobExecutionContext context, JobDefinitionType value, String gridftpEndpoint, HpcApplicationDeploymentType appDepType)
+ throws Exception {
+
+ // TODO set data directory
+ Map<String, Object> inputParams = context.getInMessageContext()
+ .getParameters();
+
+ for (String paramKey : inputParams.keySet()) {
+
+ ActualParameter inParam = (ActualParameter) inputParams
+ .get(paramKey);
+
+ // if single urls then convert each url into jsdl source
+ // elements,
+ // that are formed by concat of gridftpurl+inputdir+filename
+
+ String paramDataType = inParam.getType().getType().toString();
+
+ if ("URI".equals(paramDataType)) {
+ createInURIElement(value, gridftpEndpoint,
+ appDepType.getInputDataDirectory(), inParam);
+ }
+
+ // string params are converted into the job arguments
+
+ else if ("String".equals(paramDataType)) {
+ String stringPrm = ((StringParameterType) inParam.getType())
+ .getValue();
+ ApplicationProcessor.addApplicationArgument(value, appDepType, stringPrm);
+ }
+ }
+
+ }
+
+
+ public static boolean isUnicoreEndpoint(JobExecutionContext context) {
+ return ( (context.getApplicationContext().getHostDescription().getType() instanceof UnicoreHostType)?true:false );
+ }
+
+
+}
Modified: airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/JSDLGenerator.java
URL: http://svn.apache.org/viewvc/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/JSDLGenerator.java?rev=1451054&r1=1451053&r2=1451054&view=diff
==============================================================================
--- airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/JSDLGenerator.java (original)
+++ airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/JSDLGenerator.java Thu Feb 28 00:26:09 2013
@@ -20,29 +20,11 @@
*/
package org.apache.airavata.gfac.provider.utils;
-import java.io.File;
-import java.net.URI;
-import java.util.Map;
-import org.apache.airavata.commons.gfac.type.ActualParameter;
import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.utils.GFacUtils;
import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
-import org.apache.airavata.schemas.gfac.JobTypeType;
-import org.apache.airavata.schemas.gfac.NameValuePairType;
-import org.apache.airavata.schemas.gfac.StdErrParameterType;
-import org.apache.airavata.schemas.gfac.StdOutParameterType;
-import org.apache.airavata.schemas.gfac.StringArrayType;
-import org.apache.airavata.schemas.gfac.StringParameterType;
-import org.apache.airavata.schemas.gfac.URIArrayType;
-import org.apache.airavata.schemas.gfac.URIParameterType;
-import org.apache.airavata.schemas.gfac.UnicoreHostType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.ApplicationType;
import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionDocument;
import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.EnvironmentType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.FileNameType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.POSIXApplicationType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,47 +38,32 @@ import org.slf4j.LoggerFactory;
public class JSDLGenerator {
protected final Logger log = LoggerFactory.getLogger(this.getClass());
-
+
+
public synchronized static JobDefinitionDocument buildJSDLInstance(JobExecutionContext context) throws Exception {
JobDefinitionDocument jobDefDoc = JobDefinitionDocument.Factory
.newInstance();
JobDefinitionType value = jobDefDoc.addNewJobDefinition();
-
-
+
HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
.getApplicationContext().getApplicationDeploymentDescription()
.getType();
+
+ // build Identification
+ createJobIdentification(value, appDepType);
+ ResourceProcessor.generateResourceElements(value, context);
- // TODO: here need to create spmd element
- if (appDepType.getJobType() != null) {
- // TODO set data output directory
- int status = appDepType.getJobType().intValue();
-
- switch (status) {
- // TODO: this check should be done outside this class
- case JobTypeType.INT_MPI:
- // use spmd:openmpi
- break;
-
- case JobTypeType.INT_OPEN_MP:
- // use spmd:openmp
- break;
-
- case JobTypeType.INT_SERIAL:
- break;
-
- case JobTypeType.INT_SINGLE:
- break;
-
- default:
- break;
- }
- }
-
+ ApplicationProcessor.generateJobSpecificAppElements(value, context);
+ DataStagingProcessor.generateDataStagingElements(value, context);
+
+ return jobDefDoc;
+ }
+
+ private static void createJobIdentification(JobDefinitionType value, HpcApplicationDeploymentType appDepType){
if( appDepType.getProjectAccount() != null ){
if (appDepType.getProjectAccount().getProjectAccountNumber() != null)
@@ -108,311 +75,6 @@ public class JSDLGenerator {
appDepType.getProjectAccount()
.getProjectAccountDescription());
}
-
-
- ApplicationType appType = JSDLUtils.getOrCreateApplication(value);
-
- String gridftpEndpoint = ((UnicoreHostType) context.getApplicationContext().getHostDescription().getType())
- .getGridFTPEndPointArray()[0];
-
-
- String stdout = null, stderr = null;
-
- POSIXApplicationType posixType = JSDLUtils .getOrCreatePOSIXApplication(value);
-
-
-
- // TODO: here need to create spmd element
- if (appDepType.getJobType() != null) {
- // TODO set data output directory
- int status = appDepType.getJobType().intValue();
-
- switch (status) {
- // TODO: this check should be done outside this class
- case JobTypeType.INT_MPI:
- // use spmd:openmpi
- break;
-
- case JobTypeType.INT_OPEN_MP:
- // use spmd:openmp
- break;
-
- case JobTypeType.INT_SERIAL:
- break;
-
- case JobTypeType.INT_SINGLE:
- break;
-
- default:
- break;
- }
- }
-
- if(appDepType.getStandardOutput() != null) {
- stdout = new File(appDepType.getStandardOutput()).getName();
- createStdOut(value, stdout);
- }
-
- if(appDepType.getStandardError() != null) {
- stderr = new File(appDepType.getStandardError()).getName();
- createStdErr(value, stderr);
- }
-
- if (appDepType.getApplicationName() != null) {
- String appName = appDepType.getApplicationName()
- .getStringValue();
- appType.setApplicationName(appName);
- JSDLUtils.getOrCreateJobIdentification(value).setJobName(appName);
- }
-
- if (appDepType.getExecutableLocation() != null) {
- FileNameType fNameType = FileNameType.Factory.newInstance();
- fNameType.setStringValue(appDepType.getExecutableLocation());
- posixType.setExecutable(fNameType);
-
- }
-
- if (appDepType.getMaxWallTime() > 0) {
- RangeValueType rangeType = new RangeValueType();
- rangeType.setLowerBound(appDepType.getMaxWallTime());
- rangeType.setIncludeLowerBound(true);
- JSDLUtils.setIndividualCPUTimeRequirements(value, rangeType);
- }
-
- if (appDepType.getCpuCount() > 0) {
- RangeValueType rangeType = new RangeValueType();
- rangeType.setLowerBound(Double.NaN);
- rangeType.setUpperBound(Double.NaN);
- rangeType.setExact(appDepType.getCpuCount());
- JSDLUtils.setTotalCPUCountRequirements(value, rangeType);
- }
-
- if (appDepType.getMinMemory() > 0 && appDepType.getMaxMemory() > 0) {
- RangeValueType rangeType = new RangeValueType();
- rangeType.setLowerBound(appDepType.getMinMemory());
- rangeType.setUpperBound(appDepType.getMaxMemory());
- JSDLUtils.setIndividualPhysicalMemoryRequirements(value, rangeType);
- }
-
- if (appDepType.getMinMemory() > 0) {
- // TODO set Wall time
- }
-
- if (appDepType.getProcessorsPerNode() > 0) {
- RangeValueType rangeType = new RangeValueType();
- rangeType.setLowerBound(Double.NaN);
- rangeType.setUpperBound(Double.NaN);
- rangeType.setExact(appDepType.getProcessorsPerNode());
- JSDLUtils.setIndividualCPUCountRequirements(value, rangeType);
- }
-
- if (appDepType.getApplicationEnvironmentArray().length > 0) {
- createApplicationEnvironment(value,
- appDepType.getApplicationEnvironmentArray());
- }
-
- if (context.getInMessageContext().getParameters().size() > 0) {
- buildFromInputContext(context, value, gridftpEndpoint, appDepType);
- }
-
- if (context.getOutMessageContext().getParameters().size() > 0) {
- buildFromOutputContext(context, value, gridftpEndpoint, appDepType);
- }
-
- createStdOutURIs(value, appDepType, gridftpEndpoint, stdout, stderr, isUnicoreEndpoint(context));
-
- return jobDefDoc;
- }
-
- private static void createStdOut(JobDefinitionType value, String stdout) {
- FileNameType fName = FileNameType.Factory.newInstance();
- fName.setStringValue(stdout);
- JSDLUtils.getOrCreatePOSIXApplication(value).setOutput(fName);
- }
-
- private static void createStdErr(JobDefinitionType value, String stderr) {
- FileNameType fName = FileNameType.Factory.newInstance();
- fName.setStringValue(stderr);
- JSDLUtils.getOrCreatePOSIXApplication(value).setError(fName);
-
- }
-
- private static void createApplicationEnvironment(JobDefinitionType value,
- NameValuePairType[] nameValuePairs) {
- for (NameValuePairType nv : nameValuePairs) {
- EnvironmentType envType = JSDLUtils.getPOSIXApplication(value)
- .addNewEnvironment();
- envType.setName(nv.getName());
- envType.setStringValue(nv.getValue());
- }
-
- }
-
- private static void createInURIElement(JobDefinitionType value,
- String endpoint, String inputDir, ActualParameter inParam)
- throws Exception {
-
- String uri = ((URIParameterType) inParam.getType()).getValue();
- String fileName = new File(uri).getName();
- if (uri.startsWith("file")) {
- URI gridFTPInputDirURI = GFacUtils.createGsiftpURI(endpoint,
- inputDir);
- String filePath = gridFTPInputDirURI.toString() + File.separator
- + fileName;
- JSDLUtils
- .addDataStagingSourceElement(value, filePath, null, fileName);
- } else if (uri.startsWith("gsiftp") || uri.startsWith("http")
- || uri.startsWith("rns")) {
- // no need to stage-in those files to the input
- // directory
- JSDLUtils.addDataStagingSourceElement(value, uri, null, fileName);
- }
-
- }
-
- private static void createStdOutURIs(JobDefinitionType value,
- HpcApplicationDeploymentType appDeptype, String endpoint,
- String stdout, String stderr, boolean isUnicore) throws Exception {
-
- URI remoteOutputDir = GFacUtils.createGsiftpURI(endpoint,
- appDeptype.getOutputDataDirectory());
-
- String stdoutFileName = (stdout == null || stdout.equals("")) ? "stdout"
- : stdout;
- String stdoutURI = GFacUtils.createGsiftpURIAsString(
- remoteOutputDir.toString(), stdoutFileName);
- JSDLUtils.addDataStagingTargetElement(value, null, stdoutFileName,
- stdoutURI);
-
- String stderrFileName = (stdout == null || stderr.equals("")) ? "stderr"
- : stderr;
- String stderrURI = GFacUtils.createGsiftpURIAsString(
- remoteOutputDir.toString(), stderrFileName);
- JSDLUtils.addDataStagingTargetElement(value, null, stderrFileName,
- stderrURI);
-
- if(isUnicore) {
- String scriptExitCodeFName = "UNICORE_SCRIPT_EXIT_CODE";
- String scriptExitCode = GFacUtils.createGsiftpURIAsString(
- remoteOutputDir.toString(), scriptExitCodeFName);
- JSDLUtils.addDataStagingTargetElement(value, null,
- scriptExitCodeFName, scriptExitCode.toString());
- }
-
- }
-
-
- private static void createOutStringElements(JobDefinitionType value,
- HpcApplicationDeploymentType appDeptype, String endpoint, String prmValue) throws Exception {
-
- String outputUri = GFacUtils.createGsiftpURIAsString(endpoint, appDeptype.getOutputDataDirectory());
-
- URI finalOutputUri = GFacUtils.createGsiftpURI(outputUri, prmValue);
- JSDLUtils.addDataStagingTargetElement(value, null, prmValue, finalOutputUri.toString());
- }
-
-
- private static void createOutURIElement(JobDefinitionType value,
- String prmValue) throws Exception {
- String fileName = new File(prmValue.toString()).getName();
- JSDLUtils.addDataStagingTargetElement(value, null, fileName, prmValue);
- }
-
-
- private static JobDefinitionType buildFromOutputContext(JobExecutionContext context,
- JobDefinitionType value, String gridftpEndpoint,
- HpcApplicationDeploymentType appDepType) throws Exception {
-
- Map<String, Object> outputParams = context.getOutMessageContext()
- .getParameters();
-
- for (String paramKey : outputParams.keySet()) {
-
- ActualParameter outParam = (ActualParameter) outputParams
- .get(paramKey);
-
- // if single urls then convert each url into jsdl source
- // elements,
- // that are formed by concat of gridftpurl+inputdir+filename
-
- String paramDataType = outParam.getType().getType().toString();
-
- if ("URI".equals(paramDataType)) {
- String uriPrm = ((URIParameterType) outParam.getType())
- .getValue();
- createOutURIElement(value, uriPrm);
- }
-
- // string params are converted into the job arguments
-
- else if (("URIArray").equals(paramDataType)) {
- String[] uriArray = ((URIArrayType) outParam.getType())
- .getValueArray();
- for (String u : uriArray) {
- createOutURIElement(value, u);
- }
-
- }
- else if ("String".equals(paramDataType)) {
- String stringPrm = ((StringParameterType) outParam
- .getType()).getValue();
- createOutStringElements(value, appDepType, gridftpEndpoint, stringPrm);
- }
-
- else if ("StringArray".equals(paramDataType)) {
- String[] valueArray = ((StringArrayType) outParam.getType())
- .getValueArray();
- for (String v : valueArray) {
- createOutStringElements(value, appDepType, gridftpEndpoint, v);
- }
- }
- }
-
- return value;
- }
-
-
- private static void buildFromInputContext(JobExecutionContext context, JobDefinitionType value, String gridftpEndpoint, HpcApplicationDeploymentType appDepType)
- throws Exception {
-
- // TODO set data directory
- Map<String, Object> inputParams = context.getInMessageContext()
- .getParameters();
-
- for (String paramKey : inputParams.keySet()) {
-
- ActualParameter inParam = (ActualParameter) inputParams
- .get(paramKey);
-
- // if single urls then convert each url into jsdl source
- // elements,
- // that are formed by concat of gridftpurl+inputdir+filename
-
- String paramDataType = inParam.getType().getType().toString();
-
- if ("URI".equals(paramDataType)) {
- createInURIElement(value, gridftpEndpoint,
- appDepType.getInputDataDirectory(), inParam);
- }
-
- // string params are converted into the job arguments
-
- else if ("String".equals(paramDataType)) {
- String stringPrm = ((StringParameterType) inParam.getType())
- .getValue();
-
- JSDLUtils.getOrCreatePOSIXApplication(value)
- .addNewArgument().setStringValue(stringPrm);
-
- }
- }
-
- }
-
-
-
- public static boolean isUnicoreEndpoint(JobExecutionContext context) {
- return ( (context.getApplicationContext().getHostDescription().getType() instanceof UnicoreHostType)?true:false );
}
}
Modified: airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/JSDLUtils.java
URL: http://svn.apache.org/viewvc/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/JSDLUtils.java?rev=1451054&r1=1451053&r2=1451054&view=diff
==============================================================================
--- airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/JSDLUtils.java (original)
+++ airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/JSDLUtils.java Thu Feb 28 00:26:09 2013
@@ -27,6 +27,8 @@ import org.ggf.schemas.jsdl.x2005.x11.js
import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.POSIXApplicationType;
import org.ggf.schemas.jsdl.x2006.x07.jsdlHpcpa.HPCProfileApplicationDocument;
import org.ggf.schemas.jsdl.x2006.x07.jsdlHpcpa.HPCProfileApplicationType;
+import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.SPMDApplicationDocument;
+import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.SPMDApplicationType;
/**
@@ -45,6 +47,14 @@ public class JSDLUtils
public static final QName HPC_PROFILE_APPLICATION=HPCProfileApplicationDocument.type.getDocumentElementName();
+ public static final QName SPMD_APPLICATION=SPMDApplicationDocument.type.getDocumentElementName();
+
+ public static final String PROCESSESPERHOST = "ProcessesPerHost";
+ public static final String NUMBEROFPROCESSES = "NumberOfProcesses";
+ public static final String THREADSPERHOST = "ThreadsPerHost";
+
+
+
public static EnvironmentType addEnvVariable(JobDefinitionType def,String name, String value) {
POSIXApplicationType posixApp = getOrCreatePOSIXApplication(def);
EnvironmentType newEnv = posixApp.addNewEnvironment();
@@ -269,6 +279,46 @@ public class JSDLUtils
return getPOSIXApplication(value);
}
+
+ public static SPMDApplicationType getOrCreateSPMDApplication(JobDefinitionType value) {
+
+ ApplicationType application = getOrCreateApplication(value);
+
+ if (getSPMDApplication(value) == null) {
+ XmlCursor acursor = application.newCursor();
+ acursor.toEndToken();
+ acursor.insertElement(SPMD_APPLICATION);
+ acursor.dispose();
+ }
+ return getSPMDApplication(value);
+ }
+
+ public static SPMDApplicationType getSPMDApplication(JobDefinitionType value) {
+ if (value != null &&
+ value.getJobDescription() != null &&
+ value.getJobDescription().isSetApplication() ) {
+ XmlCursor acursor = value.getJobDescription().getApplication().newCursor();
+ if (acursor.toFirstChild()) {
+ do {
+ if(acursor.getName().equals(SPMD_APPLICATION)) {
+ XmlObject result = acursor.getObject();
+ acursor.dispose();
+ return (SPMDApplicationType) result;
+ }
+ } while (acursor.toNextSibling());
+ acursor.dispose();
+ return null;
+ } else {
+ acursor.dispose();
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+
+
+
public static POSIXApplicationType getPOSIXApplication(JobDefinitionType value) {
if (value != null &&
value.getJobDescription() != null &&
@@ -293,6 +343,8 @@ public class JSDLUtils
}
}
+
+
public static HPCProfileApplicationType getOrCreateHPCProfileApplication(JobDefinitionType value) {
ApplicationType application = getOrCreateApplication(value);
Added: airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/ResourceProcessor.java
URL: http://svn.apache.org/viewvc/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/ResourceProcessor.java?rev=1451054&view=auto
==============================================================================
--- airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/ResourceProcessor.java (added)
+++ airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/ResourceProcessor.java Thu Feb 28 00:26:09 2013
@@ -0,0 +1,73 @@
+package org.apache.airavata.gfac.provider.utils;
+
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
+
+public class ResourceProcessor {
+
+
+ public static void generateResourceElements(JobDefinitionType value, JobExecutionContext context) throws Exception{
+
+ HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
+ .getApplicationContext().getApplicationDeploymentDescription()
+ .getType();
+
+ createMemory(value, appDepType);
+
+ if (appDepType.getCpuCount() > 0) {
+ RangeValueType rangeType = new RangeValueType();
+ rangeType.setLowerBound(Double.NaN);
+ rangeType.setUpperBound(Double.NaN);
+ rangeType.setExact(appDepType.getCpuCount());
+ JSDLUtils.setTotalCPUCountRequirements(value, rangeType);
+ }
+
+ if (appDepType.getProcessorsPerNode() > 0) {
+ RangeValueType rangeType = new RangeValueType();
+ rangeType.setLowerBound(Double.NaN);
+ rangeType.setUpperBound(Double.NaN);
+ rangeType.setExact(appDepType.getProcessorsPerNode());
+ JSDLUtils.setIndividualCPUCountRequirements(value, rangeType);
+ }
+
+ if (appDepType.getNodeCount() > 0) {
+ RangeValueType rangeType = new RangeValueType();
+ rangeType.setLowerBound(Double.NaN);
+ rangeType.setUpperBound(Double.NaN);
+ rangeType.setExact(appDepType.getNodeCount());
+ JSDLUtils.setTotalResourceCountRequirements(value, rangeType);
+ }
+
+ }
+
+
+ private static void createMemory(JobDefinitionType value, HpcApplicationDeploymentType appDepType){
+ if (appDepType.getMinMemory() > 0 && appDepType.getMaxMemory() > 0) {
+ RangeValueType rangeType = new RangeValueType();
+ rangeType.setLowerBound(appDepType.getMinMemory());
+ rangeType.setUpperBound(appDepType.getMaxMemory());
+ JSDLUtils.setIndividualPhysicalMemoryRequirements(value, rangeType);
+ }
+
+ else if (appDepType.getMinMemory() > 0 && appDepType.getMaxMemory() <= 0) {
+ // TODO set Wall time
+ RangeValueType rangeType = new RangeValueType();
+ rangeType.setLowerBound(appDepType.getMinMemory());
+ JSDLUtils.setIndividualPhysicalMemoryRequirements(value, rangeType);
+ }
+
+ else if (appDepType.getMinMemory() <= 0 && appDepType.getMaxMemory() > 0) {
+ // TODO set Wall time
+ RangeValueType rangeType = new RangeValueType();
+ rangeType.setUpperBound(appDepType.getMinMemory());
+ JSDLUtils.setIndividualPhysicalMemoryRequirements(value, rangeType);
+ }
+
+ }
+
+
+
+
+
+}
Added: airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/SPMDProcessor.java
URL: http://svn.apache.org/viewvc/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/SPMDProcessor.java?rev=1451054&view=auto
==============================================================================
--- airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/SPMDProcessor.java (added)
+++ airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/SPMDProcessor.java Thu Feb 28 00:26:09 2013
@@ -0,0 +1,12 @@
+package org.apache.airavata.gfac.provider.utils;
+
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
+
+public class SPMDProcessor {
+
+ public static void generateSPMDElements(JobDefinitionType value, JobExecutionContext context) {
+
+ }
+
+}
Added: airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/SPMDVariations.java
URL: http://svn.apache.org/viewvc/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/SPMDVariations.java?rev=1451054&view=auto
==============================================================================
--- airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/SPMDVariations.java (added)
+++ airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/gfac/provider/utils/SPMDVariations.java Thu Feb 28 00:26:09 2013
@@ -0,0 +1,31 @@
+package org.apache.airavata.gfac.provider.utils;
+
+public enum SPMDVariations {
+
+ MPI ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MPI"),
+ GridMPI ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/GridMPI"),
+ IntelMPI ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/IntelMPI"),
+ LAMMPI ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/LAM-MPI"),
+ MPICH1 ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MPICH1"),
+ MPICH2 ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MPICH2"),
+ MPICHGM ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MPICH-GM"),
+ MPICHMX ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MPICH-MX"),
+ MVAPICH ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MVAPICH"),
+ MVAPICH2 ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MVAPICH2"),
+ OpenMPI ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/OpenMPI"),
+ POE ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/POE"),
+ PVM ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/PVM");
+
+ private final String variation;
+
+ private SPMDVariations(String variation) {
+ this.variation = variation;
+ }
+
+ public String value(){
+ return variation;
+ }
+
+}
+
+
Added: airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/AbstractBESTest.java
URL: http://svn.apache.org/viewvc/airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/AbstractBESTest.java?rev=1451054&view=auto
==============================================================================
--- airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/AbstractBESTest.java (added)
+++ airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/AbstractBESTest.java Thu Feb 28 00:26:09 2013
@@ -0,0 +1,104 @@
+package org.apache.airavata.core.gfac.services.impl;
+
+import java.io.File;
+import java.net.URL;
+import java.util.Date;
+import java.util.UUID;
+
+import org.apache.airavata.commons.gfac.type.ApplicationDescription;
+import org.apache.airavata.commons.gfac.type.HostDescription;
+import org.apache.airavata.commons.gfac.type.ServiceDescription;
+import org.apache.airavata.gfac.GFacConfiguration;
+import org.apache.airavata.gfac.context.ApplicationContext;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.context.MessageContext;
+import org.apache.airavata.schemas.gfac.JobTypeType;
+import org.apache.airavata.schemas.gfac.UnicoreHostType;
+import org.apache.log4j.PropertyConfigurator;
+
+public abstract class AbstractBESTest {
+
+ protected JobExecutionContext jobExecutionContext;
+
+ public static final String[] hostArray = new String[] { "https://zam1161v01.zam.kfa-juelich.de:8002/INTEROP1/services/BESFactory?res=default_bes_factory" };
+
+
+ //directory where data will be copy into and copied out to unicore resources
+
+ // private static final String scratchDir = "/brashear/msmemon/airavata";
+
+ public static final String gridftpAddress = "gsiftp://gridftp1.ls4.tacc.utexas.edu:2811";
+ public static final String scratchDir = "/scratch/02055/msmemon/airavata";
+
+ protected String remoteTempDir = null;
+
+
+
+ protected void initTest() throws Exception {
+ PropertyConfigurator.configure("src/test/resources/logging.properties");
+
+ /*
+ * Default tmp location
+ */
+ String date = (new Date()).toString();
+ date = date.replaceAll(" ", "_");
+ date = date.replaceAll(":", "_");
+
+
+ remoteTempDir = scratchDir + File.separator + "BESJOB" + "_" + date + "_"
+ + UUID.randomUUID();
+ jobExecutionContext = new JobExecutionContext(getGFACConfig(), getServiceDesc("BES-APP-Service").getType().getName());
+ jobExecutionContext.setApplicationContext(getApplicationContext());
+ jobExecutionContext.setInMessageContext(getInMessageContext());
+ jobExecutionContext.setOutMessageContext(getOutMessageContext());
+
+ }
+
+ protected abstract void submitJob() throws Exception;
+
+ protected GFacConfiguration getGFACConfig() throws Exception{
+ URL resource = this.getClass().getClassLoader().getResource("gfac-config.xml");
+ System.out.println(resource.getFile());
+ GFacConfiguration gFacConfiguration = GFacConfiguration.create(new File(resource.getPath()),null,null);
+ gFacConfiguration.setMyProxyLifeCycle(3600);
+ gFacConfiguration.setMyProxyServer("myproxy.teragrid.org");
+ gFacConfiguration.setMyProxyUser("msmemon");
+ gFacConfiguration.setMyProxyPassphrase("*******");
+ gFacConfiguration.setTrustedCertLocation("/home/m.memon/.globus/certificates");
+ return gFacConfiguration;
+ }
+
+ protected ApplicationContext getApplicationContext() {
+ ApplicationContext applicationContext = new ApplicationContext();
+ applicationContext.setHostDescription(getHostDesc());
+ return applicationContext;
+ }
+
+ protected HostDescription getHostDesc() {
+ HostDescription host = new HostDescription(UnicoreHostType.type);
+ host.getType().setHostAddress("zam1161v01.zam.kfa-juelich.de");
+ host.getType().setHostName("DEMO-INTEROP-SITE");
+ ((UnicoreHostType) host.getType()).setUnicoreHostAddressArray(hostArray);
+ ((UnicoreHostType) host.getType()).setGridFTPEndPointArray(new String[]{gridftpAddress});
+ return host;
+ }
+
+
+ protected abstract ApplicationDescription getApplicationDesc(JobTypeType jobType);
+
+
+ protected ServiceDescription getServiceDesc(String serviceName) {
+ ServiceDescription serv = new ServiceDescription();
+ serv.getType().setName(serviceName);
+ return serv;
+ }
+
+
+ protected abstract MessageContext getInMessageContext();
+
+
+
+ protected abstract MessageContext getOutMessageContext();
+
+
+}
Added: airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/BESParallelJobTest.java
URL: http://svn.apache.org/viewvc/airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/BESParallelJobTest.java?rev=1451054&view=auto
==============================================================================
--- airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/BESParallelJobTest.java (added)
+++ airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/BESParallelJobTest.java Thu Feb 28 00:26:09 2013
@@ -0,0 +1,121 @@
+package org.apache.airavata.core.gfac.services.impl;
+
+import java.io.File;
+import java.util.Date;
+import java.util.UUID;
+
+import org.apache.airavata.commons.gfac.type.ApplicationDescription;
+import org.apache.airavata.gfac.GFacAPI;
+import org.apache.airavata.gfac.context.ApplicationContext;
+import org.apache.airavata.gfac.context.MessageContext;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.ExtendedKeyValueType;
+import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
+import org.apache.airavata.schemas.gfac.JobTypeType;
+import org.apache.airavata.schemas.gfac.ProjectAccountType;
+import org.apache.airavata.schemas.gfac.QueueType;
+import org.junit.Before;
+import org.junit.Test;
+
+public class BESParallelJobTest extends AbstractBESTest{
+
+
+ @Before
+ public void initJobContext() throws Exception {
+ initTest();
+ }
+
+
+ @Test
+ public void submitJob() throws Exception {
+ JobTypeType jobType = JobTypeType.Factory.newInstance();
+ jobType.set(JobTypeType.MPI);
+ ApplicationContext appContext = getApplicationContext();
+ appContext.setApplicationDeploymentDescription(getApplicationDesc(jobType));
+ jobExecutionContext.setApplicationContext(appContext);
+ GFacAPI gFacAPI = new GFacAPI();
+ gFacAPI.submitJob(jobExecutionContext);
+ }
+
+
+
+
+ protected ApplicationDescription getApplicationDesc(JobTypeType jobType) {
+ ApplicationDescription appDesc = new ApplicationDescription(
+ HpcApplicationDeploymentType.type);
+ HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) appDesc
+ .getType();
+ ApplicationDeploymentDescriptionType.ApplicationName name = ApplicationDeploymentDescriptionType.ApplicationName.Factory
+ .newInstance();
+ name.setStringValue("MPIRemote");
+ appDepType.setApplicationName(name);
+ ProjectAccountType projectAccountType = appDepType.addNewProjectAccount();
+ projectAccountType.setProjectAccountNumber("TG-AST110064");
+
+ QueueType queueType = appDepType.addNewQueue();
+ queueType.setQueueName("development");
+
+ // TODO: also handle parallel jobs
+ if((jobType.enumValue() == JobTypeType.SERIAL) || (jobType.enumValue() == JobTypeType.SINGLE)) {
+ appDepType.setJobType(JobTypeType.SERIAL);
+ }
+ else if (jobType.enumValue() == JobTypeType.MPI) {
+ appDepType.setJobType(JobTypeType.MPI);
+ }
+ else {
+ appDepType.setJobType(JobTypeType.OPEN_MP);
+ }
+
+ appDepType.setNodeCount(1);
+ appDepType.setProcessorsPerNode(1);
+
+ /*
+ * Use bat file if it is compiled on Windows
+ */
+ appDepType.setExecutableLocation("/home/bes/mpiexamples/a.out");
+
+ ExtendedKeyValueType extKV = appDepType.addNewKeyValuePairs();
+ // using jsdl spmd standard
+ extKV.setName("NumberOfProcesses");
+ // this will be transformed into mpiexec -n 4
+ extKV.setStringValue("4");
+
+ /*
+ * Default tmp location
+ */
+ String date = (new Date()).toString();
+ date = date.replaceAll(" ", "_");
+ date = date.replaceAll(":", "_");
+
+ String remoteTempDir = scratchDir + File.separator + "SimpleEcho" + "_" + date + "_"
+ + UUID.randomUUID();
+
+ System.out.println(remoteTempDir);
+
+ // no need of these parameters, as unicore manages by itself
+ appDepType.setScratchWorkingDirectory(remoteTempDir);
+ appDepType.setStaticWorkingDirectory(remoteTempDir);
+ appDepType.setInputDataDirectory(remoteTempDir + File.separator + "inputData");
+ appDepType.setOutputDataDirectory(remoteTempDir + File.separator + "outputData");
+
+ appDepType.setStandardOutput(appDepType.getOutputDataDirectory()+"/jsdl_stdout");
+
+ appDepType.setStandardError(appDepType.getOutputDataDirectory()+"/jsdl_stderr");
+
+ return appDesc;
+ }
+ protected MessageContext getInMessageContext() {
+ MessageContext inMessage = new MessageContext();
+ return inMessage;
+ }
+
+
+
+ protected MessageContext getOutMessageContext() {
+ MessageContext outMessage = new MessageContext();
+ return outMessage;
+ }
+
+
+
+}
Modified: airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/BESProviderTest.java
URL: http://svn.apache.org/viewvc/airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/BESProviderTest.java?rev=1451054&r1=1451053&r2=1451054&view=diff
==============================================================================
--- airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/BESProviderTest.java (original)
+++ airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/BESProviderTest.java Thu Feb 28 00:26:09 2013
@@ -1,217 +1,217 @@
-//package org.apache.airavata.core.gfac.services.impl;
-//
-//import java.io.File;
-//import java.net.URL;
-//import java.util.ArrayList;
-//import java.util.Date;
-//import java.util.List;
-//import java.util.UUID;
-//
-//import org.apache.airavata.commons.gfac.type.ActualParameter;
-//import org.apache.airavata.commons.gfac.type.ApplicationDescription;
-//import org.apache.airavata.commons.gfac.type.HostDescription;
-//import org.apache.airavata.commons.gfac.type.ServiceDescription;
-//import org.apache.airavata.gfac.GFacAPI;
-//import org.apache.airavata.gfac.GFacConfiguration;
-//import org.apache.airavata.gfac.GFacException;
-//import org.apache.airavata.gfac.context.ApplicationContext;
-//import org.apache.airavata.gfac.context.JobExecutionContext;
-//import org.apache.airavata.gfac.context.MessageContext;
-//import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-//import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
-//import org.apache.airavata.schemas.gfac.InputParameterType;
-//import org.apache.airavata.schemas.gfac.JobTypeType;
-//import org.apache.airavata.schemas.gfac.OutputParameterType;
-//import org.apache.airavata.schemas.gfac.ProjectAccountType;
-//import org.apache.airavata.schemas.gfac.QueueType;
-//import org.apache.airavata.schemas.gfac.StringParameterType;
-//import org.apache.airavata.schemas.gfac.URIParameterType;
-//import org.apache.airavata.schemas.gfac.UnicoreHostType;
-//import org.apache.log4j.PropertyConfigurator;
-//import org.junit.Before;
-//import org.junit.Test;
-//
-//public class BESProviderTest {
-// private JobExecutionContext jobExecutionContext;
-//
-// private static final String[] hostArray = new String[] { "https://zam1161v01.zam.kfa-juelich.de:8002/INTEROP1/services/BESFactory?res=default_bes_factory" };
-// private static final String gridftpAddress = "gsiftp://gridftp1.ls4.tacc.utexas.edu:2811";
-//
-// //directory where data will be copy into and copied out to unicore resources
-//
-// // private static final String scratchDir = "/brashear/msmemon/airavata";
-//
-// private static final String scratchDir = "/scratch/02055/msmemon/airavata";
-//
-// private String remoteTempDir = null;
-//
-// @Before
-// public void initJobContext() throws Exception {
-// PropertyConfigurator.configure("src/test/resources/logging.properties");
-//
-// /*
-// * Default tmp location
-// */
-// String date = (new Date()).toString();
-// date = date.replaceAll(" ", "_");
-// date = date.replaceAll(":", "_");
-//
-//
-// remoteTempDir = scratchDir + File.separator + "SimpleEcho" + "_" + date + "_"
-// + UUID.randomUUID();
-//
-// jobExecutionContext = new JobExecutionContext(getGFACConfig(), getServiceDesc().getType().getName());
-// jobExecutionContext.setApplicationContext(getApplicationContext());
-// jobExecutionContext.setInMessageContext(getInMessageContext());
-// jobExecutionContext.setOutMessageContext(getOutMessageContext());
-// }
-//
-// @Test
-// public void testBESProvider() throws GFacException {
-//// GFacAPI gFacAPI = new GFacAPI();
-//// gFacAPI.submitJob(jobExecutionContext);
-// }
-//
-// private GFacConfiguration getGFACConfig() throws Exception{
-// URL resource = BESProviderTest.class.getClassLoader().getResource("gfac-config.xml");
-// System.out.println(resource.getFile());
-// GFacConfiguration gFacConfiguration = GFacConfiguration.create(new File(resource.getPath()),null,null);
-// gFacConfiguration.setMyProxyLifeCycle(3600);
-// gFacConfiguration.setMyProxyServer("myproxy.teragrid.org");
-// gFacConfiguration.setMyProxyUser("msmemon");
-// gFacConfiguration.setMyProxyPassphrase("******");
-// gFacConfiguration.setTrustedCertLocation("/home/m.memon/.globus/certificates");
-// return gFacConfiguration;
-// }
-//
-// private ApplicationContext getApplicationContext() {
-// ApplicationContext applicationContext = new ApplicationContext();
-// applicationContext.setHostDescription(getHostDesc());
-// applicationContext
-// .setApplicationDeploymentDescription(getApplicationDesc());
-// applicationContext.setServiceDescription(getServiceDesc());
-// return applicationContext;
-// }
-//
-// private ApplicationDescription getApplicationDesc() {
-// ApplicationDescription appDesc = new ApplicationDescription(
-// HpcApplicationDeploymentType.type);
-// HpcApplicationDeploymentType app = (HpcApplicationDeploymentType) appDesc
-// .getType();
-// ApplicationDeploymentDescriptionType.ApplicationName name = ApplicationDeploymentDescriptionType.ApplicationName.Factory
-// .newInstance();
-// name.setStringValue("Simple Cat");
-// app.setApplicationName(name);
-// ProjectAccountType projectAccountType = app.addNewProjectAccount();
-// projectAccountType.setProjectAccountNumber("TG-AST110064");
-//
-// QueueType queueType = app.addNewQueue();
-// queueType.setQueueName("development");
-//
-// app.setCpuCount(1);
-// // TODO: also handle parallel jobs
-// app.setJobType(JobTypeType.SERIAL);
-// app.setNodeCount(1);
-// app.setProcessorsPerNode(1);
-//
-// /*
-// * Use bat file if it is compiled on Windows
-// */
-// app.setExecutableLocation("/bin/cat");
-//
-//
-// System.out.println(remoteTempDir);
-//
-//
-// app.setScratchWorkingDirectory(remoteTempDir);
-// app.setStaticWorkingDirectory(remoteTempDir);
-//
-//
-// // this is required
-// app.setInputDataDirectory(remoteTempDir + File.separator + "inputData");
-// app.setOutputDataDirectory(remoteTempDir + File.separator + "outputData");
-//
-// app.setStandardOutput(app.getOutputDataDirectory()+ File.separator + "stdout");
-// app.setStandardError(app.getOutputDataDirectory()+ File.separator + "stderr");
-//
-// return appDesc;
-// }
-//
-// private HostDescription getHostDesc() {
-// HostDescription host = new HostDescription(UnicoreHostType.type);
-// host.getType().setHostAddress("zam1161v01.zam.kfa-juelich.de");
-// host.getType().setHostName("DEMO-INTEROP-SITE");
-// ((UnicoreHostType) host.getType()).setUnicoreHostAddressArray(hostArray);
-// ((UnicoreHostType) host.getType()).setGridFTPEndPointArray(new String[]{gridftpAddress});
-// return host;
-// }
-//
-// private ServiceDescription getServiceDesc() {
-// ServiceDescription serv = new ServiceDescription();
-// serv.getType().setName("SimpleCat");
-//
-// List<InputParameterType> inputList = new ArrayList<InputParameterType>();
-// InputParameterType input = InputParameterType.Factory.newInstance();
-// input.setParameterName("copy_input");
-// input.setParameterType(URIParameterType.Factory.newInstance());
-// inputList.add(input);
-// InputParameterType[] inputParamList = inputList
-// .toArray(new InputParameterType[inputList.size()]);
-//
-// List<OutputParameterType> outputList = new ArrayList<OutputParameterType>();
-// OutputParameterType output = OutputParameterType.Factory.newInstance();
-// output.setParameterName("echo_output");
-// output.setParameterType(StringParameterType.Factory.newInstance());
-// outputList.add(output);
-// OutputParameterType[] outputParamList = outputList
-// .toArray(new OutputParameterType[outputList.size()]);
-//
-// serv.getType().setInputParametersArray(inputParamList);
-// serv.getType().setOutputParametersArray(outputParamList);
-// return serv;
-// }
-//
-// private MessageContext getInMessageContext() {
-// MessageContext inMessage = new MessageContext();
-//
-// ActualParameter copy_input = new ActualParameter();
-// copy_input.getType().changeType(URIParameterType.type);
-// ((URIParameterType)copy_input.getType()).setValue("file:///tmp/tmpstrace");
-// inMessage.addParameter("f1", copy_input);
-//
-// ActualParameter f2 = new ActualParameter();
-// f2.getType().changeType(URIParameterType.type);
-// ((URIParameterType)f2.getType()).setValue("http://unicore-dev.zam.kfa-juelich.de/maven/cog-globus/cog-jglobus/1.4/cog-jglobus-1.4.jar");
-// inMessage.addParameter("f2", f2);
-//
-// ActualParameter f3 = new ActualParameter();
-// f3.getType().changeType(URIParameterType.type);
-// ((URIParameterType)f3.getType()).setValue(gridftpAddress+"/"+scratchDir+"/file1.txt");
-// inMessage.addParameter("f3", f3);
-//
-// ActualParameter a1 = new ActualParameter();
-// a1.getType().changeType(StringParameterType.type);
-// ((StringParameterType)a1.getType()).setValue("tmpstrace");
-// inMessage.addParameter("arg1", a1);
-//
-// return inMessage;
-// }
-//
-// private MessageContext getOutMessageContext() {
-//
-// MessageContext outMessage = new MessageContext();
-// ActualParameter a1 = new ActualParameter();
-// a1.getType().changeType(StringParameterType.type);
-// ((StringParameterType)a1.getType()).setValue("file1.txt");
-// outMessage.addParameter("echo_output", a1);
-//
-// ActualParameter o1 = new ActualParameter();
-// o1.getType().changeType(URIParameterType.type);
-// // this may be any gridftp / ftp directory
-// ((URIParameterType)o1.getType()).setValue(gridftpAddress+"/"+remoteTempDir + "/" + "outputData"+"/"+"cog-jglobus-1.4.jar");
-// outMessage.addParameter("o1", o1);
-//
-// return outMessage;
-// }
-//
-//}
+package org.apache.airavata.core.gfac.services.impl;
+
+import java.io.File;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.airavata.commons.gfac.type.ActualParameter;
+import org.apache.airavata.commons.gfac.type.ApplicationDescription;
+import org.apache.airavata.commons.gfac.type.HostDescription;
+import org.apache.airavata.commons.gfac.type.ServiceDescription;
+import org.apache.airavata.gfac.GFacAPI;
+import org.apache.airavata.gfac.GFacConfiguration;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.context.ApplicationContext;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.context.MessageContext;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
+import org.apache.airavata.schemas.gfac.InputParameterType;
+import org.apache.airavata.schemas.gfac.JobTypeType;
+import org.apache.airavata.schemas.gfac.OutputParameterType;
+import org.apache.airavata.schemas.gfac.ProjectAccountType;
+import org.apache.airavata.schemas.gfac.QueueType;
+import org.apache.airavata.schemas.gfac.StringParameterType;
+import org.apache.airavata.schemas.gfac.URIParameterType;
+import org.apache.airavata.schemas.gfac.UnicoreHostType;
+import org.apache.log4j.PropertyConfigurator;
+import org.junit.Before;
+import org.junit.Test;
+
+public class BESProviderTest {
+ private JobExecutionContext jobExecutionContext;
+
+ private static final String[] hostArray = new String[] { "https://zam1161v01.zam.kfa-juelich.de:8002/INTEROP1/services/BESFactory?res=default_bes_factory" };
+ private static final String gridftpAddress = "gsiftp://gridftp1.ls4.tacc.utexas.edu:2811";
+
+ //directory where data will be copy into and copied out to unicore resources
+
+ // private static final String scratchDir = "/brashear/msmemon/airavata";
+ //FIXME: to move all the configurations to property files.
+ private static final String scratchDir = "/scratch/01437/ogce/test/";
+
+ private String remoteTempDir = null;
+
+ @Before
+ public void initJobContext() throws Exception {
+ PropertyConfigurator.configure("src/test/resources/logging.properties");
+
+ /*
+ * Default tmp location
+ */
+ String date = (new Date()).toString();
+ date = date.replaceAll(" ", "_");
+ date = date.replaceAll(":", "_");
+
+
+ remoteTempDir = scratchDir + File.separator + "SimpleEcho" + "_" + date + "_"
+ + UUID.randomUUID();
+
+ jobExecutionContext = new JobExecutionContext(getGFACConfig(), getServiceDesc().getType().getName());
+ jobExecutionContext.setApplicationContext(getApplicationContext());
+ jobExecutionContext.setInMessageContext(getInMessageContext());
+ jobExecutionContext.setOutMessageContext(getOutMessageContext());
+ }
+
+ @Test
+ public void testBESProvider() throws GFacException {
+ GFacAPI gFacAPI = new GFacAPI();
+ gFacAPI.submitJob(jobExecutionContext);
+ }
+
+ private GFacConfiguration getGFACConfig() throws Exception{
+ URL resource = BESProviderTest.class.getClassLoader().getResource("gfac-config.xml");
+ System.out.println(resource.getFile());
+ GFacConfiguration gFacConfiguration = GFacConfiguration.create(new File(resource.getPath()),null,null);
+ gFacConfiguration.setMyProxyLifeCycle(3600);
+ gFacConfiguration.setMyProxyServer("myproxy.teragrid.org");
+ gFacConfiguration.setMyProxyUser("*****");
+ gFacConfiguration.setMyProxyPassphrase("*****");
+ gFacConfiguration.setTrustedCertLocation("./certificates");
+ return gFacConfiguration;
+ }
+
+ private ApplicationContext getApplicationContext() {
+ ApplicationContext applicationContext = new ApplicationContext();
+ applicationContext.setHostDescription(getHostDesc());
+ applicationContext
+ .setApplicationDeploymentDescription(getApplicationDesc());
+ applicationContext.setServiceDescription(getServiceDesc());
+ return applicationContext;
+ }
+
+ private ApplicationDescription getApplicationDesc() {
+ ApplicationDescription appDesc = new ApplicationDescription(
+ HpcApplicationDeploymentType.type);
+ HpcApplicationDeploymentType app = (HpcApplicationDeploymentType) appDesc
+ .getType();
+ ApplicationDeploymentDescriptionType.ApplicationName name = ApplicationDeploymentDescriptionType.ApplicationName.Factory
+ .newInstance();
+ name.setStringValue("Simple Cat");
+ app.setApplicationName(name);
+ ProjectAccountType projectAccountType = app.addNewProjectAccount();
+ projectAccountType.setProjectAccountNumber("TG-AST110064");
+
+ QueueType queueType = app.addNewQueue();
+ queueType.setQueueName("development");
+
+ app.setCpuCount(1);
+ // TODO: also handle parallel jobs
+ app.setJobType(JobTypeType.SERIAL);
+ app.setNodeCount(1);
+ app.setProcessorsPerNode(1);
+
+ /*
+ * Use bat file if it is compiled on Windows
+ */
+ app.setExecutableLocation("/bin/cat");
+
+
+ System.out.println(remoteTempDir);
+
+
+ app.setScratchWorkingDirectory(remoteTempDir);
+ app.setStaticWorkingDirectory(remoteTempDir);
+
+
+ // this is required
+ app.setInputDataDirectory(remoteTempDir + File.separator + "inputData");
+ app.setOutputDataDirectory(remoteTempDir + File.separator + "outputData");
+
+ app.setStandardOutput(app.getOutputDataDirectory()+ File.separator + "stdout");
+ app.setStandardError(app.getOutputDataDirectory()+ File.separator + "stderr");
+
+ return appDesc;
+ }
+
+ private HostDescription getHostDesc() {
+ HostDescription host = new HostDescription(UnicoreHostType.type);
+ host.getType().setHostAddress("zam1161v01.zam.kfa-juelich.de");
+ host.getType().setHostName("DEMO-INTEROP-SITE");
+ ((UnicoreHostType) host.getType()).setUnicoreHostAddressArray(hostArray);
+ ((UnicoreHostType) host.getType()).setGridFTPEndPointArray(new String[]{gridftpAddress});
+ return host;
+ }
+
+ private ServiceDescription getServiceDesc() {
+ ServiceDescription serv = new ServiceDescription();
+ serv.getType().setName("SimpleCat");
+
+ List<InputParameterType> inputList = new ArrayList<InputParameterType>();
+ InputParameterType input = InputParameterType.Factory.newInstance();
+ input.setParameterName("copy_input");
+ input.setParameterType(URIParameterType.Factory.newInstance());
+ inputList.add(input);
+ InputParameterType[] inputParamList = inputList
+ .toArray(new InputParameterType[inputList.size()]);
+
+ List<OutputParameterType> outputList = new ArrayList<OutputParameterType>();
+ OutputParameterType output = OutputParameterType.Factory.newInstance();
+ output.setParameterName("echo_output");
+ output.setParameterType(StringParameterType.Factory.newInstance());
+ outputList.add(output);
+ OutputParameterType[] outputParamList = outputList
+ .toArray(new OutputParameterType[outputList.size()]);
+
+ serv.getType().setInputParametersArray(inputParamList);
+ serv.getType().setOutputParametersArray(outputParamList);
+ return serv;
+ }
+
+ private MessageContext getInMessageContext() {
+ MessageContext inMessage = new MessageContext();
+
+ ActualParameter copy_input = new ActualParameter();
+ copy_input.getType().changeType(URIParameterType.type);
+ ((URIParameterType)copy_input.getType()).setValue("file:///tmp/rave_db.h2.db ");
+ inMessage.addParameter("f1", copy_input);
+
+ ActualParameter f2 = new ActualParameter();
+ f2.getType().changeType(URIParameterType.type);
+ ((URIParameterType)f2.getType()).setValue("http://unicore-dev.zam.kfa-juelich.de/maven/cog-globus/cog-jglobus/1.4/cog-jglobus-1.4.jar");
+ inMessage.addParameter("f2", f2);
+
+ ActualParameter f3 = new ActualParameter();
+ f3.getType().changeType(URIParameterType.type);
+ ((URIParameterType)f3.getType()).setValue(gridftpAddress+"/"+scratchDir+"/README");
+ inMessage.addParameter("f3", f3);
+
+ ActualParameter a1 = new ActualParameter();
+ a1.getType().changeType(StringParameterType.type);
+ ((StringParameterType)a1.getType()).setValue("tmpstrace");
+ inMessage.addParameter("arg1", a1);
+
+ return inMessage;
+ }
+
+ private MessageContext getOutMessageContext() {
+
+ MessageContext outMessage = new MessageContext();
+ ActualParameter a1 = new ActualParameter();
+ a1.getType().changeType(StringParameterType.type);
+ ((StringParameterType)a1.getType()).setValue("README");
+ outMessage.addParameter("echo_output", a1);
+
+ ActualParameter o1 = new ActualParameter();
+ o1.getType().changeType(URIParameterType.type);
+ // this may be any gridftp / ftp directory
+ ((URIParameterType)o1.getType()).setValue(gridftpAddress+"/"+remoteTempDir + "/" + "outputData"+"/"+"cog-jglobus-1.4.jar");
+ outMessage.addParameter("o1", o1);
+
+ return outMessage;
+ }
+
+}