You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by pa...@apache.org on 2011/10/27 23:39:00 UTC
svn commit: r1190045 [2/2] - in /incubator/airavata/trunk/modules:
commons/gfac-schema/schemas/
commons/gfac-schema/src/main/java/org/apache/airavata/commons/gfac/type/
commons/registry-api/src/main/java/org/apache/airavata/registry/api/impl/
commons/r...
Modified: incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/provider/impl/SSHProvider.java
URL: http://svn.apache.org/viewvc/incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/provider/impl/SSHProvider.java?rev=1190045&r1=1190044&r2=1190045&view=diff
==============================================================================
--- incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/provider/impl/SSHProvider.java (original)
+++ incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/provider/impl/SSHProvider.java Thu Oct 27 21:38:59 2011
@@ -24,6 +24,7 @@ package org.apache.airavata.core.gfac.pr
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -48,7 +49,8 @@ import org.apache.airavata.core.gfac.uti
import org.apache.airavata.core.gfac.utils.GfacUtils;
import org.apache.airavata.core.gfac.utils.InputUtils;
import org.apache.airavata.core.gfac.utils.OutputUtils;
-import org.apache.airavata.schemas.gfac.ShellApplicationDeploymentType;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.NameValuePairType;
import org.apache.xmlbeans.XmlException;
/**
@@ -85,7 +87,7 @@ public class SSHProvider extends Abstrac
ssh.loadKnownHosts();
ssh.authPublickey(sshContext.getUsername(), pkey);
- ssh.connect(context.getExecutionDescription().getHost().getType().getAddress());
+ ssh.connect(context.getExecutionDescription().getHost().getType().getHostAddress());
return ssh.startSession();
} catch (NullPointerException ne) {
@@ -104,7 +106,7 @@ public class SSHProvider extends Abstrac
}
public void makeDirectory(InvocationContext context) throws ProviderException {
- ShellApplicationDeploymentType app = (ShellApplicationDeploymentType) context.getExecutionDescription().getApp().getType();
+ ApplicationDeploymentDescriptionType app = context.getExecutionDescription().getApp().getType();
Session session = null;
try {
@@ -113,16 +115,16 @@ public class SSHProvider extends Abstrac
StringBuilder commandString = new StringBuilder();
commandString.append("mkdir -p ");
- commandString.append(app.getTmpDir());
+ commandString.append(app.getScratchWorkingDirectory());
commandString.append(" ; ");
commandString.append("mkdir -p ");
- commandString.append(app.getWorkingDir());
+ commandString.append(app.getStaticWorkingDirectory());
commandString.append(" ; ");
commandString.append("mkdir -p ");
- commandString.append(app.getInputDir());
+ commandString.append(app.getInputDataDirectory());
commandString.append(" ; ");
commandString.append("mkdir -p ");
- commandString.append(app.getOutputDir());
+ commandString.append(app.getOutputDataDirectory());
Command cmd = session.exec(commandString.toString());
cmd.join(COMMAND_EXECUTION_TIMEOUT, TimeUnit.SECONDS);
@@ -138,7 +140,7 @@ public class SSHProvider extends Abstrac
}
public void setupEnvironment(InvocationContext context) throws ProviderException {
- ShellApplicationDeploymentType app = (ShellApplicationDeploymentType) context.getExecutionDescription().getApp().getType();
+ ApplicationDeploymentDescriptionType app = context.getExecutionDescription().getApp().getType();
// input parameter
ArrayList<String> tmp = new ArrayList<String>();
@@ -152,7 +154,7 @@ public class SSHProvider extends Abstrac
/*
* Builder Command
*/
- cmdList.add(app.getExecutable());
+ cmdList.add(app.getExecutableLocation());
cmdList.addAll(tmp);
// create process builder from command
@@ -162,12 +164,12 @@ public class SSHProvider extends Abstrac
// TODO: Make 1> and 2> into static constants.
// TODO: This only works for the BASH shell. CSH and TCSH will be
// different.
- command += SPACE + "1>" + SPACE + app.getStdOut();
- command += SPACE + "2>" + SPACE + app.getStdErr();
+ command += SPACE + "1>" + SPACE + app.getStandardOutput();
+ command += SPACE + "2>" + SPACE + app.getStandardError();
}
public void executeApplication(InvocationContext context) throws ProviderException {
- ShellApplicationDeploymentType app = (ShellApplicationDeploymentType) context.getExecutionDescription().getApp().getType();
+ ApplicationDeploymentDescriptionType app = context.getExecutionDescription().getApp().getType();
Session session = null;
try {
@@ -176,21 +178,22 @@ public class SSHProvider extends Abstrac
/*
* Going to working Directory
*/
- session.exec("cd " + app.getWorkingDir());
+ session.exec("cd " + app.getStaticWorkingDirectory());
// get the env of the host and the application
- ShellApplicationDeploymentType.Env.Entry[] env = app.getEnv().getEntryArray();
+ NameValuePairType[] env = app.getApplicationEnvironmentArray();
- Map<String, String> nv = null;
- for (int i = 0; i < env.length; i++) {
- String key = app.getEnv().getEntryArray(i).getKey();
- String value = app.getEnv().getEntryArray(i).getValue();
- nv.put(key, value);
+ Map<String, String> nv = new HashMap<String, String>();
+ if (env != null) {
+ for (int i = 0; i < env.length; i++) {
+ String key = env[i].getName();
+ String value = env[i].getValue();
+ nv.put(key, value);
+ }
}
-
// extra env's
- nv.put(GFacConstants.INPUT_DATA_DIR_VAR_NAME, app.getInputDir());
- nv.put(GFacConstants.OUTPUT_DATA_DIR_VAR_NAME, app.getOutputDir());
+ nv.put(GFacConstants.INPUT_DATA_DIR_VAR_NAME, app.getInputDataDirectory());
+ nv.put(GFacConstants.OUTPUT_DATA_DIR_VAR_NAME, app.getOutputDataDirectory());
/*
* Set environment
@@ -230,7 +233,7 @@ public class SSHProvider extends Abstrac
}
public Map<String, ?> processOutput(InvocationContext context) throws ProviderException {
- ShellApplicationDeploymentType app = (ShellApplicationDeploymentType) context.getExecutionDescription().getApp().getType();
+ ApplicationDeploymentDescriptionType app = context.getExecutionDescription().getApp().getType();
try {
// Get the Stdouts and StdErrs
@@ -239,15 +242,15 @@ public class SSHProvider extends Abstrac
File localStdErrFile = File.createTempFile(timeStampedServiceName, "stderr");
SCPFileTransfer fileTransfer = ssh.newSCPFileTransfer();
- fileTransfer.download(app.getStdOut(), localStdOutFile.getAbsolutePath());
- fileTransfer.download(app.getStdErr(), localStdErrFile.getAbsolutePath());
+ fileTransfer.download(app.getStandardOutput(), localStdOutFile.getAbsolutePath());
+ fileTransfer.download(app.getStandardError(), localStdErrFile.getAbsolutePath());
String stdOutStr = GfacUtils.readFileToString(localStdOutFile.getAbsolutePath());
String stdErrStr = GfacUtils.readFileToString(localStdErrFile.getAbsolutePath());
return OutputUtils.fillOutputFromStdout(context.<ActualParameter> getOutput(), stdOutStr);
-
- } catch (XmlException e){
+
+ } catch (XmlException e) {
throw new ProviderException("Cannot read output:" + e.getMessage(), e);
} catch (ConnectionException e) {
throw new ProviderException(e.getMessage(), e);
Modified: incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/provider/utils/GramRSLGenerator.java
URL: http://svn.apache.org/viewvc/incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/provider/utils/GramRSLGenerator.java?rev=1190045&r1=1190044&r2=1190045&view=diff
==============================================================================
--- incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/provider/utils/GramRSLGenerator.java (original)
+++ incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/provider/utils/GramRSLGenerator.java Thu Oct 27 21:38:59 2011
@@ -21,6 +21,7 @@
package org.apache.airavata.core.gfac.provider.utils;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
@@ -29,7 +30,7 @@ import org.apache.airavata.core.gfac.con
import org.apache.airavata.core.gfac.exception.ToolsException;
import org.apache.airavata.core.gfac.utils.GFacConstants;
import org.apache.airavata.schemas.gfac.GramApplicationDeploymentType;
-import org.apache.airavata.schemas.gfac.ShellApplicationDeploymentType;
+import org.apache.airavata.schemas.gfac.NameValuePairType;
import org.globus.gram.GramAttributes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,21 +49,21 @@ public class GramRSLGenerator {
GramApplicationDeploymentType app = (GramApplicationDeploymentType) context.getExecutionDescription().getApp().getType();
GramAttributes jobAttr = new GramAttributes();
- jobAttr.setExecutable(app.getExecutable());
- jobAttr.setDirectory(app.getWorkingDir());
- jobAttr.setStdout(app.getStdOut());
- jobAttr.setStderr(app.getStdErr());
+ jobAttr.setExecutable(app.getExecutableLocation());
+ jobAttr.setDirectory(app.getStaticWorkingDirectory());
+ jobAttr.setStdout(app.getStandardOutput());
+ jobAttr.setStderr(app.getStandardError());
/*
* The env here contains the env of the host and the application. i.e the env specified in the host description
* and application description documents
*/
- ShellApplicationDeploymentType.Env.Entry[] env = app.getEnv().getEntryArray();
+ NameValuePairType[] env = app.getApplicationEnvironmentArray();
if(env.length != 0){
- Map<String, String> nv = null;
+ Map<String, String> nv =new HashMap<String, String>();
for (int i = 0; i < env.length; i++) {
- String key = app.getEnv().getEntryArray(i).getKey();
- String value = app.getEnv().getEntryArray(i).getValue();
+ String key = env[i].getName();
+ String value = env[i].getValue();
nv.put(key, value);
}
@@ -70,24 +71,24 @@ public class GramRSLGenerator {
jobAttr.addEnvVariable(entry.getKey(), entry.getValue());
}
}
- jobAttr.addEnvVariable(GFacConstants.INPUT_DATA_DIR_VAR_NAME, app.getInputDir());
- jobAttr.addEnvVariable(GFacConstants.INPUT_DATA_DIR_VAR_NAME, app.getOutputDir());
+ jobAttr.addEnvVariable(GFacConstants.INPUT_DATA_DIR_VAR_NAME, app.getInputDataDirectory());
+ jobAttr.addEnvVariable(GFacConstants.INPUT_DATA_DIR_VAR_NAME, app.getOutputDataDirectory());
- if (app.getWallTime() > 0) {
- log.info("Setting max wall clock time to " + app.getWallTime());
+ if (app.getMaxWallTime() > 0) {
+ log.info("Setting max wall clock time to " + app.getMaxWallTime());
- if (app.getWallTime() > 30 && app.getQueueName() != null && app.getQueueName().equals("debug")) {
+ if (app.getMaxWallTime() > 30 && app.getQueue() != null && app.getQueue().getQueueName().equals("debug")) {
throw new ToolsException("NCSA debug Queue only support jobs < 30 minutes");
}
- jobAttr.setMaxWallTime(app.getWallTime());
+ jobAttr.setMaxWallTime(app.getMaxWallTime());
jobAttr.set("proxy_timeout", "1");
} else {
jobAttr.setMaxWallTime(29);
}
- if (app.getStdIn() != null) {
- jobAttr.setStdin(app.getStdIn());
+ if (app.getStandardInput() != null) {
+ jobAttr.setStdin(app.getStandardInput());
} else {
// input parameter
for (Iterator<String> iterator = context.getInput().getNames(); iterator.hasNext();) {
@@ -103,18 +104,18 @@ public class GramRSLGenerator {
log.info("Setting number of procs to " + app.getCpuCount());
jobAttr.setNumProcs(app.getCpuCount());
}
- if (app.getProjectName() != null) {
- log.info("Setting project to " + app.getProjectName());
- jobAttr.setProject(app.getProjectName());
- }
- if (app.getQueueName() != null) {
- log.info("Setting job queue to " + app.getQueueName());
- jobAttr.setQueue(app.getQueueName());
+ if (app.getProjectAccount().getProjectAccountNumber() != null) {
+ log.info("Setting project to " + app.getProjectAccount().getProjectAccountNumber());
+ jobAttr.setProject(app.getProjectAccount().getProjectAccountNumber());
+ }
+ if (app.getQueue().getQueueName() != null) {
+ log.info("Setting job queue to " + app.getQueue().getQueueName());
+ jobAttr.setQueue(app.getQueue().getQueueName());
}
String jobType = JobType.SINGLE.toString();
if (app.getJobType() != null) {
- jobType = app.getJobType();
+ jobType = app.getJobType().toString();
}
if (jobType.equalsIgnoreCase(JobType.SINGLE.toString())) {
Modified: incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/scheduler/impl/SchedulerImpl.java
URL: http://svn.apache.org/viewvc/incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/scheduler/impl/SchedulerImpl.java?rev=1190045&r1=1190044&r2=1190045&view=diff
==============================================================================
--- incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/scheduler/impl/SchedulerImpl.java (original)
+++ incubator/airavata/trunk/modules/gfac-core/src/main/java/org/apache/airavata/core/gfac/scheduler/impl/SchedulerImpl.java Thu Oct 27 21:38:59 2011
@@ -80,7 +80,7 @@ public class SchedulerImpl implements Sc
*/
ApplicationDeploymentDescription app = null;
try {
- app = registryService.getDeploymentDescription(context.getServiceName(), host.getType().getName());
+ app = registryService.getDeploymentDescription(context.getServiceName(), host.getType().getHostName());
} catch (RegistryException e2) {
e2.printStackTrace();
}
@@ -103,7 +103,7 @@ public class SchedulerImpl implements Sc
/*
* Determine provider
*/
- String hostName = host.getType().getAddress();
+ String hostName = host.getType().getHostAddress();
try {
if (GfacUtils.isLocalHost(hostName)) {
return new LocalProvider();
@@ -125,7 +125,7 @@ public class SchedulerImpl implements Sc
for (Iterator<HostDescription> iterator = hosts.iterator(); iterator.hasNext();) {
result = iterator.next();
- log.info("Found service on: " + result.getType().getAddress());
+ log.info("Found service on: " + result.getType().getHostAddress());
}
return result;
} else {
Modified: incubator/airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/PropertiesBasedServiceImplTest.java
URL: http://svn.apache.org/viewvc/incubator/airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/PropertiesBasedServiceImplTest.java?rev=1190045&r1=1190044&r2=1190045&view=diff
==============================================================================
--- incubator/airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/PropertiesBasedServiceImplTest.java (original)
+++ incubator/airavata/trunk/modules/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/PropertiesBasedServiceImplTest.java Thu Oct 27 21:38:59 2011
@@ -35,9 +35,11 @@ import org.apache.airavata.core.gfac.con
import org.apache.airavata.core.gfac.context.message.impl.ParameterContextImpl;
import org.apache.airavata.core.gfac.notification.impl.LoggingNotification;
import org.apache.airavata.registry.api.impl.JCRRegistry;
-import org.apache.airavata.schemas.gfac.Parameter;
-import org.apache.airavata.schemas.gfac.ShellApplicationDeploymentType;
-import org.apache.airavata.schemas.gfac.StringParameter;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType.ApplicationName;
+import org.apache.airavata.schemas.gfac.InputParameterType;
+import org.apache.airavata.schemas.gfac.OutputParameterType;
+import org.apache.airavata.schemas.gfac.StringParameterType;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -56,25 +58,24 @@ public class PropertiesBasedServiceImplT
* Host
*/
HostDescription host = new HostDescription();
- host.getType().setName("localhost");
- host.getType().setAddress("localhost");
+ host.getType().setHostName("localhost");
+ host.getType().setHostAddress("localhost");
/*
* App
*/
- ApplicationDeploymentDescription appDesc = new ApplicationDeploymentDescription(
- ShellApplicationDeploymentType.type);
- ShellApplicationDeploymentType app = (ShellApplicationDeploymentType) appDesc
- .getType();
- app.setName("EchoLocal");
- app.setExecutable("/bin/echo");
- app.setTmpDir("/tmp");
- app.setWorkingDir("/tmp");
- app.setInputDir("/tmp/input");
- app.setOutputDir("/tmp/output");
- app.setStdOut("/tmp/echo.stdout");
- app.setStdErr("/tmp/echo.stdout");
- app.setEnv(app.addNewEnv());
+ ApplicationDeploymentDescription appDesc = new ApplicationDeploymentDescription();
+ ApplicationDeploymentDescriptionType app = appDesc.getType();
+ ApplicationName name = ApplicationName.Factory.newInstance();
+ name.setStringValue("EchoLocal");
+ app.setApplicationName(name);
+ app.setExecutableLocation("/bin/echo");
+ app.setScratchWorkingDirectory("/tmp");
+ app.setStaticWorkingDirectory("/tmp");
+ app.setInputDataDirectory("/tmp/input");
+ app.setOutputDataDirectory("/tmp/output");
+ app.setStandardOutput("/tmp/echo.stdout");
+ app.setStandardError("/tmp/echo.stdout");
/*
* Service
@@ -82,21 +83,22 @@ public class PropertiesBasedServiceImplT
ServiceDescription serv = new ServiceDescription();
serv.getType().setName("SimpleEcho");
- Parameter input = Parameter.Factory.newInstance();
- input.setName("echo_input");
- input.addNewType();
- List<Parameter> inputList = new ArrayList<Parameter>();
+ List<InputParameterType> inputList = new ArrayList<InputParameterType>();
+ InputParameterType input = InputParameterType.Factory.newInstance();
+ input.setParameterName("echo_input");
+ input.setParameterType(StringParameterType.Factory.newInstance());
inputList.add(input);
- Parameter[] inputParamList = inputList.toArray(new Parameter[inputList
+ InputParameterType[] inputParamList = inputList.toArray(new InputParameterType[inputList
.size()]);
-
- Parameter output = Parameter.Factory.newInstance();
- output.setName("echo_output");
- output.addNewType();
- List<Parameter> outputList = new ArrayList<Parameter>();
+
+ List<OutputParameterType> outputList = new ArrayList<OutputParameterType>();
+ OutputParameterType output = OutputParameterType.Factory.newInstance();
+ output.setParameterName("echo_output");
+ input.setParameterType(StringParameterType.Factory.newInstance());
outputList.add(output);
- Parameter[] outputParamList = outputList
- .toArray(new Parameter[outputList.size()]);
+ OutputParameterType[] outputParamList = outputList
+ .toArray(new OutputParameterType[outputList.size()]);
+
serv.getType().setInputParametersArray(inputParamList);
serv.getType().setOutputParametersArray(outputParamList);
@@ -105,10 +107,10 @@ public class PropertiesBasedServiceImplT
*/
jcrRegistry.saveHostDescription(host);
jcrRegistry.saveDeploymentDescription(serv.getType().getName(), host
- .getType().getName(), appDesc);
+ .getType().getHostName(), appDesc);
jcrRegistry.saveServiceDescription(serv);
jcrRegistry.deployServiceOnHost(serv.getType().getName(), host
- .getType().getName());
+ .getType().getHostName());
}
@Test
@@ -126,15 +128,15 @@ public class PropertiesBasedServiceImplT
* Input
*/
ParameterContextImpl input = new ParameterContextImpl();
- ActualParameter echo_input = new ActualParameter(StringParameter.type);
- ((StringParameter)echo_input.getType()).setValue("echo_output=hello");
+ ActualParameter echo_input = new ActualParameter();
+ ((StringParameterType)echo_input.getType()).setValue("echo_output=hello");
input.add("echo_input", echo_input);
/*
* Output
*/
ParameterContextImpl output = new ParameterContextImpl();
- ActualParameter echo_output = new ActualParameter(StringParameter.type);
+ ActualParameter echo_output = new ActualParameter();
output.add("echo_output", echo_output);
// parameter
@@ -147,7 +149,7 @@ public class PropertiesBasedServiceImplT
Assert.assertNotNull(ct.getOutput());
Assert.assertNotNull(ct.getOutput().getValue("echo_output"));
- Assert.assertEquals("hello", ((StringParameter)((ActualParameter)ct.getOutput().getValue("echo_output")).getType()).getValue());
+ Assert.assertEquals("hello", ((StringParameterType)((ActualParameter)ct.getOutput().getValue("echo_output")).getType()).getValue());
} catch (Exception e) {
e.printStackTrace();