You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2015/03/19 16:02:56 UTC
[30/62] [abbrv] airavata git commit: adding lsf support with tests
adding lsf support with tests
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/5ff650f4
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/5ff650f4
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/5ff650f4
Branch: refs/heads/queue-gfac-rabbitmq
Commit: 5ff650f4c61a9ee7bfcf6f15ea984d74d545ec32
Parents: e9468ca
Author: Lahiru Gunathilake <gl...@gmail.com>
Authored: Sat Mar 7 00:37:06 2015 -0500
Committer: Lahiru Gunathilake <gl...@gmail.com>
Committed: Sat Mar 7 00:37:06 2015 -0500
----------------------------------------------------------------------
.../client/samples/CreateLaunchExperiment.java | 73 +++++-
.../tools/RegisterSampleApplications.java | 53 +++-
.../server/src/main/resources/LSFTemplate.xslt | 43 ++--
.../server/src/main/resources/PBSTemplate.xslt | 3 +-
.../credential/store/client/TestSSLClient.java | 2 +-
.../store/store/impl/db/SSHCredentialTest.java | 6 +-
.../test/resources/airavata-server.properties | 254 +++++++++++++++++++
.../server/src/main/assembly/bin-assembly.xml | 1 +
.../airavata/gfac/server/GfacServerHandler.java | 1 -
.../org/apache/airavata/gfac/RequestData.java | 2 +
.../gfac/gsissh/util/GFACGSISSHUtils.java | 10 +-
.../airavata/gfac/ssh/util/GFACSSHUtils.java | 23 +-
.../ssh/api/job/JobManagerConfiguration.java | 9 +-
.../gsi/ssh/api/job/LSFJobConfiguration.java | 18 +-
.../gsi/ssh/api/job/LSFOutputParser.java | 1 +
.../gsi/ssh/api/job/PBSJobConfiguration.java | 15 ++
.../gsi/ssh/api/job/SlurmJobConfiguration.java | 15 ++
.../gsi/ssh/impl/GSISSHAbstractCluster.java | 10 +-
.../airavata/gsi/ssh/impl/RawCommandInfo.java | 8 +-
.../impl/DefaultSSHApiTestWithMyProxyAuth.java | 15 +-
20 files changed, 500 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
index 3fc74eb..bb2ae6e 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
@@ -58,7 +58,7 @@ public class CreateLaunchExperiment {
private static final String DEFAULT_GATEWAY = "php_reference_gateway";
private static Airavata.Client airavataClient;
- private static String echoAppId = "Echo_f828a575-7f17-4149-9d45-abe2aa9c6109";
+ private static String echoAppId = "Echo_802454e5-6358-4371-9a04-3d5d59cecbc7";
private static String mpiAppId = "HelloMPI_720e159f-198f-4daa-96ca-9f5eafee92c9";
private static String wrfAppId = "WRF_7ad5da38-c08b-417c-a9ea-da9298839762";
private static String amberAppId = "Amber_a56d457c-f239-4c0b-ba00-66bda936f7bc";
@@ -69,15 +69,15 @@ public class CreateLaunchExperiment {
private static String trinityAppId = "Trinity_e894acf5-9bca-46e8-a1bd-7e2d5155191a";
private static String autodockAppId = "AutoDock_43d9fdd0-c404-49f4-b913-3abf9080a8c9";
-
private static String localHost = "localhost";
private static String trestlesHostName = "trestles.sdsc.xsede.org";
private static String unicoreHostName = "fsd-cloud15.zam.kfa-juelich.de";
private static String stampedeHostName = "stampede.tacc.xsede.org";
private static String br2HostName = "bigred2.uits.iu.edu";
+ private static String umassrcHostName = "ghpcc06.umassrc.org";
private static String gatewayId;
-
+
// unicore service endpoint url
private static final String unicoreEndPointURL = "https://fsd-cloud15.zam.kfa-juelich.de:7000/INTEROP1/services/BESFactory?res=default_bes_factory";
@@ -88,7 +88,7 @@ public class CreateLaunchExperiment {
// createGateway();
// getGateway("testGatewayId");
// registerApplications(); // run this only the first time
- createAndLaunchExp();
+ createAndLaunchExp();
}
private static String fsdResourceId;
@@ -154,13 +154,14 @@ public class CreateLaunchExperiment {
// final String expId = createExperimentForBR2Amber(airavataClient);
// final String expId = createExperimentWRFStampede(airavataClient);
// final String expId = createExperimentForStampedeAmber(airavataClient);
- final String expId = createExperimentForTrestlesAmber(airavataClient);
+// final String expId = createExperimentForTrestlesAmber(airavataClient);
// final String expId = createExperimentGROMACSStampede(airavataClient);
// final String expId = createExperimentESPRESSOStampede(airavataClient);
// final String expId = createExperimentLAMMPSStampede(airavataClient);
// final String expId = createExperimentNWCHEMStampede(airavataClient);
// final String expId = createExperimentTRINITYStampede(airavataClient);
// final String expId = createExperimentAUTODOCKStampede(airavataClient); // this is not working , we need to register AutoDock app on stampede
+ final String expId = createExperimentForLSF(airavataClient);
// final String expId = "Ultrascan_ln_eb029947-391a-4ccf-8ace-9bafebe07cc0";
System.out.println("Experiment ID : " + expId);
// updateExperiment(airavata, expId);
@@ -185,6 +186,8 @@ public class CreateLaunchExperiment {
//Register all compute hosts
registerSampleApplications.registerXSEDEHosts();
+ registerSampleApplications.registerNonXSEDEHosts();
+
//Register Gateway Resource Preferences
registerSampleApplications.registerGatewayResourceProfile();
@@ -1343,6 +1346,63 @@ public class CreateLaunchExperiment {
return null;
}
+ public static String createExperimentForLSF(Airavata.Client client) throws TException {
+ try {
+ List<InputDataObjectType> exInputs = new ArrayList<InputDataObjectType>();
+ InputDataObjectType input = new InputDataObjectType();
+ input.setName("Input_to_Echo");
+ input.setType(DataType.STRING);
+ input.setValue("Echoed_Output=Hello World");
+ input.setRequiredToAddedToCommandLine(true);
+ exInputs.add(input);
+
+ List<OutputDataObjectType> exOut = new ArrayList<OutputDataObjectType>();
+ OutputDataObjectType output = new OutputDataObjectType();
+ output.setName("output_file");
+ output.setType(DataType.URI);
+ output.setValue("");
+ exOut.add(output);
+
+ Project project = ProjectModelUtil.createProject("default", "lg11w", "test project");
+ String projectId = client.createProject(DEFAULT_GATEWAY, project);
+
+ Experiment simpleExperiment =
+ ExperimentModelUtil.createSimpleExperiment(projectId, "lg11w", "sshEchoExperiment", "StressMem", echoAppId, exInputs);
+ simpleExperiment.setExperimentOutputs(exOut);
+
+ Map<String, String> computeResources = airavataClient.getAvailableAppInterfaceComputeResources(echoAppId);
+ if (computeResources != null && computeResources.size() != 0) {
+ for (String id : computeResources.keySet()) {
+ String resourceName = computeResources.get(id);
+ if (resourceName.equals(umassrcHostName)) {
+ ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling(id, 10, 1, 1, "long", 60, 0, 1000, "airavata");
+ UserConfigurationData userConfigurationData = new UserConfigurationData();
+ userConfigurationData.setAiravataAutoSchedule(false);
+ userConfigurationData.setOverrideManualScheduledParams(false);
+ userConfigurationData.setComputationalResourceScheduling(scheduling);
+ simpleExperiment.setUserConfigurationData(userConfigurationData);
+ simpleExperiment.setEmailAddresses(Arrays.asList(new String[]{"test@umassmed.edu"}));
+ return client.createExperiment(DEFAULT_GATEWAY, simpleExperiment);
+ }
+ }
+ }
+ } catch (AiravataSystemException e) {
+ logger.error("Error occured while creating the experiment...", e.getMessage());
+ throw new AiravataSystemException(e);
+ } catch (InvalidRequestException e) {
+ logger.error("Error occured while creating the experiment...", e.getMessage());
+ throw new InvalidRequestException(e);
+ } catch (AiravataClientException e) {
+ logger.error("Error occured while creating the experiment...", e.getMessage());
+ throw new AiravataClientException(e);
+ } catch (TException e) {
+ logger.error("Error occured while creating the experiment...", e.getMessage());
+ throw new TException(e);
+ }
+ return null;
+ }
+
+
public static String createExperimentForBR2Amber(Airavata.Client client) throws TException {
try {
List<InputDataObjectType> exInputs = client.getApplicationInputs(amberAppId);
@@ -1538,8 +1598,7 @@ public class CreateLaunchExperiment {
public static void launchExperiment(Airavata.Client client, String expId)
throws TException {
try {
-// String tokenId = "5f116091-0ad3-4ab6-9df7-6ac909f21f8b";
- String tokenId ="aaaaaa";
+ String tokenId ="aa-dcdb-48e3-9cd5-ac90b710d55e";
client.launchExperiment(expId, tokenId);
} catch (ExperimentNotFoundException e) {
logger.error("Error occured while launching the experiment...", e.getMessage());
http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
index 7f7ac1d..8ae2dd2 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
@@ -54,6 +54,8 @@ public class RegisterSampleApplications {
private static String stampedeResourceId = "stampede.tacc.xsede.org_92ac5ed6-35a5-4910-82ef-48f128f9245a";
private static String trestlesResourceId = "trestles.sdsc.xsede.org_db29986e-5a27-4949-ae7f-04a6012d0d35";
private static String bigredResourceId = "bigred2.uits.iu.edu_3eae6e9d-a1a7-44ec-ac85-3796ef726ef1";
+ private static String lsfResourceId = "lsf_3eae6e9d-a1a7-44ec-ac85-3796ef726ef1";
+
private static String fsdResourceId;
// unicore service endpoint url
private static final String unicoreEndPointURL = "https://fsd-cloud15.zam.kfa-juelich.de:7000/INTEROP1/services/BESFactory?res=default_bes_factory";
@@ -73,6 +75,7 @@ public class RegisterSampleApplications {
private static final String monteXName = "TinkerMonte";
private static final String gaussianName = "Gaussian";
private static final String gamessName = "Gamess";
+ private static final String stressMemName = "StressMem";
//Appplication Descriptions
private static final String echoDescription = "A Simple Echo Application";
@@ -90,6 +93,7 @@ public class RegisterSampleApplications {
private static final String gaussianDescription = "Grid Chem Gaussian Application";
private static final String gamessDescription = "A Gamess Application";
+
//App Module Id's
private static String echoModuleId;
private static String amberModuleId;
@@ -136,6 +140,7 @@ public class RegisterSampleApplications {
//Register all compute hosts
registerSampleApplications.registerXSEDEHosts();
+ registerSampleApplications.registerNonXSEDEHosts();
//Register Gateway Resource Preferences
registerSampleApplications.registerGatewayResourceProfile();
@@ -200,10 +205,24 @@ public class RegisterSampleApplications {
bigredResourceId = registerComputeHost("bigred2.uits.iu.edu", "IU BigRed II Cluster",
ResourceJobManagerType.PBS, "push", "/opt/torque/torque-4.2.3.1/bin/", SecurityProtocol.SSH_KEYS, 22, "aprun -n");
System.out.println("BigredII Resource Id is " + bigredResourceId);
-
+
fsdResourceId = registerUnicoreEndpoint("fsd-cloud15.zam.kfa-juelich.de", "interop host", JobSubmissionProtocol.UNICORE, SecurityProtocol.GSI);
System.out.println("FSd Resource Id: "+fsdResourceId);
-
+
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+ public void registerNonXSEDEHosts() {
+ try {
+ System.out.println("\n #### Registering Non-XSEDE Computational Resources #### \n");
+
+ //Register LSF resource
+ lsfResourceId = registerComputeHost("ghpcc06.umassrc.org", "LSF Cluster",
+ ResourceJobManagerType.LSF, "push", "source /etc/bashrc;/lsf/9.1/linux2.6-glibc2.3-x86_64/bin", SecurityProtocol.SSH_KEYS, 22, null);
+ System.out.println("LSF Resource Id is " + lsfResourceId);
} catch (TException e) {
e.printStackTrace();
@@ -317,7 +336,6 @@ public class RegisterSampleApplications {
gamessName, "17May13", gamessDescription));
System.out.println("Gamess Module Id " + gamessModuleId);
-
} catch (TException e) {
e.printStackTrace();
}
@@ -341,6 +359,7 @@ public class RegisterSampleApplications {
//Registering FSD Apps
registerFSDApps();
+ registerLSFApps();
}
@@ -493,8 +512,8 @@ public class RegisterSampleApplications {
e.printStackTrace();
}
}
-
-
+
+
public void registerMPIInterface() {
try {
System.out.println("#### Registering MPI Interface #### \n");
@@ -1084,6 +1103,22 @@ public class RegisterSampleApplications {
}
}
+ public void registerLSFApps() {
+ try {
+ System.out.println("#### Registering Application Deployments on Trestles #### \n");
+
+ //Register Echo
+ String echoAppDeployId = airavataClient.registerApplicationDeployment(DEFAULT_GATEWAY,
+ RegisterSampleApplicationsUtils.createApplicationDeployment(echoModuleId, lsfResourceId,
+ "/home/lg11w/executables/echo.sh", ApplicationParallelismType.SERIAL,
+ echoDescription, null, null, null));
+ System.out.println("Echo on trestles deployment Id " + echoAppDeployId);
+
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+ }
+
public void registerBigRedApps() {
try {
System.out.println("#### Registering Application Deployments on BigRed II #### \n");
@@ -1127,7 +1162,7 @@ public class RegisterSampleApplications {
e.printStackTrace();
}
}
-
+
public void registerFSDApps() {
try {
System.out.println("#### Registering Application Deployments on FSD #### \n");
@@ -1207,6 +1242,10 @@ public class RegisterSampleApplications {
ComputeResourcePreference bigRedResourcePreferences = RegisterSampleApplicationsUtils.
createComputeResourcePreference(bigredResourceId, "TG-STA110014S", false, null, null, null,
"/N/dc2/scratch/cgateway/gta-work-dirs");
+
+ ComputeResourcePreference lsfResourcePreferences = RegisterSampleApplicationsUtils.
+ createComputeResourcePreference(lsfResourceId, "airavata", false, null, null, null,
+ "/home/lg11w/mywork");
ComputeResourcePreference fsdResourcePreferences = RegisterSampleApplicationsUtils.
createComputeResourcePreference(fsdResourceId, null, false, null, JobSubmissionProtocol.UNICORE, DataMovementProtocol.UNICORE_STORAGE_SERVICE,null);
@@ -1217,6 +1256,7 @@ public class RegisterSampleApplications {
gatewayResourceProfile.addToComputeResourcePreferences(trestlesResourcePreferences);
gatewayResourceProfile.addToComputeResourcePreferences(bigRedResourcePreferences);
gatewayResourceProfile.addToComputeResourcePreferences(fsdResourcePreferences);
+ gatewayResourceProfile.addToComputeResourcePreferences(lsfResourcePreferences);
String gatewayProfile = airavataClient.registerGatewayResourceProfile(gatewayResourceProfile);
System.out.println("Gateway Profile is registered with Id " + gatewayProfile);
@@ -1232,6 +1272,7 @@ public class RegisterSampleApplications {
properties.setProperty("stampedeResourceId", stampedeResourceId);
properties.setProperty("trestlesResourceId", trestlesResourceId);
properties.setProperty("bigredResourceId", bigredResourceId);
+ properties.setProperty("lsfResourceId", lsfResourceId);
properties.setProperty("echoInterfaceId", echoInterfaceId);
properties.setProperty("amberInterfaceId", amberInterfaceId);
http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/modules/configuration/server/src/main/resources/LSFTemplate.xslt
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/LSFTemplate.xslt b/modules/configuration/server/src/main/resources/LSFTemplate.xslt
index ab9fbbd..7081260 100644
--- a/modules/configuration/server/src/main/resources/LSFTemplate.xslt
+++ b/modules/configuration/server/src/main/resources/LSFTemplate.xslt
@@ -9,47 +9,61 @@
<xsl:output method="text" />
<xsl:template match="/ns:JobDescriptor">
<xsl:param name="quote">"</xsl:param>
- #! /bin/bash
- # LSF batch job submission script generated by Apache Airavata
- #
+#! /bin/bash
+# LSF batch job submission script generated by Apache Airavata
+#
+ <xsl:choose>
+ <xsl:when test="ns:shellName">
+#BSUB -L <xsl:value-of select="ns:shellName"/>
+ </xsl:when></xsl:choose>
<xsl:choose>
<xsl:when test="ns:queueName">
- #BSUB -n <xsl:value-of select="ns:queueName"/>
+#BSUB -q <xsl:value-of select="ns:queueName"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+ <xsl:when test="ns:nodes">
+#BSUB -n <xsl:value-of select="ns:nodes"/>
</xsl:when>
</xsl:choose>
<xsl:choose>
<xsl:when test="ns:mailAddress">
- #BSUB -u <xsl:value-of select="ns:mailAddress"/>
+#BSUB -u <xsl:value-of select="ns:mailAddress"/>
</xsl:when>
</xsl:choose>
<xsl:choose>
<xsl:when test="ns:jobName">
- #BSUB -J <xsl:value-of select="ns:jobName"/>
+#BSUB -J <xsl:value-of select="ns:jobName"/>
</xsl:when>
</xsl:choose>
<xsl:choose>
- <xsl:when test="ns:acountString">
- #BSUB -P <xsl:value-of select="ns:acountString"/>
- </xsl:when>
- </xsl:choose>
+ <xsl:when test="ns:acountString">
+#BSUB -P <xsl:value-of select="ns:acountString"/>
+ </xsl:when>
+ </xsl:choose>
<xsl:choose>
<xsl:when test="ns:maxWallTime">
- #BSUB -W<xsl:value-of select="ns:maxWallTime"/>
+#BSUB -W <xsl:value-of select="ns:maxWallTime"/>
</xsl:when>
</xsl:choose>
<xsl:choose>
<xsl:when test="ns:standardOutFile">
- #BSUB -o <xsl:value-of select="ns:standardOutFile"/>
+#BSUB -o "<xsl:value-of select="ns:standardOutFile"/>"
</xsl:when>
</xsl:choose>
<xsl:choose>
<xsl:when test="ns:standardOutFile">
- #BSUB -e <xsl:value-of select="ns:standardErrorFile"/>
+#BSUB -e "<xsl:value-of select="ns:standardErrorFile"/>"
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+ <xsl:when test="ns:chassisName">
+#BSUB -m c<xsl:value-of select="ns:chassisName"/>
</xsl:when>
</xsl:choose>
<xsl:choose>
<xsl:when test="ns:usedMem">
- #BSUB -R rusage[mem=<xsl:value-of select="ns:usedMem"/>]
+#BSUB -R rusage[mem=<xsl:value-of select="ns:usedMem"/>]
</xsl:when>
</xsl:choose>
@@ -74,7 +88,6 @@
<xsl:value-of select="."/><xsl:text> </xsl:text>
</xsl:for-each>
<xsl:text>
</xsl:text>
- ~/rabbitmq-java-client-bin-3.3.5/runjava.sh com.rabbitmq.examples.SimpleProducer amqp://<xsl:value-of select="ns:callBackIp"/><xsl:text> </xsl:text><xsl:value-of select="ns:userName"/>,<xsl:value-of select="ns:jobName"/><xsl:text> </xsl:text><xsl:value-of select="$quote"/><xsl:value-of select="$quote"/><xsl:text> </xsl:text><xsl:value-of select="ns:callBackPort"/>
</xsl:template>
</xsl:stylesheet>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/modules/configuration/server/src/main/resources/PBSTemplate.xslt
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/PBSTemplate.xslt b/modules/configuration/server/src/main/resources/PBSTemplate.xslt
index aa3cccc..7676883 100644
--- a/modules/configuration/server/src/main/resources/PBSTemplate.xslt
+++ b/modules/configuration/server/src/main/resources/PBSTemplate.xslt
@@ -96,7 +96,6 @@ cd <xsl:text> </xsl:text><xsl:value-of select="ns:workingDirectory"/><xsl:text
<xsl:value-of select="."/><xsl:text> </xsl:text>
</xsl:for-each>
<xsl:text>
</xsl:text>
-~/rabbitmq-java-client-bin-3.3.5/runjava.sh com.rabbitmq.examples.SimpleProducer amqp://<xsl:value-of select="ns:callBackIp"/><xsl:text> </xsl:text><xsl:value-of select="ns:userName"/>,<xsl:value-of select="ns:jobName"/><xsl:text> </xsl:text><xsl:value-of select="$quote"/><xsl:value-of select="$quote"/><xsl:text> </xsl:text><xsl:value-of select="ns:callBackPort"/>
-</xsl:template>
+R</xsl:template>
</xsl:stylesheet>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/modules/credential-store/credential-store-service/src/main/java/org/apache/airavata/credential/store/client/TestSSLClient.java
----------------------------------------------------------------------
diff --git a/modules/credential-store/credential-store-service/src/main/java/org/apache/airavata/credential/store/client/TestSSLClient.java b/modules/credential-store/credential-store-service/src/main/java/org/apache/airavata/credential/store/client/TestSSLClient.java
index cc5ebb6..1d9f4d9 100644
--- a/modules/credential-store/credential-store-service/src/main/java/org/apache/airavata/credential/store/client/TestSSLClient.java
+++ b/modules/credential-store/credential-store-service/src/main/java/org/apache/airavata/credential/store/client/TestSSLClient.java
@@ -63,7 +63,7 @@ public class TestSSLClient {
TProtocol protocol = new TBinaryProtocol(transport);
CredentialStoreService.Client client = new CredentialStoreService.Client(protocol);
-// testSSHCredential(client);
+ testSSHCredential(client);
testCertificateCredential(client);
transport.close();
} catch (TTransportException e) {
http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/modules/credential-store/credential-store-service/src/test/java/org/apache/airavata/credential/store/store/impl/db/SSHCredentialTest.java
----------------------------------------------------------------------
diff --git a/modules/credential-store/credential-store-service/src/test/java/org/apache/airavata/credential/store/store/impl/db/SSHCredentialTest.java b/modules/credential-store/credential-store-service/src/test/java/org/apache/airavata/credential/store/store/impl/db/SSHCredentialTest.java
index 7f44125..4f72b46 100644
--- a/modules/credential-store/credential-store-service/src/test/java/org/apache/airavata/credential/store/store/impl/db/SSHCredentialTest.java
+++ b/modules/credential-store/credential-store-service/src/test/java/org/apache/airavata/credential/store/store/impl/db/SSHCredentialTest.java
@@ -43,8 +43,8 @@ public class SSHCredentialTest {
String userName = "airavata";
String password = "airavata";
String gatewayId = "default";
- String privateKeyPath = "/Users/chathuri/.ssh/id_dsa";
- String pubKeyPath = "/Users/chathuri/.ssh/id_dsa.pub";
+ String privateKeyPath = "/Users/lginnali/.ssh/id_dsa";
+ String pubKeyPath = "/Users/lginnali/.ssh/id_dsa.pub";
try {
AiravataUtils.setExecutionAsServer();
@@ -67,7 +67,7 @@ public class SSHCredentialTest {
pubKeyStream.close();
sshCredential.setPrivateKey(bFilePub);
sshCredential.setPublicKey(bFilePub);
- sshCredential.setPassphrase("test");
+ sshCredential.setPassphrase("gjtlmiJdas7wph");
writer.writeCredentials(sshCredential);
System.out.println(token);
} catch (ClassNotFoundException e) {
http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/modules/credential-store/credential-store-service/src/test/resources/airavata-server.properties
----------------------------------------------------------------------
diff --git a/modules/credential-store/credential-store-service/src/test/resources/airavata-server.properties b/modules/credential-store/credential-store-service/src/test/resources/airavata-server.properties
new file mode 100644
index 0000000..9bdce87
--- /dev/null
+++ b/modules/credential-store/credential-store-service/src/test/resources/airavata-server.properties
@@ -0,0 +1,254 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+###########################################################################
+#
+# This properties file provides configuration for all Airavata Services:
+# API Server, Registry, Workflow Interpreter, GFac, Orchestrator
+#
+###########################################################################
+
+###########################################################################
+# API Server Registry Configuration
+###########################################################################
+
+#for derby [AiravataJPARegistry]
+registry.jdbc.driver=org.apache.derby.jdbc.ClientDriver
+registry.jdbc.url=jdbc:derby://localhost:1527/persistent_data;create=true;user=airavata;password=airavata
+# MySql database configuration
+#registry.jdbc.driver=com.mysql.jdbc.Driver
+#registry.jdbc.url=jdbc:mysql://localhost:3306/persistent_data
+registry.jdbc.user=airavata
+registry.jdbc.password=airavata
+start.derby.server.mode=true
+validationQuery=SELECT 1 from CONFIGURATION
+jpa.cache.size=5000
+#jpa.connection.properties=MaxActive=10,MaxIdle=5,MinIdle=2,MaxWait=60000,testWhileIdle=true,testOnBorrow=true
+
+# Properties for default user mode
+default.registry.user=admin
+default.registry.password=admin
+default.registry.password.hash.method=SHA
+default.registry.gateway=php_reference_gateway
+
+#ip=127.0.0.1
+
+###########################################################################
+# Application Catalog DB Configuration
+###########################################################################
+#for derby [AiravataJPARegistry]
+appcatalog.jdbc.driver=org.apache.derby.jdbc.ClientDriver
+appcatalog.jdbc.url=jdbc:derby://localhost:1527/app_catalog;create=true;user=airavata;password=airavata
+# MySql database configuration
+#appcatalog.jdbc.driver=com.mysql.jdbc.Driver
+#appcatalog.jdbc.url=jdbc:mysql://localhost:3306/app_catalog
+appcatalog.jdbc.user=airavata
+appcatalog.jdbc.password=airavata
+appcatalog.validationQuery=SELECT 1 from CONFIGURATION
+
+###########################################################################
+# Server module Configuration
+###########################################################################
+
+servers=apiserver,orchestrator,gfac,credentialstore
+#shutdown.trategy=NONE
+shutdown.trategy=SELF_TERMINATE
+
+
+apiserver.server.host=localhost
+apiserver.server.port=8930
+apiserver.server.min.threads=50
+orchestrator.server.host=localhost
+orchestrator.server.port=8940
+gfac.server.host=localhost
+gfac.server.port=8950
+orchestrator.server.min.threads=50
+
+###########################################################################
+# Job Scheduler can send informative email messages to you about the status of your job.
+# Specify a string which consists of either the single character "n" (no mail), or one or more
+# of the characters "a" (send mail when job is aborted), "b" (send mail when job begins),
+# and "e" (send mail when job terminates). The default is "a" if not specified.
+###########################################################################
+
+job.notification.enable=true
+#Provide comma separated email ids as a string if more than one
+job.notification.emailids=
+job.notification.flags=abe
+
+###########################################################################
+# Credential Store module Configuration
+###########################################################################
+start.credential.store=false
+credential.store.keystore.url=/Users/lginnali/Downloads/airavata_sym.jks
+credential.store.keystore.alias=airavata
+credential.store.keystore.password=airavata
+credential.store.jdbc.url=jdbc:derby://localhost:1527/persistent_data;create=true;user=airavata;password=airavata
+credential.store.jdbc.user=airavata
+credential.store.jdbc.password=airavata
+credential.store.jdbc.driver=org.apache.derby.jdbc.ClientDriver
+credential.store.server.host=localhost
+credential.store.server.port=8960
+credentialstore=org.apache.airavata.credential.store.server.CredentialStoreServer
+credential.store.thrift.server.keystore=/Users/chathuri/dev/airavata/credential-store/oa4mp/airavata.jks
+credential.store.thrift.server.keystore.password=airavata
+
+notifier.enabled=false
+#period in milliseconds
+notifier.duration=5000
+
+email.server=smtp.googlemail.com
+email.server.port=465
+email.user=airavata
+email.password=xxx
+email.ssl=true
+email.from=airavata@apache.org
+
+###########################################################################
+# Airavata GFac MyProxy GSI credentials to access Grid Resources.
+###########################################################################
+#
+# Security Configuration used by Airavata Generic Factory Service
+# to interact with Computational Resources.
+#
+gfac.thread.pool.size=50
+airavata.server.thread.pool.size=50
+gfac=org.apache.airavata.gfac.server.GfacServer
+myproxy.server=myproxy.teragrid.org
+myproxy.username=ogce
+myproxy.password=
+myproxy.life=3600
+# XSEDE Trusted certificates can be downloaded from https://software.xsede.org/security/xsede-certs.tar.gz
+trusted.cert.location=/Users/lahirugunathilake/Downloads/certificates
+gfac.passive=true
+# SSH PKI key pair or ssh password can be used SSH based authentication is used.
+# if user specify both password authentication gets the higher preference
+
+################# ---------- For ssh key pair authentication ------------------- ################
+#public.ssh.key=/path to public key for ssh
+#private.ssh.key=/path to private key file for ssh
+#ssh.keypass=passphrase for the private key
+#ssh.username=username for ssh connection
+### Incase of password authentication.
+#ssh.password=Password for ssh connection
+
+
+
+###########################################################################
+# Airavata Workflow Interpreter Configurations
+###########################################################################
+
+#runInThread=true
+#provenance=true
+#provenanceWriterThreadPoolSize=20
+#gfac.embedded=true
+#workflowserver=org.apache.airavata.api.server.WorkflowServer
+
+
+###########################################################################
+# API Server module Configuration
+###########################################################################
+apiserver=org.apache.airavata.api.server.AiravataAPIServer
+
+###########################################################################
+# Workflow Server module Configuration
+###########################################################################
+
+workflowserver=org.apache.airavata.api.server.WorkflowServer
+
+###########################################################################
+# Advance configuration to change service implementations
+###########################################################################
+# If false, disables two phase commit when submitting jobs
+TwoPhase=true
+#
+# Class which implemented HostScheduler interface. It will determine the which host to submit the request
+#
+host.scheduler=org.apache.airavata.gfac.core.scheduler.impl.SimpleHostScheduler
+
+###########################################################################
+# Monitoring module Configuration
+###########################################################################
+
+#This will be the primary monitoring tool which runs in airavata, in future there will be multiple monitoring
+#mechanisms and one would be able to start a monitor
+monitors=org.apache.airavata.gfac.monitor.impl.pull.qstat.QstatMonitor,org.apache.airavata.gfac.monitor.impl.LocalJobMonitor
+
+
+###########################################################################
+# AMQP Notification Configuration
+###########################################################################
+
+
+amqp.notification.enable=1
+
+amqp.broker.host=localhost
+amqp.broker.port=5672
+amqp.broker.username=guest
+amqp.broker.password=guest
+
+amqp.sender=org.apache.airavata.wsmg.client.amqp.rabbitmq.AMQPSenderImpl
+amqp.topic.sender=org.apache.airavata.wsmg.client.amqp.rabbitmq.AMQPTopicSenderImpl
+amqp.broadcast.sender=org.apache.airavata.wsmg.client.amqp.rabbitmq.AMQPBroadcastSenderImpl
+
+#,org.apache.airavata.gfac.monitor.impl.push.amqp.AMQPMonitor
+#This is the amqp related configuration and this lists down the Rabbitmq host, this is an xsede specific configuration
+amqp.hosts=info1.dyn.teragrid.org,info2.dyn.teragrid.org
+proxy.file.path=/Users/lahirugunathilake/Downloads/x509up_u503876
+connection.name=xsede
+#publisher
+activity.listeners=org.apache.airavata.gfac.core.monitor.AiravataJobStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataTaskStatusUpdator,org.apache.airavata.gfac.core.monitor.AiravataWorkflowNodeStatusUpdator,org.apache.airavata.api.server.listener.AiravataExperimentStatusUpdator,org.apache.airavata.gfac.core.monitor.GfacInternalStatusUpdator,org.apache.airavata.workflow.engine.util.ProxyMonitorPublisher
+publish.rabbitmq=false
+status.publisher=org.apache.airavata.messaging.core.impl.RabbitMQStatusPublisher
+task.launch.publisher=org.apache.airavata.messaging.core.impl.RabbitMQTaskLaunchPublisher
+rabbitmq.broker.url=amqp://localhost:5672
+rabbitmq.status.exchange.name=airavata_rabbitmq_exchange
+rabbitmq.task.launch.exchange.name=airavata_task_launch_rabbitmq_exchange
+
+
+###########################################################################
+# Orchestrator module Configuration
+###########################################################################
+
+#job.submitter=org.apache.airavata.orchestrator.core.impl.GFACEmbeddedJobSubmitter
+job.submitter=org.apache.airavata.orchestrator.core.impl.GFACPassiveJobSubmitter
+#job.submitter=org.apache.airavata.orchestrator.core.impl.GFACRPCJobSubmitter
+job.validators=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppDataValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator
+submitter.interval=10000
+threadpool.size=10
+start.submitter=true
+embedded.mode=true
+enable.validation=true
+orchestrator=org.apache.airavata.orchestrator.server.OrchestratorServer
+
+###########################################################################
+# Zookeeper Server Configuration
+###########################################################################
+
+embedded.zk=false
+zookeeper.server.host=localhost
+zookeeper.server.port=2181
+airavata-server=/api-server
+orchestrator-server=/orchestrator-server
+gfac-server=/gfac-server
+gfac-experiments=/gfac-experiments
+gfac-server-name=gfac-node0
+orchestrator-server-name=orch-node0
+airavata-server-name=api-node0
http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/modules/distribution/server/src/main/assembly/bin-assembly.xml
----------------------------------------------------------------------
diff --git a/modules/distribution/server/src/main/assembly/bin-assembly.xml b/modules/distribution/server/src/main/assembly/bin-assembly.xml
index 4f061e4..18ccad6 100644
--- a/modules/distribution/server/src/main/assembly/bin-assembly.xml
+++ b/modules/distribution/server/src/main/assembly/bin-assembly.xml
@@ -127,6 +127,7 @@
<include>gfac-config.xml</include>
<include>PBSTemplate.xslt</include>
<include>SLURMTemplate.xslt</include>
+ <include>LSFTemplate.xslt</include>
<include>SGETemplate.xslt</include>
<include>gsissh.properties</include>
</includes>
http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
index 583ec07..1687462 100644
--- a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
+++ b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
@@ -31,7 +31,6 @@ import org.apache.airavata.common.utils.AiravataZKUtils;
import org.apache.airavata.common.utils.Constants;
import org.apache.airavata.common.utils.MonitorPublisher;
import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.gfac.GFacException;
import org.apache.airavata.gfac.core.cpi.BetterGfacImpl;
import org.apache.airavata.gfac.core.cpi.GFac;
import org.apache.airavata.gfac.core.utils.GFacThreadPoolExecutor;
http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/RequestData.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/RequestData.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/RequestData.java
index 9f45ce6..000ea7d 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/RequestData.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/RequestData.java
@@ -49,6 +49,8 @@ public class RequestData {
private int myProxyLifeTime = DEFAULT_LIFE_TIME;
+
+
public RequestData() {
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java
index 7a432e8..631f59c 100644
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java
@@ -60,6 +60,8 @@ public class GFACGSISSHUtils {
public static final String PBS_JOB_MANAGER = "pbs";
public static final String SLURM_JOB_MANAGER = "slurm";
public static final String SUN_GRID_ENGINE_JOB_MANAGER = "UGE";
+ public static final String LSF_JOB_MANAGER = "lsf";
+
public static int maxClusterCount = 5;
public static Map<String, List<Cluster>> clusters = new HashMap<String, List<Cluster>>();
@@ -138,6 +140,8 @@ public class GFACGSISSHUtils {
jConfig = CommonUtils.getSLURMJobManager(installedParentPath);
} else if (SUN_GRID_ENGINE_JOB_MANAGER.equalsIgnoreCase(jobManager)) {
jConfig = CommonUtils.getSGEJobManager(installedParentPath);
+ }else if(LSF_JOB_MANAGER.equalsIgnoreCase(jobManager)) {
+ jConfig = CommonUtils.getLSFJobManager(installedParentPath);
}
}
pbsCluster = new PBSCluster(serverInfo, tokenizedMyProxyAuthInfo, jConfig);
@@ -177,7 +181,7 @@ public class GFACGSISSHUtils {
if(ServerSettings.getSetting(ServerSettings.JOB_NOTIFICATION_ENABLE).equalsIgnoreCase("true")){
jobDescriptor.setMailOptions(ServerSettings.getSetting(ServerSettings.JOB_NOTIFICATION_FLAGS));
String emailids = ServerSettings.getSetting(ServerSettings.JOB_NOTIFICATION_EMAILIDS);
-
+
if(jobExecutionContext.getTaskData().isSetEmailAddresses()){
List<String> emailList = jobExecutionContext.getTaskData().getEmailAddresses();
String elist = GFacUtils.listToCsv(emailList, ',');
@@ -204,6 +208,7 @@ public class GFACGSISSHUtils {
jobDescriptor.setStandardOutFile(jobExecutionContext.getStandardOutput());
jobDescriptor.setStandardErrorFile(jobExecutionContext.getStandardError());
String computationalProjectAccount = taskData.getTaskScheduling().getComputationalProjectAccount();
+ taskData.getEmailAddresses();
if (computationalProjectAccount == null){
ComputeResourcePreference computeResourcePreference = jobExecutionContext.getApplicationContext().getComputeResourcePreference();
if (computeResourcePreference != null) {
@@ -311,6 +316,9 @@ public class GFACGSISSHUtils {
}
if (taskScheduling.getWallTimeLimit() > 0) {
jobDescriptor.setMaxWallTime(String.valueOf(taskScheduling.getWallTimeLimit()));
+ if(resourceJobManager.getResourceJobManagerType().equals(ResourceJobManagerType.LSF)){
+ jobDescriptor.setMaxWallTimeForLSF(String.valueOf(taskScheduling.getWallTimeLimit()));
+ }
}
if (taskScheduling.getTotalPhysicalMemory() > 0) {
http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java
index a9e08d3..c4d25f5 100644
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java
+++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java
@@ -25,6 +25,7 @@ import org.airavata.appcatalog.cpi.AppCatalogException;
import org.apache.airavata.common.exception.ApplicationSettingsException;
import org.apache.airavata.common.utils.ServerSettings;
import org.apache.airavata.credential.store.credential.impl.ssh.SSHCredential;
+import org.apache.airavata.gfac.Constants;
import org.apache.airavata.gfac.GFacException;
import org.apache.airavata.gfac.RequestData;
import org.apache.airavata.gfac.core.context.JobExecutionContext;
@@ -41,6 +42,7 @@ import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
import org.apache.airavata.gsi.ssh.api.job.JobManagerConfiguration;
import org.apache.airavata.gsi.ssh.impl.GSISSHAbstractCluster;
import org.apache.airavata.gsi.ssh.impl.PBSCluster;
+import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPasswordAuthenticationInfo;
import org.apache.airavata.gsi.ssh.util.CommonUtils;
import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
import org.apache.airavata.model.appcatalog.appinterface.DataType;
@@ -96,15 +98,21 @@ public class GFACSSHUtils {
Cluster pbsCluster = null;
try {
- TokenizedSSHAuthInfo tokenizedSSHAuthInfo = new TokenizedSSHAuthInfo(requestData);
+ AuthenticationInfo tokenizedSSHAuthInfo = new TokenizedSSHAuthInfo(requestData);
String installedParentPath = jobExecutionContext.getResourceJobManager().getJobManagerBinPath();
if (installedParentPath == null) {
installedParentPath = "/";
}
- SSHCredential credentials = tokenizedSSHAuthInfo.getCredentials();// this is just a call to get and set credentials in to this object,data will be used
+ SSHCredential credentials =((TokenizedSSHAuthInfo)tokenizedSSHAuthInfo).getCredentials();// this is just a call to get and set credentials in to this object,data will be used
+ if(credentials.getPrivateKey()==null || credentials.getPublicKey()==null){
+ // now we fall back to username password authentication
+ Properties configurationProperties = ServerSettings.getProperties();
+ tokenizedSSHAuthInfo = new DefaultPasswordAuthenticationInfo(configurationProperties.getProperty(Constants.SSH_PASSWORD));
+ }
serverInfo.setUserName(credentials.getPortalUserName());
jobExecutionContext.getExperiment().setUserName(credentials.getPortalUserName());
+
// inside the pbsCluser object
String key = credentials.getPortalUserName() + jobExecutionContext.getHostName() + serverInfo.getPort();
@@ -147,8 +155,11 @@ public class GFACSSHUtils {
jConfig = CommonUtils.getSLURMJobManager(installedParentPath);
} else if (SUN_GRID_ENGINE_JOB_MANAGER.equalsIgnoreCase(jobManager)) {
jConfig = CommonUtils.getSGEJobManager(installedParentPath);
+ } else if (LSF_JOB_MANAGER.equalsIgnoreCase(jobManager)) {
+ jConfig = CommonUtils.getLSFJobManager(installedParentPath);
}
}
+
pbsCluster = new PBSCluster(serverInfo, tokenizedSSHAuthInfo,jConfig);
List<Cluster> pbsClusters = null;
if (!(clusters.containsKey(key))) {
@@ -322,7 +333,7 @@ public class GFACSSHUtils {
List<String> inputValues = new ArrayList<String>();
MessageContext input = jobExecutionContext.getInMessageContext();
- // sort the inputs first and then build the command List
+ // sort the inputs first and then build the command ListR
Comparator<InputDataObjectType> inputOrderComparator = new Comparator<InputDataObjectType>() {
@Override
public int compare(InputDataObjectType inputDataObjectType, InputDataObjectType t1) {
@@ -411,6 +422,12 @@ public class GFACSSHUtils {
}
if (taskScheduling.getWallTimeLimit() > 0) {
jobDescriptor.setMaxWallTime(String.valueOf(taskScheduling.getWallTimeLimit()));
+ if(resourceJobManager.getResourceJobManagerType().equals(ResourceJobManagerType.LSF)){
+ jobDescriptor.setMaxWallTimeForLSF(String.valueOf(taskScheduling.getWallTimeLimit()));
+ }
+ }
+ if (taskScheduling.getTotalPhysicalMemory() > 0) {
+ jobDescriptor.setUsedMemory(taskScheduling.getTotalPhysicalMemory() + "");
}
} else {
logger.error("Task scheduling cannot be null at this point..");
http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobManagerConfiguration.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobManagerConfiguration.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobManagerConfiguration.java
index f68ba29..85a843e 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobManagerConfiguration.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobManagerConfiguration.java
@@ -37,6 +37,13 @@ public interface JobManagerConfiguration {
public RawCommandInfo getSubmitCommand(String workingDirectory, String pbsFilePath);
public OutputParser getParser();
-
+
public String getInstalledPath();
+
+ public String getBaseCancelCommand();
+
+ public String getBaseMonitorCommand();
+
+ public String getBaseSubmitCommand();
+
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/LSFJobConfiguration.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/LSFJobConfiguration.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/LSFJobConfiguration.java
index 018d49d..46fe9ad 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/LSFJobConfiguration.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/LSFJobConfiguration.java
@@ -80,7 +80,7 @@ public class LSFJobConfiguration implements JobManagerConfiguration {
@Override
public RawCommandInfo getSubmitCommand(String workingDirectory, String pbsFilePath) {
- return new RawCommandInfo(this.installedPath + "bsub <" +
+ return new RawCommandInfo(this.installedPath + "bsub < " +
workingDirectory + File.separator + FilenameUtils.getName(pbsFilePath));
}
@@ -97,4 +97,20 @@ public class LSFJobConfiguration implements JobManagerConfiguration {
public String getInstalledPath() {
return installedPath;
}
+
+
+ @Override
+ public String getBaseCancelCommand() {
+ return "bkill";
+ }
+
+ @Override
+ public String getBaseMonitorCommand() {
+ return "bjobs";
+ }
+
+ @Override
+ public String getBaseSubmitCommand() {
+ return "bsub";
+ }
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/LSFOutputParser.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/LSFOutputParser.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/LSFOutputParser.java
index 5490ff6..bd02e37 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/LSFOutputParser.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/LSFOutputParser.java
@@ -57,6 +57,7 @@ public class LSFOutputParser implements OutputParser {
@Override
public void parseJobStatuses(String userName, Map<String, JobStatus> statusMap, String rawOutput) throws SSHApiException {
logger.debug(rawOutput);
+
String[] info = rawOutput.split("\n");
// int lastStop = 0;
for (String jobID : statusMap.keySet()) {
http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/PBSJobConfiguration.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/PBSJobConfiguration.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/PBSJobConfiguration.java
index 18ea772..e935dfb 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/PBSJobConfiguration.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/PBSJobConfiguration.java
@@ -95,4 +95,19 @@ public class PBSJobConfiguration implements JobManagerConfiguration {
public RawCommandInfo getUserBasedMonitorCommand(String userName) {
return new RawCommandInfo(this.installedPath + "qstat -u " + userName);
}
+
+ @Override
+ public String getBaseCancelCommand() {
+ return "qdel";
+ }
+
+ @Override
+ public String getBaseMonitorCommand() {
+ return "qstat";
+ }
+
+ @Override
+ public String getBaseSubmitCommand() {
+ return "qsub ";
+ }
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/SlurmJobConfiguration.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/SlurmJobConfiguration.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/SlurmJobConfiguration.java
index 0b774ed..807ac42 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/SlurmJobConfiguration.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/SlurmJobConfiguration.java
@@ -94,4 +94,19 @@ public class SlurmJobConfiguration implements JobManagerConfiguration{
public RawCommandInfo getUserBasedMonitorCommand(String userName) {
return new RawCommandInfo(this.installedPath + "squeue -u " + userName);
}
+
+ @Override
+ public String getBaseCancelCommand() {
+ return "scancel";
+ }
+
+ @Override
+ public String getBaseMonitorCommand() {
+ return "squeue";
+ }
+
+ @Override
+ public String getBaseSubmitCommand() {
+ return "sbatch";
+ }
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
index dd7f2d9..0420cff 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
@@ -241,7 +241,7 @@ public class GSISSHAbstractCluster implements Cluster {
StandardOutReader stdOutReader = new StandardOutReader();
CommandExecutor.executeCommand(rawCommandInfo, this.getSession(), stdOutReader);
- String outputifAvailable = getOutputifAvailable(stdOutReader, "Error reading output of job submission",rawCommandInfo.getBaseCommand(jobManagerConfiguration.getInstalledPath()));
+ String outputifAvailable = getOutputifAvailable(stdOutReader, "Error reading output of job submission",jobManagerConfiguration.getBaseCancelCommand());
// this might not be the case for all teh resources, if so Cluster implementation can override this method
// because here after cancelling we try to get the job description and return it back
try {
@@ -274,7 +274,7 @@ public class GSISSHAbstractCluster implements Cluster {
//Check whether pbs submission is successful or not, if it failed throw and exception in submitJob method
// with the error thrown in qsub command
//
- String outputifAvailable = getOutputifAvailable(standardOutReader,"Error reading output of job submission",rawCommandInfo.getBaseCommand(jobManagerConfiguration.getInstalledPath()));
+ String outputifAvailable = getOutputifAvailable(standardOutReader,"Error reading output of job submission",jobManagerConfiguration.getBaseSubmitCommand());
OutputParser outputParser = jobManagerConfiguration.getParser();
return outputParser.parseJobSubmission(outputifAvailable);
}
@@ -411,7 +411,7 @@ public class GSISSHAbstractCluster implements Cluster {
RawCommandInfo rawCommandInfo = jobManagerConfiguration.getMonitorCommand(jobID);
StandardOutReader stdOutReader = new StandardOutReader();
CommandExecutor.executeCommand(rawCommandInfo, this.getSession(), stdOutReader);
- String result = getOutputifAvailable(stdOutReader, "Error getting job information from the resource !",rawCommandInfo.getBaseCommand(jobManagerConfiguration.getInstalledPath()));
+ String result = getOutputifAvailable(stdOutReader, "Error getting job information from the resource !",jobManagerConfiguration.getBaseMonitorCommand());
JobDescriptor jobDescriptor = new JobDescriptor();
jobManagerConfiguration.getParser().parseSingleJob(jobDescriptor, result);
return jobDescriptor;
@@ -421,7 +421,7 @@ public class GSISSHAbstractCluster implements Cluster {
RawCommandInfo rawCommandInfo = jobManagerConfiguration.getMonitorCommand(jobID);
StandardOutReader stdOutReader = new StandardOutReader();
CommandExecutor.executeCommand(rawCommandInfo, this.getSession(), stdOutReader);
- String result = getOutputifAvailable(stdOutReader, "Error getting job information from the resource !", rawCommandInfo.getBaseCommand(jobManagerConfiguration.getInstalledPath()));
+ String result = getOutputifAvailable(stdOutReader, "Error getting job information from the resource !", jobManagerConfiguration.getBaseMonitorCommand());
return jobManagerConfiguration.getParser().parseJobStatus(jobID, result);
}
@@ -638,7 +638,7 @@ public class GSISSHAbstractCluster implements Cluster {
}
}
}
- String result = getOutputifAvailable(stdOutReader, "Error getting job information from the resource !", rawCommandInfo.getBaseCommand(jobManagerConfiguration.getInstalledPath()));
+ String result = getOutputifAvailable(stdOutReader, "Error getting job information from the resource !", jobManagerConfiguration.getBaseMonitorCommand());
jobManagerConfiguration.getParser().parseJobStatuses(userName, jobIDs, result);
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/RawCommandInfo.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/RawCommandInfo.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/RawCommandInfo.java
index ac88fe7..0e9d16e 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/RawCommandInfo.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/RawCommandInfo.java
@@ -48,13 +48,7 @@ public class RawCommandInfo implements CommandInfo {
public String getRawCommand() {
return rawCommand;
}
- /*
- * To get command without the path and other parameters. This is required to read errors
- */
- public String getBaseCommand(String installPath) {
- return rawCommand.substring(rawCommand.lastIndexOf(installPath)+1, rawCommand.indexOf(" "));
- }
-
+
public void setRawCommand(String rawCommand) {
this.rawCommand = rawCommand;
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/5ff650f4/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTestWithMyProxyAuth.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTestWithMyProxyAuth.java b/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTestWithMyProxyAuth.java
index 5a12723..0ca7546 100644
--- a/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTestWithMyProxyAuth.java
+++ b/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTestWithMyProxyAuth.java
@@ -26,6 +26,7 @@ import org.apache.airavata.gsi.ssh.api.*;
import org.apache.airavata.gsi.ssh.api.authentication.GSIAuthenticationInfo;
import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
import org.apache.airavata.gsi.ssh.config.ConfigReader;
+import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPasswordAuthenticationInfo;
import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo;
import org.apache.airavata.gsi.ssh.util.CommonUtils;
import org.slf4j.Logger;
@@ -49,17 +50,13 @@ public class DefaultSSHApiTestWithMyProxyAuth {
public static void main(String[]ars){
- String myProxyUserName = "ogce";
- String myProxyPassword = "OGCE@xsede14";
- String certificateLocation = "/Users/raminder/.globus/certificates";
+ String myProxyUserName = "lg11w";
-
- GSIAuthenticationInfo authenticationInfo
- = new MyProxyAuthenticationInfo(myProxyUserName, myProxyPassword, "myproxy.teragrid.org",
- 7512, 17280000, certificateLocation);
+ DefaultPasswordAuthenticationInfo authenticationInfo
+ = new DefaultPasswordAuthenticationInfo("");
// Create command
- CommandInfo commandInfo = new RawCommandInfo("/bin/ls");
+ CommandInfo commandInfo = new RawCommandInfo("source /etc/bashrc; bsub </home/lg11w/mywork/sshEchoExperiment_9d267072-ca65-4ca8-847a-cd3d130f6050/366787899.lsf");
// Server info
//Stampede
@@ -68,7 +65,7 @@ public class DefaultSSHApiTestWithMyProxyAuth {
// ServerInfo serverInfo = new ServerInfo(myProxyUserName, "trestles.sdsc.xsede.org", 22);
//Lonestar
- ServerInfo serverInfo = new ServerInfo(myProxyUserName, "lonestar.tacc.utexas.edu", 22);
+ ServerInfo serverInfo = new ServerInfo(myProxyUserName, "ghpcc06.umassrc.org", 22);
// Output
CommandOutput commandOutput = new SystemCommandOutput();