You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2014/03/07 04:29:04 UTC
[1/2] git commit: More changes to slurm implementation - AIRAVATA-1057
Repository: airavata
Updated Branches:
refs/heads/master 532f2691e -> c57378bda
More changes to slurm implementation - AIRAVATA-1057
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/0315de48
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/0315de48
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/0315de48
Branch: refs/heads/master
Commit: 0315de482a079dc97d1a128f08d554629ad47721
Parents: 532f269
Author: lahiru <la...@apache.org>
Authored: Thu Mar 6 22:27:38 2014 -0500
Committer: lahiru <la...@apache.org>
Committed: Thu Mar 6 22:27:38 2014 -0500
----------------------------------------------------------------------
.../client/samples/CreateLaunchExperiment.java | 12 ++--
.../airavata/job/monitor/AMQPMonitorTest.java | 4 +-
.../airavata/job/monitor/QstatMonitorTest.java | 3 +-
.../src/main/assembly/bin-assembly.xml | 1 +
.../src/main/resources/SLURMTemplate.xslt | 69 ++++++++++++++++++++
.../org/apache/airavata/gfac/cpi/GFacImpl.java | 16 +++--
.../core/gfac/services/impl/BigRed2Test.java | 7 +-
.../gfac/services/impl/GSISSHProviderTest.java | 3 +-
8 files changed, 98 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/0315de48/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
index 63184d3..3bad86b 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
@@ -123,7 +123,7 @@ public class CreateLaunchExperiment {
DocumentCreator documentCreator = new DocumentCreator(getAiravataAPI());
documentCreator.createLocalHostDocs();
documentCreator.createGramDocs();
- documentCreator.createGSISSHDocs();
+ documentCreator.createSlurmDocs();
} catch (AiravataAPIInvocationException e) {
logger.error("Unable to create airavata API", e.getMessage());
throw new AiravataAPIInvocationException(e);
@@ -165,11 +165,13 @@ public class CreateLaunchExperiment {
output.setValue("");
exOut.add(output);
- Experiment simpleExperiment = ExperimentModelUtil.createSimpleExperiment("project1", "admin", "echoExperiment", "SimpleEcho2", "SimpleEcho2", exInputs);
+ Experiment simpleExperiment =
+ ExperimentModelUtil.createSimpleExperiment("project1", "admin", "echoExperiment", "SimpleEcho2", "SimpleEcho2", exInputs);
simpleExperiment.setExperimentOutputs(exOut);
- ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling("trestles.sdsc.edu", 1, 1, 1, "normal", 0, 0, 1, "sds128");
- scheduling.setResourceHostId("gsissh-trestles");
+ ComputationalResourceScheduling scheduling =
+ ExperimentModelUtil.createComputationResourceScheduling("stampede.tacc.xsede.org", 1, 1, 1, "normal", 0, 0, 1, "TG-STA110014S");
+ scheduling.setResourceHostId("stampede-host");
UserConfigurationData userConfigurationData = new UserConfigurationData();
userConfigurationData.setAiravataAutoSchedule(false);
userConfigurationData.setOverrideManualScheduledParams(false);
@@ -192,7 +194,7 @@ public class CreateLaunchExperiment {
}
public static void launchExperiment (Airavata.Client client, String expId)
- throws ExperimentNotFoundException, AiravataSystemException, InvalidRequestException,AiravataClientException, TException{
+ throws TException{
try {
client.launchExperiment(expId, "testToken");
} catch (ExperimentNotFoundException e) {
http://git-wip-us.apache.org/repos/asf/airavata/blob/0315de48/modules/airavata-job-monitor/src/test/java/org/apache/airavata/job/monitor/AMQPMonitorTest.java
----------------------------------------------------------------------
diff --git a/modules/airavata-job-monitor/src/test/java/org/apache/airavata/job/monitor/AMQPMonitorTest.java b/modules/airavata-job-monitor/src/test/java/org/apache/airavata/job/monitor/AMQPMonitorTest.java
index 8f920c1..a6ef7ea 100644
--- a/modules/airavata-job-monitor/src/test/java/org/apache/airavata/job/monitor/AMQPMonitorTest.java
+++ b/modules/airavata-job-monitor/src/test/java/org/apache/airavata/job/monitor/AMQPMonitorTest.java
@@ -31,6 +31,7 @@ import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo
import org.apache.airavata.job.monitor.exception.AiravataMonitorException;
import org.apache.airavata.job.monitor.impl.push.amqp.AMQPMonitor;
import org.apache.airavata.schemas.gfac.GsisshHostType;
+import org.apache.axiom.om.util.CommonUtils;
import org.junit.Before;
import org.junit.Test;
@@ -93,7 +94,8 @@ public class AMQPMonitorTest {
ServerInfo serverInfo = new ServerInfo("ogce", "trestles.sdsc.edu");
- Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, "/opt/torque/bin/");
+ Cluster pbsCluster = new
+ PBSCluster(serverInfo, authenticationInfo, org.apache.airavata.gsi.ssh.util.CommonUtils.getPBSJobManager("/opt/torque/bin/"));
// Execute command
http://git-wip-us.apache.org/repos/asf/airavata/blob/0315de48/modules/airavata-job-monitor/src/test/java/org/apache/airavata/job/monitor/QstatMonitorTest.java
----------------------------------------------------------------------
diff --git a/modules/airavata-job-monitor/src/test/java/org/apache/airavata/job/monitor/QstatMonitorTest.java b/modules/airavata-job-monitor/src/test/java/org/apache/airavata/job/monitor/QstatMonitorTest.java
index 6b1074a..aabc0d7 100644
--- a/modules/airavata-job-monitor/src/test/java/org/apache/airavata/job/monitor/QstatMonitorTest.java
+++ b/modules/airavata-job-monitor/src/test/java/org/apache/airavata/job/monitor/QstatMonitorTest.java
@@ -28,6 +28,7 @@ import org.apache.airavata.gsi.ssh.api.authentication.GSIAuthenticationInfo;
import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
import org.apache.airavata.gsi.ssh.impl.PBSCluster;
import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo;
+import org.apache.airavata.gsi.ssh.util.CommonUtils;
import org.apache.airavata.job.monitor.exception.AiravataMonitorException;
import org.apache.airavata.job.monitor.impl.pull.qstat.QstatMonitor;
import org.apache.airavata.schemas.gfac.GsisshHostType;
@@ -92,7 +93,7 @@ public class QstatMonitorTest {
ServerInfo serverInfo = new ServerInfo("ogce", hostDescription.getType().getHostAddress());
- Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, "/opt/torque/bin/");
+ Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, CommonUtils.getPBSJobManager("/opt/torque/bin/"));
// Execute command
http://git-wip-us.apache.org/repos/asf/airavata/blob/0315de48/modules/distribution/airavata-server/src/main/assembly/bin-assembly.xml
----------------------------------------------------------------------
diff --git a/modules/distribution/airavata-server/src/main/assembly/bin-assembly.xml b/modules/distribution/airavata-server/src/main/assembly/bin-assembly.xml
index 9b9f67c..8509c77 100644
--- a/modules/distribution/airavata-server/src/main/assembly/bin-assembly.xml
+++ b/modules/distribution/airavata-server/src/main/assembly/bin-assembly.xml
@@ -126,6 +126,7 @@
<include>provenance.sql</include>
<include>gfac-config.xml</include>
<include>PBSTemplate.xslt</include>
+ <include>SLURMTemplate.xslt</include>
<include>gsissh.properties</include>
</includes>
</fileSet>
http://git-wip-us.apache.org/repos/asf/airavata/blob/0315de48/modules/distribution/airavata-server/src/main/resources/SLURMTemplate.xslt
----------------------------------------------------------------------
diff --git a/modules/distribution/airavata-server/src/main/resources/SLURMTemplate.xslt b/modules/distribution/airavata-server/src/main/resources/SLURMTemplate.xslt
new file mode 100644
index 0000000..169dbbf
--- /dev/null
+++ b/modules/distribution/airavata-server/src/main/resources/SLURMTemplate.xslt
@@ -0,0 +1,69 @@
+<!--Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to you under
+ the Apache License, Version 2.0 (theÏ "License"); you may not use this file except in compliance with the License. You may
+ obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to
+ in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+ ANY ~ KIND, either express or implied. See the License for the specific language governing permissions and limitations under
+ the License. -->
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:ns="http://airavata.apache.org/gsi/ssh/2012/12">
+<xsl:output method="text" />
+<xsl:template match="/ns:JobDescriptor">
+<xsl:choose>
+<xsl:when test="ns:shellName">
+#!<xsl:value-of select="ns:shellName"/>
+ </xsl:when>
+ </xsl:choose>
+<xsl:choose>
+ <xsl:when test="ns:queueName">
+#SBATCH -p <xsl:value-of select="ns:queueName"/>
+ </xsl:when>
+ </xsl:choose>
+<xsl:choose>
+ <xsl:when test="ns:nodes">
+#SBATCH -n <xsl:value-of select="ns:nodes"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+ <xsl:when test="ns:mailAddress">
+#SBATCH -mail-user=<xsl:value-of select="ns:mailAddress"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+ <xsl:when test="ns:mailType">
+#SBATCH -mail-type=<xsl:value-of select="ns:mailType"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+<xsl:when test="ns:acountString">
+#SBATCH -A <xsl:value-of select="ns:acountString"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+ <xsl:when test="ns:maxWallTime">
+#SBATCH -t <xsl:value-of select="ns:maxWallTime"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+ <xsl:when test="ns:jobName">
+#SBATCH -J <xsl:value-of select="ns:jobName"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+ <xsl:when test="ns:standardOutFile">
+#SBATCH -o <xsl:value-of select="ns:standardOutFile"/>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:choose>
+ <xsl:when test="ns:standardOutFile">
+#SBATCH -e <xsl:value-of select="ns:standardErrorFile"/>
+ </xsl:when>
+ </xsl:choose>
+cd <xsl:text> </xsl:text><xsl:value-of select="ns:workingDirectory"/><xsl:text>
</xsl:text>
+ <xsl:choose><xsl:when test="ns:jobSubmitterCommand">
+<xsl:value-of select="ns:jobSubmitterCommand"/><xsl:text> </xsl:text></xsl:when></xsl:choose><xsl:value-of select="ns:executablePath"/><xsl:text> </xsl:text>
+<xsl:for-each select="ns:inputs/ns:input">
+ <xsl:value-of select="."/><xsl:text> </xsl:text>
+ </xsl:for-each>
+</xsl:template>
+
+</xsl:stylesheet>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/0315de48/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/cpi/GFacImpl.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/cpi/GFacImpl.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/cpi/GFacImpl.java
index 8988554..47dee57 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/cpi/GFacImpl.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/cpi/GFacImpl.java
@@ -375,7 +375,9 @@ public class GFacImpl implements GFac {
GSIAuthenticationInfo authenticationInfo
= new MyProxyAuthenticationInfo(requestData.getMyProxyUserName(), requestData.getMyProxyPassword(), requestData.getMyProxyServerUrl(),
requestData.getMyProxyPort(), requestData.getMyProxyLifeTime(), System.getProperty(Constants.TRUSTED_CERTIFICATE_SYSTEM_PROPERTY));
- ServerInfo serverInfo = new ServerInfo(requestData.getMyProxyUserName(), registeredHost.getType().getHostAddress());
+ GsisshHostType gsisshHostType = (GsisshHostType)registeredHost.getType();
+ ServerInfo serverInfo = new ServerInfo(requestData.getMyProxyUserName(), registeredHost.getType().getHostAddress(),
+ gsisshHostType.getPort());
Cluster pbsCluster = null;
try {
@@ -383,17 +385,17 @@ public class GFacImpl implements GFac {
String installedParentPath = ((HpcApplicationDeploymentType)
jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType()).getInstalledParentPath();
String jobManager = ((GsisshHostType) registeredHost.getType()).getJobManager();
- if(jobManager == null){
- log.error("No Job Manager is configured, so we are picking pbs as the default job manager");
+ if (jobManager == null) {
+ log.error("No Job Manager is configured, so we are picking pbs as the default job manager");
jConfig = CommonUtils.getPBSJobManager(installedParentPath);
- }else{
- if("pbs".equalsIgnoreCase(jobManager)){
+ } else {
+ if ("pbs".equalsIgnoreCase(jobManager)) {
jConfig = CommonUtils.getPBSJobManager(installedParentPath);
- }else if("slurm".equalsIgnoreCase(jobManager)){
+ } else if ("slurm".equalsIgnoreCase(jobManager)) {
jConfig = CommonUtils.getSLURMJobManager(installedParentPath);
}
}
- pbsCluster = new PBSCluster(serverInfo, authenticationInfo,jConfig);
+ pbsCluster = new PBSCluster(serverInfo, authenticationInfo, jConfig);
} catch (SSHApiException e) {
e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/0315de48/modules/gfac/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2Test.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2Test.java b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2Test.java
index bca3b2b..feeefe2 100644
--- a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2Test.java
+++ b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2Test.java
@@ -36,8 +36,11 @@ import org.apache.airavata.gsi.ssh.api.Cluster;
import org.apache.airavata.gsi.ssh.api.SSHApiException;
import org.apache.airavata.gsi.ssh.api.ServerInfo;
import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
+import org.apache.airavata.gsi.ssh.api.job.JobManagerConfiguration;
+import org.apache.airavata.gsi.ssh.api.job.PBSJobConfiguration;
import org.apache.airavata.gsi.ssh.impl.PBSCluster;
import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPasswordAuthenticationInfo;
+import org.apache.airavata.gsi.ssh.util.CommonUtils;
import org.apache.airavata.schemas.gfac.*;
import org.junit.Before;
import org.junit.Test;
@@ -189,8 +192,8 @@ public class BigRed2Test {
Cluster pbsCluster = null;
try {
- pbsCluster = new PBSCluster(serverInfo, authenticationInfo,
- (app.getInstalledParentPath()));
+ JobManagerConfiguration pbsJobManager = CommonUtils.getPBSJobManager(app.getInstalledParentPath());
+ pbsCluster = new PBSCluster(serverInfo, authenticationInfo,pbsJobManager);
} catch (SSHApiException e) {
e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/0315de48/modules/gfac/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTest.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTest.java b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTest.java
index abc41b9..1041540 100644
--- a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTest.java
+++ b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTest.java
@@ -39,6 +39,7 @@ import org.apache.airavata.gsi.ssh.api.ServerInfo;
import org.apache.airavata.gsi.ssh.api.authentication.GSIAuthenticationInfo;
import org.apache.airavata.gsi.ssh.impl.PBSCluster;
import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo;
+import org.apache.airavata.gsi.ssh.util.CommonUtils;
import org.apache.airavata.schemas.gfac.*;
import org.junit.Before;
import org.junit.Test;
@@ -191,7 +192,7 @@ public class GSISSHProviderTest {
ServerInfo serverInfo = new ServerInfo("ogce", "trestles.sdsc.edu");
Cluster pbsCluster = null;
try {
- pbsCluster = new PBSCluster(serverInfo, authenticationInfo, app.getInstalledParentPath());
+ pbsCluster = new PBSCluster(serverInfo, authenticationInfo, CommonUtils.getPBSJobManager(app.getInstalledParentPath()));
} catch (SSHApiException e) {
e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
}
[2/2] git commit: More changes
Posted by la...@apache.org.
More changes
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/c57378bd
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/c57378bd
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/c57378bd
Branch: refs/heads/master
Commit: c57378bda550db91d24bf6c596c7cc3765969895
Parents: 0315de4
Author: lahiru <la...@apache.org>
Authored: Thu Mar 6 22:28:32 2014 -0500
Committer: lahiru <la...@apache.org>
Committed: Thu Mar 6 22:28:32 2014 -0500
----------------------------------------------------------------------
.../java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/c57378bd/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java b/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java
index c339d9d..eb8d1f2 100644
--- a/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java
+++ b/tools/gsissh/src/test/java/org/apache/airavata/gsi/ssh/impl/DefaultSSHApiTest.java
@@ -51,7 +51,7 @@ public class DefaultSSHApiTest {
@BeforeTest
public void setUp() throws Exception {
System.setProperty("myproxy.user", "ogce");
- System.setProperty("myproxy.password", "0Gce3098");
+ System.setProperty("myproxy.password", "");
System.setProperty("basedir", "/Users/lahirugunathilake/Downloads");
System.setProperty("gsi.working.directory", "/home/ogce");
myProxyUserName = System.getProperty("myproxy.user");