You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2014/09/26 06:44:41 UTC
[1/2] git commit: Fixing more issues with push monitoring
Repository: airavata
Updated Branches:
refs/heads/master 0df74e911 -> 72f3524f7
Fixing more issues with push monitoring
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/aaba6d6e
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/aaba6d6e
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/aaba6d6e
Branch: refs/heads/master
Commit: aaba6d6ec687ee132c6166f1ee7efa75b73b0d83
Parents: 21c9a92
Author: lahiru <la...@apache.org>
Authored: Thu Sep 25 17:02:56 2014 -0400
Committer: lahiru <la...@apache.org>
Committed: Thu Sep 25 17:02:56 2014 -0400
----------------------------------------------------------------------
.../airavata/client/samples/CreateLaunchExperiment.java | 10 ++++++----
.../server/src/main/resources/PBSTemplate.xslt | 3 ++-
.../server/src/main/resources/SGETemplate.xslt | 6 +++---
.../server/src/main/resources/SLURMTemplate.xslt | 5 +++--
.../apache/airavata/gfac/server/GfacServerHandler.java | 7 ++++---
.../apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java | 1 +
.../gfac/monitor/impl/pull/qstat/HPCPullMonitor.java | 8 --------
.../monitor/impl/push/amqp/SimpleJobFinishConsumer.java | 4 +++-
.../org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java | 1 +
.../apache/airavata/gsi/ssh/api/job/JobDescriptor.java | 10 ++++++++++
.../src/main/resources/schemas/PBSJobDescriptor.xsd | 1 +
11 files changed, 34 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/aaba6d6e/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
index a3dd336..bbfc873 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperiment.java
@@ -46,7 +46,7 @@ public class CreateLaunchExperiment {
private static final String DEFAULT_USER = "default.registry.user";
private static final String DEFAULT_GATEWAY = "default.registry.gateway";
private static Airavata.Client airavataClient;
- private static String echoAppId = "Echo_c6e6aaac-7d9d-44fc-aba2-63b5100528e8";
+ private static String echoAppId = "Echo_38c5ad74-3a27-46d3-963b-d63f1c37b194";
private static String wrfAppId = "WRF_5f097c9c-7066-49ec-aed7-4e39607b3adc";
private static String amberAppId = "Amber_89906be6-5678-49a6-9d04-a0604fbdef2e";
@@ -56,13 +56,14 @@ public class CreateLaunchExperiment {
private static String br2HostName = "bigred2.uits.iu.edu";
public static void main(String[] args) {
- try {
+ for (int i = 0; i < 1; i++)
+ try {
airavataClient = AiravataClientFactory.createAiravataClient(THRIFT_SERVER_HOST, THRIFT_SERVER_PORT);
System.out.println("API version is " + airavataClient.getAPIVersion());
// registerApplications(); // run this only the first time
// final String expId = createExperimentForSSHHost(airavata);
-// final String expId = createEchoExperimentForTrestles(airavataClient);
+ final String expId = createEchoExperimentForTrestles(airavataClient);
// final String expId = createEchoExperimentForStampede(airavataClient);
// final String expId = createExperimentEchoForLocalHost(airavataClient);
// final String expId = createExperimentWRFTrestles(airavataClient);
@@ -74,7 +75,8 @@ public class CreateLaunchExperiment {
// System.out.println("Experiment ID : " + expId);
// updateExperiment(airavata, expId);
-// launchExperiment(airavataClient, expId);
+ launchExperiment(airavataClient, expId);
+
// System.out.println(expId);
} catch (Exception e) {
logger.error("Error while connecting with server", e.getMessage());
http://git-wip-us.apache.org/repos/asf/airavata/blob/aaba6d6e/modules/configuration/server/src/main/resources/PBSTemplate.xslt
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/PBSTemplate.xslt b/modules/configuration/server/src/main/resources/PBSTemplate.xslt
index 5c266f4..ca02d79 100644
--- a/modules/configuration/server/src/main/resources/PBSTemplate.xslt
+++ b/modules/configuration/server/src/main/resources/PBSTemplate.xslt
@@ -8,6 +8,7 @@
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:ns="http://airavata.apache.org/gsi/ssh/2012/12">
<xsl:output method="text" />
<xsl:template match="/ns:JobDescriptor">
+ <xsl:param name="quote">"</xsl:param>
#! /bin/sh
# PBS batch job submission script generated by Apache Airavata
# <xsl:choose>
@@ -74,7 +75,7 @@ cd <xsl:text> </xsl:text><xsl:value-of select="ns:workingDirectory"/><xsl:text
<xsl:for-each select="ns:postJobCommands/ns:command">
<xsl:value-of select="."/><xsl:text> </xsl:text>
</xsl:for-each>
-~/rabbitmq-java-client-bin-3.3.5/runjava.sh com.rabbitmq.examples.SimpleProducer amqp://<xsl:value-of select="ns:callBackIp"/><xsl:text> </xsl:text><xsl:value-of select="ns:userName"/>,<xsl:value-of select="ns:jobName"/>
+~/rabbitmq-java-client-bin-3.3.5/runjava.sh com.rabbitmq.examples.SimpleProducer amqp://<xsl:value-of select="ns:callBackIp"/><xsl:text> </xsl:text><xsl:value-of select="ns:userName"/>,<xsl:value-of select="ns:jobName"/><xsl:text> </xsl:text><xsl:value-of select="$quote"/><xsl:value-of select="$quote"/><xsl:text> </xsl:text><xsl:value-of select="ns:callBackPort"/>
</xsl:template>
</xsl:stylesheet>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/aaba6d6e/modules/configuration/server/src/main/resources/SGETemplate.xslt
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/SGETemplate.xslt b/modules/configuration/server/src/main/resources/SGETemplate.xslt
index bcf8948..d2bf871 100644
--- a/modules/configuration/server/src/main/resources/SGETemplate.xslt
+++ b/modules/configuration/server/src/main/resources/SGETemplate.xslt
@@ -8,7 +8,8 @@
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:ns="http://airavata.apache.org/gsi/ssh/2012/12">
<xsl:output method="text" />
<xsl:template match="/ns:JobDescriptor">
-#! /bin/bash
+ <xsl:param name="quote">"</xsl:param>
+ #! /bin/bash
# Grid Engine batch job script built by Apache Airavata
# <xsl:choose>
<xsl:when test="ns:shellName">
@@ -73,7 +74,6 @@ cd <xsl:text> </xsl:text><xsl:value-of select="ns:workingDirectory"/><xsl:text
<xsl:for-each select="ns:postJobCommands/ns:command">
<xsl:value-of select="."/><xsl:text> </xsl:text>
</xsl:for-each>
-~/rabbitmq-java-client-bin-3.3.5/runjava.sh com.rabbitmq.examples.SimpleProducer amqp://<xsl:value-of select="ns:callBackIp"/><xsl:text> </xsl:text><xsl:value-of select="ns:userName"/>,<xsl:value-of select="ns:jobName"/>
+ ~/rabbitmq-java-client-bin-3.3.5/runjava.sh com.rabbitmq.examples.SimpleProducer amqp://<xsl:value-of select="ns:callBackIp"/><xsl:text> </xsl:text><xsl:value-of select="ns:userName"/>,<xsl:value-of select="ns:jobName"/><xsl:text> </xsl:text><xsl:value-of select="$quote"/><xsl:value-of select="$quote"/><xsl:text> </xsl:text><xsl:value-of select="ns:callBackPort"/>
</xsl:template>
-
</xsl:stylesheet>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/aaba6d6e/modules/configuration/server/src/main/resources/SLURMTemplate.xslt
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/SLURMTemplate.xslt b/modules/configuration/server/src/main/resources/SLURMTemplate.xslt
index 4444ec2..b494cc0 100644
--- a/modules/configuration/server/src/main/resources/SLURMTemplate.xslt
+++ b/modules/configuration/server/src/main/resources/SLURMTemplate.xslt
@@ -8,7 +8,8 @@
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:ns="http://airavata.apache.org/gsi/ssh/2012/12">
<xsl:output method="text" />
<xsl:template match="/ns:JobDescriptor">
-<xsl:choose>
+ <xsl:param name="quote">"</xsl:param>
+ <xsl:choose>
<xsl:when test="ns:shellName">
#!<xsl:value-of select="ns:shellName"/>
</xsl:when>
@@ -69,7 +70,7 @@ cd <xsl:text> </xsl:text><xsl:value-of select="ns:workingDirectory"/><xsl:text
<xsl:for-each select="ns:inputs/ns:input">
<xsl:value-of select="."/><xsl:text> </xsl:text>
</xsl:for-each>
-~/rabbitmq-java-client-bin-3.3.5/runjava.sh com.rabbitmq.examples.SimpleProducer amqp://<xsl:value-of select="ns:callBackIp"/><xsl:text> </xsl:text><xsl:value-of select="ns:userName"/>,<xsl:value-of select="ns:jobName"/>
+ ~/rabbitmq-java-client-bin-3.3.5/runjava.sh com.rabbitmq.examples.SimpleProducer amqp://<xsl:value-of select="ns:callBackIp"/><xsl:text> </xsl:text><xsl:value-of select="ns:userName"/>,<xsl:value-of select="ns:jobName"/><xsl:text> </xsl:text><xsl:value-of select="$quote"/><xsl:value-of select="$quote"/><xsl:text> </xsl:text><xsl:value-of select="ns:callBackPort"/>
</xsl:template>
</xsl:stylesheet>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/aaba6d6e/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
index f7c3cb8..3d86dff 100644
--- a/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
+++ b/modules/gfac/airavata-gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
@@ -63,7 +63,7 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
private boolean connected = false;
- private static Integer mutex = new Integer(-1);
+ private static Integer mutex = -1;
private MonitorPublisher publisher;
@@ -153,8 +153,9 @@ public class GfacServerHandler implements GfacService.Iface, Watcher{
} else if(state == Event.KeeperState.Expired ||
state == Event.KeeperState.Disconnected){
try {
- zk = new ZooKeeper(AiravataZKUtils.getZKhostPort(),6000,this);
- synchronized(mutex){
+ mutex = -1;
+ zk = new ZooKeeper(AiravataZKUtils.getZKhostPort(), 6000, this);
+ synchronized (mutex) {
mutex.wait(); // waiting for the syncConnected event
}
storeServerConfig();
http://git-wip-us.apache.org/repos/asf/airavata/blob/aaba6d6e/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java
index dcdbb4d..40784df 100644
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java
@@ -115,6 +115,7 @@ public class GFACGSISSHUtils {
JobDescriptor jobDescriptor = new JobDescriptor();
// this is common for any application descriptor
jobDescriptor.setCallBackIp(ServerSettings.getIp());
+ jobDescriptor.setCallBackPort(ServerSettings.getSetting(org.apache.airavata.common.utils.Constants.GFAC_SERVER_PORT, "8950"));
jobDescriptor.setInputDirectory(app.getInputDataDirectory());
jobDescriptor.setOutputDirectory(app.getOutputDataDirectory());
jobDescriptor.setExecutablePath(app.getExecutableLocation());
http://git-wip-us.apache.org/repos/asf/airavata/blob/aaba6d6e/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
index bac27bf..b588894 100644
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
+++ b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java
@@ -243,14 +243,6 @@ public class HPCPullMonitor extends PullMonitor {
" 3 times, so skip this Job from Monitor");
iMonitorID.setLastMonitored(new Timestamp((new Date()).getTime()));
completedJobs.put(iMonitorID.getJobName(), iMonitorID);
- try {
- logger.error("Launching outflow handlers to check output are genereated or not");
- gfac.invokeOutFlowHandlers(iMonitorID.getJobExecutionContext());
- } catch (GFacException e) {
- publisher.publish(new TaskStatusChangeRequest(new TaskIdentity(iMonitorID.getExperimentID(), iMonitorID.getWorkflowNodeID(),
- iMonitorID.getTaskID()), TaskState.FAILED));
- logger.info(e.getLocalizedMessage(), e);
- }
} else {
// Evey
iMonitorID.setLastMonitored(new Timestamp((new Date()).getTime()));
http://git-wip-us.apache.org/repos/asf/airavata/blob/aaba6d6e/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/SimpleJobFinishConsumer.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/SimpleJobFinishConsumer.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/SimpleJobFinishConsumer.java
index 3a6aae7..64f8d21 100644
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/SimpleJobFinishConsumer.java
+++ b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/push/amqp/SimpleJobFinishConsumer.java
@@ -24,6 +24,8 @@ import com.rabbitmq.client.Channel;
import com.rabbitmq.client.Connection;
import com.rabbitmq.client.ConnectionFactory;
import com.rabbitmq.client.QueueingConsumer;
+import org.apache.airavata.common.utils.Constants;
+import org.apache.airavata.common.utils.ServerSettings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,8 +42,8 @@ public class SimpleJobFinishConsumer {
public void listen() {
try {
+ String queueName = ServerSettings.getSetting(Constants.GFAC_SERVER_PORT, "8950");
String uri = "amqp://localhost";
- String queueName = "SimpleQueue";
ConnectionFactory connFactory = new ConnectionFactory();
connFactory.setUri(uri);
http://git-wip-us.apache.org/repos/asf/airavata/blob/aaba6d6e/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java
index d5dd97b..8235725 100644
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java
+++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java
@@ -96,6 +96,7 @@ public class GFACSSHUtils {
JobDescriptor jobDescriptor = new JobDescriptor();
// this is common for any application descriptor
jobDescriptor.setCallBackIp(ServerSettings.getIp());
+ jobDescriptor.setCallBackPort(ServerSettings.getSetting(org.apache.airavata.common.utils.Constants.GFAC_SERVER_PORT, "8950"));
jobDescriptor.setInputDirectory(app.getInputDataDirectory());
jobDescriptor.setOutputDirectory(app.getOutputDataDirectory());
jobDescriptor.setExecutablePath(app.getExecutableLocation());
http://git-wip-us.apache.org/repos/asf/airavata/blob/aaba6d6e/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java
index 3890a09..9a0639b 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java
@@ -399,8 +399,18 @@ public class JobDescriptor {
this.jobDescriptionDocument.getJobDescriptor().setCallBackIp(ip);
}
+ public void setCallBackPort(String ip){
+ this.jobDescriptionDocument.getJobDescriptor().setCallBackPort(ip);
+ }
+
+
public String getCallBackIp(){
return this.jobDescriptionDocument.getJobDescriptor().getCallBackIp();
}
+ public String getCallBackPort(){
+ return this.jobDescriptionDocument.getJobDescriptor().getCallBackPort();
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/aaba6d6e/tools/gsissh/src/main/resources/schemas/PBSJobDescriptor.xsd
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/resources/schemas/PBSJobDescriptor.xsd b/tools/gsissh/src/main/resources/schemas/PBSJobDescriptor.xsd
index a9917bc..7749769 100644
--- a/tools/gsissh/src/main/resources/schemas/PBSJobDescriptor.xsd
+++ b/tools/gsissh/src/main/resources/schemas/PBSJobDescriptor.xsd
@@ -57,6 +57,7 @@
<element name="postJobCommands" type="gsissh:postJobCommands" minOccurs="0" maxOccurs="1"/>
<element name="jobSubmitterCommand" type="xsd:string" minOccurs="0" maxOccurs="1"/>
<element name="callBackIp" type="xsd:string" minOccurs="0" maxOccurs="1"/>
+ <element name="callBackPort" type="xsd:string" minOccurs="0" maxOccurs="1"/>
</sequence>
</complexType>
[2/2] git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/airavata
Posted by la...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/airavata
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/72f3524f
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/72f3524f
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/72f3524f
Branch: refs/heads/master
Commit: 72f3524f722979ef3af4858de7fc4ae20e81a065
Parents: aaba6d6 0df74e9
Author: lahiru <la...@apache.org>
Authored: Fri Sep 26 00:44:28 2014 -0400
Committer: lahiru <la...@apache.org>
Committed: Fri Sep 26 00:44:28 2014 -0400
----------------------------------------------------------------------
.../handler/GSISSHDirectorySetupHandler.java | 29 ++++++++++-------
.../gfac/gsissh/handler/GSISSHInputHandler.java | 5 +++
.../gsissh/handler/GSISSHOutputHandler.java | 34 ++++++++++----------
.../gsissh/provider/impl/GSISSHProvider.java | 7 ----
.../ssh/handler/AdvancedSCPInputHandler.java | 12 +++++++
.../ssh/handler/AdvancedSCPOutputHandler.java | 21 +++++++++++-
.../ssh/handler/SSHDirectorySetupHandler.java | 14 ++++----
.../gfac/ssh/handler/SSHInputHandler.java | 5 +++
.../gfac/ssh/handler/SSHOutputHandler.java | 28 ++++++++--------
9 files changed, 98 insertions(+), 57 deletions(-)
----------------------------------------------------------------------