You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sh...@apache.org on 2015/09/10 22:55:32 UTC
[1/2] airavata git commit: Fixed packet corrupt issue with jsch by
creating new session if previous one is not in connected state. Increased
jobId verification call by 10, trim job command before use
Repository: airavata
Updated Branches:
refs/heads/master 8ea1f0a59 -> 7c498d064
Fixed packet corrupt issue with jsch by creating new session if previous one is not in connected state. Increased jobId verification call by 10, trim job command before use
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/7ad88f70
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/7ad88f70
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/7ad88f70
Branch: refs/heads/master
Commit: 7ad88f70f55790d0227446e08301634bc4e9dd73
Parents: 369b353
Author: Shameera Rathanyaka <sh...@gmail.com>
Authored: Thu Sep 10 16:55:14 2015 -0400
Committer: Shameera Rathanyaka <sh...@gmail.com>
Committed: Thu Sep 10 16:55:14 2015 -0400
----------------------------------------------------------------------
.../airavata/gfac/impl/HPCRemoteCluster.java | 37 +++++++++++---------
.../gfac/impl/job/ForkJobConfiguration.java | 5 +--
.../gfac/impl/job/PBSJobConfiguration.java | 17 ++++-----
.../gfac/impl/job/SlurmJobConfiguration.java | 17 ++++-----
.../gfac/impl/task/SSHJobSubmissionTask.java | 4 +--
5 files changed, 44 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/7ad88f70/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
index 8eae217..f15eb99 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
@@ -67,19 +67,24 @@ public class HPCRemoteCluster extends AbstractRemoteCluster{
jSch = new JSch();
jSch.addIdentity(authentication.getPrivateKeyFilePath(), authentication.getPublicKeyFilePath(),
authentication.getPassphrase().getBytes());
- session = jSch.getSession(serverInfo.getUserName(), serverInfo.getHost(), serverInfo.getPort());
- session.setUserInfo(new DefaultUserInfo(serverInfo.getUserName(), null, authentication.getPassphrase()));
- if (authentication.getStrictHostKeyChecking().equals("yes")) {
- jSch.setKnownHosts(authentication.getKnownHostsFilePath());
- } else {
- session.setConfig("StrictHostKeyChecking","no");
- }
- session.connect(); // 0 connection timeout
+ session = getOpenSession();
} catch (JSchException e) {
throw new AiravataException("JSch initialization error ", e);
}
}
+ private Session getOpenSession() throws JSchException {
+ Session newSession = jSch.getSession(serverInfo.getUserName(), serverInfo.getHost(), serverInfo.getPort());
+ newSession.setUserInfo(new DefaultUserInfo(serverInfo.getUserName(), null, authentication.getPassphrase()));
+ if (authentication.getStrictHostKeyChecking().equals("yes")) {
+ jSch.setKnownHosts(authentication.getKnownHostsFilePath());
+ } else {
+ newSession.setConfig("StrictHostKeyChecking", "no");
+ }
+ newSession.connect(); // 0 connection timeout
+ return newSession;
+ }
+
@Override
public JobSubmissionOutput submitBatchJob(String jobScriptFilePath, String workingDirectory) throws SSHApiException {
JobSubmissionOutput jsoutput = new JobSubmissionOutput();
@@ -102,7 +107,7 @@ public class HPCRemoteCluster extends AbstractRemoteCluster{
while (retry > 0) {
try {
if (!session.isConnected()) {
- session.connect();
+ session = getOpenSession();
}
log.info("Transferring localhost:" + localFile + " to " + serverInfo.getHost() + ":" + remoteFile);
SSHUtils.scpTo(localFile, remoteFile, session);
@@ -111,7 +116,7 @@ public class HPCRemoteCluster extends AbstractRemoteCluster{
retry--;
if (!session.isConnected()) {
try {
- session.connect();
+ session = getOpenSession();
} catch (JSchException e1) {
throw new SSHApiException("JSch Session connection failed", e1);
}
@@ -133,7 +138,7 @@ public class HPCRemoteCluster extends AbstractRemoteCluster{
while(retry>0) {
try {
if (!session.isConnected()) {
- session.connect();
+ session = getOpenSession();
}
log.info("Transferring " + serverInfo.getHost() + ":" + remoteFile + " To localhost:" + localFile);
SSHUtils.scpFrom(remoteFile, localFile, session);
@@ -142,7 +147,7 @@ public class HPCRemoteCluster extends AbstractRemoteCluster{
retry--;
if (!session.isConnected()) {
try {
- session.connect();
+ session = getOpenSession();
} catch (JSchException e1) {
throw new SSHApiException("JSch Session connection failed", e1);
}
@@ -161,7 +166,7 @@ public class HPCRemoteCluster extends AbstractRemoteCluster{
public void scpThirdParty(String remoteFileSource, String remoteFileTarget) throws SSHApiException {
try {
if(!session.isConnected()){
- session.connect();
+ session = getOpenSession();
}
log.info("Transferring from:" + remoteFileSource + " To: " + remoteFileTarget);
SSHUtils.scpThirdParty(remoteFileSource, remoteFileTarget, session);
@@ -175,7 +180,7 @@ public class HPCRemoteCluster extends AbstractRemoteCluster{
public void makeDirectory(String directoryPath) throws SSHApiException {
try {
if (!session.isConnected()) {
- session.connect();
+ session = getOpenSession();
}
log.info("Creating directory: " + serverInfo.getHost() + ":" + directoryPath);
SSHUtils.makeDirectory(directoryPath, session);
@@ -224,7 +229,7 @@ public class HPCRemoteCluster extends AbstractRemoteCluster{
public List<String> listDirectory(String directoryPath) throws SSHApiException {
try {
if (!session.isConnected()) {
- session.connect();
+ session = getOpenSession();
}
log.info("Creating directory: " + serverInfo.getHost() + ":" + directoryPath);
return SSHUtils.listDirectory(directoryPath, session);
@@ -268,7 +273,7 @@ public class HPCRemoteCluster extends AbstractRemoteCluster{
ChannelExec channelExec = null;
try {
if (!session.isConnected()) {
- session.connect();
+ session = getOpenSession();
}
channelExec = ((ChannelExec) session.openChannel("exec"));
channelExec.setCommand(command);
http://git-wip-us.apache.org/repos/asf/airavata/blob/7ad88f70/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/ForkJobConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/ForkJobConfiguration.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/ForkJobConfiguration.java
index d53ce45..15d3165 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/ForkJobConfiguration.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/ForkJobConfiguration.java
@@ -42,6 +42,7 @@ public class ForkJobConfiguration implements JobManagerConfiguration {
this.jobDescriptionTemplateName = jobDescriptionTemplateName;
this.scriptExtension = scriptExtension;
this.parser = parser;
+ installedPath = installedPath.trim();
if (installedPath.endsWith("/")) {
this.installedPath = installedPath;
} else {
@@ -52,7 +53,7 @@ public class ForkJobConfiguration implements JobManagerConfiguration {
@Override
public RawCommandInfo getCancelCommand(String jobID) {
- return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.DELETION) + " " +
+ return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.DELETION).trim() + " " +
jobID);
}
@@ -83,7 +84,7 @@ public class ForkJobConfiguration implements JobManagerConfiguration {
@Override
public RawCommandInfo getSubmitCommand(String workingDirectory, String forkFilePath) {
- return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.SUBMISSION) + " " +
+ return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.SUBMISSION).trim() + " " +
workingDirectory + File.separator + FilenameUtils.getName(forkFilePath));
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/7ad88f70/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/PBSJobConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/PBSJobConfiguration.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/PBSJobConfiguration.java
index a80137f..8d19b56 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/PBSJobConfiguration.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/PBSJobConfiguration.java
@@ -42,6 +42,7 @@ public class PBSJobConfiguration implements JobManagerConfiguration {
this.jobDescriptionTemplateName = jobDescriptionTemplateName;
this.scriptExtension = scriptExtension;
this.parser = parser;
+ installedPath = installedPath.trim();
if (installedPath.endsWith("/")) {
this.installedPath = installedPath;
} else {
@@ -51,7 +52,7 @@ public class PBSJobConfiguration implements JobManagerConfiguration {
}
public RawCommandInfo getCancelCommand(String jobID) {
- return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.DELETION) + " " +
+ return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.DELETION).trim() + " " +
jobID);
}
@@ -64,7 +65,7 @@ public class PBSJobConfiguration implements JobManagerConfiguration {
}
public RawCommandInfo getMonitorCommand(String jobID) {
- return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.JOB_MONITORING)
+ return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.JOB_MONITORING).trim()
+ " -f " + jobID);
}
@@ -73,7 +74,7 @@ public class PBSJobConfiguration implements JobManagerConfiguration {
}
public RawCommandInfo getSubmitCommand(String workingDirectory, String pbsFilePath) {
- return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.SUBMISSION) + " " +
+ return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.SUBMISSION).trim() + " " +
workingDirectory + File.separator + FilenameUtils.getName(pbsFilePath));
}
@@ -94,29 +95,29 @@ public class PBSJobConfiguration implements JobManagerConfiguration {
}
public RawCommandInfo getUserBasedMonitorCommand(String userName) {
- return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.JOB_MONITORING)
+ return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.JOB_MONITORING).trim()
+ " -u " + userName);
}
@Override
public RawCommandInfo getJobIdMonitorCommand(String jobName, String userName) {
// For PBS there is no option to get jobDetails by JobName, so we search with userName
- return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.JOB_MONITORING)
+ return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.JOB_MONITORING).trim()
+ " -u " + userName + " -f | grep \"Job_Name = " + jobName + "\" -B1");
}
@Override
public String getBaseCancelCommand() {
- return jobManagerCommands.get(JobManagerCommand.DELETION);
+ return jobManagerCommands.get(JobManagerCommand.DELETION).trim();
}
@Override
public String getBaseMonitorCommand() {
- return jobManagerCommands.get(JobManagerCommand.JOB_MONITORING);
+ return jobManagerCommands.get(JobManagerCommand.JOB_MONITORING).trim();
}
@Override
public String getBaseSubmitCommand() {
- return jobManagerCommands.get(JobManagerCommand.SUBMISSION);
+ return jobManagerCommands.get(JobManagerCommand.SUBMISSION).trim();
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/7ad88f70/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/SlurmJobConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/SlurmJobConfiguration.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/SlurmJobConfiguration.java
index 0feac74..089a9a3 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/SlurmJobConfiguration.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/SlurmJobConfiguration.java
@@ -42,6 +42,7 @@ public class SlurmJobConfiguration implements JobManagerConfiguration {
this.jobDescriptionTemplateName = jobDescriptionTemplateName;
this.scriptExtension = scriptExtension;
this.parser = parser;
+ installedPath = installedPath.trim();
if (installedPath.endsWith("/")) {
this.installedPath = installedPath;
} else {
@@ -51,7 +52,7 @@ public class SlurmJobConfiguration implements JobManagerConfiguration {
}
public RawCommandInfo getCancelCommand(String jobID) {
- return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.DELETION) + " " + jobID);
+ return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.DELETION).trim() + " " + jobID);
}
public String getJobDescriptionTemplateName() {
@@ -63,7 +64,7 @@ public class SlurmJobConfiguration implements JobManagerConfiguration {
}
public RawCommandInfo getMonitorCommand(String jobID) {
- return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.JOB_MONITORING) + " -j " + jobID);
+ return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.JOB_MONITORING).trim() + " -j " + jobID);
}
public String getScriptExtension() {
@@ -71,7 +72,7 @@ public class SlurmJobConfiguration implements JobManagerConfiguration {
}
public RawCommandInfo getSubmitCommand(String workingDirectory,String pbsFilePath) {
- return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.SUBMISSION) + " " +
+ return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.SUBMISSION).trim() + " " +
workingDirectory + File.separator + FilenameUtils.getName(pbsFilePath));
}
@@ -92,26 +93,26 @@ public class SlurmJobConfiguration implements JobManagerConfiguration {
}
public RawCommandInfo getUserBasedMonitorCommand(String userName) {
- return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.JOB_MONITORING) + " -u " + userName);
+ return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.JOB_MONITORING).trim() + " -u " + userName);
}
@Override
public RawCommandInfo getJobIdMonitorCommand(String jobName, String userName) {
- return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.JOB_MONITORING) + " -n " + jobName + " -u " + userName);
+ return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.JOB_MONITORING).trim() + " -n " + jobName + " -u " + userName);
}
@Override
public String getBaseCancelCommand() {
- return jMCommands.get(JobManagerCommand.DELETION);
+ return jMCommands.get(JobManagerCommand.DELETION).trim();
}
@Override
public String getBaseMonitorCommand() {
- return jMCommands.get(JobManagerCommand.JOB_MONITORING);
+ return jMCommands.get(JobManagerCommand.JOB_MONITORING).trim();
}
@Override
public String getBaseSubmitCommand() {
- return jMCommands.get(JobManagerCommand.SUBMISSION);
+ return jMCommands.get(JobManagerCommand.SUBMISSION).trim();
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/7ad88f70/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SSHJobSubmissionTask.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SSHJobSubmissionTask.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SSHJobSubmissionTask.java
index 1cd068f..f26c4e0 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SSHJobSubmissionTask.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SSHJobSubmissionTask.java
@@ -113,8 +113,8 @@ public class SSHJobSubmissionTask implements JobSubmissionTask {
taskStatus.setReason("Submitted job to compute resource");
break;
}
- log.info("Verify step return invalid jobId, retry verification step in {} secs", verificationTryCount);
- Thread.sleep(verificationTryCount * 1000);
+ log.info("Verify step return invalid jobId, retry verification step in {} secs", verificationTryCount * 10);
+ Thread.sleep(verificationTryCount * 10000);
}
}
[2/2] airavata git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/airavata
Posted by sh...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/airavata
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/7c498d06
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/7c498d06
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/7c498d06
Branch: refs/heads/master
Commit: 7c498d064f4608c890d1124a1ba13d2651624aea
Parents: 7ad88f7 8ea1f0a
Author: Shameera Rathanyaka <sh...@gmail.com>
Authored: Thu Sep 10 16:55:21 2015 -0400
Committer: Shameera Rathanyaka <sh...@gmail.com>
Committed: Thu Sep 10 16:55:21 2015 -0400
----------------------------------------------------------------------
.../catalog/impl/ExperimentRegistry.java | 55 ++++++++++++++------
.../experiment/catalog/resources/Utils.java | 3 ++
2 files changed, 41 insertions(+), 17 deletions(-)
----------------------------------------------------------------------