You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ch...@apache.org on 2016/02/08 17:48:52 UTC
[10/50] [abbrv] airavata git commit: ignoring empty output files when
output staging
ignoring empty output files when output staging
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/462843a4
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/462843a4
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/462843a4
Branch: refs/heads/master
Commit: 462843a47c276806a57bd197400f0664eef13dcd
Parents: d79055a
Author: Chathuri Wimalasena <ch...@apache.org>
Authored: Wed Jan 20 14:44:27 2016 -0500
Committer: Chathuri Wimalasena <ch...@apache.org>
Committed: Wed Jan 20 14:44:27 2016 -0500
----------------------------------------------------------------------
.../apache/airavata/gfac/core/cluster/RemoteCluster.java | 2 +-
.../org/apache/airavata/gfac/impl/BESRemoteCluster.java | 2 +-
.../org/apache/airavata/gfac/impl/HPCRemoteCluster.java | 6 +++---
.../apache/airavata/gfac/impl/LocalRemoteCluster.java | 2 +-
.../java/org/apache/airavata/gfac/impl/SSHUtils.java | 4 ++--
.../apache/airavata/gfac/impl/task/SCPDataStageTask.java | 11 ++---------
.../apache/airavata/gfac/impl/task/utils/StreamData.java | 2 +-
7 files changed, 11 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java
index 9e4544c..f50d660 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java
@@ -70,7 +70,7 @@ public interface RemoteCluster { // FIXME: replace SSHApiException with suitable
* @param inOrOut direction to file transfer , to the remote cluster(DIRECTION.IN) or from the remote cluster(DIRECTION.OUT)
*
*/
- public void scpThirdParty(String sourceFile, String destinationFile ,Session session , DIRECTION inOrOut) throws SSHApiException;
+ public void scpThirdParty(String sourceFile, String destinationFile ,Session session , DIRECTION inOrOut, boolean ignoreEmptyFile) throws SSHApiException;
/**
* This will create directories in computing resources
http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java
index 1e41eda..0dd69fe 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java
@@ -52,7 +52,7 @@ public class BESRemoteCluster extends AbstractRemoteCluster{
}
@Override
- public void scpThirdParty(String sourceFile, String destinationFile, Session session, DIRECTION inOrOut) throws SSHApiException {
+ public void scpThirdParty(String sourceFile, String destinationFile, Session session, DIRECTION inOrOut, boolean ignoreEmptyFile) throws SSHApiException {
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
index 022c8bc..d4988b5 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
@@ -165,16 +165,16 @@ public class HPCRemoteCluster extends AbstractRemoteCluster{
}
@Override
- public void scpThirdParty(String sourceFile, String destinationFile, Session clientSession, DIRECTION direction) throws SSHApiException {
+ public void scpThirdParty(String sourceFile, String destinationFile, Session clientSession, DIRECTION direction, boolean ignoreEmptyFile) throws SSHApiException {
try {
if(!session.isConnected()){
session = getOpenSession();
}
log.info("Transferring from:" + sourceFile + " To: " + destinationFile);
if (direction == DIRECTION.TO) {
- SSHUtils.scpThirdParty(sourceFile, clientSession, destinationFile, session);
+ SSHUtils.scpThirdParty(sourceFile, clientSession, destinationFile, session, ignoreEmptyFile);
} else {
- SSHUtils.scpThirdParty(sourceFile, session, destinationFile, clientSession);
+ SSHUtils.scpThirdParty(sourceFile, session, destinationFile, clientSession, ignoreEmptyFile);
}
} catch (IOException | JSchException e) {
throw new SSHApiException("Failed scp file:" + sourceFile + " to remote file "
http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java
index d1aa6e0..1bab250 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java
@@ -71,7 +71,7 @@ public class LocalRemoteCluster extends AbstractRemoteCluster {
}
@Override
- public void scpThirdParty(String sourceFile, String destinationFile, Session session, DIRECTION inOrOut) throws SSHApiException {
+ public void scpThirdParty(String sourceFile, String destinationFile, Session session, DIRECTION inOrOut, boolean ignoreEmptyFile) throws SSHApiException {
throw new UnsupportedOperationException("Scp third party is not support with LocalRemoteCluster");
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/SSHUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/SSHUtils.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/SSHUtils.java
index 2281ebc..a8e1017 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/SSHUtils.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/SSHUtils.java
@@ -272,7 +272,7 @@ public class SSHUtils {
* @param destinationSession JSch Session for target
* @return returns the final local file path of the new file came from the remote resource
*/
- public static void scpThirdParty(String sourceFile, Session sourceSession, String destinationFile, Session destinationSession) throws
+ public static void scpThirdParty(String sourceFile, Session sourceSession, String destinationFile, Session destinationSession, boolean ignoreEmptyFile) throws
IOException, JSchException {
OutputStream sout = null;
InputStream sin = null;
@@ -346,7 +346,7 @@ public class SSHUtils {
break;
}
}
- if (fileSize == 0L){
+ if (fileSize == 0L && !ignoreEmptyFile){
String error = "Input file is empty...";
log.error(error);
throw new JSchException(error);
http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java
index 605b972..71f689a 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java
@@ -25,16 +25,10 @@ import com.jcraft.jsch.Session;
import org.apache.airavata.common.exception.AiravataException;
import org.apache.airavata.common.exception.ApplicationSettingsException;
import org.apache.airavata.common.utils.ThriftUtils;
-import org.apache.airavata.credential.store.credential.Credential;
-import org.apache.airavata.credential.store.credential.impl.ssh.SSHCredential;
-import org.apache.airavata.credential.store.store.CredentialReader;
-import org.apache.airavata.credential.store.store.CredentialStoreException;
import org.apache.airavata.gfac.core.GFacException;
import org.apache.airavata.gfac.core.GFacUtils;
import org.apache.airavata.gfac.core.SSHApiException;
import org.apache.airavata.gfac.core.authentication.AuthenticationInfo;
-import org.apache.airavata.gfac.core.authentication.SSHKeyAuthentication;
-import org.apache.airavata.gfac.core.authentication.SSHPasswordAuthentication;
import org.apache.airavata.gfac.core.cluster.CommandInfo;
import org.apache.airavata.gfac.core.cluster.RawCommandInfo;
import org.apache.airavata.gfac.core.cluster.RemoteCluster;
@@ -60,7 +54,6 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -258,7 +251,7 @@ public class SCPDataStageTask implements Task {
* scp third party file transfer 'to' compute resource.
*/
taskContext.getParentProcessContext().getDataMovementRemoteCluster().scpThirdParty(sourceURI.getPath(),
- destinationURI.getPath(), sshSession, RemoteCluster.DIRECTION.TO);
+ destinationURI.getPath(), sshSession, RemoteCluster.DIRECTION.TO, false);
}
private void outputDataStaging(TaskContext taskContext, Session sshSession, URI sourceURI, URI destinationURI)
@@ -268,7 +261,7 @@ public class SCPDataStageTask implements Task {
* scp third party file transfer 'from' comute resource.
*/
taskContext.getParentProcessContext().getDataMovementRemoteCluster().scpThirdParty(sourceURI.getPath(),
- destinationURI.getPath(), sshSession, RemoteCluster.DIRECTION.FROM);
+ destinationURI.getPath(), sshSession, RemoteCluster.DIRECTION.FROM, true);
// update output locations
GFacUtils.saveExperimentOutput(taskContext.getParentProcessContext(), taskContext.getProcessOutput().getName(), destinationURI.getPath());
GFacUtils.saveProcessOutput(taskContext.getParentProcessContext(), taskContext.getProcessOutput().getName(), destinationURI.getPath());
http://git-wip-us.apache.org/repos/asf/airavata/blob/462843a4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/StreamData.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/StreamData.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/StreamData.java
index d89c7d3..76b678a 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/StreamData.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/StreamData.java
@@ -158,7 +158,7 @@ public class StreamData extends TimerTask {
* scp third party file transfer 'from' comute resource.
*/
taskContext.getParentProcessContext().getDataMovementRemoteCluster().scpThirdParty(sourceURI.getPath(),
- destinationURI.getPath(), sshSession, RemoteCluster.DIRECTION.FROM);
+ destinationURI.getPath(), sshSession, RemoteCluster.DIRECTION.FROM, true);
// update output locations
GFacUtils.saveExperimentOutput(taskContext.getParentProcessContext(), taskContext.getProcessOutput().getName(), destinationURI.getPath());
GFacUtils.saveProcessOutput(taskContext.getParentProcessContext(), taskContext.getProcessOutput().getName(), destinationURI.getPath());