You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ra...@apache.org on 2014/06/23 23:56:57 UTC
git commit: Made changes to SCP connection creator and add
third-party transfer.AIRAVATA-1300
Repository: airavata
Updated Branches:
refs/heads/master bc20a06d1 -> 64582600a
Made changes to SCP connection creator and add third-party
transfer.AIRAVATA-1300
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/64582600
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/64582600
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/64582600
Branch: refs/heads/master
Commit: 64582600a429fc71fdf6d823a3a83cba0d880578
Parents: bc20a06
Author: raminder <ra...@apache.org>
Authored: Mon Jun 23 17:56:33 2014 -0400
Committer: raminder <ra...@apache.org>
Committed: Mon Jun 23 17:56:33 2014 -0400
----------------------------------------------------------------------
.../gfac/ssh/handler/SSHInputHandler.java | 5 +
.../apache/airavata/gsi/ssh/api/Cluster.java | 8 ++
.../gsi/ssh/impl/GSISSHAbstractCluster.java | 18 ++-
.../apache/airavata/gsi/ssh/util/SSHUtils.java | 117 +++++++++++++++++++
4 files changed, 147 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/64582600/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHInputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHInputHandler.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHInputHandler.java
index b237a81..4c1345d 100644
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHInputHandler.java
+++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHInputHandler.java
@@ -123,10 +123,15 @@ public class SSHInputHandler extends AbstractHandler {
String substring = paramValue.substring(i + 1);
try {
String targetFile = app.getInputDataDirectory() + File.separator + substring;
+ if(paramValue.startsWith("scp:")){
+ paramValue = paramValue.substring(paramValue.indexOf(":") + 1, paramValue.length());
+ cluster.scpThirdParty(paramValue, targetFile);
+ }else{
if(paramValue.startsWith("file")){
paramValue = paramValue.substring(paramValue.indexOf(":") + 1, paramValue.length());
}
cluster.scpTo(targetFile, paramValue);
+ }
return targetFile;
} catch (SSHApiException e) {
throw new GFacHandlerException("Error while input File Staging", e, e.getLocalizedMessage());
http://git-wip-us.apache.org/repos/asf/airavata/blob/64582600/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
index 4454624..b3ecc16 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
@@ -72,6 +72,14 @@ public interface Cluster {
public void scpFrom(String remoteFile, String localFile) throws SSHApiException;
/**
+ * This will copy a remote file in path rFile to local file lFile
+ * @param remoteFile remote file path, this has to be a full qualified path
+ * @param localFile This is the local file to copy, this can be a directory too
+ * @throws SSHApiException
+ */
+ public void scpThirdParty(String remoteFileSorce, String remoteFileTarget) throws SSHApiException;
+
+ /**
* This will create directories in computing resources
* @param directoryPath the full qualified path for the directory user wants to create
* @throws SSHApiException throws during error
http://git-wip-us.apache.org/repos/asf/airavata/blob/64582600/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
index 0ddea0f..118bf91 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
@@ -135,7 +135,7 @@ public class GSISSHAbstractCluster implements Cluster {
logDebug("The private key file for vanilla SSH " + privateKeyFile);
String publicKeyFile = sshPublicKeyFileAuthentication.
- getPrivateKeyFile(serverInfo.getUserName(), serverInfo.getHost());
+ getPublicKeyFile(serverInfo.getUserName(), serverInfo.getHost());
logDebug("The public key file for vanilla SSH " + publicKeyFile);
@@ -374,6 +374,22 @@ public class GSISSHAbstractCluster implements Cluster {
+ serverInfo.getHost() + ":rFile", e);
}
}
+
+ public void scpThirdParty(String remoteFileSource, String remoteFileTarget) throws SSHApiException {
+ try {
+ if(!session.isConnected()){
+ session.connect();
+ }
+ log.info("Transfering from:" + remoteFileSource + " To: " + remoteFileTarget);
+ SSHUtils.scpThirdParty(remoteFileSource, remoteFileTarget, session);
+ } catch (IOException e) {
+ throw new SSHApiException("Failed during scping file:" + remoteFileSource + " to remote file "
+ +remoteFileTarget , e);
+ } catch (JSchException e) {
+ throw new SSHApiException("Failed during scping file:" + remoteFileSource + " to remote file "
+ +remoteFileTarget, e);
+ }
+ }
public void makeDirectory(String directoryPath) throws SSHApiException {
try {
http://git-wip-us.apache.org/repos/asf/airavata/blob/64582600/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java
index 02547a0..f36d2d0 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java
@@ -583,6 +583,123 @@ public class SSHUtils {
}
}
+ /**
+ * This method will copy a remote file to a local directory
+ *
+ * @param remoteFile remote file path, this has to be a full qualified path
+ * @param localFile This is the local file to copy, this can be a directory too
+ * @param session
+ * @return returns the final local file path of the new file came from the remote resource
+ */
+ public static void scpThirdParty(String remoteFileSource, String remoteFileTarget, Session session) throws IOException, JSchException, SSHApiException {
+ FileOutputStream fos = null;
+ try {
+ String prefix = null;
+
+ // exec 'scp -f remotefile' remotely
+ String command = "scp -3 " + remoteFileSource + " " + remoteFileTarget;
+ Channel channel = session.openChannel("exec");
+ ((ChannelExec) channel).setCommand(command);
+
+ StandardOutReader stdOutReader = new StandardOutReader();
+ ((ChannelExec) channel).setErrStream(stdOutReader.getStandardError());
+ // get I/O streams for remote scp
+ OutputStream out = channel.getOutputStream();
+ InputStream in = channel.getInputStream();
+
+ channel.connect();
+
+ byte[] buf = new byte[1024];
+
+ // send '\0'
+ buf[0] = 0;
+ out.write(buf, 0, 1);
+ out.flush();
+
+ while (true) {
+ int c = checkAck(in);
+ if (c != 'C') {
+ break;
+ }
+
+ // read '0644 '
+ in.read(buf, 0, 5);
+
+ long filesize = 0L;
+ while (true) {
+ if (in.read(buf, 0, 1) < 0) {
+ // error
+ break;
+ }
+ if (buf[0] == ' ') break;
+ filesize = filesize * 10L + (long) (buf[0] - '0');
+ }
+
+// String file = null;
+// for (int i = 0; ; i++) {
+// in.read(buf, i, 1);
+// if (buf[i] == (byte) 0x0a) {
+// file = new String(buf, 0, i);
+// break;
+// }
+// }
+
+ //System.out.println("filesize="+filesize+", file="+file);
+
+ // send '\0'
+ buf[0] = 0;
+ out.write(buf, 0, 1);
+ out.flush();
+
+ while (true) {
+ int len = in.read(buf, 0, buf.length);
+ if (len <= 0) break;
+ out.write(buf, 0, len);
+ }
+// // read a content of lfile
+// fos = new FileOutputStream(prefix == null ? localFile : prefix + file);
+// int foo;
+// while (true) {
+// if (buf.length < filesize) foo = buf.length;
+// else foo = (int) filesize;
+// foo = in.read(buf, 0, foo);
+// if (foo < 0) {
+// // error
+// break;
+// }
+// fos.write(buf, 0, foo);
+// filesize -= foo;
+// if (filesize == 0L) break;
+// }
+// fos.close();
+// fos = null;
+
+ if (checkAck(in) != 0) {
+ String error = "Error transfering the file content";
+ log.error(error);
+ throw new SSHApiException(error);
+ }
+
+ // send '\0'
+ buf[0] = 0;
+ out.write(buf, 0, 1);
+ out.flush();
+ }
+ stdOutReader.onOutput(channel);
+ if (!stdOutReader.getStdErrorString().equals("")) {
+ throw new SSHApiException(stdOutReader.getStdErrorString());
+ }
+
+ } catch (Exception e) {
+ log.error(e.getMessage(), e);
+ } finally {
+ try {
+ if (fos != null) fos.close();
+ } catch (Exception ee) {
+ }
+ }
+ }
+
public static void makeDirectory(String path, Session session) throws IOException, JSchException, SSHApiException {
// exec 'scp -t rfile' remotely