You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2017/05/23 14:21:05 UTC
[04/13] airavata git commit: fixing bug in moving checkpoint files
fixing bug in moving checkpoint files
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/1108b57c
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/1108b57c
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/1108b57c
Branch: refs/heads/develop
Commit: 1108b57c961ad2ffd684ed4bef6a8e7f84bf7088
Parents: 868fc55
Author: scnakandala <su...@gmail.com>
Authored: Wed May 3 16:04:16 2017 -0400
Committer: scnakandala <su...@gmail.com>
Committed: Wed May 3 17:19:16 2017 -0400
----------------------------------------------------------------------
.../gfac/core/cluster/RemoteCluster.java | 12 ++++++++-
.../airavata/gfac/impl/BESRemoteCluster.java | 22 +++++++++++++---
.../airavata/gfac/impl/HPCRemoteCluster.java | 27 ++++++++++++++++++++
.../airavata/gfac/impl/LocalRemoteCluster.java | 18 ++++++++++---
.../gfac/impl/task/SCPDataStageTask.java | 18 ++++++++++---
5 files changed, 87 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/1108b57c/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java
index 9a5c757..ebf5f6b 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/RemoteCluster.java
@@ -23,7 +23,6 @@ import com.jcraft.jsch.Session;
import org.apache.airavata.gfac.core.GFacException;
import org.apache.airavata.gfac.core.SSHApiException;
import org.apache.airavata.gfac.core.authentication.AuthenticationInfo;
-import org.apache.airavata.gfac.core.authentication.SSHKeyAuthentication;
import org.apache.airavata.model.status.JobStatus;
import java.util.List;
@@ -79,6 +78,17 @@ public interface RemoteCluster { // FIXME: replace SSHApiException with suitable
boolean ignoreEmptyFile) throws GFacException;
/**
+ * This method can be used to get the file name of a file giving the extension. It assumes that there will be only
+ * one file with that extension. In case if there are more than one file one random file name from the matching ones
+ * will be returned.
+ * @param fileExtension
+ * @param parentPath
+ * @param session
+ * @return
+ */
+ public String getFileNameFromExtension(String fileExtension, String parentPath, Session session) throws GFacException;
+
+ /**
* This will create directories in computing resources
*
* @param directoryPath the full qualified path for the directory user wants to create
http://git-wip-us.apache.org/repos/asf/airavata/blob/1108b57c/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java
index de0c70f..4f22c6a 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/BESRemoteCluster.java
@@ -22,10 +22,11 @@ package org.apache.airavata.gfac.impl;
import com.jcraft.jsch.Session;
import org.apache.airavata.gfac.core.GFacException;
import org.apache.airavata.gfac.core.JobManagerConfiguration;
-import org.apache.airavata.gfac.core.SSHApiException;
import org.apache.airavata.gfac.core.authentication.AuthenticationInfo;
-import org.apache.airavata.gfac.core.authentication.SSHKeyAuthentication;
-import org.apache.airavata.gfac.core.cluster.*;
+import org.apache.airavata.gfac.core.cluster.AbstractRemoteCluster;
+import org.apache.airavata.gfac.core.cluster.CommandInfo;
+import org.apache.airavata.gfac.core.cluster.JobSubmissionOutput;
+import org.apache.airavata.gfac.core.cluster.ServerInfo;
import org.apache.airavata.model.status.JobStatus;
import java.util.List;
@@ -56,6 +57,21 @@ public class BESRemoteCluster extends AbstractRemoteCluster{
}
+ /**
+ * This method can be used to get the file name of a file giving the extension. It assumes that there will be only
+ * one file with that extension. In case if there are more than one file one random file name from the matching ones
+ * will be returned.
+ *
+ * @param fileExtension
+ * @param parentPath
+ * @param session
+ * @return
+ */
+ @Override
+ public String getFileNameFromExtension(String fileExtension, String parentPath, Session session) {
+ return null;
+ }
+
@Override
public void makeDirectory(String directoryPath) throws GFacException {
http://git-wip-us.apache.org/repos/asf/airavata/blob/1108b57c/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
index c1bca29..3d972b0 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
@@ -190,6 +190,33 @@ public class HPCRemoteCluster extends AbstractRemoteCluster{
}
}
+ /**
+ * This method can be used to get the file name of a file giving the extension. It assumes that there will be only
+ * one file with that extension. In case if there are more than one file one random file name from the matching ones
+ * will be returned.
+ *
+ * @param fileExtension
+ * @param parentPath
+ * @param session
+ * @return
+ */
+ @Override
+ public String getFileNameFromExtension(String fileExtension, String parentPath, Session session) throws GFacException {
+ try {
+ List<String> fileNames = SSHUtils.listDirectory(parentPath, session);
+ for(String fileName : fileNames){
+ if(fileName.endsWith(fileExtension)){
+ return fileName;
+ }
+ }
+ log.warn("No matching file found for extension: " + fileExtension + " in the " + parentPath + " directory");
+ return null;
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw new GFacException("Failed to list directory " + parentPath);
+ }
+ }
+
@Override
public void makeDirectory(String directoryPath) throws GFacException {
int retryCount = 0;
http://git-wip-us.apache.org/repos/asf/airavata/blob/1108b57c/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java
index 878e3b6..64a96db 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/LocalRemoteCluster.java
@@ -21,12 +21,9 @@ package org.apache.airavata.gfac.impl;
import com.jcraft.jsch.JSchException;
import com.jcraft.jsch.Session;
-import org.apache.airavata.common.exception.AiravataException;
import org.apache.airavata.gfac.core.GFacException;
import org.apache.airavata.gfac.core.JobManagerConfiguration;
-import org.apache.airavata.gfac.core.SSHApiException;
import org.apache.airavata.gfac.core.authentication.AuthenticationInfo;
-import org.apache.airavata.gfac.core.authentication.SSHKeyAuthentication;
import org.apache.airavata.gfac.core.cluster.*;
import org.apache.airavata.model.status.JobStatus;
import org.slf4j.Logger;
@@ -131,6 +128,21 @@ public class LocalRemoteCluster extends AbstractRemoteCluster {
}
}
+ /**
+ * This method can be used to get the file name of a file giving the extension. It assumes that there will be only
+ * one file with that extension. In case if there are more than one file one random file name from the matching ones
+ * will be returned.
+ *
+ * @param fileExtension
+ * @param parentPath
+ * @param session
+ * @return
+ */
+ @Override
+ public String getFileNameFromExtension(String fileExtension, String parentPath, Session session) {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public void makeDirectory(String directoryPath) throws GFacException {
Path dirPath = Paths.get(directoryPath);
http://git-wip-us.apache.org/repos/asf/airavata/blob/1108b57c/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java
index 4e4aa86..9a42124 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/SCPDataStageTask.java
@@ -133,6 +133,20 @@ public class SCPDataStageTask implements Task {
URI sourceURI = new URI(subTaskModel.getSource());
String fileName = sourceURI.getPath().substring(sourceURI.getPath().lastIndexOf(File.separator) + 1,
sourceURI.getPath().length());
+
+ authenticationInfo = Factory.getComputerResourceSSHKeyAuthentication(processContext);
+ ServerInfo serverInfo = processContext.getComputeResourceServerInfo();
+ Session sshSession = Factory.getSSHSession(authenticationInfo, serverInfo);
+
+ //Wildcard for file name. Has to find the correct name.
+ if(fileName.startsWith("*.")){
+ String temp = taskContext.getParentProcessContext().getDataMovementRemoteCluster()
+ .getFileNameFromExtension(fileName.substring(2), inputPath, sshSession);
+ if(temp != null && temp != ""){
+ fileName = temp;
+ }
+ }
+
URI destinationURI = null;
if (subTaskModel.getDestination().startsWith("dummy")) {
destinationURI = TaskUtils.getDestinationURI(taskContext, hostName, inputPath, fileName);
@@ -149,11 +163,9 @@ public class SCPDataStageTask implements Task {
return status;
}
- authenticationInfo = Factory.getComputerResourceSSHKeyAuthentication(processContext);
+
status = new TaskStatus(TaskState.COMPLETED);
- ServerInfo serverInfo = processContext.getComputeResourceServerInfo();
- Session sshSession = Factory.getSSHSession(authenticationInfo, serverInfo);
if (processState == ProcessState.INPUT_DATA_STAGING) {
inputDataStaging(taskContext, sshSession, sourceURI, destinationURI);
status.setReason("Successfully staged input data");