You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ms...@apache.org on 2015/06/22 18:22:42 UTC
airavata git commit: minor changes and verbose logging
Repository: airavata
Updated Branches:
refs/heads/airavata-0.15-release-branch c5194d06e -> f18e34307
minor changes and verbose logging
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/f18e3430
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/f18e3430
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/f18e3430
Branch: refs/heads/airavata-0.15-release-branch
Commit: f18e343079023b8afac8f28572146202bea25c85
Parents: c5194d0
Author: msmemon <sh...@gmail.com>
Authored: Mon Jun 22 18:22:31 2015 +0200
Committer: msmemon <sh...@gmail.com>
Committed: Mon Jun 22 18:22:31 2015 +0200
----------------------------------------------------------------------
.../gfac/bes/provider/impl/BESProvider.java | 17 +----
.../gfac/bes/utils/DataTransferrer.java | 66 ++++++++++++++++----
.../gfac/bes/utils/ResourceProcessor.java | 2 -
3 files changed, 55 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/f18e3430/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java
index 40baa22..3ed08cb 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java
@@ -131,7 +131,6 @@ public class BESProvider extends AbstractProvider implements GFacProvider,
userDN = "CN=zdv575, O=Ultrascan Gateway, C=DE";
}
CreateActivityDocument cad = CreateActivityDocument.Factory.newInstance();
- JobDefinitionDocument jobDefDoc = JobDefinitionDocument.Factory.newInstance();
// create storage
StorageCreator storageCreator = new StorageCreator(secProperties, factoryUrl, 5, null);
@@ -140,7 +139,7 @@ public class BESProvider extends AbstractProvider implements GFacProvider,
JobDefinitionType jobDefinition = JSDLGenerator.buildJSDLInstance(jobExecutionContext, sc.getUrl()).getJobDefinition();
cad.addNewCreateActivity().addNewActivityDocument().setJobDefinition(jobDefinition);
- log.debug("Submitted JSDL: " + jobDefDoc.getJobDefinition().getJobDescription());
+ log.info("Submitted JSDL: " + jobDefinition.getJobDescription());
@@ -229,19 +228,7 @@ public class BESProvider extends AbstractProvider implements GFacProvider,
} catch (Exception e) {
log.error("Cannot create storage..");
throw new GFacProviderException("Cannot create storage..", e);
- } finally {
- // destroy sms instance
- try {
- if (sc != null) {
- sc.destroy();
- }
- } catch (Exception e) {
- log.warn(
- "Cannot destroy temporary SMS instance:" + sc.getUrl(),
- e);
- }
- }
-
+ }
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/f18e3430/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java
index d94cb20..2dbf637 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java
@@ -26,6 +26,10 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -119,6 +123,10 @@ public class DataTransferrer {
// }
// }
+ if(log.isDebugEnabled()) {
+ log.debug("Download location is:"+downloadLocation);
+ }
+
List<OutputDataObjectType> applicationOutputs = jobContext.getTaskData().getApplicationOutputs();
if (applicationOutputs != null && !applicationOutputs.isEmpty()){
for (OutputDataObjectType output : applicationOutputs){
@@ -134,17 +142,13 @@ public class DataTransferrer {
String outputPath = downloadLocation + File.separator + value;
jobContext.addOutputFile(outputPath);
} catch (Exception e) {
+ log.error("Cannot download remote files..");
throw new GFacProviderException(e.getLocalizedMessage(),e);
}
}
}
}
-
-
-
-
-
downloadStdOuts();
}
@@ -199,6 +203,7 @@ public class DataTransferrer {
jobContext.setStandardError(stderrLocation);
log.info("Stderr downloaded to -> "+stderrLocation);
} catch (Exception e) {
+
throw new GFacProviderException(e.getLocalizedMessage(),e);
}
@@ -255,14 +260,49 @@ public class DataTransferrer {
private String getDownloadLocation() {
TaskDetails taskData = jobContext.getTaskData();
- //In case of third party transfer this will not work.
-// if (taskData != null && taskData.getAdvancedOutputDataHandling() != null) {
-// String outputDataDirectory = taskData.getAdvancedOutputDataHandling().getOutputDataDir();
-// return outputDataDirectory;
-// }
- String outputDataDir = File.separator + "tmp";
- outputDataDir = outputDataDir + File.separator + jobContext.getExperimentID();
- (new File(outputDataDir)).mkdirs();
+ String outputDataDir = "";
+
+ if (taskData != null
+ && taskData.getAdvancedOutputDataHandling() != null ) {
+
+ outputDataDir = taskData.getAdvancedOutputDataHandling().getOutputDataDir();
+
+
+ if (outputDataDir == null || "".equals(outputDataDir)) {
+ outputDataDir = getTempPath(jobContext.getExperimentID());
+ }
+
+ else {
+
+ try {
+ URI u = new URI(outputDataDir);
+ } catch (URISyntaxException e) {
+ return getTempPath(jobContext.getExperimentID());
+ }
+ // in case of remote locations use the tmp location
+ if (outputDataDir.startsWith("scp:") ||
+ outputDataDir.startsWith("ftp:") ||
+ outputDataDir.startsWith("gsiftp:")) {
+ outputDataDir = getTempPath(jobContext.getExperimentID());
+ } else if ( outputDataDir.startsWith("file:") &&
+ outputDataDir.contains("@")){
+ outputDataDir = getTempPath(jobContext.getExperimentID());
+
+ } else {
+ outputDataDir = taskData.getAdvancedOutputDataHandling()
+ .getOutputDataDir();
+ }
+ }
+ }
+
return outputDataDir;
}
+
+ private String getTempPath(String experimentID) {
+ String tmpOutputDir = File.separator + "tmp" + File.separator
+ + jobContext.getExperimentID();
+ (new File(tmpOutputDir)).mkdirs();
+ return tmpOutputDir;
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/f18e3430/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java
index cc3ad8b..296ef9f 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java
@@ -34,8 +34,6 @@ public class ResourceProcessor {
TaskDetails taskData = context.getTaskData();
-
-
if(taskData != null && taskData.isSetTaskScheduling()){
try {