You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sh...@apache.org on 2014/10/31 22:41:55 UTC
[1/2] git commit: Ingegrated appCatalog thrift model with GSISSH
input and output handlers and inprove job execution context
Repository: airavata
Updated Branches:
refs/heads/gfac_appcatalog_int 04f09e7d4 -> bb4fe1226
Ingegrated appCatalog thrift model with GSISSH input and output handlers and inprove job execution context
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/ad2b1d3a
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/ad2b1d3a
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/ad2b1d3a
Branch: refs/heads/gfac_appcatalog_int
Commit: ad2b1d3a97ec38cb98c970fa815b4863c9e8e898
Parents: f29dfbe
Author: shamrath <sh...@gmail.com>
Authored: Fri Oct 31 17:41:22 2014 -0400
Committer: shamrath <sh...@gmail.com>
Committed: Fri Oct 31 17:41:22 2014 -0400
----------------------------------------------------------------------
.../model/workspace/experiment/JobDetails.java | 11 ++-
.../gfac/core/context/JobExecutionContext.java | 27 +++++-
.../airavata/gfac/core/cpi/BetterGfacImpl.java | 91 ++++++++++++++++----
.../handler/GSISSHDirectorySetupHandler.java | 7 +-
.../gfac/gsissh/handler/GSISSHInputHandler.java | 18 ++--
.../gsissh/handler/GSISSHOutputHandler.java | 53 +++---------
.../airavata/gsi/ssh/api/job/JobDescriptor.java | 7 ++
7 files changed, 143 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/ad2b1d3a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/workspace/experiment/JobDetails.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/workspace/experiment/JobDetails.java b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/workspace/experiment/JobDetails.java
index d1cbe5e..c1034a0 100644
--- a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/workspace/experiment/JobDetails.java
+++ b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/workspace/experiment/JobDetails.java
@@ -271,9 +271,14 @@ import org.slf4j.LoggerFactory;
}
}
- public String getJobDescription() {
- return this.jobDescription;
- }
+ /**
+ * this method is deprecated after we introduce new thirft model with appcatalog
+ * @return
+ */
+ @Deprecated
+ public String getJobDescription() {
+ return this.jobDescription;
+ }
public void setJobDescription(String jobDescription) {
this.jobDescription = jobDescription;
http://git-wip-us.apache.org/repos/asf/airavata/blob/ad2b1d3a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/JobExecutionContext.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/JobExecutionContext.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/JobExecutionContext.java
index 891bece..d5e8daa 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/JobExecutionContext.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/JobExecutionContext.java
@@ -33,6 +33,7 @@ import org.apache.airavata.gfac.SecurityContext;
import org.apache.airavata.gfac.core.cpi.GFac;
import org.apache.airavata.gfac.core.notification.GFacNotifier;
import org.apache.airavata.gfac.core.provider.GFacProvider;
+import org.apache.airavata.model.appcatalog.computeresource.DataMovementInterface;
import org.apache.airavata.model.appcatalog.computeresource.DataMovementProtocol;
import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionInterface;
import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
@@ -100,12 +101,20 @@ public class JobExecutionContext extends AbstractContext implements Serializable
private DataMovementProtocol preferredDataMovementProtocol;
/**
* List of job submission protocols sorted by priority order.
- */
+ */
private List<JobSubmissionInterface> hostPrioritizedJobSubmissionInterfaces;
/**
* use preferred job submission protocol.
*/
private JobSubmissionInterface preferredJobSubmissionInterface;
+ /**
+ * List of job submission protocols sorted by priority order.
+ */
+ private List<DataMovementInterface> hostPrioritizedDataMovementInterfaces;
+ /**
+ * use preferred job submission protocol.
+ */
+ private DataMovementInterface preferredDataMovementInterface;
// private ContextHeaderDocument.ContextHeader contextHeader;
@@ -434,4 +443,20 @@ public class JobExecutionContext extends AbstractContext implements Serializable
public String getHostName() {
return applicationContext.getComputeResourceDescription().getHostName();
}
+
+ public List<DataMovementInterface> getHostPrioritizedDataMovementInterfaces() {
+ return hostPrioritizedDataMovementInterfaces;
+ }
+
+ public void setHostPrioritizedDataMovementInterfaces(List<DataMovementInterface> hostPrioritizedDataMovementInterfaces) {
+ this.hostPrioritizedDataMovementInterfaces = hostPrioritizedDataMovementInterfaces;
+ }
+
+ public DataMovementInterface getPreferredDataMovementInterface() {
+ return preferredDataMovementInterface;
+ }
+
+ public void setPreferredDataMovementInterface(DataMovementInterface preferredDataMovementInterface) {
+ this.preferredDataMovementInterface = preferredDataMovementInterface;
+ }
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/ad2b1d3a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
index e8e4c66..656a291 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
@@ -52,8 +52,9 @@ import org.apache.airavata.messaging.core.PublisherFactory;
import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
import org.apache.airavata.model.appcatalog.appinterface.ApplicationInterfaceDescription;
import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
+import org.apache.airavata.model.appcatalog.computeresource.DataMovementInterface;
+import org.apache.airavata.model.appcatalog.computeresource.FileSystems;
import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionInterface;
-import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference;
import org.apache.airavata.model.messaging.event.*;
import org.apache.airavata.model.workspace.experiment.*;
@@ -74,6 +75,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
/**
@@ -303,6 +305,7 @@ public class BetterGfacImpl implements GFac,Watcher {
jobExecutionContext.setZk(zk);
jobExecutionContext.setCredentialStoreToken(AiravataZKUtils.getExpTokenId(zk, experimentID, taskID));
+ // handle job submission protocol
List<JobSubmissionInterface> jobSubmissionInterfaces = computeResource.getJobSubmissionInterfaces();
if (jobSubmissionInterfaces != null && !jobSubmissionInterfaces.isEmpty()){
Collections.sort(jobSubmissionInterfaces, new Comparator<JobSubmissionInterface>() {
@@ -316,36 +319,92 @@ public class BetterGfacImpl implements GFac,Watcher {
}else {
throw new GFacException("Compute resource should have at least one job submission interface defined...");
}
+ // handle data movement protocol
+ List<DataMovementInterface> dataMovementInterfaces = computeResource.getDataMovementInterfaces();
+ if (dataMovementInterfaces != null && !dataMovementInterfaces.isEmpty()) {
+ Collections.sort(dataMovementInterfaces, new Comparator<DataMovementInterface>() {
+ @Override
+ public int compare(DataMovementInterface dataMovementInterface, DataMovementInterface dataMovementInterface2) {
+ return dataMovementInterface.getPriorityOrder() - dataMovementInterface2.getPriorityOrder();
+ }
+ });
+ jobExecutionContext.setHostPrioritizedDataMovementInterfaces(dataMovementInterfaces);
+ }
+
+ // set compute resource configuration as default preferred values, after that replace those with gateway user preferences.
+ populateDefaultComputeResourceConfiguration(jobExecutionContext, applicationInterface, computeResource);
+ // if gateway resource preference is set
if (gatewayResourcePreferences != null ) {
if (gatewayResourcePreferences.getScratchLocation() == null) {
gatewayResourcePreferences.setScratchLocation("/tmp");
}
+ setUpWorkingLocation(jobExecutionContext, applicationInterface, gatewayResourcePreferences.getScratchLocation());
- /**
- * Working dir
- */
- String workingDir = gatewayResourcePreferences.getScratchLocation() + File.separator + jobExecutionContext.getExperimentID();
- jobExecutionContext.setWorkingDir(workingDir);
+ jobExecutionContext.setPreferredJobSubmissionProtocol(gatewayResourcePreferences.getPreferredJobSubmissionProtocol());
+ if (gatewayResourcePreferences.getPreferredJobSubmissionProtocol() == null) {
+ jobExecutionContext.setPreferredJobSubmissionInterface(jobExecutionContext.getHostPrioritizedJobSubmissionInterfaces().get(0));
+ jobExecutionContext.setPreferredJobSubmissionProtocol(jobExecutionContext.getPreferredJobSubmissionInterface().getJobSubmissionProtocol());
+ } else {
+ for (JobSubmissionInterface jobSubmissionInterface : jobSubmissionInterfaces) {
+ if (gatewayResourcePreferences.getPreferredJobSubmissionProtocol() == jobSubmissionInterface.getJobSubmissionProtocol()) {
+ jobExecutionContext.setPreferredJobSubmissionInterface(jobSubmissionInterface);
+ break;
+ }
+ }
+ }
+
+ // set gatewayUserPreferred data movement protocol and interface
+ jobExecutionContext.setPreferredDataMovementProtocol(gatewayResourcePreferences.getPreferredDataMovementProtocol());
+ if (gatewayResourcePreferences.getPreferredJobSubmissionProtocol() == null) {
+ jobExecutionContext.setPreferredDataMovementInterface(jobExecutionContext.getHostPrioritizedDataMovementInterfaces().get(0));
+ jobExecutionContext.setPreferredDataMovementProtocol(jobExecutionContext.getPreferredDataMovementInterface().getDataMovementProtocol());
+ } else {
+ for (DataMovementInterface dataMovementInterface : dataMovementInterfaces) {
+ if (gatewayResourcePreferences.getPreferredDataMovementProtocol() == dataMovementInterface.getDataMovementProtocol()) {
+ jobExecutionContext.setPreferredDataMovementInterface(dataMovementInterface);
+ break;
+ }
+ }
+ }
+ }
+ return jobExecutionContext;
+ }
+
+ private void setUpWorkingLocation(JobExecutionContext jobExecutionContext, ApplicationInterfaceDescription applicationInterface, String scratchLocation) {
+
+ /**
+ * Working dir
+ */
+ String workingDir = scratchLocation + File.separator + jobExecutionContext.getExperimentID();
+ jobExecutionContext.setWorkingDir(workingDir);
/*
* Input and Output Directory
*/
- jobExecutionContext.setInputDir(workingDir + File.separator + Constants.INPUT_DATA_DIR_VAR_NAME);
- jobExecutionContext.setOutputDir(workingDir + File.separator + Constants.OUTPUT_DATA_DIR_VAR_NAME);
+ jobExecutionContext.setInputDir(workingDir + File.separator + Constants.INPUT_DATA_DIR_VAR_NAME);
+ jobExecutionContext.setOutputDir(workingDir + File.separator + Constants.OUTPUT_DATA_DIR_VAR_NAME);
/*
* Stdout and Stderr for Shell
*/
- jobExecutionContext.setStandardOutput(workingDir + File.separator + applicationInterface.getApplicationName().replaceAll("\\s+", "") + ".stdout");
- jobExecutionContext.setStandardError(workingDir + File.separator + applicationInterface.getApplicationName().replaceAll("\\s+", "") + ".stderr");
+ jobExecutionContext.setStandardOutput(workingDir + File.separator + applicationInterface.getApplicationName().replaceAll("\\s+", "") + ".stdout");
+ jobExecutionContext.setStandardError(workingDir + File.separator + applicationInterface.getApplicationName().replaceAll("\\s+", "") + ".stderr");
+ }
- jobExecutionContext.setPreferredJobSubmissionProtocol(gatewayResourcePreferences.getPreferredJobSubmissionProtocol());
- if (gatewayResourcePreferences.getPreferredJobSubmissionProtocol() == null) {
- jobExecutionContext.setPreferredJobSubmissionInterface(jobExecutionContext.getHostPrioritizedJobSubmissionInterfaces().get(0));
- jobExecutionContext.setPreferredJobSubmissionProtocol(jobExecutionContext.getPreferredJobSubmissionInterface().getJobSubmissionProtocol());
- }
+ private void populateDefaultComputeResourceConfiguration(JobExecutionContext jobExecutionContext, ApplicationInterfaceDescription applicationInterface, ComputeResourceDescription computeResource) {
+ Map<FileSystems, String> fileSystems = computeResource.getFileSystems();
+ String scratchLocation = fileSystems.get(FileSystems.SCRATCH);
+ if (scratchLocation != null) {
+ setUpWorkingLocation(jobExecutionContext, applicationInterface, scratchLocation);
+ }
+
+ jobExecutionContext.setPreferredJobSubmissionInterface(jobExecutionContext.getHostPrioritizedJobSubmissionInterfaces().get(0));
+ jobExecutionContext.setPreferredJobSubmissionProtocol(jobExecutionContext.getPreferredJobSubmissionInterface().getJobSubmissionProtocol());
+
+ if (jobExecutionContext.getHostPrioritizedDataMovementInterfaces() != null) {
+ jobExecutionContext.setPreferredDataMovementInterface(jobExecutionContext.getHostPrioritizedDataMovementInterfaces().get(0));
+ jobExecutionContext.setPreferredDataMovementProtocol(jobExecutionContext.getPreferredDataMovementInterface().getDataMovementProtocol());
}
- return jobExecutionContext;
}
private boolean submitJob(JobExecutionContext jobExecutionContext) throws GFacException {
http://git-wip-us.apache.org/repos/asf/airavata/blob/ad2b1d3a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHDirectorySetupHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHDirectorySetupHandler.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHDirectorySetupHandler.java
index b87f99a..b2790c9 100644
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHDirectorySetupHandler.java
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHDirectorySetupHandler.java
@@ -77,12 +77,11 @@ public class GSISSHDirectorySetupHandler extends AbstractRecoverableHandler {
} else {
log.info("Successfully retrieved the Security Context");
}
- ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
- String workingDirectory = app.getScratchWorkingDirectory();
+ String workingDirectory = jobExecutionContext.getWorkingDir();
cluster.makeDirectory(workingDirectory);
- cluster.makeDirectory(app.getInputDataDirectory());
- cluster.makeDirectory(app.getOutputDataDirectory());
+ cluster.makeDirectory(jobExecutionContext.getInputDir());
+ cluster.makeDirectory(jobExecutionContext.getOutputDir());
DataTransferDetails detail = new DataTransferDetails();
TransferStatus status = new TransferStatus();
status.setTransferState(TransferState.DIRECTORY_SETUP);
http://git-wip-us.apache.org/repos/asf/airavata/blob/ad2b1d3a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHInputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHInputHandler.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHInputHandler.java
index 5665b5b..b882be6 100644
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHInputHandler.java
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHInputHandler.java
@@ -27,17 +27,18 @@ import org.apache.airavata.commons.gfac.type.MappingFactory;
import org.apache.airavata.gfac.GFacException;
import org.apache.airavata.gfac.core.context.JobExecutionContext;
import org.apache.airavata.gfac.core.context.MessageContext;
-import org.apache.airavata.gfac.core.handler.AbstractHandler;
import org.apache.airavata.gfac.core.handler.AbstractRecoverableHandler;
import org.apache.airavata.gfac.core.handler.GFacHandlerException;
import org.apache.airavata.gfac.core.utils.GFacUtils;
import org.apache.airavata.gfac.gsissh.security.GSISecurityContext;
import org.apache.airavata.gfac.gsissh.util.GFACGSISSHUtils;
import org.apache.airavata.gsi.ssh.api.Cluster;
-import org.apache.airavata.gsi.ssh.api.SSHApiException;
-import org.apache.airavata.model.workspace.experiment.*;
+import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
+import org.apache.airavata.model.workspace.experiment.DataTransferDetails;
+import org.apache.airavata.model.workspace.experiment.ErrorCategory;
+import org.apache.airavata.model.workspace.experiment.TransferState;
+import org.apache.airavata.model.workspace.experiment.TransferStatus;
import org.apache.airavata.registry.cpi.ChildDataType;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
import org.apache.airavata.schemas.gfac.URIArrayType;
import org.apache.airavata.schemas.gfac.URIParameterType;
import org.slf4j.Logger;
@@ -45,7 +46,11 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
/**
* Recoverability for this handler assumes the same input values will come in the second
@@ -171,11 +176,10 @@ public class GSISSHInputHandler extends AbstractRecoverableHandler {
}
private static String stageInputFiles(Cluster cluster, JobExecutionContext jobExecutionContext, String paramValue) throws IOException, GFacException {
- ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
int i = paramValue.lastIndexOf(File.separator);
String substring = paramValue.substring(i + 1);
try {
- String targetFile = app.getInputDataDirectory() + File.separator + substring;
+ String targetFile = jobExecutionContext.getInputDir() + File.separator + substring;
if (paramValue.startsWith("file")) {
paramValue = paramValue.substring(paramValue.indexOf(":") + 1, paramValue.length());
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/ad2b1d3a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java
index ac9bf3c..a714099 100644
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java
@@ -27,6 +27,7 @@ import java.util.*;
import net.schmizz.sshj.connection.ConnectionException;
import net.schmizz.sshj.transport.TransportException;
+import org.apache.aiaravata.application.catalog.data.impl.AppCatalogFactory;
import org.apache.airavata.common.exception.ApplicationSettingsException;
import org.apache.airavata.common.utils.Constants;
import org.apache.airavata.commons.gfac.type.ActualParameter;
@@ -46,6 +47,10 @@ import org.apache.airavata.gfac.gsissh.util.GFACGSISSHUtils;
import org.apache.airavata.gsi.ssh.api.Cluster;
import org.apache.airavata.gsi.ssh.api.SSHApiException;
import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
+import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
+import org.apache.airavata.model.appcatalog.computeresource.MonitorMode;
+import org.apache.airavata.model.appcatalog.computeresource.SSHJobSubmission;
+import org.apache.airavata.model.appcatalog.computeresource.SecurityProtocol;
import org.apache.airavata.model.messaging.event.TaskIdentifier;
import org.apache.airavata.model.messaging.event.TaskOutputChangeEvent;
import org.apache.airavata.model.workspace.experiment.*;
@@ -67,36 +72,6 @@ public class GSISSHOutputHandler extends AbstractRecoverableHandler {
int oldIndex = 0;
List<String> oldFiles = new ArrayList<String>();
StringBuffer data = new StringBuffer("|");
- if (jobExecutionContext.getApplicationContext().getHostDescription().getType() instanceof GsisshHostType) { // this is because we don't have the right jobexecution context
- // so attempting to get it from the registry
- if (Constants.PUSH.equals(((GsisshHostType) jobExecutionContext.getApplicationContext().getHostDescription().getType()).getMonitorMode())) {
- log.warn("During the out handler chain jobExecution context came null, so trying to handler");
- ApplicationDescription applicationDeploymentDescription = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription();
- TaskDetails taskData = null;
- try {
- taskData = (TaskDetails) jobExecutionContext.getRegistry().get(RegistryModelType.TASK_DETAIL, jobExecutionContext.getTaskData().getTaskID());
- } catch (RegistryException e) {
- log.error("Error retrieving job details from Registry");
- throw new GFacHandlerException("Error retrieving job details from Registry", e);
- }
- JobDetails jobDetails = taskData.getJobDetailsList().get(0);
- String jobDescription = jobDetails.getJobDescription();
- if (jobDescription != null) {
- JobDescriptor jobDescriptor = null;
- try {
- jobDescriptor = JobDescriptor.fromXML(jobDescription);
- } catch (XmlException e1) {
- e1.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
- }
- applicationDeploymentDescription.getType().setScratchWorkingDirectory(
- jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getWorkingDirectory());
- applicationDeploymentDescription.getType().setInputDataDirectory(jobDescriptor.getInputDirectory());
- applicationDeploymentDescription.getType().setOutputDataDirectory(jobDescriptor.getOutputDirectory());
- applicationDeploymentDescription.getType().setStandardError(jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getStandardErrorFile());
- applicationDeploymentDescription.getType().setStandardOutput(jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getStandardOutFile());
- }
- }
- }
try {
if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) == null) {
@@ -114,8 +89,6 @@ public class GSISSHOutputHandler extends AbstractRecoverableHandler {
DataTransferDetails detail = new DataTransferDetails();
TransferStatus status = new TransferStatus();
- ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext()
- .getApplicationDeploymentDescription().getType();
Cluster cluster = null;
try {
@@ -174,7 +147,7 @@ public class GSISSHOutputHandler extends AbstractRecoverableHandler {
localStdOutFile = new File(outputDataDir + File.separator + timeStampedExperimentID + "stdout");
while(stdOutStr.isEmpty()){
try {
- cluster.scpFrom(app.getStandardOutput(), localStdOutFile.getAbsolutePath());
+ cluster.scpFrom(jobExecutionContext.getStandardOutput(), localStdOutFile.getAbsolutePath());
stdOutStr = GFacUtils.readFileToString(localStdOutFile.getAbsolutePath());
} catch (Exception e) {
log.error(e.getLocalizedMessage());
@@ -192,7 +165,7 @@ public class GSISSHOutputHandler extends AbstractRecoverableHandler {
data.append(oldFiles.get(index++)).append(",");
} else {
localStdErrFile = new File(outputDataDir + File.separator + timeStampedExperimentID + "stderr");
- cluster.scpFrom(app.getStandardError(), localStdErrFile.getAbsolutePath());
+ cluster.scpFrom(jobExecutionContext.getStandardError(), localStdErrFile.getAbsolutePath());
StringBuffer temp = new StringBuffer(data.append(localStdErrFile.getAbsolutePath()).append(",").toString());
GFacUtils.savePluginData(jobExecutionContext, temp.insert(0, ++index), this.getClass().getName());
}
@@ -219,7 +192,7 @@ public class GSISSHOutputHandler extends AbstractRecoverableHandler {
List<String> outputList = null;
int retry=3;
while(retry>0){
- outputList = cluster.listDirectory(app.getOutputDataDirectory());
+ outputList = cluster.listDirectory(jobExecutionContext.getOutputDir());
if (outputList.size() == 1 && outputList.get(0).isEmpty()) {
Thread.sleep(10000);
} else if (outputList.size() > 0) {
@@ -229,7 +202,6 @@ public class GSISSHOutputHandler extends AbstractRecoverableHandler {
}
retry--;
if(retry==0){
-// log.info("Ohhhhhhh shitttttttOhhhhhhh shitttttttOhhhhhhh shitttttttOhhhhhhh shitttttttOhhhhhhh shitttttttOhhhhhhh shittttttt");
}
Thread.sleep(10000);
}
@@ -269,7 +241,7 @@ public class GSISSHOutputHandler extends AbstractRecoverableHandler {
outputFile = oldFiles.get(index);
data.append(oldFiles.get(index++)).append(",");
} else {
- cluster.scpFrom(app.getOutputDataDirectory() + File.separator + valueList, outputDataDir);
+ cluster.scpFrom(jobExecutionContext.getOutputDir() + File.separator + valueList, outputDataDir);
outputFile = outputDataDir + File.separator + valueList;
jobExecutionContext.addOutputFile(outputFile);
StringBuffer temp = new StringBuffer(data.append(outputFile).append(",").toString());
@@ -296,9 +268,10 @@ public class GSISSHOutputHandler extends AbstractRecoverableHandler {
);
}
}
- app.setStandardError(localStdErrFile.getAbsolutePath());
- app.setStandardOutput(localStdOutFile.getAbsolutePath());
- app.setOutputDataDirectory(outputDataDir);
+ // Why we set following?
+// app.setStandardError(localStdErrFile.getAbsolutePath());
+// app.setStandardOutput(localStdOutFile.getAbsolutePath());
+// app.setOutputDataDirectory(outputDataDir);
status.setTransferState(TransferState.DOWNLOAD);
detail.setTransferStatus(status);
detail.setTransferDescription(outputDataDir);
http://git-wip-us.apache.org/repos/asf/airavata/blob/ad2b1d3a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java
index 9a0639b..9b7102b 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/job/JobDescriptor.java
@@ -60,6 +60,13 @@ public class JobDescriptor {
return this.jobDescriptionDocument;
}
+ /**
+ * With new app catalog thrift object integration, we don't use this
+ * @param xml
+ * @return
+ * @throws XmlException
+ */
+ @Deprecated
public static JobDescriptor fromXML(String xml)
throws XmlException {
JobDescriptorDocument parse = JobDescriptorDocument.Factory
[2/2] git commit: Merge branch 'gfac_appcatalog_int' of
https://git-wip-us.apache.org/repos/asf/airavata into gfac_appcatalog_int
Posted by sh...@apache.org.
Merge branch 'gfac_appcatalog_int' of https://git-wip-us.apache.org/repos/asf/airavata into gfac_appcatalog_int
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/bb4fe122
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/bb4fe122
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/bb4fe122
Branch: refs/heads/gfac_appcatalog_int
Commit: bb4fe1226ae1a170f59e3992116bcea70f19c3f6
Parents: ad2b1d3 04f09e7
Author: shamrath <sh...@gmail.com>
Authored: Fri Oct 31 17:41:42 2014 -0400
Committer: shamrath <sh...@gmail.com>
Committed: Fri Oct 31 17:41:42 2014 -0400
----------------------------------------------------------------------
.../gfac/bes/handlers/AbstractSMSHandler.java | 74 ++--
.../gfac/bes/provider/impl/BESProvider.java | 378 +++++++++----------
.../bes/security/UNICORESecurityContext.java | 4 +-
.../gfac/bes/utils/ApplicationProcessor.java | 212 ++++-------
.../airavata/gfac/core/utils/GFacUtils.java | 23 +-
.../apache/airavata/gfac/ec2/EC2Provider.java | 15 +-
6 files changed, 306 insertions(+), 400 deletions(-)
----------------------------------------------------------------------