You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sh...@apache.org on 2015/06/23 23:32:05 UTC

[1/2] airavata git commit: Added part of Process context population logic

Repository: airavata
Updated Branches:
  refs/heads/master e5d861e70 -> 013e8c090


Added part of Process context population logic


Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/726370bd
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/726370bd
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/726370bd

Branch: refs/heads/master
Commit: 726370bdee8f67a54f2017c8f8feecabd8ff37ad
Parents: 9f2c086
Author: Shameera Rathanyaka <sh...@gmail.com>
Authored: Tue Jun 23 17:31:30 2015 -0400
Committer: Shameera Rathanyaka <sh...@gmail.com>
Committed: Tue Jun 23 17:31:30 2015 -0400

----------------------------------------------------------------------
 .../core/cluster/AbstractRemoteCluster.java     |  17 ++-
 .../gfac/core/context/ProcessContext.java       |  16 +++
 .../org/apache/airavata/gfac/impl/Factory.java  | 138 ++++++++++++++++---
 .../airavata/gfac/impl/GFacEngineImpl.java      |  69 +++++-----
 .../airavata/gfac/impl/HPCRemoteCluster.java    |  20 ++-
 .../gfac/impl/job/LSFJobConfiguration.java      |  14 +-
 .../gfac/impl/job/PBSJobConfiguration.java      |  35 +++--
 .../gfac/impl/job/SlurmJobConfiguration.java    |  14 +-
 .../gfac/impl/job/UGEJobConfiguration.java      |  15 +-
 .../airavata/gfac/server/GfacServerHandler.java |   2 +-
 10 files changed, 224 insertions(+), 116 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/airavata/blob/726370bd/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/AbstractRemoteCluster.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/AbstractRemoteCluster.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/AbstractRemoteCluster.java
index 3487224..12711f7 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/AbstractRemoteCluster.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cluster/AbstractRemoteCluster.java
@@ -21,19 +21,22 @@
 package org.apache.airavata.gfac.core.cluster;
 
 import org.apache.airavata.gfac.core.JobManagerConfiguration;
+import org.apache.airavata.gfac.core.authentication.AuthenticationInfo;
 
 import java.util.Map;
 
-public class AbstractRemoteCluster {
+public abstract class AbstractRemoteCluster implements RemoteCluster {
 
-	ServerInfo serverInfo;
-	JobManagerConfiguration jobManagerConfiguration;
-	Map<String,String> authenticationParam;
+	protected final OutputParser outputParser;
+	protected final AuthenticationInfo authenticationInfo;
+	protected final ServerInfo serverInfo;
+	protected final JobManagerConfiguration jobManagerConfiguration;
 
-	public AbstractRemoteCluster(ServerInfo serverInfo, JobManagerConfiguration jobManagerConfiguration, Map<String,
-			String> authenticationParam) {
+	public AbstractRemoteCluster(ServerInfo serverInfo, JobManagerConfiguration jobManagerConfiguration, AuthenticationInfo
+			authenticationInfo) {
 		this.serverInfo = serverInfo;
 		this.jobManagerConfiguration = jobManagerConfiguration;
-		this.authenticationParam = authenticationParam;
+		this.authenticationInfo = authenticationInfo;
+		this.outputParser = jobManagerConfiguration.getParser();
 	}
 }

http://git-wip-us.apache.org/repos/asf/airavata/blob/726370bd/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
index 0ae69b8..3f8cf02 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
@@ -29,6 +29,7 @@ import org.apache.airavata.model.appcatalog.appinterface.ApplicationInterfaceDes
 import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
 import org.apache.airavata.model.appcatalog.computeresource.DataMovementProtocol;
 import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
+import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference;
 import org.apache.airavata.model.appcatalog.gatewayprofile.GatewayResourceProfile;
 import org.apache.airavata.model.job.JobModel;
 import org.apache.airavata.model.process.ProcessModel;
@@ -64,6 +65,7 @@ public class ProcessContext {
 	private JobSubmissionProtocol jobSubmissionProtocol;
 	private DataMovementProtocol dataMovementProtocol;
     private JobModel jobModel;
+	private ComputeResourcePreference computeResourcePreference;
 
 	public ProcessContext(String processId, String gatewayId, String tokenId) {
 		this.processId = processId;
@@ -222,6 +224,9 @@ public class ProcessContext {
     }
 
 	public JobSubmissionProtocol getJobSubmissionProtocol() {
+		if (jobSubmissionProtocol == null) {
+			jobSubmissionProtocol = computeResourcePreference.getPreferredJobSubmissionProtocol();
+		}
 		return jobSubmissionProtocol;
 	}
 
@@ -230,6 +235,9 @@ public class ProcessContext {
 	}
 
 	public DataMovementProtocol getDataMovementProtocol() {
+		if (dataMovementProtocol == null) {
+			dataMovementProtocol = computeResourcePreference.getPreferredDataMovementProtocol();
+		}
 		return dataMovementProtocol;
 	}
 
@@ -244,4 +252,12 @@ public class ProcessContext {
     public void setJobModel(JobModel jobModel) {
         this.jobModel = jobModel;
     }
+
+	public ComputeResourcePreference getComputeResourcePreference() {
+		return computeResourcePreference;
+	}
+
+	public void setComputeResourcePreference(ComputeResourcePreference computeResourcePreference) {
+		this.computeResourcePreference = computeResourcePreference;
+	}
 }

http://git-wip-us.apache.org/repos/asf/airavata/blob/726370bd/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java
index 93c8de9..a2bebc3 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java
@@ -28,10 +28,17 @@ import org.apache.airavata.common.utils.ServerSettings;
 import org.apache.airavata.gfac.core.GFacEngine;
 import org.apache.airavata.gfac.core.GFacException;
 import org.apache.airavata.gfac.core.JobManagerConfiguration;
+import org.apache.airavata.gfac.core.authentication.AuthenticationInfo;
 import org.apache.airavata.gfac.core.cluster.RemoteCluster;
 import org.apache.airavata.gfac.core.cluster.ServerInfo;
+import org.apache.airavata.gfac.core.config.DataTransferTaskConfig;
+import org.apache.airavata.gfac.core.config.GFacYamlConfigruation;
+import org.apache.airavata.gfac.core.config.JobSubmitterTaskConfig;
+import org.apache.airavata.gfac.core.config.ResourceConfig;
 import org.apache.airavata.gfac.core.monitor.JobMonitor;
 import org.apache.airavata.gfac.core.scheduler.HostScheduler;
+import org.apache.airavata.gfac.core.task.JobSubmissionTask;
+import org.apache.airavata.gfac.core.task.Task;
 import org.apache.airavata.gfac.impl.job.LSFJobConfiguration;
 import org.apache.airavata.gfac.impl.job.LSFOutputParser;
 import org.apache.airavata.gfac.impl.job.PBSJobConfiguration;
@@ -41,7 +48,14 @@ import org.apache.airavata.gfac.impl.job.SlurmOutputParser;
 import org.apache.airavata.gfac.impl.job.UGEJobConfiguration;
 import org.apache.airavata.gfac.impl.job.UGEOutputParser;
 import org.apache.airavata.gfac.monitor.email.EmailBasedMonitor;
+import org.apache.airavata.model.appcatalog.computeresource.DataMovementProtocol;
+import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionInterface;
+import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
+import org.apache.airavata.model.appcatalog.computeresource.LOCALSubmission;
+import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManager;
 import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManagerType;
+import org.apache.airavata.model.appcatalog.computeresource.SSHJobSubmission;
+import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference;
 import org.apache.airavata.registry.core.experiment.catalog.impl.RegistryFactory;
 import org.apache.airavata.registry.cpi.AppCatalog;
 import org.apache.airavata.registry.cpi.AppCatalogException;
@@ -52,20 +66,23 @@ import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.CuratorFrameworkFactory;
 import org.apache.curator.retry.ExponentialBackoffRetry;
 
-import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 public abstract class Factory {
 
 	private static GFacEngine engine;
-	private static Map<String, RemoteCluster> remoteClusterMap;
 	private static LocalEventPublisher localEventPublisher;
 	private static CuratorFramework curatorClient;
 	private static EmailBasedMonitor emailBasedMonitor;
 	private static Date startMonitorDate = Calendar.getInstance().getTime();
+	private static Map<String, RemoteCluster> remoteClusterMap = new HashMap<>();
+	private static Map<JobSubmissionProtocol, JobSubmissionTask> jobSubmissionTask = new HashMap<>();
+	private static Map<DataMovementProtocol, Task> dataMovementTask = new HashMap<>();
+	private static Map<ResourceJobManagerType, ResourceConfig> resources = new HashMap<>();
 
 	public static GFacEngine getGFacEngine() throws GFacException {
 		if (engine == null) {
@@ -78,10 +95,6 @@ public abstract class Factory {
 		return engine;
 	}
 
-	public static RemoteCluster getRemoteCluster(ServerInfo serverInfo) {
-		return remoteClusterMap.get(serverInfo.getHost());
-	}
-
 	public static ExperimentCatalog getDefaultExpCatalog() throws RegistryException {
 		return RegistryFactory.getDefaultExpCatalog();
 	}
@@ -119,7 +132,7 @@ public abstract class Factory {
 			return null; // TODO write a job monitor for this.
 		} else {
 			if (emailBasedMonitor == null) {
-				synchronized (EmailBasedMonitor.class){
+				synchronized (EmailBasedMonitor.class) {
 					if (emailBasedMonitor == null) {
 						emailBasedMonitor = new EmailBasedMonitor(resourceJobManagerType);
 						emailBasedMonitor.setDate(startMonitorDate);
@@ -131,24 +144,115 @@ public abstract class Factory {
 		}
 	}
 
-	public static JobManagerConfiguration getPBSJobManager(String installedPath) {
-		return new PBSJobConfiguration("PBSTemplate.xslt",".pbs", installedPath, new PBSOutputParser());
+	public static JobManagerConfiguration getJobManagerConfiguration(ResourceJobManager resourceJobManager) {
+		switch (resourceJobManager.getResourceJobManagerType()) {
+			case PBS:
+				return new PBSJobConfiguration("PBSTemplate.xslt", ".pbs", resourceJobManager.getJobManagerBinPath(),
+						resourceJobManager.getJobManagerCommands(), new PBSOutputParser());
+			case SLURM:
+				return new SlurmJobConfiguration("SLURMTemplate.xslt", ".slurm", resourceJobManager
+						.getJobManagerBinPath(), resourceJobManager.getJobManagerCommands(), new SlurmOutputParser());
+			case LSF:
+				return new LSFJobConfiguration("LSFTemplate.xslt", ".lsf", resourceJobManager.getJobManagerBinPath(),
+						resourceJobManager.getJobManagerCommands(), new LSFOutputParser());
+			case UGE:
+				return new UGEJobConfiguration("UGETemplate.xslt", ".pbs", resourceJobManager.getJobManagerBinPath(),
+						resourceJobManager.getJobManagerCommands(), new UGEOutputParser());
+
+			default:
+				return null;
+		}
+	}
+
+	public static HostScheduler getHostScheduler() {
+		return new DefaultHostScheduler();
+	}
+
+
+	public static RemoteCluster getRemoteCluster(ComputeResourcePreference cRP) throws GFacException,
+			AppCatalogException, AiravataException {
+
+		String key = cRP.getPreferredJobSubmissionProtocol().toString() + ":" + cRP.getComputeResourceId();
+		RemoteCluster remoteCluster = remoteClusterMap.get(key);
+		if (remoteCluster == null) {
+			String hostName = Factory.getDefaultAppCatalog().getComputeResource().getComputeResource(cRP
+					.getComputeResourceId()).getHostName();
+			ServerInfo serverInfo = new ServerInfo(cRP.getLoginUserName(), hostName);
+
+			List<JobSubmissionInterface> jobSubmissionInterfaces = Factory.getDefaultAppCatalog().getComputeResource()
+					.getComputeResource(cRP.getComputeResourceId())
+					.getJobSubmissionInterfaces();
+
+			ResourceJobManager resourceJobManager = null;
+			JobSubmissionInterface jsInterface = null;
+			for (JobSubmissionInterface jobSubmissionInterface : jobSubmissionInterfaces) {
+				if (jobSubmissionInterface.getJobSubmissionProtocol() == cRP.getPreferredJobSubmissionProtocol()) {
+					jsInterface = jobSubmissionInterface;
+				}
+			}
+			if (jsInterface == null) {
+				// TODO: throw an exception.
+			} else if (jsInterface.getJobSubmissionProtocol() == JobSubmissionProtocol.SSH) {
+				SSHJobSubmission sshJobSubmission = getDefaultAppCatalog().getComputeResource().getSSHJobSubmission
+						(jsInterface.getJobSubmissionInterfaceId());
+				resourceJobManager = sshJobSubmission.getResourceJobManager();
+			} else if (jsInterface.getJobSubmissionProtocol() == JobSubmissionProtocol.LOCAL) {
+				LOCALSubmission localSubmission = getDefaultAppCatalog().getComputeResource().getLocalJobSubmission
+						(jsInterface.getJobSubmissionInterfaceId());
+				resourceJobManager = localSubmission.getResourceJobManager();
+			} else {
+				// TODO : throw an not supported jobsubmission protocol exception. we only support SSH and LOCAL
+			}
+
+			if (resourceJobManager == null) {
+				// TODO throw an exception
+			}
+
+			JobManagerConfiguration jobManagerConfiguration = getJobManagerConfiguration(resourceJobManager);
+			AuthenticationInfo authenticationInfo = null;
+			remoteCluster = new HPCRemoteCluster(serverInfo, jobManagerConfiguration, null);
+			remoteClusterMap.put(key, remoteCluster);
+		}
+		return remoteCluster;
 	}
 
-	public static JobManagerConfiguration getSLURMJobManager(String installedPath) {
-		return new SlurmJobConfiguration("SLURMTemplate.xslt", ".slurm", installedPath, new SlurmOutputParser());
+	public static JobSubmissionTask getJobSubmissionTask(JobSubmissionProtocol jobSubmissionProtocol) throws
+			GFacException {
+		if (jobSubmissionTask == null) {
+			loadConfiguration();
+		}
+		return jobSubmissionTask.get(jobSubmissionProtocol);
 	}
 
-	public static JobManagerConfiguration getUGEJobManager(String installedPath) {
-		return new UGEJobConfiguration("UGETemplate.xslt", ".pbs", installedPath, new UGEOutputParser());
+	public static Task getDataMovementTask(DataMovementProtocol dataMovementProtocol) throws GFacException {
+		if (dataMovementTask == null) {
+			loadConfiguration();
+		}
+		return dataMovementTask.get(dataMovementProtocol);
 	}
 
-	public static JobManagerConfiguration getLSFJobManager(String installedPath) {
-		return new LSFJobConfiguration("LSFTemplate.xslt", ".lsf", installedPath, new LSFOutputParser());
+	public static ResourceConfig getResourceConfig(ResourceJobManagerType resourceJobManagerType) throws
+			GFacException {
+		if (resources == null) {
+			loadConfiguration();
+		}
+		return resources.get(resourceJobManagerType);
 	}
 
-	public static HostScheduler getHostScheduler() {
-		return new DefaultHostScheduler();
+	private static void loadConfiguration() throws GFacException {
+		GFacYamlConfigruation config = new GFacYamlConfigruation();
+		for (JobSubmitterTaskConfig jobSubmitterTaskConfig : config.getJobSbumitters()) {
+			jobSubmissionTask.put(jobSubmitterTaskConfig.getSubmissionProtocol(), null);
+		}
+
+		for (DataTransferTaskConfig dataTransferTaskConfig : config.getFileTransferTasks()) {
+			dataMovementTask.put(dataTransferTaskConfig.getTransferProtocol(), null);
+		}
+
+		for (ResourceConfig resourceConfig : config.getResourceConfiguration()) {
+			resources.put(resourceConfig.getJobManagerType(), resourceConfig);
+		}
 	}
 
+
 }

http://git-wip-us.apache.org/repos/asf/airavata/blob/726370bd/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/GFacEngineImpl.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/GFacEngineImpl.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/GFacEngineImpl.java
index 0819b4f..bcb569d 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/GFacEngineImpl.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/GFacEngineImpl.java
@@ -21,61 +21,55 @@
 
 package org.apache.airavata.gfac.impl;
 
+import org.apache.airavata.common.exception.AiravataException;
 import org.apache.airavata.gfac.core.GFacEngine;
 import org.apache.airavata.gfac.core.GFacException;
-import org.apache.airavata.gfac.core.config.DataTransferTaskConfig;
-import org.apache.airavata.gfac.core.config.GFacYamlConfigruation;
-import org.apache.airavata.gfac.core.config.JobSubmitterTaskConfig;
-import org.apache.airavata.gfac.core.config.ResourceConfig;
 import org.apache.airavata.gfac.core.context.ProcessContext;
 import org.apache.airavata.gfac.core.task.Task;
-import org.apache.airavata.model.appcatalog.computeresource.DataMovementProtocol;
-import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
-import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManagerType;
+import org.apache.airavata.model.appcatalog.gatewayprofile.GatewayResourceProfile;
 import org.apache.airavata.model.application.io.DataType;
 import org.apache.airavata.model.application.io.InputDataObjectType;
 import org.apache.airavata.model.application.io.OutputDataObjectType;
 import org.apache.airavata.model.process.ProcessModel;
-import org.apache.airavata.model.task.TaskModel;
+import org.apache.airavata.registry.cpi.AppCatalog;
+import org.apache.airavata.registry.cpi.AppCatalogException;
+import org.apache.airavata.registry.cpi.ExperimentCatalog;
+import org.apache.airavata.registry.cpi.ExperimentCatalogModelType;
+import org.apache.airavata.registry.cpi.RegistryException;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
-import java.util.Map;
 
 public class GFacEngineImpl implements GFacEngine {
-	private static GFacEngineImpl engine;
-	Map<JobSubmissionProtocol, Task> jobSubmissionTask;
-	Map<DataMovementProtocol, Task> dataMovementTask;
-	Map<ResourceJobManagerType, ResourceConfig> resources;
-
 
 	public GFacEngineImpl() throws GFacException {
-		GFacYamlConfigruation config = new GFacYamlConfigruation();
-		for (JobSubmitterTaskConfig jobSubmitterTaskConfig : config.getJobSbumitters()) {
-			jobSubmissionTask.put(jobSubmitterTaskConfig.getSubmissionProtocol(), null);
-		}
-
-		for (DataTransferTaskConfig dataTransferTaskConfig : config.getFileTransferTasks()) {
-			dataMovementTask.put(dataTransferTaskConfig.getTransferProtocol(), null);
-		}
 
-		for (ResourceConfig resourceConfig : config.getResourceConfiguration()) {
-			resources.put(resourceConfig.getJobManagerType(), resourceConfig);
-		}
 	}
 
 	@Override
 	public ProcessContext populateProcessContext(String experimentId, String processId, String gatewayId, String
 			tokenId) throws GFacException {
-		ProcessContext processContext = new ProcessContext(processId, gatewayId, tokenId);
-		processContext.setProcessModel(new ProcessModel());
-		// TODO: get process model from app catalog
-		// TODO: set datamovement protocol and jobsubmission protocol
-		// TODO: set up gatewayResourceProfile.
-		// TODO: set RemoteCluster
-		return processContext;
+		try {
+			ProcessContext processContext = new ProcessContext(processId, gatewayId, tokenId);
+			AppCatalog appCatalog = Factory.getDefaultAppCatalog();
+			ExperimentCatalog expCatalog = Factory.getDefaultExpCatalog();
+			processContext.setProcessModel((ProcessModel) expCatalog.get(ExperimentCatalogModelType.PROCESS, processId));
+			GatewayResourceProfile gatewayProfile = appCatalog.getGatewayProfile().getGatewayProfile(gatewayId);
+			processContext.setGatewayResourceProfile(gatewayProfile);
+			processContext.setComputeResourcePreference(appCatalog.getGatewayProfile().getComputeResourcePreference
+					(gatewayId, processContext.getProcessModel().getComputeResourceId()));
+			processContext.setRemoteCluster(Factory.getRemoteCluster(processContext.getComputeResourcePreference()));
+			//
+			return processContext;
+		} catch (AppCatalogException e) {
+			throw new GFacException("App catalog access exception ", e);
+		} catch (RegistryException e) {
+			throw new GFacException("Registry access exception", e);
+		} catch (AiravataException e) {
+			throw new GFacException("Remote cluster initialization error", e);
+		}
 	}
 
 	@Override
@@ -95,7 +89,7 @@ public class GFacEngineImpl implements GFacEngine {
 						break;
 					case URI:
 						// TODO : provide data staging data model
-						taskChain.add(dataMovementTask.get(processContext.getDataMovementProtocol()));
+						taskChain.add(Factory.getDataMovementTask(processContext.getDataMovementProtocol()));
 						break;
 					default:
 						// nothing to do
@@ -103,7 +97,7 @@ public class GFacEngineImpl implements GFacEngine {
 				}
 			}
 		}
-		taskChain.add(jobSubmissionTask.get(processContext.getJobSubmissionProtocol()));
+		taskChain.add(Factory.getJobSubmissionTask(processContext.getJobSubmissionProtocol()));
 		List<OutputDataObjectType> processOutputs = processContext.getProcessModel().getProcessOutputs();
 		for (OutputDataObjectType processOutput : processOutputs) {
 			DataType type = processOutput.getType();
@@ -114,7 +108,7 @@ public class GFacEngineImpl implements GFacEngine {
 					break;
 				case URI:
 					// TODO : Provide data staging data model
-					taskChain.add(dataMovementTask.get(processContext.getDataMovementProtocol()));
+					taskChain.add(Factory.getDataMovementTask(processContext.getDataMovementProtocol()));
 					break;
 			}
 		}
@@ -160,4 +154,9 @@ public class GFacEngineImpl implements GFacEngine {
 			}
 		});
 	}
+
+
+
+
+
 }

http://git-wip-us.apache.org/repos/asf/airavata/blob/726370bd/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
index 645cb30..d847a4d 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
@@ -26,16 +26,16 @@ import com.jcraft.jsch.JSchException;
 import com.jcraft.jsch.Session;
 import com.jcraft.jsch.UserInfo;
 import org.apache.airavata.common.exception.AiravataException;
-import org.apache.airavata.gfac.core.authentication.AuthenticationInfo;
+import org.apache.airavata.gfac.core.JobManagerConfiguration;
 import org.apache.airavata.gfac.core.SSHApiException;
+import org.apache.airavata.gfac.core.authentication.AuthenticationInfo;
 import org.apache.airavata.gfac.core.authentication.SSHKeyAuthentication;
+import org.apache.airavata.gfac.core.cluster.AbstractRemoteCluster;
 import org.apache.airavata.gfac.core.cluster.CommandInfo;
 import org.apache.airavata.gfac.core.cluster.CommandOutput;
 import org.apache.airavata.gfac.core.cluster.OutputParser;
 import org.apache.airavata.gfac.core.cluster.RawCommandInfo;
-import org.apache.airavata.gfac.core.cluster.RemoteCluster;
 import org.apache.airavata.gfac.core.cluster.ServerInfo;
-import org.apache.airavata.gfac.core.JobManagerConfiguration;
 import org.apache.airavata.model.status.JobStatus;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -49,28 +49,24 @@ import java.util.Map;
 /**
  * One Remote cluster instance for each compute resource.
  */
-public class HPCRemoteCluster implements RemoteCluster{
+public class HPCRemoteCluster extends AbstractRemoteCluster{
     private static final Logger log = LoggerFactory.getLogger(HPCRemoteCluster.class);
 	private final SSHKeyAuthentication authentication;
-	private final ServerInfo serverInfo;
-	private final JobManagerConfiguration jobManagerConfiguration;
 	private final JSch jSch;
 	private Session session;
-	private OutputParser outputParser;
 
 	public HPCRemoteCluster(ServerInfo serverInfo, JobManagerConfiguration jobManagerConfiguration, AuthenticationInfo
-			authenticationInfo, OutputParser outputParser) throws AiravataException {
+			authenticationInfo) throws AiravataException {
+		super(serverInfo, jobManagerConfiguration, authenticationInfo);
 		try {
-			this.serverInfo = serverInfo;
-			this.jobManagerConfiguration = jobManagerConfiguration;
 			if (authenticationInfo instanceof SSHKeyAuthentication) {
 				authentication = (SSHKeyAuthentication) authenticationInfo;
 			} else {
 				throw new AiravataException("Support ssh key authentication only");
 			}
-			this.outputParser = outputParser;
 			jSch = new JSch();
-			jSch.addIdentity(authentication.getPrivateKeyFilePath(), authentication.getPublicKeyFilePath(), authentication
+			jSch.addIdentity(authentication.getPrivateKeyFilePath(), authentication.getPublicKeyFilePath(),
+					authentication
 					.getPassphrase().getBytes());
 			session = jSch.getSession(serverInfo.getUserName(), serverInfo.getHost(), serverInfo.getPort());
 			session.setUserInfo(new DefaultUserInfo(serverInfo.getUserName(), null, authentication.getPassphrase()));

http://git-wip-us.apache.org/repos/asf/airavata/blob/726370bd/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/LSFJobConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/LSFJobConfiguration.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/LSFJobConfiguration.java
index b84b8ff..783fc56 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/LSFJobConfiguration.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/LSFJobConfiguration.java
@@ -23,28 +23,25 @@ package org.apache.airavata.gfac.impl.job;
 import org.apache.airavata.gfac.core.JobManagerConfiguration;
 import org.apache.airavata.gfac.core.cluster.OutputParser;
 import org.apache.airavata.gfac.core.cluster.RawCommandInfo;
+import org.apache.airavata.model.appcatalog.computeresource.JobManagerCommand;
 import org.apache.commons.io.FilenameUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.File;
+import java.util.Map;
 
 public class LSFJobConfiguration implements JobManagerConfiguration {
     private final static Logger logger = LoggerFactory.getLogger(LSFJobConfiguration.class);
-
+	private final Map<JobManagerCommand, String> jobMangerCommands;
     private String jobDescriptionTemplateName;
-
     private String scriptExtension;
-
     private String installedPath;
-
     private OutputParser parser;
 
-    public LSFJobConfiguration(){
-        // this can be used to construct and use setter methods to set all the params in order
-    }
     public LSFJobConfiguration(String jobDescriptionTemplateName,
-                                 String scriptExtension,String installedPath,OutputParser parser) {
+                               String scriptExtension, String installedPath, Map<JobManagerCommand, String>
+		                               jobManagerCommands, OutputParser parser) {
         this.jobDescriptionTemplateName = jobDescriptionTemplateName;
         this.scriptExtension = scriptExtension;
         this.parser = parser;
@@ -53,6 +50,7 @@ public class LSFJobConfiguration implements JobManagerConfiguration {
         } else {
             this.installedPath = installedPath + "/";
         }
+	    this.jobMangerCommands = jobManagerCommands;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/airavata/blob/726370bd/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/PBSJobConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/PBSJobConfiguration.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/PBSJobConfiguration.java
index d709514..1867095 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/PBSJobConfiguration.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/PBSJobConfiguration.java
@@ -23,35 +23,32 @@ package org.apache.airavata.gfac.impl.job;
 import org.apache.airavata.gfac.core.JobManagerConfiguration;
 import org.apache.airavata.gfac.core.cluster.OutputParser;
 import org.apache.airavata.gfac.core.cluster.RawCommandInfo;
+import org.apache.airavata.model.appcatalog.computeresource.JobManagerCommand;
 import org.apache.commons.io.FilenameUtils;
 
 import java.io.File;
+import java.util.Map;
 
 public class PBSJobConfiguration implements JobManagerConfiguration {
 
-    private String jobDescriptionTemplateName;
-
+	private final Map<JobManagerCommand, String> jobManagerCommands;
+	private String jobDescriptionTemplateName;
     private String scriptExtension;
-
     private String installedPath;
-
     private OutputParser parser;
 
-    public PBSJobConfiguration() {
-        // this can be used to construct and use setter methods to set all the params in order
-    }
-
-    public PBSJobConfiguration(String jobDescriptionTemplateName,
-                               String scriptExtension, String installedPath, OutputParser parser) {
-        this.jobDescriptionTemplateName = jobDescriptionTemplateName;
-        this.scriptExtension = scriptExtension;
-        this.parser = parser;
-        if (installedPath.endsWith("/")) {
-            this.installedPath = installedPath;
-        } else {
-            this.installedPath = installedPath + "/";
-        }
-    }
+	public PBSJobConfiguration(String jobDescriptionTemplateName, String scriptExtension, String installedPath,
+	                           Map<JobManagerCommand, String> jobManagerCommands, OutputParser parser) {
+		this.jobDescriptionTemplateName = jobDescriptionTemplateName;
+		this.scriptExtension = scriptExtension;
+		this.parser = parser;
+		if (installedPath.endsWith("/")) {
+			this.installedPath = installedPath;
+		} else {
+			this.installedPath = installedPath + "/";
+		}
+		this.jobManagerCommands = jobManagerCommands;
+	}
 
     public RawCommandInfo getCancelCommand(String jobID) {
         return new RawCommandInfo(this.installedPath + "qdel " + jobID);

http://git-wip-us.apache.org/repos/asf/airavata/blob/726370bd/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/SlurmJobConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/SlurmJobConfiguration.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/SlurmJobConfiguration.java
index 354db8a..157b77c 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/SlurmJobConfiguration.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/SlurmJobConfiguration.java
@@ -23,25 +23,22 @@ package org.apache.airavata.gfac.impl.job;
 import org.apache.airavata.gfac.core.JobManagerConfiguration;
 import org.apache.airavata.gfac.core.cluster.OutputParser;
 import org.apache.airavata.gfac.core.cluster.RawCommandInfo;
+import org.apache.airavata.model.appcatalog.computeresource.JobManagerCommand;
 import org.apache.commons.io.FilenameUtils;
 
 import java.io.File;
+import java.util.Map;
 
 public class SlurmJobConfiguration implements JobManagerConfiguration {
-
+	private final Map<JobManagerCommand, String> jobManagerCommands;
     private String jobDescriptionTemplateName;
-
     private String scriptExtension;
-
     private String installedPath;
-
     private OutputParser parser;
 
-    public SlurmJobConfiguration(){
-        // this can be used to construct and use setter methods to set all the params in order
-    }
     public SlurmJobConfiguration(String jobDescriptionTemplateName,
-                                   String scriptExtension,String installedPath,OutputParser parser) {
+                                 String scriptExtension, String installedPath, Map<JobManagerCommand, String>
+		                                 jobManagerCommands, OutputParser parser) {
         this.jobDescriptionTemplateName = jobDescriptionTemplateName;
         this.scriptExtension = scriptExtension;
         this.parser = parser;
@@ -50,6 +47,7 @@ public class SlurmJobConfiguration implements JobManagerConfiguration {
         } else {
             this.installedPath = installedPath + "/";
         }
+	    this.jobManagerCommands = jobManagerCommands;
     }
 
     public RawCommandInfo getCancelCommand(String jobID) {

http://git-wip-us.apache.org/repos/asf/airavata/blob/726370bd/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/UGEJobConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/UGEJobConfiguration.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/UGEJobConfiguration.java
index f9c60cb..aabdffb 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/UGEJobConfiguration.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/UGEJobConfiguration.java
@@ -23,26 +23,22 @@ package org.apache.airavata.gfac.impl.job;
 import org.apache.airavata.gfac.core.JobManagerConfiguration;
 import org.apache.airavata.gfac.core.cluster.OutputParser;
 import org.apache.airavata.gfac.core.cluster.RawCommandInfo;
+import org.apache.airavata.model.appcatalog.computeresource.JobManagerCommand;
 import org.apache.commons.io.FilenameUtils;
 
 import java.io.File;
+import java.util.Map;
 
 public class UGEJobConfiguration implements JobManagerConfiguration {
-
+	private final Map<JobManagerCommand, String> jobManagerCommands;
     private String jobDescriptionTemplateName;
-
     private String scriptExtension;
-
     private String installedPath;
-
     private OutputParser parser;
 
-    public UGEJobConfiguration() {
-        // this can be used to construct and use setter methods to set all the params in order
-    }
-
     public UGEJobConfiguration(String jobDescriptionTemplateName,
-                               String scriptExtension, String installedPath, OutputParser parser) {
+                               String scriptExtension, String installedPath, Map<JobManagerCommand, String>
+		                               jobManagerCommands, OutputParser parser) {
         this.jobDescriptionTemplateName = jobDescriptionTemplateName;
         this.scriptExtension = scriptExtension;
         this.parser = parser;
@@ -51,6 +47,7 @@ public class UGEJobConfiguration implements JobManagerConfiguration {
         } else {
             this.installedPath = installedPath + "/";
         }
+	    this.jobManagerCommands = jobManagerCommands;
     }
 
     public RawCommandInfo getCancelCommand(String jobID) {

http://git-wip-us.apache.org/repos/asf/airavata/blob/726370bd/modules/gfac/gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java b/modules/gfac/gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
index a4a4874..8583a67 100644
--- a/modules/gfac/gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
+++ b/modules/gfac/gfac-service/src/main/java/org/apache/airavata/gfac/server/GfacServerHandler.java
@@ -237,7 +237,7 @@ public class GfacServerHandler implements GfacService.Iface {
                     ExperimentStatus status = new ExperimentStatus();
                     status.setState(ExperimentState.EXECUTING);
                     status.setTimeOfStateChange(Calendar.getInstance().getTimeInMillis());
-                    experimentCatalog.update(ExperimentCatalogModelType.EXPERIMENT_STATUS, status, event.getExperimentId());
+                    Factory.getDefaultExpCatalog().update(ExperimentCatalogModelType.EXPERIMENT_STATUS, status, event.getExperimentId());
                     try {
 	                    GFacUtils.createExperimentNode(curatorClient, gfacServerName, event.getExperimentId(), message.getDeliveryTag(),
 			                    event.getTokenId());

[2/2] airavata git commit: Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/airavata

Posted by sh...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/airavata


Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/013e8c09
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/013e8c09
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/013e8c09

Branch: refs/heads/master
Commit: 013e8c0905659ecb5170c29b81e698f1b999c1d7
Parents: 726370b e5d861e
Author: Shameera Rathanyaka <sh...@gmail.com>
Authored: Tue Jun 23 17:31:35 2015 -0400
Committer: Shameera Rathanyaka <sh...@gmail.com>
Committed: Tue Jun 23 17:31:35 2015 -0400

----------------------------------------------------------------------
 .../src/main/resources/lib/airavata/Airavata.h  | 152 ++++-----
 .../model/experiment/ExperimentModel.java       | 213 +++++++++----
 .../experiment/ExperimentSummaryModel.java      | 149 +++++++--
 .../event/ProcessStatusChangeRequestEvent.java  |  22 +-
 .../apache/airavata/gfac/core/GFacUtils.java    |  67 +++-
 .../gfac/impl/AiravataJobStatusUpdator.java     | 240 +++++++-------
 .../gfac/impl/AiravataProcessStatusUpdator.java | 254 ++++++++-------
 .../gfac/impl/AiravataTaskStatusUpdator.java    | 318 +++++++++----------
 .../core/impl/RabbitMQStatusConsumer.java       |   2 +-
 .../utils/ThriftDataModelConversion.java        |   6 +-
 .../airavata-api/experiment_model.thrift        |  42 +--
 .../airavata-api/messaging_events.thrift        |   2 +-
 12 files changed, 860 insertions(+), 607 deletions(-)
----------------------------------------------------------------------