You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2014/05/05 16:44:27 UTC
[3/8] changing package names of gfac implementations
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/provider/utils/HadoopUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/provider/utils/HadoopUtils.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/provider/utils/HadoopUtils.java
new file mode 100644
index 0000000..2947fc3
--- /dev/null
+++ b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/provider/utils/HadoopUtils.java
@@ -0,0 +1,60 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+
+package org.apache.airavata.gfac.hadoop.provider.utils;
+
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.context.MessageContext;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.util.Collection;
+
+public class HadoopUtils {
+ public static Configuration createHadoopConfiguration(
+ JobExecutionContext jobExecutionContext,
+ boolean isWhirrBasedDeployment,
+ File hadoopConfigDir) throws FileNotFoundException {
+ MessageContext inMessageContext = jobExecutionContext.getInMessageContext();
+ Configuration hadoopConf = new Configuration();
+
+ if(isWhirrBasedDeployment){
+ hadoopConf.addResource(new FileInputStream(
+ new File((String)inMessageContext.getParameter("HADOOP_SITE_XML"))));
+ } else {
+ readHadoopClusterConfigurationFromDirectory(hadoopConfigDir, hadoopConf);
+ }
+
+ return hadoopConf;
+ }
+
+ private static void readHadoopClusterConfigurationFromDirectory(File localHadoopConfigurationDirectory, Configuration hadoopConf)
+ throws FileNotFoundException {
+ Collection hadoopConfigurationFiles =
+ FileUtils.listFiles(localHadoopConfigurationDirectory, null, false);
+ for (Object f : hadoopConfigurationFiles) {
+ hadoopConf.addResource(new FileInputStream((File)f));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HDFSDataMovementHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HDFSDataMovementHandler.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HDFSDataMovementHandler.java
deleted file mode 100644
index 49eadf4..0000000
--- a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HDFSDataMovementHandler.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-
-package org.apache.airavata.gfac.handler;
-
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.gfac.provider.utils.HadoopUtils;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.apache.airavata.schemas.gfac.HadoopApplicationDeploymentDescriptionType;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Map;
-
-public class HDFSDataMovementHandler implements GFacHandler {
- private static final Logger logger = LoggerFactory.getLogger(HDFSDataMovementHandler.class);
-
- private boolean isWhirrBasedDeployment = false;
- private File hadoopConfigDir;
-
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- MessageContext inMessageContext = jobExecutionContext.getInMessageContext();
- if(inMessageContext.getParameter("HADOOP_DEPLOYMENT_TYPE").equals("WHIRR")){
- isWhirrBasedDeployment = true;
- } else {
- String hadoopConfigDirPath = (String)inMessageContext.getParameter("HADOOP_CONFIG_DIR");
- File hadoopConfigDir = new File(hadoopConfigDirPath);
- if (!hadoopConfigDir.exists()){
- throw new GFacHandlerException("Specified hadoop configuration directory doesn't exist.");
- } else if (FileUtils.listFiles(hadoopConfigDir, null, null).size() <= 0){
- throw new GFacHandlerException("Cannot find any hadoop configuration files inside specified directory.");
- }
-
- this.hadoopConfigDir = hadoopConfigDir;
- }
-
- if(jobExecutionContext.isInPath()){
- try {
- handleInPath(jobExecutionContext);
- } catch (IOException e) {
- throw new GFacHandlerException("Error while copying input data from local file system to HDFS.",e);
- }
- } else {
- handleOutPath(jobExecutionContext);
- }
- }
-
- private void handleInPath(JobExecutionContext jobExecutionContext) throws GFacHandlerException, IOException {
- ApplicationDeploymentDescriptionType appDepDesc =
- jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
- HadoopApplicationDeploymentDescriptionType hadoopAppDesc =
- (HadoopApplicationDeploymentDescriptionType)appDepDesc;
- if(appDepDesc.isSetInputDataDirectory() && isInputDataDirectoryLocal(appDepDesc)){
- Configuration hadoopConf = HadoopUtils.createHadoopConfiguration(jobExecutionContext, isWhirrBasedDeployment, hadoopConfigDir);
- FileSystem hdfs = FileSystem.get(hadoopConf);
- hdfs.copyFromLocalFile(new Path(appDepDesc.getInputDataDirectory()),
- new Path(hadoopAppDesc.getHadoopJobConfiguration().getHdfsInputDirectory()));
- }
- }
-
- private boolean isInputDataDirectoryLocal(ApplicationDeploymentDescriptionType appDepDesc){
- String inputDataDirectoryPath = appDepDesc.getInputDataDirectory();
- File inputDataDirectory = new File(inputDataDirectoryPath);
- if(inputDataDirectory.exists() && FileUtils.listFiles(inputDataDirectory, null, null).size() > 0){
- return true;
- }
-
- return false;
- }
-
- private void handleOutPath(JobExecutionContext jobExecutionContext){}
-
- public void initProperties(Map<String, String> properties) throws GFacHandlerException {
-
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HadoopDeploymentHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HadoopDeploymentHandler.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HadoopDeploymentHandler.java
deleted file mode 100644
index c7de3f9..0000000
--- a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HadoopDeploymentHandler.java
+++ /dev/null
@@ -1,274 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-
-package org.apache.airavata.gfac.handler;
-
-import com.google.common.io.Files;
-import org.apache.airavata.commons.gfac.type.HostDescription;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.schemas.gfac.HadoopHostType;
-import org.apache.commons.configuration.CompositeConfiguration;
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.commons.io.FileUtils;
-import org.apache.whirr.Cluster;
-import org.apache.whirr.ClusterController;
-import org.apache.whirr.ClusterControllerFactory;
-import org.apache.whirr.ClusterSpec;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.*;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.Map;
-import java.util.Properties;
-
-import static org.apache.whirr.ClusterSpec.Property.*;
-import static org.apache.whirr.ClusterSpec.Property.INSTANCE_TEMPLATES;
-import static org.apache.whirr.ClusterSpec.Property.PRIVATE_KEY_FILE;
-
-/**
- * This handler takes care of deploying hadoop in cloud(in cloud bursting scenarios) and
- * deploying hadoop in local cluster. In case of existing hadoop cluster this will ignore
- * cluster setup just use the hadoop configuration provided by user.
- */
-public class HadoopDeploymentHandler implements GFacHandler {
- private static final Logger logger = LoggerFactory.getLogger("hadoop-dep-handler");
-
- /**
- * Once invoked this method will deploy Hadoop in a local cluster or cloud based on the
- * configuration provided. If there is a already deployed hadoop cluster this will skip
- * deployment.
- *
- *
- * @param jobExecutionContext job execution context containing all the required configurations
- * and runtime information.
- * @throws GFacHandlerException
- */
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- if(jobExecutionContext.isInPath()){
- handleInPath(jobExecutionContext);
- } else {
- handleOutPath(jobExecutionContext);
- }
- }
-
- private void handleInPath(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- HostDescription hostDescription =
- jobExecutionContext.getApplicationContext().getHostDescription();
- if (!isHadoopDeploymentAvailable(hostDescription)) {
- // Temp directory to keep generated configuration files.
- File tempDirectory = Files.createTempDir();
- try {
- File hadoopSiteXML = launchHadoopCluster(hostDescription, tempDirectory);
- jobExecutionContext.getInMessageContext().addParameter("HADOOP_SITE_XML", hadoopSiteXML.getAbsolutePath());
- jobExecutionContext.getInMessageContext().addParameter("HADOOP_DEPLOYMENT_TYPE", "WHIRR");
- // TODO: Add hadoop-site.xml to job execution context.
- } catch (IOException e) {
- throw new GFacHandlerException("IO Error while processing configurations.",e);
- } catch (ConfigurationException e) {
- throw new GFacHandlerException("Whirr configuration error.", e);
- } catch (InterruptedException e) {
- throw new GFacHandlerException("Hadoop cluster launch interrupted.", e);
- } catch (TransformerException e) {
- throw new GFacHandlerException("Error while creating hadoop-site.xml", e);
- } catch (ParserConfigurationException e) {
- throw new GFacHandlerException("Error while creating hadoop-site.xml", e);
- }
- } else {
- jobExecutionContext.getInMessageContext().addParameter("HADOOP_DEPLOYMENT_TYPE",
- "MANUAL");
- jobExecutionContext.getInMessageContext().addParameter("HADOOP_CONFIG_DIR",
- ((HadoopHostType)hostDescription.getType()).getHadoopConfigurationDirectory());
- logger.info("Hadoop configuration is available. Skipping hadoop deployment.");
- if(logger.isDebugEnabled()){
- logger.debug("Hadoop configuration directory: " +
- getHadoopConfigDirectory(hostDescription));
- }
- }
- }
-
- private void handleOutPath(JobExecutionContext jobExecutionContext){
- MessageContext inMessageContext = jobExecutionContext.getInMessageContext();
- if(((String)inMessageContext.getParameter("HADOOP_DEPLOYMENT_TYPE")).equals("WHIRR")){
- // TODO: Shutdown hadoop cluster.
- logger.info("Shutdown hadoop cluster.");
- }
- }
-
- private File launchHadoopCluster(HostDescription hostDescription, File workingDirectory)
- throws IOException, GFacHandlerException, ConfigurationException, InterruptedException, TransformerException, ParserConfigurationException {
- ClusterSpec hadoopClusterSpec =
- whirrConfigurationToClusterSpec(hostDescription, workingDirectory);
- ClusterController hadoopClusterController =
- createClusterController(hadoopClusterSpec.getServiceName());
- Cluster hadoopCluster = hadoopClusterController.launchCluster(hadoopClusterSpec);
-
- logger.info(String.format("Started cluster of %s instances.\n",
- hadoopCluster.getInstances().size()));
-
- File siteXML = new File(workingDirectory, "hadoop-site.xml");
- clusterPropertiesToHadoopSiteXml(hadoopCluster.getConfiguration(), siteXML);
-
- return siteXML;
- }
-
- private ClusterController createClusterController(String serviceName){
- ClusterControllerFactory factory = new ClusterControllerFactory();
- ClusterController controller = factory.create(serviceName);
-
- if(controller == null){
- logger.warn("Unable to find the service {0}, using default.", serviceName);
- controller = factory.create(null);
- }
-
- return controller;
- }
-
- private ClusterSpec whirrConfigurationToClusterSpec(HostDescription hostDescription,
- File workingDirectory) throws IOException, GFacHandlerException, ConfigurationException {
- File whirrConfig = getWhirrConfigurationFile(hostDescription, workingDirectory);
- CompositeConfiguration compositeConfiguration = new CompositeConfiguration();
- Configuration configuration = new PropertiesConfiguration(whirrConfig);
- compositeConfiguration.addConfiguration(configuration);
-
- ClusterSpec hadoopClusterSpec = new ClusterSpec(compositeConfiguration);
-
- for (ClusterSpec.Property required : EnumSet.of(CLUSTER_NAME, PROVIDER, IDENTITY, CREDENTIAL,
- INSTANCE_TEMPLATES, PRIVATE_KEY_FILE)) {
- if (hadoopClusterSpec.getConfiguration().getString(required.getConfigName()) == null) {
- throw new IllegalArgumentException(String.format("Option '%s' not set.",
- required.getSimpleName()));
- }
- }
-
- return hadoopClusterSpec;
- }
-
- private File getWhirrConfigurationFile(HostDescription hostDescription, File workingDirectory)
- throws GFacHandlerException, IOException {
- HadoopHostType hadoopHostDesc = (HadoopHostType)hostDescription;
- if(hadoopHostDesc.isSetWhirrConfiguration()){
- HadoopHostType.WhirrConfiguration whirrConfig = hadoopHostDesc.getWhirrConfiguration();
- if(whirrConfig.isSetConfigurationFile()){
- File whirrConfigFile = new File(whirrConfig.getConfigurationFile());
- if(!whirrConfigFile.exists()){
- throw new GFacHandlerException(
- "Specified whirr configuration file doesn't exists.");
- }
-
- FileUtils.copyFileToDirectory(whirrConfigFile, workingDirectory);
-
- return new File(workingDirectory, whirrConfigFile.getName());
- } else if(whirrConfig.isSetConfiguration()){
- Properties whirrConfigProps =
- whirrConfigurationsToProperties(whirrConfig.getConfiguration());
- File whirrConfigFile = new File(workingDirectory, "whirr-hadoop.config");
- whirrConfigProps.store(
- new FileOutputStream(whirrConfigFile), null);
-
- return whirrConfigFile;
- }
- }
-
- throw new GFacHandlerException("Cannot find Whirr configurations. Whirr configuration "
- + "is required if you don't have already running Hadoop deployment.");
- }
-
- private Properties whirrConfigurationsToProperties(
- HadoopHostType.WhirrConfiguration.Configuration configuration){
- Properties whirrConfigProps = new Properties();
-
- for(HadoopHostType.WhirrConfiguration.Configuration.Property property:
- configuration.getPropertyArray()) {
- whirrConfigProps.put(property.getName(), property.getValue());
- }
-
- return whirrConfigProps;
- }
-
- private void clusterPropertiesToHadoopSiteXml(Properties props, File hadoopSiteXml) throws ParserConfigurationException, TransformerException {
- DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance();
- DocumentBuilder documentBuilder = domFactory.newDocumentBuilder();
-
- Document hadoopSiteXmlDoc = documentBuilder.newDocument();
-
- hadoopSiteXmlDoc.setXmlVersion("1.0");
- hadoopSiteXmlDoc.setXmlStandalone(true);
- hadoopSiteXmlDoc.createProcessingInstruction("xml-stylesheet", "type=\"text/xsl\" href=\"configuration.xsl\"");
-
- Element configEle = hadoopSiteXmlDoc.createElement("configuration");
-
- hadoopSiteXmlDoc.appendChild(configEle);
-
- for(Map.Entry<Object, Object> entry : props.entrySet()){
- addPropertyToConfiguration(entry, configEle, hadoopSiteXmlDoc);
- }
-
- saveDomToFile(hadoopSiteXmlDoc, hadoopSiteXml);
- }
-
- private void saveDomToFile(Document dom, File destFile) throws TransformerException {
- Source source = new DOMSource(dom);
-
- Result result = new StreamResult(destFile);
-
- Transformer transformer = TransformerFactory.newInstance().newTransformer();
- transformer.transform(source, result);
- }
-
- private void addPropertyToConfiguration(Map.Entry<Object, Object> entry, Element configElement, Document doc){
- Element property = doc.createElement("property");
- configElement.appendChild(property);
-
- Element nameEle = doc.createElement("name");
- nameEle.setTextContent(entry.getKey().toString());
- property.appendChild(nameEle);
-
- Element valueEle = doc.createElement("value");
- valueEle.setTextContent(entry.getValue().toString());
- property.appendChild(valueEle);
- }
-
- private boolean isHadoopDeploymentAvailable(HostDescription hostDescription) {
- return ((HadoopHostType) hostDescription.getType()).isSetHadoopConfigurationDirectory();
- }
-
- private String getHadoopConfigDirectory(HostDescription hostDescription){
- return ((HadoopHostType)hostDescription.getType()).getHadoopConfigurationDirectory();
- }
-
- public void initProperties(Map<String, String> properties) throws GFacHandlerException {
-
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/impl/HadoopProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/impl/HadoopProvider.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/impl/HadoopProvider.java
deleted file mode 100644
index 0fc8b47..0000000
--- a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/impl/HadoopProvider.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-
-package org.apache.airavata.gfac.provider.impl;
-
-import java.io.File;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.ArrayList;
-import java.util.Map;
-
-import org.apache.airavata.commons.gfac.type.ActualParameter;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.gfac.provider.AbstractProvider;
-import org.apache.airavata.gfac.provider.GFacProviderException;
-import org.apache.airavata.gfac.provider.utils.HadoopUtils;
-import org.apache.airavata.schemas.gfac.HadoopApplicationDeploymentDescriptionType;
-import org.apache.airavata.schemas.gfac.OutputParameterType;
-import org.apache.airavata.schemas.gfac.StringParameterType;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import sun.reflect.generics.reflectiveObjects.NotImplementedException;
-
-/**
- * Executes hadoop job using the cluster configuration provided by handlers in
- * in-flow.
- */
-public class HadoopProvider extends AbstractProvider {
- private static final Logger logger = LoggerFactory.getLogger(HadoopProvider.class);
-
- private boolean isWhirrBasedDeployment = false;
- private File hadoopConfigDir;
-
- public void initialize(JobExecutionContext jobExecutionContext) throws GFacProviderException {
- MessageContext inMessageContext = jobExecutionContext.getInMessageContext();
- if(inMessageContext.getParameter("HADOOP_DEPLOYMENT_TYPE").equals("WHIRR")){
- isWhirrBasedDeployment = true;
- } else {
- String hadoopConfigDirPath = (String)inMessageContext.getParameter("HADOOP_CONFIG_DIR");
- File hadoopConfigDir = new File(hadoopConfigDirPath);
- if (!hadoopConfigDir.exists()){
- throw new GFacProviderException("Specified hadoop configuration directory doesn't exist.");
- } else if (FileUtils.listFiles(hadoopConfigDir, null, null).size() <= 0){
- throw new GFacProviderException("Cannot find any hadoop configuration files inside specified directory.");
- }
-
- this.hadoopConfigDir = hadoopConfigDir;
- }
- }
-
- public void execute(JobExecutionContext jobExecutionContext) throws GFacProviderException {
- HadoopApplicationDeploymentDescriptionType hadoopAppDesc =
- (HadoopApplicationDeploymentDescriptionType)jobExecutionContext
- .getApplicationContext().getApplicationDeploymentDescription().getType();
- MessageContext inMessageContext = jobExecutionContext.getInMessageContext();
- HadoopApplicationDeploymentDescriptionType.HadoopJobConfiguration jobConf = hadoopAppDesc.getHadoopJobConfiguration();
-
- try{
- // Preparing Hadoop configuration
- Configuration hadoopConf = HadoopUtils.createHadoopConfiguration(
- jobExecutionContext, isWhirrBasedDeployment, hadoopConfigDir);
-
- // Load jar containing map-reduce job implementation
- ArrayList<URL> mapRedJars = new ArrayList<URL>();
- mapRedJars.add(new File(jobConf.getJarLocation()).toURL());
- URLClassLoader childClassLoader = new URLClassLoader(mapRedJars.toArray(new URL[mapRedJars.size()]),
- this.getClass().getClassLoader());
-
- Job job = new Job(hadoopConf);
-
- job.setJobName(jobConf.getJobName());
-
- job.setOutputKeyClass(Class.forName(jobConf.getOutputKeyClass(), true, childClassLoader));
- job.setOutputValueClass(Class.forName(jobConf.getOutputValueClass(), true, childClassLoader));
-
- job.setMapperClass((Class<? extends Mapper>)Class.forName(jobConf.getMapperClass(), true, childClassLoader));
- job.setCombinerClass((Class<? extends Reducer>) Class.forName(jobConf.getCombinerClass(), true, childClassLoader));
- job.setReducerClass((Class<? extends Reducer>) Class.forName(jobConf.getCombinerClass(), true, childClassLoader));
-
- job.setInputFormatClass((Class<? extends InputFormat>)Class.forName(jobConf.getInputFormatClass(), true, childClassLoader));
- job.setOutputFormatClass((Class<? extends OutputFormat>) Class.forName(jobConf.getOutputFormatClass(), true, childClassLoader));
-
- FileInputFormat.setInputPaths(job, new Path(hadoopAppDesc.getInputDataDirectory()));
- FileOutputFormat.setOutputPath(job, new Path(hadoopAppDesc.getOutputDataDirectory()));
-
- job.waitForCompletion(true);
- System.out.println(job.getTrackingURL());
- if(jobExecutionContext.getOutMessageContext() == null){
- jobExecutionContext.setOutMessageContext(new MessageContext());
- }
-
- OutputParameterType[] outputParametersArray = jobExecutionContext.getApplicationContext().
- getServiceDescription().getType().getOutputParametersArray();
- for(OutputParameterType outparamType : outputParametersArray){
- String paramName = outparamType.getParameterName();
- if(paramName.equals("test-hadoop")){
- ActualParameter outParam = new ActualParameter();
- outParam.getType().changeType(StringParameterType.type);
- ((StringParameterType) outParam.getType()).setValue(job.getTrackingURL());
- jobExecutionContext.getOutMessageContext().addParameter("test-hadoop", outParam);
- }
- }
- } catch (Exception e) {
- String errMessage = "Error occurred during Map-Reduce job execution.";
- logger.error(errMessage, e);
- throw new GFacProviderException(errMessage, e);
- }
- }
-
- public void dispose(JobExecutionContext jobExecutionContext) throws GFacProviderException {
- // TODO: How to handle cluster shutdown. Best way is to introduce inPath/outPath to handler.
- }
-
- @Override
- public void cancelJob(String jobId, JobExecutionContext jobExecutionContext) throws GFacException {
- throw new NotImplementedException();
- }
-
-
- public void initProperties(Map<String, String> properties) throws GFacProviderException, GFacException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/utils/HadoopUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/utils/HadoopUtils.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/utils/HadoopUtils.java
deleted file mode 100644
index c3053d1..0000000
--- a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/utils/HadoopUtils.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-
-package org.apache.airavata.gfac.provider.utils;
-
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.util.Collection;
-
-public class HadoopUtils {
- public static Configuration createHadoopConfiguration(
- JobExecutionContext jobExecutionContext,
- boolean isWhirrBasedDeployment,
- File hadoopConfigDir) throws FileNotFoundException {
- MessageContext inMessageContext = jobExecutionContext.getInMessageContext();
- Configuration hadoopConf = new Configuration();
-
- if(isWhirrBasedDeployment){
- hadoopConf.addResource(new FileInputStream(
- new File((String)inMessageContext.getParameter("HADOOP_SITE_XML"))));
- } else {
- readHadoopClusterConfigurationFromDirectory(hadoopConfigDir, hadoopConf);
- }
-
- return hadoopConf;
- }
-
- private static void readHadoopClusterConfigurationFromDirectory(File localHadoopConfigurationDirectory, Configuration hadoopConf)
- throws FileNotFoundException {
- Collection hadoopConfigurationFiles =
- FileUtils.listFiles(localHadoopConfigurationDirectory, null, false);
- for (Object f : hadoopConfigurationFiles) {
- hadoopConf.addResource(new FileInputStream((File)f));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/test/resources/gfac-config.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-hadoop/src/test/resources/gfac-config.xml b/modules/gfac/gfac-hadoop/src/test/resources/gfac-config.xml
index f43a5fb..9c4774c 100644
--- a/modules/gfac/gfac-hadoop/src/test/resources/gfac-config.xml
+++ b/modules/gfac/gfac-hadoop/src/test/resources/gfac-config.xml
@@ -20,21 +20,11 @@
<OutHandlers></OutHandlers>
</GlobalHandlers>
- <Provider class="org.apache.airavata.gfac.provider.impl.HadoopProvider" host="org.apache.airavata.schemas.gfac.impl.HadoopHostTypeImpl">
+ <Provider class="org.apache.airavata.gfac.hadoop.provider.impl.HadoopProvider" host="org.apache.airavata.schemas.gfac.impl.HadoopHostTypeImpl">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.HadoopDeploymentHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.HDFSDataMovementHandler"/>
+ <Handler class="org.apache.airavata.gfac.hadoop.handler.HadoopDeploymentHandler"/>
+ <Handler class="org.apache.airavata.gfac.hadoop.handler.HDFSDataMovementHandler"/>
</InHandlers>
<OutHandlers/>
</Provider>
-
- <Application name="UltraScan">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
- </OutHandlers>
- </Application>
</GFac>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/handler/LocalDirectorySetupHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/handler/LocalDirectorySetupHandler.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/handler/LocalDirectorySetupHandler.java
deleted file mode 100644
index 8a11c2a..0000000
--- a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/handler/LocalDirectorySetupHandler.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.handler;
-
-import org.apache.airavata.commons.gfac.type.ApplicationDescription;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.apache.airavata.schemas.gfac.HostDescriptionType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.Map;
-
-public class LocalDirectorySetupHandler implements GFacHandler{
- private static final Logger log = LoggerFactory.getLogger(LocalDirectorySetupHandler.class);
-
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- log.info("Invoking GramDirectorySetupHandler ...");
- HostDescriptionType type = jobExecutionContext.getApplicationContext().getHostDescription().getType();
- ApplicationDescription applicationDeploymentDescription = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription();
- ApplicationDeploymentDescriptionType app = applicationDeploymentDescription.getType();
- log.debug("working diectroy = " + app.getStaticWorkingDirectory());
- log.debug("temp directory = " + app.getScratchWorkingDirectory());
-
- makeFileSystemDir(app.getStaticWorkingDirectory(),jobExecutionContext);
- makeFileSystemDir(app.getScratchWorkingDirectory(),jobExecutionContext);
- makeFileSystemDir(app.getInputDataDirectory(),jobExecutionContext);
- makeFileSystemDir(app.getOutputDataDirectory(),jobExecutionContext);
- }
- private void makeFileSystemDir(String dir, JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- File f = new File(dir);
- if (f.isDirectory() && f.exists()) {
- return;
- } else if (!new File(dir).mkdir()) {
- throw new GFacHandlerException("Cannot make directory "+dir);
- }
- }
-
- public void initProperties(Map<String, String> properties) throws GFacHandlerException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/handler/LocalDirectorySetupHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/handler/LocalDirectorySetupHandler.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/handler/LocalDirectorySetupHandler.java
new file mode 100644
index 0000000..81493fb
--- /dev/null
+++ b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/handler/LocalDirectorySetupHandler.java
@@ -0,0 +1,63 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.local.handler;
+
+import org.apache.airavata.commons.gfac.type.ApplicationDescription;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.handler.GFacHandler;
+import org.apache.airavata.gfac.handler.GFacHandlerException;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.HostDescriptionType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.Map;
+
+public class LocalDirectorySetupHandler implements GFacHandler {
+ private static final Logger log = LoggerFactory.getLogger(LocalDirectorySetupHandler.class);
+
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ log.info("Invoking GramDirectorySetupHandler ...");
+ HostDescriptionType type = jobExecutionContext.getApplicationContext().getHostDescription().getType();
+ ApplicationDescription applicationDeploymentDescription = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription();
+ ApplicationDeploymentDescriptionType app = applicationDeploymentDescription.getType();
+ log.debug("working diectroy = " + app.getStaticWorkingDirectory());
+ log.debug("temp directory = " + app.getScratchWorkingDirectory());
+
+ makeFileSystemDir(app.getStaticWorkingDirectory(),jobExecutionContext);
+ makeFileSystemDir(app.getScratchWorkingDirectory(),jobExecutionContext);
+ makeFileSystemDir(app.getInputDataDirectory(),jobExecutionContext);
+ makeFileSystemDir(app.getOutputDataDirectory(),jobExecutionContext);
+ }
+ private void makeFileSystemDir(String dir, JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ File f = new File(dir);
+ if (f.isDirectory() && f.exists()) {
+ return;
+ } else if (!new File(dir).mkdir()) {
+ throw new GFacHandlerException("Cannot make directory "+dir);
+ }
+ }
+
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/impl/LocalProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/impl/LocalProvider.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/impl/LocalProvider.java
new file mode 100644
index 0000000..f0a0bf9
--- /dev/null
+++ b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/impl/LocalProvider.java
@@ -0,0 +1,239 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.local.impl;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.airavata.gfac.Constants;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.local.utils.InputStreamToFileWriter;
+import org.apache.airavata.gfac.local.utils.InputUtils;
+import org.apache.airavata.gfac.notification.events.StartExecutionEvent;
+import org.apache.airavata.gfac.provider.AbstractProvider;
+import org.apache.airavata.gfac.provider.GFacProviderException;
+import org.apache.airavata.gfac.provider.utils.ProviderUtils;
+import org.apache.airavata.gfac.utils.GFacUtils;
+import org.apache.airavata.gfac.utils.OutputUtils;
+import org.apache.airavata.model.workspace.experiment.JobDetails;
+import org.apache.airavata.model.workspace.experiment.JobState;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.NameValuePairType;
+import org.apache.xmlbeans.XmlException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import sun.reflect.generics.reflectiveObjects.NotImplementedException;
+
+public class LocalProvider extends AbstractProvider {
+ private static final Logger log = LoggerFactory.getLogger(LocalProvider.class);
+ private ProcessBuilder builder;
+ private List<String> cmdList;
+ private String jobId;
+
+ public static class LocalProviderJobData{
+ private String applicationName;
+ private List<String> inputParameters;
+ private String workingDir;
+ private String inputDir;
+ private String outputDir;
+ public String getApplicationName() {
+ return applicationName;
+ }
+ public void setApplicationName(String applicationName) {
+ this.applicationName = applicationName;
+ }
+ public List<String> getInputParameters() {
+ return inputParameters;
+ }
+ public void setInputParameters(List<String> inputParameters) {
+ this.inputParameters = inputParameters;
+ }
+ public String getWorkingDir() {
+ return workingDir;
+ }
+ public void setWorkingDir(String workingDir) {
+ this.workingDir = workingDir;
+ }
+ public String getInputDir() {
+ return inputDir;
+ }
+ public void setInputDir(String inputDir) {
+ this.inputDir = inputDir;
+ }
+ public String getOutputDir() {
+ return outputDir;
+ }
+ public void setOutputDir(String outputDir) {
+ this.outputDir = outputDir;
+ }
+ }
+ public LocalProvider(){
+ cmdList = new ArrayList<String>();
+ }
+
+ public void initialize(JobExecutionContext jobExecutionContext) throws GFacProviderException,GFacException {
+ super.initialize(jobExecutionContext);
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().
+ getApplicationDeploymentDescription().getType();
+
+ buildCommand(app.getExecutableLocation(), ProviderUtils.getInputParameters(jobExecutionContext));
+ initProcessBuilder(app);
+
+ // extra environment variables
+ builder.environment().put(Constants.INPUT_DATA_DIR_VAR_NAME, app.getInputDataDirectory());
+ builder.environment().put(Constants.OUTPUT_DATA_DIR_VAR_NAME, app.getOutputDataDirectory());
+
+ // set working directory
+ builder.directory(new File(app.getStaticWorkingDirectory()));
+
+ // log info
+ log.info("Command = " + InputUtils.buildCommand(cmdList));
+ log.info("Working dir = " + builder.directory());
+ for (String key : builder.environment().keySet()) {
+ log.info("Env[" + key + "] = " + builder.environment().get(key));
+ }
+ }
+
+ public void execute(JobExecutionContext jobExecutionContext) throws GFacProviderException {
+ jobExecutionContext.getNotifier().publish(new StartExecutionEvent());
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.
+ getApplicationContext().getApplicationDeploymentDescription().getType();
+ JobDetails jobDetails = new JobDetails();
+ try {
+ jobId = jobExecutionContext.getTaskData().getTaskID();
+ jobDetails.setJobID(jobId);
+ jobDetails.setJobDescription(app.toString());
+ jobExecutionContext.setJobDetails(jobDetails);
+ jobDetails.setJobDescription(app.toString());
+ GFacUtils.saveJobStatus(jobExecutionContext,jobDetails, JobState.SETUP);
+ // running cmd
+ Process process = builder.start();
+
+ Thread standardOutWriter = new InputStreamToFileWriter(process.getInputStream(), app.getStandardOutput());
+ Thread standardErrorWriter = new InputStreamToFileWriter(process.getErrorStream(), app.getStandardError());
+
+ // start output threads
+ standardOutWriter.setDaemon(true);
+ standardErrorWriter.setDaemon(true);
+ standardOutWriter.start();
+ standardErrorWriter.start();
+
+ int returnValue = process.waitFor();
+
+ // make sure other two threads are done
+ standardOutWriter.join();
+ standardErrorWriter.join();
+
+ /*
+ * check return value. usually not very helpful to draw conclusions based on return values so don't bother.
+ * just provide warning in the log messages
+ */
+ if (returnValue != 0) {
+ log.error("Process finished with non zero return value. Process may have failed");
+ } else {
+ log.info("Process finished with return value of zero.");
+ }
+
+ StringBuffer buf = new StringBuffer();
+ buf.append("Executed ").append(InputUtils.buildCommand(cmdList))
+ .append(" on the localHost, working directory = ").append(app.getStaticWorkingDirectory())
+ .append(" tempDirectory = ").append(app.getScratchWorkingDirectory()).append(" With the status ")
+ .append(String.valueOf(returnValue));
+ log.info(buf.toString());
+ } catch (IOException io) {
+ throw new GFacProviderException(io.getMessage(), io);
+ } catch (InterruptedException e) {
+ throw new GFacProviderException(e.getMessage(), e);
+ }catch (GFacException e) {
+ throw new GFacProviderException(e.getMessage(), e);
+ }
+ }
+
+// private void saveApplicationJob(JobExecutionContext jobExecutionContext)
+// throws GFacProviderException {
+// ApplicationDeploymentDescriptionType app = jobExecutionContext.
+// getApplicationContext().getApplicationDeploymentDescription().getType();
+// ApplicationJob appJob = GFacUtils.createApplicationJob(jobExecutionContext);
+// appJob.setJobId(jobId);
+// LocalProviderJobData data = new LocalProviderJobData();
+// data.setApplicationName(app.getExecutableLocation());
+// data.setInputDir(app.getInputDataDirectory());
+// data.setOutputDir(app.getOutputDataDirectory());
+// data.setWorkingDir(builder.directory().toString());
+// data.setInputParameters(ProviderUtils.getInputParameters(jobExecutionContext));
+// ByteArrayOutputStream stream = new ByteArrayOutputStream();
+// JAXB.marshal(data, stream);
+// appJob.setJobData(stream.toString());
+// appJob.setSubmittedTime(Calendar.getInstance().getTime());
+// appJob.setStatus(ApplicationJobStatus.SUBMITTED);
+// appJob.setStatusUpdateTime(appJob.getSubmittedTime());
+// GFacUtils.recordApplicationJob(jobExecutionContext, appJob);
+// }
+
+ public void dispose(JobExecutionContext jobExecutionContext) throws GFacProviderException {
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
+
+ try {
+ String stdOutStr = GFacUtils.readFileToString(app.getStandardOutput());
+ String stdErrStr = GFacUtils.readFileToString(app.getStandardError());
+ Map<String, Object> output = jobExecutionContext.getOutMessageContext().getParameters();
+ OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr);
+ } catch (XmlException e) {
+ throw new GFacProviderException("Cannot read output:" + e.getMessage(), e);
+ } catch (IOException io) {
+ throw new GFacProviderException(io.getMessage(), io);
+ } catch (Exception e){
+ throw new GFacProviderException("Error in retrieving results",e);
+ }
+ }
+
+ public void cancelJob(String jobId, JobExecutionContext jobExecutionContext) throws GFacException {
+ throw new NotImplementedException();
+ }
+
+
+ private void buildCommand(String executable, List<String> inputParameterList){
+ cmdList.add(executable);
+ cmdList.addAll(inputParameterList);
+ }
+
+ private void initProcessBuilder(ApplicationDeploymentDescriptionType app){
+ builder = new ProcessBuilder(cmdList);
+
+ NameValuePairType[] env = app.getApplicationEnvironmentArray();
+
+ if(env != null && env.length > 0){
+ Map<String,String> builderEnv = builder.environment();
+ for (NameValuePairType entry : env) {
+ builderEnv.put(entry.getName(), entry.getValue());
+ }
+ }
+ }
+
+ public void initProperties(Map<String, String> properties) throws GFacProviderException, GFacException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputStreamToFileWriter.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputStreamToFileWriter.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputStreamToFileWriter.java
new file mode 100644
index 0000000..2c9a4d2
--- /dev/null
+++ b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputStreamToFileWriter.java
@@ -0,0 +1,68 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.local.utils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+
+public class InputStreamToFileWriter extends Thread{
+ protected final Logger log = LoggerFactory.getLogger(this.getClass());
+
+ private BufferedReader in;
+ private BufferedWriter out;
+
+ public InputStreamToFileWriter(InputStream in, String out) throws IOException {
+ this.in = new BufferedReader(new InputStreamReader(in));
+ this.out = new BufferedWriter(new FileWriter(out));
+ }
+
+ public void run() {
+ try {
+ String line = null;
+ while ((line = in.readLine()) != null) {
+ if (log.isDebugEnabled()) {
+ log.debug(line);
+ }
+ out.write(line);
+ out.newLine();
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ if (in != null) {
+ try {
+ in.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ if (out != null) {
+ try {
+ out.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputUtils.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputUtils.java
new file mode 100644
index 0000000..ba65753
--- /dev/null
+++ b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputUtils.java
@@ -0,0 +1,39 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.local.utils;
+
+import java.util.List;
+
+public class InputUtils {
+ private static final String SPACE = " ";
+
+ private InputUtils() {
+ }
+
+ public static String buildCommand(List<String> cmdList) {
+ StringBuffer buff = new StringBuffer();
+ for (String string : cmdList) {
+ buff.append(string);
+ buff.append(SPACE);
+ }
+ return buff.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/LocalProviderUtil.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/LocalProviderUtil.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/LocalProviderUtil.java
new file mode 100644
index 0000000..4a19b65
--- /dev/null
+++ b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/LocalProviderUtil.java
@@ -0,0 +1,54 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.local.utils;
+
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.provider.GFacProviderException;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+
+public class LocalProviderUtil {
+ private static final Logger log = LoggerFactory.getLogger(LocalProviderUtil.class);
+
+ private void makeFileSystemDir(String dir) throws GFacProviderException {
+ File f = new File(dir);
+ if (f.isDirectory() && f.exists()) {
+ return;
+ } else if (!new File(dir).mkdir()) {
+ throw new GFacProviderException("Cannot make directory " + dir);
+ }
+ }
+
+ public void makeDirectory(JobExecutionContext jobExecutionContext) throws GFacProviderException {
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.
+ getApplicationContext().getApplicationDeploymentDescription().getType();
+ log.info("working diectroy = " + app.getStaticWorkingDirectory());
+ log.info("temp directory = " + app.getScratchWorkingDirectory());
+ makeFileSystemDir(app.getStaticWorkingDirectory());
+ makeFileSystemDir(app.getScratchWorkingDirectory());
+ makeFileSystemDir(app.getInputDataDirectory());
+ makeFileSystemDir(app.getOutputDataDirectory());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/provider/impl/LocalProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/provider/impl/LocalProvider.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/provider/impl/LocalProvider.java
deleted file mode 100644
index 6a92e53..0000000
--- a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/provider/impl/LocalProvider.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.provider.impl;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.airavata.gfac.Constants;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.notification.events.StartExecutionEvent;
-import org.apache.airavata.gfac.provider.AbstractProvider;
-import org.apache.airavata.gfac.provider.GFacProviderException;
-import org.apache.airavata.gfac.provider.utils.ProviderUtils;
-import org.apache.airavata.gfac.utils.GFacUtils;
-import org.apache.airavata.gfac.utils.InputStreamToFileWriter;
-import org.apache.airavata.gfac.utils.InputUtils;
-import org.apache.airavata.gfac.utils.OutputUtils;
-import org.apache.airavata.model.workspace.experiment.JobDetails;
-import org.apache.airavata.model.workspace.experiment.JobState;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.apache.airavata.schemas.gfac.NameValuePairType;
-import org.apache.xmlbeans.XmlException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import sun.reflect.generics.reflectiveObjects.NotImplementedException;
-
-public class LocalProvider extends AbstractProvider {
- private static final Logger log = LoggerFactory.getLogger(LocalProvider.class);
- private ProcessBuilder builder;
- private List<String> cmdList;
- private String jobId;
-
- public static class LocalProviderJobData{
- private String applicationName;
- private List<String> inputParameters;
- private String workingDir;
- private String inputDir;
- private String outputDir;
- public String getApplicationName() {
- return applicationName;
- }
- public void setApplicationName(String applicationName) {
- this.applicationName = applicationName;
- }
- public List<String> getInputParameters() {
- return inputParameters;
- }
- public void setInputParameters(List<String> inputParameters) {
- this.inputParameters = inputParameters;
- }
- public String getWorkingDir() {
- return workingDir;
- }
- public void setWorkingDir(String workingDir) {
- this.workingDir = workingDir;
- }
- public String getInputDir() {
- return inputDir;
- }
- public void setInputDir(String inputDir) {
- this.inputDir = inputDir;
- }
- public String getOutputDir() {
- return outputDir;
- }
- public void setOutputDir(String outputDir) {
- this.outputDir = outputDir;
- }
- }
- public LocalProvider(){
- cmdList = new ArrayList<String>();
- }
-
- public void initialize(JobExecutionContext jobExecutionContext) throws GFacProviderException,GFacException {
- super.initialize(jobExecutionContext);
- ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().
- getApplicationDeploymentDescription().getType();
-
- buildCommand(app.getExecutableLocation(), ProviderUtils.getInputParameters(jobExecutionContext));
- initProcessBuilder(app);
-
- // extra environment variables
- builder.environment().put(Constants.INPUT_DATA_DIR_VAR_NAME, app.getInputDataDirectory());
- builder.environment().put(Constants.OUTPUT_DATA_DIR_VAR_NAME, app.getOutputDataDirectory());
-
- // set working directory
- builder.directory(new File(app.getStaticWorkingDirectory()));
-
- // log info
- log.info("Command = " + InputUtils.buildCommand(cmdList));
- log.info("Working dir = " + builder.directory());
- for (String key : builder.environment().keySet()) {
- log.info("Env[" + key + "] = " + builder.environment().get(key));
- }
- }
-
- public void execute(JobExecutionContext jobExecutionContext) throws GFacProviderException {
- jobExecutionContext.getNotifier().publish(new StartExecutionEvent());
- ApplicationDeploymentDescriptionType app = jobExecutionContext.
- getApplicationContext().getApplicationDeploymentDescription().getType();
- JobDetails jobDetails = new JobDetails();
- try {
- jobId = jobExecutionContext.getTaskData().getTaskID();
- jobDetails.setJobID(jobId);
- jobDetails.setJobDescription(app.toString());
- jobExecutionContext.setJobDetails(jobDetails);
- jobDetails.setJobDescription(app.toString());
- GFacUtils.saveJobStatus(jobExecutionContext,jobDetails, JobState.SETUP);
- // running cmd
- Process process = builder.start();
-
- Thread standardOutWriter = new InputStreamToFileWriter(process.getInputStream(), app.getStandardOutput());
- Thread standardErrorWriter = new InputStreamToFileWriter(process.getErrorStream(), app.getStandardError());
-
- // start output threads
- standardOutWriter.setDaemon(true);
- standardErrorWriter.setDaemon(true);
- standardOutWriter.start();
- standardErrorWriter.start();
-
- int returnValue = process.waitFor();
-
- // make sure other two threads are done
- standardOutWriter.join();
- standardErrorWriter.join();
-
- /*
- * check return value. usually not very helpful to draw conclusions based on return values so don't bother.
- * just provide warning in the log messages
- */
- if (returnValue != 0) {
- log.error("Process finished with non zero return value. Process may have failed");
- } else {
- log.info("Process finished with return value of zero.");
- }
-
- StringBuffer buf = new StringBuffer();
- buf.append("Executed ").append(InputUtils.buildCommand(cmdList))
- .append(" on the localHost, working directory = ").append(app.getStaticWorkingDirectory())
- .append(" tempDirectory = ").append(app.getScratchWorkingDirectory()).append(" With the status ")
- .append(String.valueOf(returnValue));
- log.info(buf.toString());
- } catch (IOException io) {
- throw new GFacProviderException(io.getMessage(), io);
- } catch (InterruptedException e) {
- throw new GFacProviderException(e.getMessage(), e);
- }catch (GFacException e) {
- throw new GFacProviderException(e.getMessage(), e);
- }
- }
-
-// private void saveApplicationJob(JobExecutionContext jobExecutionContext)
-// throws GFacProviderException {
-// ApplicationDeploymentDescriptionType app = jobExecutionContext.
-// getApplicationContext().getApplicationDeploymentDescription().getType();
-// ApplicationJob appJob = GFacUtils.createApplicationJob(jobExecutionContext);
-// appJob.setJobId(jobId);
-// LocalProviderJobData data = new LocalProviderJobData();
-// data.setApplicationName(app.getExecutableLocation());
-// data.setInputDir(app.getInputDataDirectory());
-// data.setOutputDir(app.getOutputDataDirectory());
-// data.setWorkingDir(builder.directory().toString());
-// data.setInputParameters(ProviderUtils.getInputParameters(jobExecutionContext));
-// ByteArrayOutputStream stream = new ByteArrayOutputStream();
-// JAXB.marshal(data, stream);
-// appJob.setJobData(stream.toString());
-// appJob.setSubmittedTime(Calendar.getInstance().getTime());
-// appJob.setStatus(ApplicationJobStatus.SUBMITTED);
-// appJob.setStatusUpdateTime(appJob.getSubmittedTime());
-// GFacUtils.recordApplicationJob(jobExecutionContext, appJob);
-// }
-
- public void dispose(JobExecutionContext jobExecutionContext) throws GFacProviderException {
- ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
-
- try {
- String stdOutStr = GFacUtils.readFileToString(app.getStandardOutput());
- String stdErrStr = GFacUtils.readFileToString(app.getStandardError());
- Map<String, Object> output = jobExecutionContext.getOutMessageContext().getParameters();
- OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr);
- } catch (XmlException e) {
- throw new GFacProviderException("Cannot read output:" + e.getMessage(), e);
- } catch (IOException io) {
- throw new GFacProviderException(io.getMessage(), io);
- } catch (Exception e){
- throw new GFacProviderException("Error in retrieving results",e);
- }
- }
-
- public void cancelJob(String jobId, JobExecutionContext jobExecutionContext) throws GFacException {
- throw new NotImplementedException();
- }
-
-
- private void buildCommand(String executable, List<String> inputParameterList){
- cmdList.add(executable);
- cmdList.addAll(inputParameterList);
- }
-
- private void initProcessBuilder(ApplicationDeploymentDescriptionType app){
- builder = new ProcessBuilder(cmdList);
-
- NameValuePairType[] env = app.getApplicationEnvironmentArray();
-
- if(env != null && env.length > 0){
- Map<String,String> builderEnv = builder.environment();
- for (NameValuePairType entry : env) {
- builderEnv.put(entry.getName(), entry.getValue());
- }
- }
- }
-
- public void initProperties(Map<String, String> properties) throws GFacProviderException, GFacException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputStreamToFileWriter.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputStreamToFileWriter.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputStreamToFileWriter.java
deleted file mode 100644
index 691e11d..0000000
--- a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputStreamToFileWriter.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.utils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.*;
-
-public class InputStreamToFileWriter extends Thread{
- protected final Logger log = LoggerFactory.getLogger(this.getClass());
-
- private BufferedReader in;
- private BufferedWriter out;
-
- public InputStreamToFileWriter(InputStream in, String out) throws IOException {
- this.in = new BufferedReader(new InputStreamReader(in));
- this.out = new BufferedWriter(new FileWriter(out));
- }
-
- public void run() {
- try {
- String line = null;
- while ((line = in.readLine()) != null) {
- if (log.isDebugEnabled()) {
- log.debug(line);
- }
- out.write(line);
- out.newLine();
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- if (in != null) {
- try {
- in.close();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- if (out != null) {
- try {
- out.close();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputUtils.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputUtils.java
deleted file mode 100644
index dedfa29..0000000
--- a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputUtils.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.utils;
-
-import java.util.List;
-
-public class InputUtils {
- private static final String SPACE = " ";
-
- private InputUtils() {
- }
-
- public static String buildCommand(List<String> cmdList) {
- StringBuffer buff = new StringBuffer();
- for (String string : cmdList) {
- buff.append(string);
- buff.append(SPACE);
- }
- return buff.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/LocalProviderUtil.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/LocalProviderUtil.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/LocalProviderUtil.java
deleted file mode 100644
index c80eeda..0000000
--- a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/LocalProviderUtil.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.utils;
-
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.provider.GFacProviderException;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-
-public class LocalProviderUtil {
- private static final Logger log = LoggerFactory.getLogger(LocalProviderUtil.class);
-
- private void makeFileSystemDir(String dir) throws GFacProviderException {
- File f = new File(dir);
- if (f.isDirectory() && f.exists()) {
- return;
- } else if (!new File(dir).mkdir()) {
- throw new GFacProviderException("Cannot make directory " + dir);
- }
- }
-
- public void makeDirectory(JobExecutionContext jobExecutionContext) throws GFacProviderException {
- ApplicationDeploymentDescriptionType app = jobExecutionContext.
- getApplicationContext().getApplicationDeploymentDescription().getType();
- log.info("working diectroy = " + app.getStaticWorkingDirectory());
- log.info("temp directory = " + app.getScratchWorkingDirectory());
- makeFileSystemDir(app.getStaticWorkingDirectory());
- makeFileSystemDir(app.getScratchWorkingDirectory());
- makeFileSystemDir(app.getInputDataDirectory());
- makeFileSystemDir(app.getOutputDataDirectory());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/test/java/org/apache/airavata/core/gfac/services/impl/LocalProviderTest.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/test/java/org/apache/airavata/core/gfac/services/impl/LocalProviderTest.java b/modules/gfac/gfac-local/src/test/java/org/apache/airavata/core/gfac/services/impl/LocalProviderTest.java
index bade5f0..902e424 100644
--- a/modules/gfac/gfac-local/src/test/java/org/apache/airavata/core/gfac/services/impl/LocalProviderTest.java
+++ b/modules/gfac/gfac-local/src/test/java/org/apache/airavata/core/gfac/services/impl/LocalProviderTest.java
@@ -26,9 +26,9 @@ import org.apache.airavata.gfac.GFacException;
import org.apache.airavata.gfac.context.ApplicationContext;
import org.apache.airavata.gfac.context.JobExecutionContext;
import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.gfac.handler.LocalDirectorySetupHandler;
+import org.apache.airavata.gfac.local.handler.LocalDirectorySetupHandler;
+import org.apache.airavata.gfac.local.impl.LocalProvider;
import org.apache.airavata.gfac.provider.GFacProviderException;
-import org.apache.airavata.gfac.provider.impl.LocalProvider;
import org.apache.airavata.model.workspace.experiment.TaskDetails;
import org.apache.airavata.persistance.registry.jpa.impl.LoggingRegistryImpl;
import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/test/resources/gfac-config.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/test/resources/gfac-config.xml b/modules/gfac/gfac-local/src/test/resources/gfac-config.xml
index f16460f..b9432da 100644
--- a/modules/gfac/gfac-local/src/test/resources/gfac-config.xml
+++ b/modules/gfac/gfac-local/src/test/resources/gfac-config.xml
@@ -19,9 +19,9 @@
</InHandlers>
<OutHandlers></OutHandlers>
</GlobalHandlers>
- <Provider class="org.apache.airavata.gfac.provider.impl.LocalProvider" host="org.apache.airavata.schemas.gfac.impl.HostDescriptionTypeImpl">
+ <Provider class="org.apache.airavata.gfac.local.impl.LocalProvider" host="org.apache.airavata.schemas.gfac.impl.HostDescriptionTypeImpl">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.LocalDirectorySetupHandler"/>
+ <Handler class="org.apache.airavata.gfac.local.handler.LocalDirectorySetupHandler"/>
</InHandlers>
</Provider>
</GFac>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java
index ec14c08..6ab86af 100644
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java
+++ b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java
@@ -21,19 +21,13 @@
package org.apache.airavata.gfac.monitor.impl.pull.qstat;
import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.SecurityContext;
-import org.apache.airavata.gfac.context.security.GSISecurityContext;
+import org.apache.airavata.gfac.gsissh.security.GSISecurityContext;
import org.apache.airavata.gfac.monitor.HostMonitorData;
import org.apache.airavata.gfac.monitor.MonitorID;
import org.apache.airavata.gsi.ssh.api.SSHApiException;
-import org.apache.airavata.gsi.ssh.api.ServerInfo;
-import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
-import org.apache.airavata.gsi.ssh.api.job.JobManagerConfiguration;
import org.apache.airavata.gsi.ssh.impl.JobStatus;
import org.apache.airavata.gsi.ssh.impl.PBSCluster;
-import org.apache.airavata.gsi.ssh.util.CommonUtils;
import org.apache.airavata.model.workspace.experiment.JobState;
-import org.apache.airavata.schemas.gfac.GsisshHostType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/context/security/SSHSecurityContext.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/context/security/SSHSecurityContext.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/context/security/SSHSecurityContext.java
deleted file mode 100644
index 2d71b7d..0000000
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/context/security/SSHSecurityContext.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package org.apache.airavata.gfac.context.security;
-
-import java.io.IOException;
-
-import net.schmizz.sshj.SSHClient;
-import net.schmizz.sshj.connection.channel.direct.Session;
-import net.schmizz.sshj.userauth.keyprovider.KeyProvider;
-
-import org.apache.airavata.gfac.SecurityContext;
-import org.apache.airavata.gsi.ssh.api.Cluster;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Handle SSH security
- */
-public class SSHSecurityContext implements SecurityContext {
- private static final Logger log = LoggerFactory.getLogger(SSHSecurityContext.class);
-
- public static final String SSH_SECURITY_CONTEXT = "ssh";
-
- private String username;
- private String privateKeyLoc;
- private String keyPass;
- private SSHClient sshClient;
- private Session session;
-
- private Cluster pbsCluster;
-
- public String getUsername() {
- return username;
- }
-
- public void setUsername(String username) {
- this.username = username;
- }
-
- public String getPrivateKeyLoc() {
- return privateKeyLoc;
- }
-
- public void setPrivateKeyLoc(String privateKeyLoc) {
- this.privateKeyLoc = privateKeyLoc;
- }
-
- public String getKeyPass() {
- return keyPass;
- }
-
- public void setKeyPass(String keyPass) {
- this.keyPass = keyPass;
- }
-
- public void closeSession(Session session) {
- if (session != null) {
- try {
- session.close();
- } catch (Exception e) {
- log.warn("Cannot Close SSH Session");
- }
- }
- }
-
- public Session getSession(String hostAddress) throws IOException {
- try {
- if (sshClient == null) {
- sshClient = new SSHClient();
- }
- if (getSSHClient().isConnected())
- return getSSHClient().startSession();
-
- KeyProvider pkey = getSSHClient().loadKeys(getPrivateKeyLoc(), getKeyPass());
-
- getSSHClient().loadKnownHosts();
-
- getSSHClient().connect(hostAddress);
- getSSHClient().authPublickey(getUsername(), pkey);
- session = getSSHClient().startSession();
- return session;
-
- } catch (NullPointerException ne) {
- throw new SecurityException("Cannot load security context for SSH", ne);
- }
- }
-
- public SSHClient getSSHClient() {
- if (sshClient == null) {
- sshClient = new SSHClient();
- }
- return sshClient;
- }
-
- public void setPbsCluster(Cluster pbsCluster) {
- this.pbsCluster = pbsCluster;
- }
-
- public Cluster getPbsCluster() {
- return this.pbsCluster;
- }
-}