You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2014/05/05 16:44:25 UTC
[1/8] changing package names of gfac implementations
Repository: airavata
Updated Branches:
refs/heads/master 0da59822c -> 9bb8c2bef
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java
new file mode 100644
index 0000000..84b41e9
--- /dev/null
+++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java
@@ -0,0 +1,220 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.ssh.handler;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import net.schmizz.sshj.connection.ConnectionException;
+import net.schmizz.sshj.transport.TransportException;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.Constants;
+import org.apache.airavata.commons.gfac.type.ActualParameter;
+import org.apache.airavata.commons.gfac.type.ApplicationDescription;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.ssh.security.SSHSecurityContext;
+import org.apache.airavata.gfac.handler.AbstractHandler;
+import org.apache.airavata.gfac.handler.GFacHandlerException;
+import org.apache.airavata.gfac.provider.GFacProviderException;
+import org.apache.airavata.gfac.ssh.util.GFACSSHUtils;
+import org.apache.airavata.gfac.utils.GFacUtils;
+import org.apache.airavata.gfac.utils.OutputUtils;
+import org.apache.airavata.gsi.ssh.api.Cluster;
+import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
+import org.apache.airavata.model.workspace.experiment.*;
+import org.apache.airavata.registry.cpi.ChildDataType;
+import org.apache.airavata.registry.cpi.DataType;
+import org.apache.airavata.registry.cpi.RegistryException;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.GsisshHostType;
+import org.apache.airavata.schemas.gfac.URIParameterType;
+import org.apache.xmlbeans.XmlException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SSHOutputHandler extends AbstractHandler {
+ private static final Logger log = LoggerFactory.getLogger(SSHOutputHandler.class);
+
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ if (jobExecutionContext.getApplicationContext().getHostDescription().getType() instanceof GsisshHostType) { // this is because we don't have the right jobexecution context
+ // so attempting to get it from the registry
+ if (Constants.PUSH.equals(((GsisshHostType) jobExecutionContext.getApplicationContext().getHostDescription().getType()).getMonitorMode())) { // this is because we don't have the right jobexecution context
+ // so attempting to get it from the registry
+ log.warn("During the out handler chain jobExecution context came null, so trying to handler");
+ ApplicationDescription applicationDeploymentDescription = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription();
+ TaskDetails taskData = null;
+ try {
+ taskData = (TaskDetails) registry.get(DataType.TASK_DETAIL, jobExecutionContext.getTaskData().getTaskID());
+ } catch (RegistryException e) {
+ log.error("Error retrieving job details from Registry");
+ throw new GFacHandlerException("Error retrieving job details from Registry", e);
+ }
+ JobDetails jobDetails = taskData.getJobDetailsList().get(0);
+ String jobDescription = jobDetails.getJobDescription();
+ if (jobDescription != null) {
+ JobDescriptor jobDescriptor = null;
+ try {
+ jobDescriptor = JobDescriptor.fromXML(jobDescription);
+ } catch (XmlException e1) {
+ e1.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+ applicationDeploymentDescription.getType().setScratchWorkingDirectory(
+ jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getWorkingDirectory());
+ applicationDeploymentDescription.getType().setInputDataDirectory(jobDescriptor.getInputDirectory());
+ applicationDeploymentDescription.getType().setOutputDataDirectory(jobDescriptor.getOutputDirectory());
+ applicationDeploymentDescription.getType().setStandardError(jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getStandardErrorFile());
+ applicationDeploymentDescription.getType().setStandardOutput(jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getStandardOutFile());
+ }
+ }
+ }
+
+ try {
+ if (jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT) == null) {
+
+ GFACSSHUtils.addSecurityContext(jobExecutionContext);
+ }
+ } catch (ApplicationSettingsException e) {
+ log.error(e.getMessage());
+ throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
+ } catch (GFacException e1) {
+ log.error(e1.getMessage());
+ throw new GFacHandlerException("Error while creating SSHSecurityContext", e1, e1.getLocalizedMessage());
+ }
+
+ super.invoke(jobExecutionContext);
+ DataTransferDetails detail = new DataTransferDetails();
+ TransferStatus status = new TransferStatus();
+
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext()
+ .getApplicationDeploymentDescription().getType();
+ try {
+ Cluster cluster = ((SSHSecurityContext) jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT)).getPbsCluster();
+ if (cluster == null) {
+ throw new GFacProviderException("Security context is not set properly");
+ } else {
+ log.info("Successfully retrieved the Security Context");
+ }
+
+ // Get the Stdouts and StdErrs
+ String timeStampedServiceName = GFacUtils.createUniqueNameForService(jobExecutionContext.getServiceName());
+
+ TaskDetails taskData = jobExecutionContext.getTaskData();
+ String outputDataDir = null;
+ File localStdOutFile;
+ File localStdErrFile;
+
+ if (taskData.getAdvancedOutputDataHandling() != null) {
+ outputDataDir = taskData.getAdvancedOutputDataHandling().getOutputDataDir();
+ }
+ if (outputDataDir == null) {
+ outputDataDir = File.separator + "tmp";
+ }
+ outputDataDir = outputDataDir + File.separator + jobExecutionContext.getExperimentID() + "-" + jobExecutionContext.getTaskData().getTaskID();
+ (new File(outputDataDir)).mkdirs();
+
+
+ localStdOutFile = new File(outputDataDir + File.separator + timeStampedServiceName + "stdout");
+ localStdErrFile = new File(outputDataDir + File.separator + timeStampedServiceName + "stderr");
+// cluster.makeDirectory(outputDataDir);
+ cluster.scpFrom(app.getStandardOutput(), localStdOutFile.getAbsolutePath());
+ Thread.sleep(1000);
+ cluster.scpFrom(app.getStandardError(), localStdErrFile.getAbsolutePath());
+ Thread.sleep(1000);
+
+ String stdOutStr = GFacUtils.readFileToString(localStdOutFile.getAbsolutePath());
+ String stdErrStr = GFacUtils.readFileToString(localStdErrFile.getAbsolutePath());
+ status.setTransferState(TransferState.COMPLETE);
+ detail.setTransferStatus(status);
+ detail.setTransferDescription("STDOUT:" + stdOutStr);
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
+
+ status.setTransferState(TransferState.COMPLETE);
+ detail.setTransferStatus(status);
+ detail.setTransferDescription("STDERR:" + stdErrStr);
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
+
+
+ Map<String, ActualParameter> stringMap = new HashMap<String, ActualParameter>();
+ Map<String, Object> output = jobExecutionContext.getOutMessageContext().getParameters();
+ Set<String> keys = output.keySet();
+ for (String paramName : keys) {
+ ActualParameter actualParameter = (ActualParameter) output.get(paramName);
+ if ("URI".equals(actualParameter.getType().getType().toString())) {
+
+ List<String> outputList = cluster.listDirectory(app.getOutputDataDirectory());
+ if (outputList.size() == 0 || outputList.get(0).isEmpty()) {
+ stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr);
+ } else {
+ String valueList = outputList.get(0);
+ cluster.scpFrom(app.getOutputDataDirectory() + File.separator + valueList, outputDataDir);
+ jobExecutionContext.addOutputFile(outputDataDir + File.separator + valueList);
+ ((URIParameterType) actualParameter.getType()).setValue(valueList);
+ stringMap = new HashMap<String, ActualParameter>();
+ stringMap.put(paramName, actualParameter);
+ }
+ } else {
+ stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr);
+ }
+ }
+ if (stringMap == null || stringMap.isEmpty()) {
+ throw new GFacHandlerException(
+ "Empty Output returned from the Application, Double check the application"
+ + "and ApplicationDescriptor output Parameter Names");
+ }
+ status.setTransferState(TransferState.DOWNLOAD);
+ detail.setTransferStatus(status);
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
+
+ app.setStandardError(localStdErrFile.getAbsolutePath());
+ app.setStandardOutput(localStdOutFile.getAbsolutePath());
+ app.setOutputDataDirectory(outputDataDir);
+ } catch (XmlException e) {
+ throw new GFacHandlerException("Cannot read output:" + e.getMessage(), e);
+ } catch (ConnectionException e) {
+ throw new GFacHandlerException(e.getMessage(), e);
+ } catch (TransportException e) {
+ throw new GFacHandlerException(e.getMessage(), e);
+ } catch (IOException e) {
+ throw new GFacHandlerException(e.getMessage(), e);
+ } catch (Exception e) {
+ try {
+ status.setTransferState(TransferState.FAILED);
+ detail.setTransferStatus(status);
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
+ } catch (Exception e1) {
+ throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
+ }
+ throw new GFacHandlerException("Error in retrieving results", e);
+ }
+
+ }
+
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/impl/SSHProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/impl/SSHProvider.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/impl/SSHProvider.java
new file mode 100644
index 0000000..ee3dcd2
--- /dev/null
+++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/impl/SSHProvider.java
@@ -0,0 +1,310 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.ssh.impl;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Calendar;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.commons.gfac.type.ActualParameter;
+import org.apache.airavata.commons.gfac.type.MappingFactory;
+import org.apache.airavata.gfac.Constants;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.context.MessageContext;
+import org.apache.airavata.gfac.ssh.security.SSHSecurityContext;
+import org.apache.airavata.gfac.handler.GFacHandlerException;
+import org.apache.airavata.gfac.notification.events.StartExecutionEvent;
+import org.apache.airavata.gfac.provider.AbstractProvider;
+import org.apache.airavata.gfac.provider.GFacProviderException;
+import org.apache.airavata.gfac.ssh.util.GFACSSHUtils;
+import org.apache.airavata.gfac.utils.GFacUtils;
+import org.apache.airavata.gsi.ssh.api.Cluster;
+import org.apache.airavata.gsi.ssh.api.CommandExecutor;
+import org.apache.airavata.gsi.ssh.api.SSHApiException;
+import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
+import org.apache.airavata.gsi.ssh.impl.RawCommandInfo;
+import org.apache.airavata.gsi.ssh.impl.StandardOutReader;
+import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
+import org.apache.airavata.model.workspace.experiment.ErrorCategory;
+import org.apache.airavata.model.workspace.experiment.JobDetails;
+import org.apache.airavata.model.workspace.experiment.JobState;
+import org.apache.airavata.schemas.gfac.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import sun.reflect.generics.reflectiveObjects.NotImplementedException;
+
+/**
+ * Execute application using remote SSH
+ */
+public class SSHProvider extends AbstractProvider {
+ private static final Logger log = LoggerFactory.getLogger(SSHProvider.class);
+ private Cluster cluster;
+ private String jobID = null;
+ private String taskID = null;
+ // we keep gsisshprovider to support qsub submission incase of hpc scenario with ssh
+ private boolean hpcType = false;
+
+ public void initialize(JobExecutionContext jobExecutionContext) throws GFacProviderException, GFacException {
+ super.initialize(jobExecutionContext);
+ if(jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT) == null){
+ try {
+ GFACSSHUtils.addSecurityContext(jobExecutionContext);
+ } catch (ApplicationSettingsException e) {
+ log.error(e.getMessage());
+ throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
+ }
+ }
+ taskID = jobExecutionContext.getTaskData().getTaskID();
+ if (!((SSHHostType) jobExecutionContext.getApplicationContext().getHostDescription().getType()).getHpcResource()) {
+ jobID = "SSH_" + jobExecutionContext.getApplicationContext().getHostDescription().getType().getHostAddress() + "_" + Calendar.getInstance().getTimeInMillis();
+ cluster = ((SSHSecurityContext) jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT)).getPbsCluster();
+
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
+ String remoteFile = app.getStaticWorkingDirectory() + File.separatorChar + Constants.EXECUTABLE_NAME;
+ details.setJobID(taskID);
+ details.setJobDescription(remoteFile);
+ jobExecutionContext.setJobDetails(details);
+ JobDescriptor jobDescriptor = GFACSSHUtils.createJobDescriptor(jobExecutionContext, app, null);
+ details.setJobDescription(jobDescriptor.toXML());
+
+ GFacUtils.saveJobStatus(jobExecutionContext, details, JobState.SETUP);
+ log.info(remoteFile);
+ try {
+ File runscript = createShellScript(jobExecutionContext);
+ cluster.scpTo(remoteFile, runscript.getAbsolutePath());
+ } catch (Exception e) {
+ throw new GFacProviderException(e.getLocalizedMessage(), e);
+ }
+ }else{
+ hpcType = true;
+ }
+ }
+
+
+ public void execute(JobExecutionContext jobExecutionContext) throws GFacProviderException {
+ if (!hpcType) {
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
+ try {
+ /*
+ * Execute
+ */
+ String execuable = app.getStaticWorkingDirectory() + File.separatorChar + Constants.EXECUTABLE_NAME;
+ details.setJobDescription(execuable);
+
+// GFacUtils.updateJobStatus(details, JobState.SUBMITTED);
+ RawCommandInfo rawCommandInfo = new RawCommandInfo("/bin/chmod 755 " + execuable + "; " + execuable);
+
+ StandardOutReader jobIDReaderCommandOutput = new StandardOutReader();
+
+ CommandExecutor.executeCommand(rawCommandInfo, cluster.getSession(), jobIDReaderCommandOutput);
+ String stdOutputString = getOutputifAvailable(jobIDReaderCommandOutput, "Error submitting job to resource");
+
+ log.info("stdout=" + stdOutputString);
+
+// GFacUtils.updateJobStatus(details, JobState.COMPLETE);
+ } catch (Exception e) {
+ throw new GFacProviderException(e.getMessage(), e);
+ } finally {
+ if (cluster != null) {
+ try {
+ cluster.disconnect();
+ } catch (SSHApiException e) {
+ throw new GFacProviderException(e.getMessage(), e);
+ }
+ }
+ }
+ } else {
+ try {
+ jobExecutionContext.getNotifier().publish(new StartExecutionEvent());
+ HostDescriptionType host = jobExecutionContext.getApplicationContext().
+ getHostDescription().getType();
+ HpcApplicationDeploymentType app = (HpcApplicationDeploymentType) jobExecutionContext.getApplicationContext().
+ getApplicationDeploymentDescription().getType();
+ JobDetails jobDetails = new JobDetails();
+ String taskID = jobExecutionContext.getTaskData().getTaskID();
+ try {
+ Cluster cluster = null;
+ if (jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT) != null) {
+ cluster = ((SSHSecurityContext) jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT)).getPbsCluster();
+ }
+ if (cluster == null) {
+ throw new GFacProviderException("Security context is not set properly");
+ } else {
+ log.info("Successfully retrieved the Security Context");
+ }
+ // This installed path is a mandetory field, because this could change based on the computing resource
+ JobDescriptor jobDescriptor = GFACSSHUtils.createJobDescriptor(jobExecutionContext, app, cluster);
+
+ log.info(jobDescriptor.toXML());
+
+ jobDetails.setJobDescription(jobDescriptor.toXML());
+
+ String jobID = cluster.submitBatchJob(jobDescriptor);
+ jobExecutionContext.setJobDetails(jobDetails);
+ if (jobID == null) {
+ jobDetails.setJobID("none");
+ GFacUtils.saveJobStatus(jobExecutionContext, jobDetails, JobState.FAILED);
+ } else {
+ jobDetails.setJobID(jobID);
+ GFacUtils.saveJobStatus(jobExecutionContext, jobDetails, JobState.SUBMITTED);
+ }
+
+ } catch (SSHApiException e) {
+ String error = "Error submitting the job to host " + host.getHostAddress() + " message: " + e.getMessage();
+ log.error(error);
+ jobDetails.setJobID("none");
+ GFacUtils.saveJobStatus(jobExecutionContext, jobDetails, JobState.FAILED);
+ GFacUtils.saveErrorDetails(jobExecutionContext, error, CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
+ throw new GFacProviderException(error, e);
+ } catch (Exception e) {
+ String error = "Error submitting the job to host " + host.getHostAddress() + " message: " + e.getMessage();
+ log.error(error);
+ jobDetails.setJobID("none");
+ GFacUtils.saveJobStatus(jobExecutionContext, jobDetails, JobState.FAILED);
+ GFacUtils.saveErrorDetails(jobExecutionContext, error, CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
+ throw new GFacProviderException(error, e);
+ }
+ } catch (GFacException e) {
+ throw new GFacProviderException(e.getMessage(), e);
+ }
+ }
+ }
+
+ public void dispose(JobExecutionContext jobExecutionContext) throws GFacProviderException {
+
+ }
+
+
+ public void cancelJob(String jobId, JobExecutionContext jobExecutionContext) throws GFacException {
+ throw new NotImplementedException();
+ }
+
+
+ private File createShellScript(JobExecutionContext context) throws IOException {
+ ApplicationDeploymentDescriptionType app = context.getApplicationContext()
+ .getApplicationDeploymentDescription().getType();
+ String uniqueDir = app.getApplicationName().getStringValue() + System.currentTimeMillis()
+ + new Random().nextLong();
+
+ File shellScript = File.createTempFile(uniqueDir, "sh");
+ OutputStream out = new FileOutputStream(shellScript);
+
+ out.write("#!/bin/bash\n".getBytes());
+ out.write(("cd " + app.getStaticWorkingDirectory() + "\n").getBytes());
+ out.write(("export " + Constants.INPUT_DATA_DIR_VAR_NAME + "=" + app.getInputDataDirectory() + "\n").getBytes());
+ out.write(("export " + Constants.OUTPUT_DATA_DIR_VAR_NAME + "=" + app.getOutputDataDirectory() + "\n")
+ .getBytes());
+ // get the env of the host and the application
+ NameValuePairType[] env = app.getApplicationEnvironmentArray();
+
+ Map<String, String> nv = new HashMap<String, String>();
+ if (env != null) {
+ for (int i = 0; i < env.length; i++) {
+ String key = env[i].getName();
+ String value = env[i].getValue();
+ nv.put(key, value);
+ }
+ }
+ for (Entry<String, String> entry : nv.entrySet()) {
+ log.debug("Env[" + entry.getKey() + "] = " + entry.getValue());
+ out.write(("export " + entry.getKey() + "=" + entry.getValue() + "\n").getBytes());
+
+ }
+
+ // prepare the command
+ final String SPACE = " ";
+ StringBuffer cmd = new StringBuffer();
+ cmd.append(app.getExecutableLocation());
+ cmd.append(SPACE);
+
+ MessageContext input = context.getInMessageContext();
+ ;
+ Map<String, Object> inputs = input.getParameters();
+ Set<String> keys = inputs.keySet();
+ for (String paramName : keys) {
+ ActualParameter actualParameter = (ActualParameter) inputs.get(paramName);
+ if ("URIArray".equals(actualParameter.getType().getType().toString())) {
+ String[] values = ((URIArrayType) actualParameter.getType()).getValueArray();
+ for (String value : values) {
+ cmd.append(value);
+ cmd.append(SPACE);
+ }
+ } else {
+ String paramValue = MappingFactory.toString(actualParameter);
+ cmd.append(paramValue);
+ cmd.append(SPACE);
+ }
+ }
+ // We redirect the error and stdout to remote files, they will be read
+ // in later
+ cmd.append(SPACE);
+ cmd.append("1>");
+ cmd.append(SPACE);
+ cmd.append(app.getStandardOutput());
+ cmd.append(SPACE);
+ cmd.append("2>");
+ cmd.append(SPACE);
+ cmd.append(app.getStandardError());
+
+ String cmdStr = cmd.toString();
+ log.info("Command = " + cmdStr);
+ out.write((cmdStr + "\n").getBytes());
+ String message = "\"execuationSuceeded\"";
+ out.write(("echo " + message + "\n").getBytes());
+ out.close();
+
+ return shellScript;
+ }
+
+ public void initProperties(Map<String, String> properties) throws GFacProviderException, GFacException {
+
+ }
+ /**
+ * This method will read standard output and if there's any it will be parsed
+ * @param jobIDReaderCommandOutput
+ * @param errorMsg
+ * @return
+ * @throws SSHApiException
+ */
+ private String getOutputifAvailable(StandardOutReader jobIDReaderCommandOutput, String errorMsg) throws SSHApiException {
+ String stdOutputString = jobIDReaderCommandOutput.getStdOutputString();
+ String stdErrorString = jobIDReaderCommandOutput.getStdErrorString();
+
+ if(stdOutputString == null || stdOutputString.isEmpty() || (stdErrorString != null && !stdErrorString.isEmpty())){
+ log.error("Standard Error output : " + stdErrorString);
+ throw new SSHApiException(errorMsg + stdErrorString);
+ }
+ return stdOutputString;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/security/SSHSecurityContext.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/security/SSHSecurityContext.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/security/SSHSecurityContext.java
new file mode 100644
index 0000000..da05cdd
--- /dev/null
+++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/security/SSHSecurityContext.java
@@ -0,0 +1,120 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.gfac.ssh.security;
+
+import java.io.IOException;
+
+import net.schmizz.sshj.SSHClient;
+import net.schmizz.sshj.connection.channel.direct.Session;
+import net.schmizz.sshj.userauth.keyprovider.KeyProvider;
+
+import org.apache.airavata.gfac.SecurityContext;
+import org.apache.airavata.gsi.ssh.api.Cluster;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Handle SSH security
+ */
+public class SSHSecurityContext implements SecurityContext {
+ private static final Logger log = LoggerFactory.getLogger(SSHSecurityContext.class);
+
+ public static final String SSH_SECURITY_CONTEXT = "ssh";
+
+ private String username;
+ private String privateKeyLoc;
+ private String keyPass;
+ private SSHClient sshClient;
+ private Session session;
+
+ private Cluster pbsCluster;
+
+ public String getUsername() {
+ return username;
+ }
+
+ public void setUsername(String username) {
+ this.username = username;
+ }
+
+ public String getPrivateKeyLoc() {
+ return privateKeyLoc;
+ }
+
+ public void setPrivateKeyLoc(String privateKeyLoc) {
+ this.privateKeyLoc = privateKeyLoc;
+ }
+
+ public String getKeyPass() {
+ return keyPass;
+ }
+
+ public void setKeyPass(String keyPass) {
+ this.keyPass = keyPass;
+ }
+
+ public void closeSession(Session session) {
+ if (session != null) {
+ try {
+ session.close();
+ } catch (Exception e) {
+ log.warn("Cannot Close SSH Session");
+ }
+ }
+ }
+
+ public Session getSession(String hostAddress) throws IOException {
+ try {
+ if (sshClient == null) {
+ sshClient = new SSHClient();
+ }
+ if (getSSHClient().isConnected())
+ return getSSHClient().startSession();
+
+ KeyProvider pkey = getSSHClient().loadKeys(getPrivateKeyLoc(), getKeyPass());
+
+ getSSHClient().loadKnownHosts();
+
+ getSSHClient().connect(hostAddress);
+ getSSHClient().authPublickey(getUsername(), pkey);
+ session = getSSHClient().startSession();
+ return session;
+
+ } catch (NullPointerException ne) {
+ throw new SecurityException("Cannot load security context for SSH", ne);
+ }
+ }
+
+ public SSHClient getSSHClient() {
+ if (sshClient == null) {
+ sshClient = new SSHClient();
+ }
+ return sshClient;
+ }
+
+ public void setPbsCluster(Cluster pbsCluster) {
+ this.pbsCluster = pbsCluster;
+ }
+
+ public Cluster getPbsCluster() {
+ return this.pbsCluster;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java
new file mode 100644
index 0000000..03f1770
--- /dev/null
+++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java
@@ -0,0 +1,179 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.ssh.util;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.common.utils.StringUtil;
+import org.apache.airavata.commons.gfac.type.ActualParameter;
+import org.apache.airavata.commons.gfac.type.HostDescription;
+import org.apache.airavata.commons.gfac.type.MappingFactory;
+import org.apache.airavata.gfac.Constants;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.context.MessageContext;
+import org.apache.airavata.gfac.ssh.security.SSHSecurityContext;
+import org.apache.airavata.gsi.ssh.api.Cluster;
+import org.apache.airavata.gsi.ssh.api.SSHApiException;
+import org.apache.airavata.gsi.ssh.api.ServerInfo;
+import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
+import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
+import org.apache.airavata.gsi.ssh.impl.PBSCluster;
+import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPasswordAuthenticationInfo;
+import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPublicKeyFileAuthentication;
+import org.apache.airavata.gsi.ssh.util.CommonUtils;
+import org.apache.airavata.model.workspace.experiment.ComputationalResourceScheduling;
+import org.apache.airavata.model.workspace.experiment.TaskDetails;
+import org.apache.airavata.schemas.gfac.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+public class GFACSSHUtils {
+ private final static Logger logger = LoggerFactory.getLogger(GFACSSHUtils.class);
+
+ public static void addSecurityContext(JobExecutionContext jobExecutionContext) throws GFacException, ApplicationSettingsException {
+ HostDescription registeredHost = jobExecutionContext.getApplicationContext().getHostDescription();
+ if (registeredHost.getType() instanceof GlobusHostType || registeredHost.getType() instanceof UnicoreHostType
+ || registeredHost.getType() instanceof GsisshHostType) {
+ logger.error("This is a wrong method to invoke to non ssh host types,please check your gfac-config.xml");
+ } else if (registeredHost.getType() instanceof SSHHostType) {
+ Properties configurationProperties = ServerSettings.getProperties();
+ String sshUserName = configurationProperties.getProperty(Constants.SSH_USER_NAME);
+ String sshPrivateKey = configurationProperties.getProperty(Constants.SSH_PRIVATE_KEY);
+ String sshPrivateKeyPass = configurationProperties.getProperty(Constants.SSH_PRIVATE_KEY_PASS);
+ String sshPassword = configurationProperties.getProperty(Constants.SSH_PASSWORD);
+ String sshPublicKey = configurationProperties.getProperty(Constants.SSH_PUBLIC_KEY);
+ SSHSecurityContext sshSecurityContext = new SSHSecurityContext();
+ AuthenticationInfo authenticationInfo = null;
+ // we give higher preference to the password over keypair ssh authentication
+ if (sshPassword != null) {
+ authenticationInfo = new DefaultPasswordAuthenticationInfo(sshPassword);
+ } else {
+ authenticationInfo = new DefaultPublicKeyFileAuthentication(sshPublicKey, sshPrivateKey, sshPrivateKeyPass);
+ }
+ ServerInfo serverInfo = new ServerInfo(sshUserName, registeredHost.getType().getHostAddress());
+
+ Cluster pbsCluster = null;
+ try {
+ String installedParentPath = "/";
+ if (((SSHHostType) registeredHost.getType()).getHpcResource()) {
+ installedParentPath = ((HpcApplicationDeploymentType)
+ jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType()).getInstalledParentPath();
+ }
+ pbsCluster = new PBSCluster(serverInfo, authenticationInfo,
+ CommonUtils.getPBSJobManager(installedParentPath));
+ } catch (SSHApiException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+ sshSecurityContext.setPbsCluster(pbsCluster);
+ sshSecurityContext.setUsername(sshUserName);
+ jobExecutionContext.addSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT, sshSecurityContext);
+ }
+ }
+ public static JobDescriptor createJobDescriptor(JobExecutionContext jobExecutionContext,
+ ApplicationDeploymentDescriptionType app, Cluster cluster) {
+ JobDescriptor jobDescriptor = new JobDescriptor();
+ // this is common for any application descriptor
+ jobDescriptor.setInputDirectory(app.getInputDataDirectory());
+ jobDescriptor.setOutputDirectory(app.getOutputDataDirectory());
+ jobDescriptor.setExecutablePath(app.getExecutableLocation());
+ jobDescriptor.setStandardOutFile(app.getStandardOutput());
+ jobDescriptor.setStandardErrorFile(app.getStandardError());
+ Random random = new Random();
+ int i = random.nextInt();
+ jobDescriptor.setJobName(app.getApplicationName().getStringValue() + String.valueOf(i));
+ jobDescriptor.setWorkingDirectory(app.getStaticWorkingDirectory());
+
+
+ List<String> inputValues = new ArrayList<String>();
+ MessageContext input = jobExecutionContext.getInMessageContext();
+ Map<String, Object> inputs = input.getParameters();
+ Set<String> keys = inputs.keySet();
+ for (String paramName : keys) {
+ ActualParameter actualParameter = (ActualParameter) inputs.get(paramName);
+ if ("URIArray".equals(actualParameter.getType().getType().toString()) || "StringArray".equals(actualParameter.getType().getType().toString())
+ || "FileArray".equals(actualParameter.getType().getType().toString())) {
+ String[] values = null;
+ if (actualParameter.getType() instanceof URIArrayType) {
+ values = ((URIArrayType) actualParameter.getType()).getValueArray();
+ } else if (actualParameter.getType() instanceof StringArrayType) {
+ values = ((StringArrayType) actualParameter.getType()).getValueArray();
+ } else if (actualParameter.getType() instanceof FileArrayType) {
+ values = ((FileArrayType) actualParameter.getType()).getValueArray();
+ }
+ String value = StringUtil.createDelimiteredString(values, " ");
+ inputValues.add(value);
+ } else {
+ String paramValue = MappingFactory.toString(actualParameter);
+ inputValues.add(paramValue);
+ }
+ }
+ jobDescriptor.setInputValues(inputValues);
+
+ // this part will fill out the hpcApplicationDescriptor
+ if (app instanceof HpcApplicationDeploymentType) {
+ HpcApplicationDeploymentType applicationDeploymentType
+ = (HpcApplicationDeploymentType) app;
+ jobDescriptor.setShellName("/bin/bash");
+ jobDescriptor.setAllEnvExport(true);
+ jobDescriptor.setMailOptions("n");
+ jobDescriptor.setNodes(applicationDeploymentType.getNodeCount());
+ jobDescriptor.setProcessesPerNode(applicationDeploymentType.getProcessorsPerNode());
+ jobDescriptor.setMaxWallTime(String.valueOf(applicationDeploymentType.getMaxWallTime()));
+ jobDescriptor.setJobSubmitter(applicationDeploymentType.getJobSubmitterCommand());
+ if (applicationDeploymentType.getProjectAccount() != null) {
+ if (applicationDeploymentType.getProjectAccount().getProjectAccountNumber() != null) {
+ jobDescriptor.setAcountString(applicationDeploymentType.getProjectAccount().getProjectAccountNumber());
+ }
+ }
+ if (applicationDeploymentType.getQueue() != null) {
+ if (applicationDeploymentType.getQueue().getQueueName() != null) {
+ jobDescriptor.setQueueName(applicationDeploymentType.getQueue().getQueueName());
+ }
+ }
+ jobDescriptor.setOwner(((PBSCluster) cluster).getServerInfo().getUserName());
+ TaskDetails taskData = jobExecutionContext.getTaskData();
+ if (taskData != null && taskData.isSetTaskScheduling()) {
+ ComputationalResourceScheduling computionnalResource = taskData.getTaskScheduling();
+ if (computionnalResource.getNodeCount() > 0) {
+ jobDescriptor.setNodes(computionnalResource.getNodeCount());
+ }
+ if (computionnalResource.getComputationalProjectAccount() != null) {
+ jobDescriptor.setAcountString(computionnalResource.getComputationalProjectAccount());
+ }
+ if (computionnalResource.getQueueName() != null) {
+ jobDescriptor.setQueueName(computionnalResource.getQueueName());
+ }
+ if (computionnalResource.getTotalCPUCount() > 0) {
+ jobDescriptor.setProcessesPerNode(computionnalResource.getTotalCPUCount());
+ }
+ if (computionnalResource.getWallTimeLimit() > 0) {
+ jobDescriptor.setMaxWallTime(String.valueOf(computionnalResource.getWallTimeLimit()));
+ }
+ }
+
+ }
+ return jobDescriptor;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/util/GFACSSHUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/util/GFACSSHUtils.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/util/GFACSSHUtils.java
deleted file mode 100644
index 9146bfd..0000000
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/util/GFACSSHUtils.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.util;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.common.utils.StringUtil;
-import org.apache.airavata.commons.gfac.type.ActualParameter;
-import org.apache.airavata.commons.gfac.type.HostDescription;
-import org.apache.airavata.commons.gfac.type.MappingFactory;
-import org.apache.airavata.gfac.Constants;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.RequestData;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.gfac.context.security.SSHSecurityContext;
-import org.apache.airavata.gsi.ssh.api.Cluster;
-import org.apache.airavata.gsi.ssh.api.SSHApiException;
-import org.apache.airavata.gsi.ssh.api.ServerInfo;
-import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
-import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
-import org.apache.airavata.gsi.ssh.impl.PBSCluster;
-import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPasswordAuthenticationInfo;
-import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPublicKeyFileAuthentication;
-import org.apache.airavata.gsi.ssh.util.CommonUtils;
-import org.apache.airavata.model.workspace.experiment.ComputationalResourceScheduling;
-import org.apache.airavata.model.workspace.experiment.TaskDetails;
-import org.apache.airavata.schemas.gfac.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.*;
-
-public class GFACSSHUtils {
- private final static Logger logger = LoggerFactory.getLogger(GFACSSHUtils.class);
-
- public static void addSecurityContext(JobExecutionContext jobExecutionContext) throws GFacException, ApplicationSettingsException {
- HostDescription registeredHost = jobExecutionContext.getApplicationContext().getHostDescription();
- if (registeredHost.getType() instanceof GlobusHostType || registeredHost.getType() instanceof UnicoreHostType
- || registeredHost.getType() instanceof GsisshHostType) {
- logger.error("This is a wrong method to invoke to non ssh host types,please check your gfac-config.xml");
- } else if (registeredHost.getType() instanceof SSHHostType) {
- Properties configurationProperties = ServerSettings.getProperties();
- String sshUserName = configurationProperties.getProperty(Constants.SSH_USER_NAME);
- String sshPrivateKey = configurationProperties.getProperty(Constants.SSH_PRIVATE_KEY);
- String sshPrivateKeyPass = configurationProperties.getProperty(Constants.SSH_PRIVATE_KEY_PASS);
- String sshPassword = configurationProperties.getProperty(Constants.SSH_PASSWORD);
- String sshPublicKey = configurationProperties.getProperty(Constants.SSH_PUBLIC_KEY);
- SSHSecurityContext sshSecurityContext = new SSHSecurityContext();
- AuthenticationInfo authenticationInfo = null;
- // we give higher preference to the password over keypair ssh authentication
- if (sshPassword != null) {
- authenticationInfo = new DefaultPasswordAuthenticationInfo(sshPassword);
- } else {
- authenticationInfo = new DefaultPublicKeyFileAuthentication(sshPublicKey, sshPrivateKey, sshPrivateKeyPass);
- }
- ServerInfo serverInfo = new ServerInfo(sshUserName, registeredHost.getType().getHostAddress());
-
- Cluster pbsCluster = null;
- try {
- String installedParentPath = "/";
- if (((SSHHostType) registeredHost.getType()).getHpcResource()) {
- installedParentPath = ((HpcApplicationDeploymentType)
- jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType()).getInstalledParentPath();
- }
- pbsCluster = new PBSCluster(serverInfo, authenticationInfo,
- CommonUtils.getPBSJobManager(installedParentPath));
- } catch (SSHApiException e) {
- e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
- }
- sshSecurityContext.setPbsCluster(pbsCluster);
- sshSecurityContext.setUsername(sshUserName);
- jobExecutionContext.addSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT, sshSecurityContext);
- }
- }
- public static JobDescriptor createJobDescriptor(JobExecutionContext jobExecutionContext,
- ApplicationDeploymentDescriptionType app, Cluster cluster) {
- JobDescriptor jobDescriptor = new JobDescriptor();
- // this is common for any application descriptor
- jobDescriptor.setInputDirectory(app.getInputDataDirectory());
- jobDescriptor.setOutputDirectory(app.getOutputDataDirectory());
- jobDescriptor.setExecutablePath(app.getExecutableLocation());
- jobDescriptor.setStandardOutFile(app.getStandardOutput());
- jobDescriptor.setStandardErrorFile(app.getStandardError());
- Random random = new Random();
- int i = random.nextInt();
- jobDescriptor.setJobName(app.getApplicationName().getStringValue() + String.valueOf(i));
- jobDescriptor.setWorkingDirectory(app.getStaticWorkingDirectory());
-
-
- List<String> inputValues = new ArrayList<String>();
- MessageContext input = jobExecutionContext.getInMessageContext();
- Map<String, Object> inputs = input.getParameters();
- Set<String> keys = inputs.keySet();
- for (String paramName : keys) {
- ActualParameter actualParameter = (ActualParameter) inputs.get(paramName);
- if ("URIArray".equals(actualParameter.getType().getType().toString()) || "StringArray".equals(actualParameter.getType().getType().toString())
- || "FileArray".equals(actualParameter.getType().getType().toString())) {
- String[] values = null;
- if (actualParameter.getType() instanceof URIArrayType) {
- values = ((URIArrayType) actualParameter.getType()).getValueArray();
- } else if (actualParameter.getType() instanceof StringArrayType) {
- values = ((StringArrayType) actualParameter.getType()).getValueArray();
- } else if (actualParameter.getType() instanceof FileArrayType) {
- values = ((FileArrayType) actualParameter.getType()).getValueArray();
- }
- String value = StringUtil.createDelimiteredString(values, " ");
- inputValues.add(value);
- } else {
- String paramValue = MappingFactory.toString(actualParameter);
- inputValues.add(paramValue);
- }
- }
- jobDescriptor.setInputValues(inputValues);
-
- // this part will fill out the hpcApplicationDescriptor
- if (app instanceof HpcApplicationDeploymentType) {
- HpcApplicationDeploymentType applicationDeploymentType
- = (HpcApplicationDeploymentType) app;
- jobDescriptor.setShellName("/bin/bash");
- jobDescriptor.setAllEnvExport(true);
- jobDescriptor.setMailOptions("n");
- jobDescriptor.setNodes(applicationDeploymentType.getNodeCount());
- jobDescriptor.setProcessesPerNode(applicationDeploymentType.getProcessorsPerNode());
- jobDescriptor.setMaxWallTime(String.valueOf(applicationDeploymentType.getMaxWallTime()));
- jobDescriptor.setJobSubmitter(applicationDeploymentType.getJobSubmitterCommand());
- if (applicationDeploymentType.getProjectAccount() != null) {
- if (applicationDeploymentType.getProjectAccount().getProjectAccountNumber() != null) {
- jobDescriptor.setAcountString(applicationDeploymentType.getProjectAccount().getProjectAccountNumber());
- }
- }
- if (applicationDeploymentType.getQueue() != null) {
- if (applicationDeploymentType.getQueue().getQueueName() != null) {
- jobDescriptor.setQueueName(applicationDeploymentType.getQueue().getQueueName());
- }
- }
- jobDescriptor.setOwner(((PBSCluster) cluster).getServerInfo().getUserName());
- TaskDetails taskData = jobExecutionContext.getTaskData();
- if (taskData != null && taskData.isSetTaskScheduling()) {
- ComputationalResourceScheduling computionnalResource = taskData.getTaskScheduling();
- if (computionnalResource.getNodeCount() > 0) {
- jobDescriptor.setNodes(computionnalResource.getNodeCount());
- }
- if (computionnalResource.getComputationalProjectAccount() != null) {
- jobDescriptor.setAcountString(computionnalResource.getComputationalProjectAccount());
- }
- if (computionnalResource.getQueueName() != null) {
- jobDescriptor.setQueueName(computionnalResource.getQueueName());
- }
- if (computionnalResource.getTotalCPUCount() > 0) {
- jobDescriptor.setProcessesPerNode(computionnalResource.getTotalCPUCount());
- }
- if (computionnalResource.getWallTimeLimit() > 0) {
- jobDescriptor.setMaxWallTime(String.valueOf(computionnalResource.getWallTimeLimit()));
- }
- }
-
- }
- return jobDescriptor;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2TestWithSSHAuth.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2TestWithSSHAuth.java b/modules/gfac/gfac-ssh/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2TestWithSSHAuth.java
index 8d83174..8c42869 100644
--- a/modules/gfac/gfac-ssh/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2TestWithSSHAuth.java
+++ b/modules/gfac/gfac-ssh/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2TestWithSSHAuth.java
@@ -30,7 +30,7 @@ import org.apache.airavata.gfac.SecurityContext;
import org.apache.airavata.gfac.context.ApplicationContext;
import org.apache.airavata.gfac.context.JobExecutionContext;
import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.gfac.context.security.SSHSecurityContext;
+import org.apache.airavata.gfac.ssh.security.SSHSecurityContext;
import org.apache.airavata.gfac.cpi.GFacImpl;
import org.apache.airavata.gsi.ssh.api.Cluster;
import org.apache.airavata.gsi.ssh.api.SSHApiException;
@@ -44,7 +44,6 @@ import org.apache.airavata.gsi.ssh.util.CommonUtils;
import org.apache.airavata.model.workspace.experiment.TaskDetails;
import org.apache.airavata.persistance.registry.jpa.impl.RegistryFactory;
import org.apache.airavata.schemas.gfac.*;
-import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/test/java/org/apache/airavata/core/gfac/services/impl/SSHProviderTestWithSSHAuth.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/test/java/org/apache/airavata/core/gfac/services/impl/SSHProviderTestWithSSHAuth.java b/modules/gfac/gfac-ssh/src/test/java/org/apache/airavata/core/gfac/services/impl/SSHProviderTestWithSSHAuth.java
index 60b941a..74a79e0 100644
--- a/modules/gfac/gfac-ssh/src/test/java/org/apache/airavata/core/gfac/services/impl/SSHProviderTestWithSSHAuth.java
+++ b/modules/gfac/gfac-ssh/src/test/java/org/apache/airavata/core/gfac/services/impl/SSHProviderTestWithSSHAuth.java
@@ -37,7 +37,7 @@ import org.apache.airavata.gfac.GFacException;
import org.apache.airavata.gfac.context.ApplicationContext;
import org.apache.airavata.gfac.context.JobExecutionContext;
import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.gfac.context.security.SSHSecurityContext;
+import org.apache.airavata.gfac.ssh.security.SSHSecurityContext;
import org.apache.airavata.gfac.cpi.GFacImpl;
import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
import org.apache.airavata.schemas.gfac.InputParameterType;
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/test/resources/gfac-config.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/test/resources/gfac-config.xml b/modules/gfac/gfac-ssh/src/test/resources/gfac-config.xml
index 85d148d..2388f64 100644
--- a/modules/gfac/gfac-ssh/src/test/resources/gfac-config.xml
+++ b/modules/gfac/gfac-ssh/src/test/resources/gfac-config.xml
@@ -21,13 +21,31 @@
</GlobalHandlers>
- <Provider class="org.apache.airavata.gfac.provider.impl.SSHProvider" host="org.apache.airavata.schemas.gfac.impl.SSHHostTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SSHDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.SSHInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SSHOutputHandler"/>
- </OutHandlers>
- </Provider>
+ <Provider class="org.apache.airavata.gfac.gsissh.provider.impl.GSISSHProvider" host="org.apache.airavata.schemas.gfac.impl.GsisshHostTypeImpl" executionMode="async">
+ <InHandlers>
+ <Handler class="org.apache.airavata.gfac.gsissh.handler.GSISSHDirectorySetupHandler"/>
+ <!--Handler class="org.apache.airavata.gfac.handler.AdvancedSCPInputHandler">
+ <property name="privateKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa"/>
+ <property name="publicKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa.pub"/>
+ <property name="userName" value="root"/>
+ <property name="hostName" value="gw98.iu.xsede.org"/>
+ <property name="inputPath" value="/tmp"/>
+ <property name="passPhrase" value="/home/airavata/outputData"/>
+ <property name="password" value="/home/airavata/outputData"/> <either we have to set password or keys, password has higher preference>
+ </Handler-->
+ <Handler class="org.apache.airavata.gfac.gsissh.handler.GSISSHInputHandler"/>
+ </InHandlers>
+ <OutHandlers>
+ <Handler class="org.apache.airavata.gfac.gsissh.handler.GSISSHOutputHandler"/>
+ <!--Handler class="org.apache.airavata.gfac.handler.AdvancedSCPOutputHandler">
+ <property name="privateKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa"/>
+ <property name="publicKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa.pub"/>
+ <property name="userName" value="root"/>
+ <property name="hostName" value="gw111.iu.xsede.org"/>
+ <property name="outputPath" value="/tmp"/>
+ <property name="passPhrase" value="/home/airavata/outputData"/>
+ <property name="password" value="/home/airavata/outputData"/> <either we have to set password or keys, password has higher preference>
+ </Handler-->
+ </OutHandlers>
+ </Provider>
</GFac>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/integration-tests/src/test/resources/gfac-config.xml
----------------------------------------------------------------------
diff --git a/modules/integration-tests/src/test/resources/gfac-config.xml b/modules/integration-tests/src/test/resources/gfac-config.xml
index 61dca4f..06afc6e 100644
--- a/modules/integration-tests/src/test/resources/gfac-config.xml
+++ b/modules/integration-tests/src/test/resources/gfac-config.xml
@@ -11,6 +11,11 @@
~ under the License. -->
<GFac>
+ <DaemonHandlers>
+ <Handler class="org.apache.airavata.gfac.monitor.handlers.GridPullMonitorHandler">
+ <property name="listeners" value="org.apache.airavata.gfac.monitor.AiravataJobStatusUpdator"/>
+ </Handler>
+ </DaemonHandlers>
<GlobalHandlers>
<InHandlers>
<Handler class="org.apache.airavata.gfac.handler.AppDescriptorCheckHandler">
@@ -19,31 +24,24 @@
</InHandlers>
<OutHandlers></OutHandlers>
</GlobalHandlers>
- <Provider class="org.apache.airavata.gfac.provider.impl.LocalProvider" host="org.apache.airavata.schemas.gfac.impl.HostDescriptionTypeImpl">
+ <Provider class="org.apache.airavata.gfac.local.impl.LocalProvider" host="org.apache.airavata.schemas.gfac.impl.HostDescriptionTypeImpl">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.LocalDirectorySetupHandler"/>
+ <Handler class="org.apache.airavata.gfac.local.handler.LocalDirectorySetupHandler"/>
</InHandlers>
</Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.GramProvider" host="org.apache.airavata.schemas.gfac.impl.GlobusHostTypeImpl">
+ <Provider class="org.apache.airavata.gfac.gram.provider.impl.GramProvider" host="org.apache.airavata.schemas.gfac.impl.GlobusHostTypeImpl">
<property name="name" value="value"/>
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler">
+ <Handler class="org.apache.airavata.gfac..gram.handler.GramDirectorySetupHandler">
<property name="name" value="value"/>
</Handler>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
+ <Handler class="org.apache.airavata.gfac..gram.handler.GridFTPInputHandler"/>
</InHandlers>
<OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
+ <Handler class="org.apache.airavata.gfac..gram.handler.GridFTPOutputHandler"/>
</OutHandlers>
</Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.BESProvider" host="org.apache.airavata.schemas.gfac.impl.UnicoreHostTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
- </OutHandlers>
+ <Provider class="org.apache.airavata.gfac.bes.provider.impl.BESProvider" host="org.apache.airavata.schemas.gfac.impl.UnicoreHostTypeImpl">
</Provider>
<Provider class="org.apache.airavata.gfac.ec2.EC2Provider" host="org.apache.airavata.schemas.gfac.impl.Ec2HostTypeImpl">
@@ -51,40 +49,58 @@
<OutHandlers/>
</Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.HadoopProvider" host="org.apache.airavata.schemas.gfac.impl.HadoopHostTypeImpl">
+ <Provider class="org.apache.airavata.gfac.hadoop.provider.impl.HadoopProvider" host="org.apache.airavata.schemas.gfac.impl.HadoopHostTypeImpl">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.HadoopDeploymentHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.HDFSDataMovementHandler"/>
+ <Handler class="org.apache.airavata.gfac.hadoop.handler.HadoopDeploymentHandler"/>
+ <Handler class="org.apache.airavata.gfac.hadoop.handler.HDFSDataMovementHandler"/>
</InHandlers>
<OutHandlers/>
</Provider>
<Application name="UltraScan">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
+ <Handler class="org.apache.airavata.gfac.handler.GfacDirectorySetupHandler"/>
+ <Handler class="org.apache.airavata.gfac.gram.handler.GridFTPInputHandler"/>
</InHandlers>
<OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
+ <Handler class="org.apache.airavata.gfac.gram.handler.GridFTPOutputHandler"/>
</OutHandlers>
</Application>
- <Provider class="org.apache.airavata.gfac.provider.impl.SSHProvider" host="org.apache.airavata.schemas.gfac.impl.SSHHostTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SCPDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.SCPInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SCPOutputHandler"/>
- </OutHandlers>
- </Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.GSISSHProvider" host="org.apache.airavata.schemas.gfac.impl.GsisshHostTypeImpl">
+ <Provider class="org.apache.airavata.gfac.ssh.provider.impl.SSHProvider" host="org.apache.airavata.schemas.gfac.impl.SSHHostTypeImpl">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SCPDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.SCPInputHandler"/>
+ <Handler class="org.apache.airavata.gfac.ssh.handler.SSHDirectorySetupHandler"/>
+ <Handler class="org.apache.airavata.gfac.ssh.handler.SSHInputHandler"/>
</InHandlers>
<OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SCPOutputHandler"/>
+ <Handler class="org.apache.airavata.gfac.ssh.handler.SSHOutputHandler"/>
</OutHandlers>
</Provider>
+ <Provider class="org.apache.airavata.gfac.gsissh.provider.impl.GSISSHProvider" host="org.apache.airavata.schemas.gfac.impl.GsisshHostTypeImpl" executionMode="async">
+ <InHandlers>
+ <Handler class="org.apache.airavata.gfac.gsissh.handler.GSISSHDirectorySetupHandler"/>
+ <!--Handler class="org.apache.airavata.gfac.handler.AdvancedSCPInputHandler">
+ <property name="privateKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa"/>
+ <property name="publicKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa.pub"/>
+ <property name="userName" value="root"/>
+ <property name="hostName" value="gw98.iu.xsede.org"/>
+ <property name="inputPath" value="/tmp"/>
+ <property name="passPhrase" value="/home/airavata/outputData"/>
+ <property name="password" value="/home/airavata/outputData"/> <either we have to set password or keys, password has higher preference>
+ </Handler-->
+ <Handler class="org.apache.airavata.gfac.gsissh.handler.GSISSHInputHandler"/>
+ </InHandlers>
+ <OutHandlers>
+ <Handler class="org.apache.airavata.gfac.gsissh.handler.GSISSHOutputHandler"/>
+ <!--Handler class="org.apache.airavata.gfac.handler.AdvancedSCPOutputHandler">
+ <property name="privateKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa"/>
+ <property name="publicKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa.pub"/>
+ <property name="userName" value="root"/>
+ <property name="hostName" value="gw111.iu.xsede.org"/>
+ <property name="outputPath" value="/tmp"/>
+ <property name="passPhrase" value="/home/airavata/outputData"/>
+ <property name="password" value="/home/airavata/outputData"/> <either we have to set password or keys, password has higher preference>
+ </Handler-->
+ </OutHandlers>
+ </Provider>
</GFac>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/orchestrator/airavata-orchestrator-service/src/main/resources/gfac-config.xml
----------------------------------------------------------------------
diff --git a/modules/orchestrator/airavata-orchestrator-service/src/main/resources/gfac-config.xml b/modules/orchestrator/airavata-orchestrator-service/src/main/resources/gfac-config.xml
index 61dca4f..06afc6e 100644
--- a/modules/orchestrator/airavata-orchestrator-service/src/main/resources/gfac-config.xml
+++ b/modules/orchestrator/airavata-orchestrator-service/src/main/resources/gfac-config.xml
@@ -11,6 +11,11 @@
~ under the License. -->
<GFac>
+ <DaemonHandlers>
+ <Handler class="org.apache.airavata.gfac.monitor.handlers.GridPullMonitorHandler">
+ <property name="listeners" value="org.apache.airavata.gfac.monitor.AiravataJobStatusUpdator"/>
+ </Handler>
+ </DaemonHandlers>
<GlobalHandlers>
<InHandlers>
<Handler class="org.apache.airavata.gfac.handler.AppDescriptorCheckHandler">
@@ -19,31 +24,24 @@
</InHandlers>
<OutHandlers></OutHandlers>
</GlobalHandlers>
- <Provider class="org.apache.airavata.gfac.provider.impl.LocalProvider" host="org.apache.airavata.schemas.gfac.impl.HostDescriptionTypeImpl">
+ <Provider class="org.apache.airavata.gfac.local.impl.LocalProvider" host="org.apache.airavata.schemas.gfac.impl.HostDescriptionTypeImpl">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.LocalDirectorySetupHandler"/>
+ <Handler class="org.apache.airavata.gfac.local.handler.LocalDirectorySetupHandler"/>
</InHandlers>
</Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.GramProvider" host="org.apache.airavata.schemas.gfac.impl.GlobusHostTypeImpl">
+ <Provider class="org.apache.airavata.gfac.gram.provider.impl.GramProvider" host="org.apache.airavata.schemas.gfac.impl.GlobusHostTypeImpl">
<property name="name" value="value"/>
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler">
+ <Handler class="org.apache.airavata.gfac..gram.handler.GramDirectorySetupHandler">
<property name="name" value="value"/>
</Handler>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
+ <Handler class="org.apache.airavata.gfac..gram.handler.GridFTPInputHandler"/>
</InHandlers>
<OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
+ <Handler class="org.apache.airavata.gfac..gram.handler.GridFTPOutputHandler"/>
</OutHandlers>
</Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.BESProvider" host="org.apache.airavata.schemas.gfac.impl.UnicoreHostTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
- </OutHandlers>
+ <Provider class="org.apache.airavata.gfac.bes.provider.impl.BESProvider" host="org.apache.airavata.schemas.gfac.impl.UnicoreHostTypeImpl">
</Provider>
<Provider class="org.apache.airavata.gfac.ec2.EC2Provider" host="org.apache.airavata.schemas.gfac.impl.Ec2HostTypeImpl">
@@ -51,40 +49,58 @@
<OutHandlers/>
</Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.HadoopProvider" host="org.apache.airavata.schemas.gfac.impl.HadoopHostTypeImpl">
+ <Provider class="org.apache.airavata.gfac.hadoop.provider.impl.HadoopProvider" host="org.apache.airavata.schemas.gfac.impl.HadoopHostTypeImpl">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.HadoopDeploymentHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.HDFSDataMovementHandler"/>
+ <Handler class="org.apache.airavata.gfac.hadoop.handler.HadoopDeploymentHandler"/>
+ <Handler class="org.apache.airavata.gfac.hadoop.handler.HDFSDataMovementHandler"/>
</InHandlers>
<OutHandlers/>
</Provider>
<Application name="UltraScan">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
+ <Handler class="org.apache.airavata.gfac.handler.GfacDirectorySetupHandler"/>
+ <Handler class="org.apache.airavata.gfac.gram.handler.GridFTPInputHandler"/>
</InHandlers>
<OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
+ <Handler class="org.apache.airavata.gfac.gram.handler.GridFTPOutputHandler"/>
</OutHandlers>
</Application>
- <Provider class="org.apache.airavata.gfac.provider.impl.SSHProvider" host="org.apache.airavata.schemas.gfac.impl.SSHHostTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SCPDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.SCPInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SCPOutputHandler"/>
- </OutHandlers>
- </Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.GSISSHProvider" host="org.apache.airavata.schemas.gfac.impl.GsisshHostTypeImpl">
+ <Provider class="org.apache.airavata.gfac.ssh.provider.impl.SSHProvider" host="org.apache.airavata.schemas.gfac.impl.SSHHostTypeImpl">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SCPDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.SCPInputHandler"/>
+ <Handler class="org.apache.airavata.gfac.ssh.handler.SSHDirectorySetupHandler"/>
+ <Handler class="org.apache.airavata.gfac.ssh.handler.SSHInputHandler"/>
</InHandlers>
<OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SCPOutputHandler"/>
+ <Handler class="org.apache.airavata.gfac.ssh.handler.SSHOutputHandler"/>
</OutHandlers>
</Provider>
+ <Provider class="org.apache.airavata.gfac.gsissh.provider.impl.GSISSHProvider" host="org.apache.airavata.schemas.gfac.impl.GsisshHostTypeImpl" executionMode="async">
+ <InHandlers>
+ <Handler class="org.apache.airavata.gfac.gsissh.handler.GSISSHDirectorySetupHandler"/>
+ <!--Handler class="org.apache.airavata.gfac.handler.AdvancedSCPInputHandler">
+ <property name="privateKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa"/>
+ <property name="publicKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa.pub"/>
+ <property name="userName" value="root"/>
+ <property name="hostName" value="gw98.iu.xsede.org"/>
+ <property name="inputPath" value="/tmp"/>
+ <property name="passPhrase" value="/home/airavata/outputData"/>
+ <property name="password" value="/home/airavata/outputData"/> <either we have to set password or keys, password has higher preference>
+ </Handler-->
+ <Handler class="org.apache.airavata.gfac.gsissh.handler.GSISSHInputHandler"/>
+ </InHandlers>
+ <OutHandlers>
+ <Handler class="org.apache.airavata.gfac.gsissh.handler.GSISSHOutputHandler"/>
+ <!--Handler class="org.apache.airavata.gfac.handler.AdvancedSCPOutputHandler">
+ <property name="privateKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa"/>
+ <property name="publicKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa.pub"/>
+ <property name="userName" value="root"/>
+ <property name="hostName" value="gw111.iu.xsede.org"/>
+ <property name="outputPath" value="/tmp"/>
+ <property name="passPhrase" value="/home/airavata/outputData"/>
+ <property name="password" value="/home/airavata/outputData"/> <either we have to set password or keys, password has higher preference>
+ </Handler-->
+ </OutHandlers>
+ </Provider>
</GFac>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/orchestrator/orchestrator-core/src/test/resources/gfac-config.xml
----------------------------------------------------------------------
diff --git a/modules/orchestrator/orchestrator-core/src/test/resources/gfac-config.xml b/modules/orchestrator/orchestrator-core/src/test/resources/gfac-config.xml
index 61dca4f..06afc6e 100644
--- a/modules/orchestrator/orchestrator-core/src/test/resources/gfac-config.xml
+++ b/modules/orchestrator/orchestrator-core/src/test/resources/gfac-config.xml
@@ -11,6 +11,11 @@
~ under the License. -->
<GFac>
+ <DaemonHandlers>
+ <Handler class="org.apache.airavata.gfac.monitor.handlers.GridPullMonitorHandler">
+ <property name="listeners" value="org.apache.airavata.gfac.monitor.AiravataJobStatusUpdator"/>
+ </Handler>
+ </DaemonHandlers>
<GlobalHandlers>
<InHandlers>
<Handler class="org.apache.airavata.gfac.handler.AppDescriptorCheckHandler">
@@ -19,31 +24,24 @@
</InHandlers>
<OutHandlers></OutHandlers>
</GlobalHandlers>
- <Provider class="org.apache.airavata.gfac.provider.impl.LocalProvider" host="org.apache.airavata.schemas.gfac.impl.HostDescriptionTypeImpl">
+ <Provider class="org.apache.airavata.gfac.local.impl.LocalProvider" host="org.apache.airavata.schemas.gfac.impl.HostDescriptionTypeImpl">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.LocalDirectorySetupHandler"/>
+ <Handler class="org.apache.airavata.gfac.local.handler.LocalDirectorySetupHandler"/>
</InHandlers>
</Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.GramProvider" host="org.apache.airavata.schemas.gfac.impl.GlobusHostTypeImpl">
+ <Provider class="org.apache.airavata.gfac.gram.provider.impl.GramProvider" host="org.apache.airavata.schemas.gfac.impl.GlobusHostTypeImpl">
<property name="name" value="value"/>
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler">
+ <Handler class="org.apache.airavata.gfac..gram.handler.GramDirectorySetupHandler">
<property name="name" value="value"/>
</Handler>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
+ <Handler class="org.apache.airavata.gfac..gram.handler.GridFTPInputHandler"/>
</InHandlers>
<OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
+ <Handler class="org.apache.airavata.gfac..gram.handler.GridFTPOutputHandler"/>
</OutHandlers>
</Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.BESProvider" host="org.apache.airavata.schemas.gfac.impl.UnicoreHostTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
- </OutHandlers>
+ <Provider class="org.apache.airavata.gfac.bes.provider.impl.BESProvider" host="org.apache.airavata.schemas.gfac.impl.UnicoreHostTypeImpl">
</Provider>
<Provider class="org.apache.airavata.gfac.ec2.EC2Provider" host="org.apache.airavata.schemas.gfac.impl.Ec2HostTypeImpl">
@@ -51,40 +49,58 @@
<OutHandlers/>
</Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.HadoopProvider" host="org.apache.airavata.schemas.gfac.impl.HadoopHostTypeImpl">
+ <Provider class="org.apache.airavata.gfac.hadoop.provider.impl.HadoopProvider" host="org.apache.airavata.schemas.gfac.impl.HadoopHostTypeImpl">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.HadoopDeploymentHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.HDFSDataMovementHandler"/>
+ <Handler class="org.apache.airavata.gfac.hadoop.handler.HadoopDeploymentHandler"/>
+ <Handler class="org.apache.airavata.gfac.hadoop.handler.HDFSDataMovementHandler"/>
</InHandlers>
<OutHandlers/>
</Provider>
<Application name="UltraScan">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
+ <Handler class="org.apache.airavata.gfac.handler.GfacDirectorySetupHandler"/>
+ <Handler class="org.apache.airavata.gfac.gram.handler.GridFTPInputHandler"/>
</InHandlers>
<OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
+ <Handler class="org.apache.airavata.gfac.gram.handler.GridFTPOutputHandler"/>
</OutHandlers>
</Application>
- <Provider class="org.apache.airavata.gfac.provider.impl.SSHProvider" host="org.apache.airavata.schemas.gfac.impl.SSHHostTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SCPDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.SCPInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SCPOutputHandler"/>
- </OutHandlers>
- </Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.GSISSHProvider" host="org.apache.airavata.schemas.gfac.impl.GsisshHostTypeImpl">
+ <Provider class="org.apache.airavata.gfac.ssh.provider.impl.SSHProvider" host="org.apache.airavata.schemas.gfac.impl.SSHHostTypeImpl">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SCPDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.SCPInputHandler"/>
+ <Handler class="org.apache.airavata.gfac.ssh.handler.SSHDirectorySetupHandler"/>
+ <Handler class="org.apache.airavata.gfac.ssh.handler.SSHInputHandler"/>
</InHandlers>
<OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SCPOutputHandler"/>
+ <Handler class="org.apache.airavata.gfac.ssh.handler.SSHOutputHandler"/>
</OutHandlers>
</Provider>
+ <Provider class="org.apache.airavata.gfac.gsissh.provider.impl.GSISSHProvider" host="org.apache.airavata.schemas.gfac.impl.GsisshHostTypeImpl" executionMode="async">
+ <InHandlers>
+ <Handler class="org.apache.airavata.gfac.gsissh.handler.GSISSHDirectorySetupHandler"/>
+ <!--Handler class="org.apache.airavata.gfac.handler.AdvancedSCPInputHandler">
+ <property name="privateKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa"/>
+ <property name="publicKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa.pub"/>
+ <property name="userName" value="root"/>
+ <property name="hostName" value="gw98.iu.xsede.org"/>
+ <property name="inputPath" value="/tmp"/>
+ <property name="passPhrase" value="/home/airavata/outputData"/>
+ <property name="password" value="/home/airavata/outputData"/> <either we have to set password or keys, password has higher preference>
+ </Handler-->
+ <Handler class="org.apache.airavata.gfac.gsissh.handler.GSISSHInputHandler"/>
+ </InHandlers>
+ <OutHandlers>
+ <Handler class="org.apache.airavata.gfac.gsissh.handler.GSISSHOutputHandler"/>
+ <!--Handler class="org.apache.airavata.gfac.handler.AdvancedSCPOutputHandler">
+ <property name="privateKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa"/>
+ <property name="publicKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa.pub"/>
+ <property name="userName" value="root"/>
+ <property name="hostName" value="gw111.iu.xsede.org"/>
+ <property name="outputPath" value="/tmp"/>
+ <property name="passPhrase" value="/home/airavata/outputData"/>
+ <property name="password" value="/home/airavata/outputData"/> <either we have to set password or keys, password has higher preference>
+ </Handler-->
+ </OutHandlers>
+ </Provider>
</GFac>
\ No newline at end of file
[7/8] changing package names of gfac implementations
Posted by la...@apache.org.
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/handler/GridFTPInputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/handler/GridFTPInputHandler.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/handler/GridFTPInputHandler.java
new file mode 100644
index 0000000..9844e6a
--- /dev/null
+++ b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/handler/GridFTPInputHandler.java
@@ -0,0 +1,207 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.gram.handler;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.StringUtil;
+import org.apache.airavata.commons.gfac.type.ActualParameter;
+import org.apache.airavata.commons.gfac.type.MappingFactory;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.ToolsException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.context.MessageContext;
+import org.apache.airavata.gfac.gram.security.GSISecurityContext;
+import org.apache.airavata.gfac.gram.external.GridFtp;
+import org.apache.airavata.gfac.gram.util.GramProviderUtils;
+import org.apache.airavata.gfac.handler.AbstractHandler;
+import org.apache.airavata.gfac.handler.AppDescriptorCheckHandler;
+import org.apache.airavata.gfac.handler.GFacHandlerException;
+import org.apache.airavata.gfac.utils.GFacUtils;
+import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
+import org.apache.airavata.model.workspace.experiment.DataTransferDetails;
+import org.apache.airavata.model.workspace.experiment.ErrorCategory;
+import org.apache.airavata.model.workspace.experiment.TransferState;
+import org.apache.airavata.model.workspace.experiment.TransferStatus;
+import org.apache.airavata.registry.cpi.ChildDataType;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.GlobusHostType;
+import org.apache.airavata.schemas.gfac.HostDescriptionType;
+import org.apache.airavata.schemas.gfac.URIArrayType;
+import org.apache.airavata.schemas.gfac.URIParameterType;
+import org.apache.airavata.schemas.gfac.UnicoreHostType;
+import org.ietf.jgss.GSSCredential;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class GridFTPInputHandler extends AbstractHandler {
+ private static final Logger log = LoggerFactory.getLogger(AppDescriptorCheckHandler.class);
+
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ log.info("Invoking GridFTPInputHandler ...");
+ super.invoke(jobExecutionContext);
+ DataTransferDetails detail = new DataTransferDetails();
+ TransferStatus status = new TransferStatus();
+
+ MessageContext inputNew = new MessageContext();
+ try {
+ MessageContext input = jobExecutionContext.getInMessageContext();
+ Set<String> parameters = input.getParameters().keySet();
+ for (String paramName : parameters) {
+ ActualParameter actualParameter = (ActualParameter) input.getParameters().get(paramName);
+ String paramValue = MappingFactory.toString(actualParameter);
+ //TODO: Review this with type
+ if ("URI".equals(actualParameter.getType().getType().toString())) {
+ ((URIParameterType) actualParameter.getType()).setValue(stageInputFiles(jobExecutionContext, paramValue));
+ } else if ("URIArray".equals(actualParameter.getType().getType().toString())) {
+ List<String> split = Arrays.asList(StringUtil.getElementsFromString(paramValue));
+ List<String> newFiles = new ArrayList<String>();
+ for (String paramValueEach : split) {
+ String stageInputFiles = stageInputFiles(jobExecutionContext, paramValueEach);
+ detail.setTransferDescription("Input Data Staged: " + stageInputFiles);
+ status.setTransferState(TransferState.UPLOAD);
+ detail.setTransferStatus(status);
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL,detail, jobExecutionContext.getTaskData().getTaskID());
+
+ newFiles.add(stageInputFiles);
+ }
+ ((URIArrayType) actualParameter.getType()).setValueArray(newFiles.toArray(new String[newFiles.size()]));
+ }
+ inputNew.getParameters().put(paramName, actualParameter);
+
+ }
+ } catch (Exception e) {
+ try {
+ status.setTransferState(TransferState.FAILED);
+ detail.setTransferStatus(status);
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL,detail, jobExecutionContext.getTaskData().getTaskID());
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
+ } catch (Exception e1) {
+ throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
+ }
+ log.error(e.getMessage());
+ throw new GFacHandlerException("Error while input File Staging", e, e.getLocalizedMessage());
+ }
+ jobExecutionContext.setInMessageContext(inputNew);
+ }
+
+ private static String stageInputFiles(JobExecutionContext jobExecutionContext, String paramValue) throws URISyntaxException, SecurityException, ToolsException, IOException,GFacException, ApplicationSettingsException {
+ URI gridftpURL = new URI(paramValue);
+
+ String[] gridFTPEndpointArray = null;
+
+ // not to download input files to the input dir if its http / gsiftp
+ // but if local then yes
+ boolean isInputNonLocal = true;
+
+ //TODO: why it is tightly coupled with gridftp
+// GlobusHostType host = (GlobusHostType) jobExecutionContext.getApplicationContext().getHostDescription().getType();
+
+ //TODO: make it more reusable
+ HostDescriptionType hostType = jobExecutionContext.getApplicationContext().getHostDescription().getType();
+
+ if(jobExecutionContext.getApplicationContext().getHostDescription().getType() instanceof GlobusHostType){
+ gridFTPEndpointArray = ((GlobusHostType) hostType).getGridFTPEndPointArray();
+ }
+ else if (jobExecutionContext.getApplicationContext().getHostDescription().getType() instanceof UnicoreHostType){
+ gridFTPEndpointArray = ((UnicoreHostType) hostType).getGridFTPEndPointArray();
+ isInputNonLocal = false;
+ }
+ else {
+ //TODO
+ }
+
+
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
+ GridFtp ftp = new GridFtp();
+ URI destURI = null;
+ GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentials();
+
+ for (String endpoint : gridFTPEndpointArray) {
+ URI inputURI = GramProviderUtils.createGsiftpURI(endpoint, app.getInputDataDirectory());
+ String fileName = new File(gridftpURL.getPath()).getName();
+ fileName = ftp.gridFTPFileExist(inputURI, fileName,gssCred);
+
+ String destLocalPath = inputURI.getPath() + File.separator + fileName;
+ //if user give a url just to refer an endpoint, not a web resource we are not doing any transfer
+ if (fileName != null && !"".equals(fileName)) {
+ destURI = GramProviderUtils.createGsiftpURI(endpoint, destLocalPath);
+ if (paramValue.startsWith("gsiftp")) {
+ // no need to do if it is unicore, as unicore will download this on user's behalf to the job space dir
+ if(isInputNonLocal) ftp.uploadFile(gridftpURL, destURI, gssCred);
+ else return paramValue;
+ } else if (paramValue.startsWith("file")) {
+ String localFile = paramValue.substring(paramValue.indexOf(":") + 1, paramValue.length());
+ FileInputStream fis = null;
+ try {
+ fis = new FileInputStream(localFile);
+ ftp.uploadFile(destURI, gssCred, fis);
+ } catch (IOException e) {
+ throw new GFacException("Unable to create file : " + localFile ,e);
+ } finally {
+ if (fis != null) {
+ fis.close();
+ }
+ }
+ } else if (paramValue.startsWith("http")) {
+ // no need to do if it is unicore
+ if(isInputNonLocal) {
+ InputStream is = null;
+ try {
+ is = gridftpURL.toURL().openStream();
+ ftp.uploadFile(destURI, gssCred, (is));
+ }finally {
+ is.close();
+ }
+ } else {
+ // don't return destUri
+ return paramValue;
+ }
+
+ } else {
+ //todo throw exception telling unsupported protocol
+ return paramValue;
+ }
+ } else {
+ // When the given input is not a web resource but a URI type input, then we don't do any transfer just keep the same value as it isin the input
+ return paramValue;
+ }
+ }
+ return destURI.getPath();
+ }
+
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/handler/GridFTPOutputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/handler/GridFTPOutputHandler.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/handler/GridFTPOutputHandler.java
new file mode 100644
index 0000000..e867b2b
--- /dev/null
+++ b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/handler/GridFTPOutputHandler.java
@@ -0,0 +1,349 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.gram.handler;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.StringUtil;
+import org.apache.airavata.commons.gfac.type.ActualParameter;
+import org.apache.airavata.commons.gfac.type.ApplicationDescription;
+import org.apache.airavata.commons.gfac.type.MappingFactory;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.ToolsException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.context.MessageContext;
+import org.apache.airavata.gfac.gram.security.GSISecurityContext;
+import org.apache.airavata.gfac.gram.external.GridFtp;
+import org.apache.airavata.gfac.gram.util.GramProviderUtils;
+import org.apache.airavata.gfac.handler.AbstractHandler;
+import org.apache.airavata.gfac.handler.GFacHandlerException;
+import org.apache.airavata.gfac.provider.GFacProviderException;
+import org.apache.airavata.gfac.utils.GFacUtils;
+import org.apache.airavata.gfac.utils.OutputUtils;
+import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
+import org.apache.airavata.model.workspace.experiment.DataTransferDetails;
+import org.apache.airavata.model.workspace.experiment.ErrorCategory;
+import org.apache.airavata.model.workspace.experiment.TaskDetails;
+import org.apache.airavata.model.workspace.experiment.TransferState;
+import org.apache.airavata.model.workspace.experiment.TransferStatus;
+import org.apache.airavata.registry.cpi.ChildDataType;
+import org.apache.airavata.registry.cpi.Registry;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.GlobusHostType;
+import org.apache.airavata.schemas.gfac.HostDescriptionType;
+import org.apache.airavata.schemas.gfac.StringArrayType;
+import org.apache.airavata.schemas.gfac.URIArrayType;
+import org.apache.airavata.schemas.gfac.URIParameterType;
+import org.apache.airavata.schemas.gfac.UnicoreHostType;
+import org.ietf.jgss.GSSCredential;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class GridFTPOutputHandler extends AbstractHandler {
+ private static final Logger log = LoggerFactory.getLogger(GridFTPOutputHandler.class);
+ private Registry registry;
+
+
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ log.info("Invoking GridFTPOutputHandler ...");
+ super.invoke(jobExecutionContext);
+
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
+
+ HostDescriptionType hostType = jobExecutionContext.getApplicationContext().getHostDescription().getType();
+ String[] gridFTPEndpointArray = null;
+ String hostName = null;
+
+ if(jobExecutionContext.getApplicationContext().getHostDescription().getType() instanceof GlobusHostType){
+ gridFTPEndpointArray = ((GlobusHostType) hostType).getGridFTPEndPointArray();
+ hostName = ((GlobusHostType) hostType).getHostName();
+
+ }
+ else if (jobExecutionContext.getApplicationContext().getHostDescription().getType() instanceof UnicoreHostType){
+ gridFTPEndpointArray = ((UnicoreHostType) hostType).getGridFTPEndPointArray();
+ hostName = ((UnicoreHostType) hostType).getHostName();
+ }
+ else {
+ //TODO
+ }
+
+ GridFtp ftp = new GridFtp();
+ File localStdErrFile = null;
+ Map<String, ActualParameter> stringMap = new HashMap<String, ActualParameter>();
+ DataTransferDetails detail = new DataTransferDetails();
+ TransferStatus status = new TransferStatus();
+
+ try {
+ GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentials();
+ String[] hostgridFTP = gridFTPEndpointArray;
+ if (hostgridFTP == null || hostgridFTP.length == 0) {
+ hostgridFTP = new String[]{hostName};
+ }
+ for (String endpoint : gridFTPEndpointArray) {
+ try {
+ /*
+ * Read Stdout and Stderror
+ */
+ URI stdoutURI = GramProviderUtils.createGsiftpURI(endpoint, app.getStandardOutput());
+ URI stderrURI = GramProviderUtils.createGsiftpURI(endpoint, app.getStandardError());
+ status.setTransferState(TransferState.COMPLETE);
+ detail.setTransferStatus(status);
+ detail.setTransferDescription("STDOUT:" + stdoutURI.toString());
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL,detail, jobExecutionContext.getTaskData().getTaskID());
+ status.setTransferState(TransferState.COMPLETE);
+ detail.setTransferStatus(status);
+ detail.setTransferDescription("STDERR:" + stderrURI.toString());
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL,detail, jobExecutionContext.getTaskData().getTaskID());
+
+ log.info("STDOUT:" + stdoutURI.toString());
+ log.info("STDERR:" + stderrURI.toString());
+
+ File logDir = new File("./service_logs");
+ if (!logDir.exists()) {
+ logDir.mkdir();
+ }
+
+ String timeStampedServiceName = GFacUtils.createUniqueNameForService(jobExecutionContext
+ .getServiceName());
+ File localStdOutFile = File.createTempFile(timeStampedServiceName, "stdout");
+ localStdErrFile = File.createTempFile(timeStampedServiceName, "stderr");
+
+
+ String stdout = null;
+ String stderr = null;
+
+ // TODO: what if job is failed
+ // and this handler is not able to find std* files?
+ try {
+ stdout = ftp.readRemoteFile(stdoutURI, gssCred, localStdOutFile);
+ stderr = ftp.readRemoteFile(stderrURI, gssCred, localStdErrFile);
+ //TODO: do we also need to set them as output parameters for another job
+ ApplicationDescription application = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription();
+ ApplicationDeploymentDescriptionType appDesc = application.getType();
+ appDesc.setStandardOutput(stdout);
+ appDesc.setStandardError(stderr);
+ jobExecutionContext.getApplicationContext().setApplicationDeploymentDescription(application);
+ }
+ catch(ToolsException e) {
+ log.error("Cannot download stdout/err files. One reason could be the job is not successfully finished: "+e.getMessage());
+ }
+
+
+ Map<String, Object> output = jobExecutionContext.getOutMessageContext().getParameters();
+ Set<String> keys = output.keySet();
+ for (String paramName : keys) {
+ ActualParameter actualParameter = (ActualParameter) output.get(paramName);
+ if ("URIArray".equals(actualParameter.getType().getType().toString())) {
+ URI outputURI = GramProviderUtils.createGsiftpURI(endpoint, app.getOutputDataDirectory());
+ List<String> outputList = ftp.listDir(outputURI, gssCred);
+ String[] valueList = outputList.toArray(new String[outputList.size()]);
+ ((URIArrayType) actualParameter.getType()).setValueArray(valueList);
+ // why to instantiate new instance?
+// stringMap = new HashMap<String, ActualParameter>();
+ stringMap.put(paramName, actualParameter);
+ }else if ("StringArray".equals(actualParameter.getType().getType().toString())) {
+ String[] valueList = OutputUtils.parseStdoutArray(stdout, paramName);
+ ((StringArrayType) actualParameter.getType()).setValueArray(valueList);
+// stringMap = new HashMap<String, ActualParameter>();
+ stringMap.put(paramName, actualParameter);
+ } else if ("URI".equals(actualParameter.getType().getType().toString())) {
+ URI outputURI = GramProviderUtils.createGsiftpURI(endpoint, app.getOutputDataDirectory());
+ List<String> outputList = ftp.listDir(outputURI, gssCred);
+ if (outputList.size() == 0 || outputList.get(0).isEmpty()) {
+ stringMap = OutputUtils.fillOutputFromStdout(output, stdout, stderr);
+ } else {
+ String valueList = outputList.get(0);
+ ((URIParameterType) actualParameter.getType()).setValue(valueList);
+ stringMap = new HashMap<String, ActualParameter>();
+ stringMap.put(paramName, actualParameter);
+ }
+ }
+ else {
+ // This is to handle exception during the output parsing.
+ stringMap = OutputUtils.fillOutputFromStdout(output, stdout, stderr);
+ }
+ status.setTransferState(TransferState.DOWNLOAD);
+ detail.setTransferStatus(status);
+ detail.setTransferDescription("Output: " + stringMap.get(paramName).toString());
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL,detail, jobExecutionContext.getTaskData().getTaskID());
+
+ }
+ if (stringMap == null || stringMap.isEmpty()) {
+ throw new GFacHandlerException("Empty Output returned from the Application, Double check the application" +
+ "and ApplicationDescriptor output Parameter Names");
+ }
+ // If users has given an output Data path to download the output files this will download the file on machine where GFac is installed
+ TaskDetails taskData = jobExecutionContext.getTaskData();
+ if(taskData != null && taskData.getAdvancedOutputDataHandling() != null){
+ String outputDataDirectory = taskData.getAdvancedOutputDataHandling().getOutputDataDir();
+ if(outputDataDirectory != null && !"".equals(outputDataDirectory)){
+ stageOutputFiles(jobExecutionContext,outputDataDirectory);
+ }
+ }
+ } catch (ToolsException e) {
+ log.error(e.getMessage());
+ throw new GFacHandlerException(e.getMessage() + "\n StdError Data: \n" +readLastLinesofStdOut(localStdErrFile.getPath(), 20),e);
+ } catch (URISyntaxException e) {
+ log.error(e.getMessage());
+ throw new GFacHandlerException("URI is malformatted:" + e.getMessage(), e, readLastLinesofStdOut(localStdErrFile.getPath(), 20));
+ }
+ }
+ } catch (Exception e) {
+ try {
+ status.setTransferState(TransferState.FAILED);
+ detail.setTransferStatus(status);
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL,detail, jobExecutionContext.getTaskData().getTaskID());
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
+ } catch (Exception e1) {
+ throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
+ }
+ log.error(e.getMessage());
+ throw new GFacHandlerException(e.getMessage(), e, readLastLinesofStdOut(localStdErrFile.getPath(), 20));
+ }
+
+ }
+
+ private static String readLastLinesofStdOut(String path, int count) {
+ StringBuffer buffer = new StringBuffer();
+ FileInputStream in = null;
+ try {
+ in = new FileInputStream(path);
+ } catch (FileNotFoundException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+ BufferedReader br = new BufferedReader(new InputStreamReader(in));
+ List<String> strLine = new ArrayList<String>();
+ String tmp = null;
+ int numberofLines = 0;
+ try {
+ while ((tmp = br.readLine()) != null) {
+ strLine.add(tmp);
+ numberofLines++;
+ }
+ } catch (IOException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+ if (numberofLines > count) {
+ for (int i = numberofLines - count; i < numberofLines; i++) {
+ buffer.append(strLine.get(i));
+ buffer.append("\n");
+ }
+ } else {
+ for (int i = 0; i < numberofLines; i++) {
+ buffer.append(strLine.get(i));
+ buffer.append("\n");
+ }
+ }
+ try {
+ in.close();
+ } catch (IOException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+ return buffer.toString();
+ }
+
+ private static void stageOutputFiles(JobExecutionContext jobExecutionContext, String outputFileStagingPath) throws GFacProviderException,GFacException, ApplicationSettingsException {
+
+
+ HostDescriptionType hostType = jobExecutionContext.getApplicationContext().getHostDescription().getType();
+ String[] gridFTPEndpointArray = null;
+
+ if(jobExecutionContext.getApplicationContext().getHostDescription().getType() instanceof GlobusHostType){
+ gridFTPEndpointArray = ((GlobusHostType) hostType).getGridFTPEndPointArray();
+ }
+ else if (jobExecutionContext.getApplicationContext().getHostDescription().getType() instanceof UnicoreHostType){
+ gridFTPEndpointArray = ((UnicoreHostType) hostType).getGridFTPEndPointArray();
+ }
+ else {
+ //TODO
+ }
+
+
+ MessageContext outputNew = new MessageContext();
+ MessageContext output = jobExecutionContext.getOutMessageContext();
+ Map<String, Object> parameters = output.getParameters();
+ for (String paramName : parameters.keySet()) {
+ ActualParameter actualParameter = (ActualParameter) parameters
+ .get(paramName);
+
+ GridFtp ftp = new GridFtp();
+ GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentials();
+ try {
+ if ("URI".equals(actualParameter.getType().getType().toString())) {
+ for (String endpoint : gridFTPEndpointArray) {
+ ((URIParameterType) actualParameter.getType()).setValue(doStaging(outputFileStagingPath,
+ MappingFactory.toString(actualParameter), ftp, gssCred, endpoint));
+ }
+ } else if ("URIArray".equals(actualParameter.getType().getType().toString())) {
+ List<String> split = Arrays.asList(StringUtil.getElementsFromString(MappingFactory.toString(actualParameter)));
+ List<String> newFiles = new ArrayList<String>();
+ for (String endpoint : gridFTPEndpointArray) {
+ for (String paramValueEach : split) {
+ newFiles.add(doStaging(outputFileStagingPath, paramValueEach, ftp, gssCred, endpoint));
+ }
+ ((URIArrayType) actualParameter.getType()).setValueArray(newFiles.toArray(new String[newFiles.size()]));
+ }
+
+ }
+ } catch (URISyntaxException e) {
+ log.error(e.getMessage());
+ throw new GFacProviderException(e.getMessage(), e);
+ } catch (ToolsException e) {
+ log.error(e.getMessage());
+ throw new GFacProviderException(e.getMessage(), e);
+ }
+ outputNew.getParameters().put(paramName, actualParameter);
+ }
+ jobExecutionContext.setOutMessageContext(outputNew);
+ }
+
+ private static String doStaging(String outputFileStagingPath, String paramValue, GridFtp ftp, GSSCredential gssCred, String endpoint) throws URISyntaxException, ToolsException {
+ URI srcURI = GramProviderUtils.createGsiftpURI(endpoint, paramValue);
+ String fileName = new File(srcURI.getPath()).getName();
+ File outputpath = new File(outputFileStagingPath);
+ if(!outputpath.exists()){
+ outputpath.mkdirs();
+ }
+ File outputFile = new File(outputpath.getAbsolutePath() + File.separator + fileName);
+ ftp.readRemoteFile(srcURI,
+ gssCred, outputFile);
+ return outputFileStagingPath + File.separator + fileName;
+ }
+
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/impl/GramProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/impl/GramProvider.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/impl/GramProvider.java
new file mode 100644
index 0000000..b675c5e
--- /dev/null
+++ b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/impl/GramProvider.java
@@ -0,0 +1,527 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.gram.impl;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.util.Map;
+import java.util.MissingResourceException;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.JobSubmissionFault;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.gram.security.GSISecurityContext;
+import org.apache.airavata.gfac.gram.util.GramJobSubmissionListener;
+import org.apache.airavata.gfac.gram.util.GramProviderUtils;
+import org.apache.airavata.gfac.notification.events.JobIDEvent;
+import org.apache.airavata.gfac.notification.events.StartExecutionEvent;
+import org.apache.airavata.gfac.provider.AbstractProvider;
+import org.apache.airavata.gfac.provider.GFacProviderException;
+import org.apache.airavata.gfac.utils.GFacUtils;
+import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
+import org.apache.airavata.model.workspace.experiment.ErrorCategory;
+import org.apache.airavata.model.workspace.experiment.JobState;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.GlobusHostType;
+import org.globus.gram.GramException;
+import org.globus.gram.GramJob;
+import org.globus.gram.WaitingForCommitException;
+import org.globus.gram.internal.GRAMConstants;
+import org.globus.gram.internal.GRAMProtocolErrorConstants;
+import org.ietf.jgss.GSSCredential;
+import org.ietf.jgss.GSSException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class GramProvider extends AbstractProvider {
+ private static final Logger log = LoggerFactory.getLogger(GramJobSubmissionListener.class);
+
+ private GramJob job;
+ private GramJobSubmissionListener listener;
+ private boolean twoPhase = true;
+
+ /**
+ * If normal job submission fail due to an authorisation failure or script failure we
+ * will re-attempt to submit the job. In-order to avoid any recursive loop during a continuous
+ * failure we track whether failure paths are tried or not. Following booleans keeps track whether
+ * we already tried a failure path or not.
+ */
+ /**
+ * To track job submissions during a authorisation failure while requesting job.
+ */
+ private boolean renewCredentialsAttempt = false;
+ /**
+ * To track job submission during a script error situation.
+ */
+ private boolean reSubmissionInProgress = false;
+ /**
+ * To track authorisation failures during status monitoring.
+ */
+ private boolean authorisationFailedAttempt = false;
+
+ private static final Map<String, GramJob> currentlyExecutingJobCache
+ = new ConcurrentHashMap<String, GramJob>();
+
+ private static Properties resources;
+
+ static {
+ try {
+
+ String propFileName = "errors.properties";
+ resources = new Properties();
+ InputStream inputStream = GramProvider.class.getClassLoader()
+ .getResourceAsStream(propFileName);
+
+ if (inputStream == null) {
+ throw new FileNotFoundException("property file '" + propFileName
+ + "' not found in the classpath");
+ }
+
+ resources.load(inputStream);
+
+ } catch (FileNotFoundException mre) {
+ log.error("errors.properties not found", mre);
+ } catch (IOException e) {
+ log.error("Error reading errors.properties file", e);
+ }
+ }
+
+
+ // This method prepare the environment before the application invocation.
+ public void initialize(JobExecutionContext jobExecutionContext) throws GFacProviderException, GFacException {
+
+ try {
+ super.initialize(jobExecutionContext);
+ String strTwoPhase = ServerSettings.getSetting("TwoPhase");
+ if (strTwoPhase != null) {
+ twoPhase = Boolean.parseBoolean(strTwoPhase);
+ log.info("Two phase commit is set to " + twoPhase);
+ }
+ } catch (ApplicationSettingsException e) {
+ log.warn("Error reading TwoPhase property from configurations.", e);
+ }
+
+ job = GramProviderUtils.setupEnvironment(jobExecutionContext, twoPhase);
+ listener = new GramJobSubmissionListener(job, jobExecutionContext);
+ job.addListener(listener);
+ }
+
+ public void execute(JobExecutionContext jobExecutionContext) throws GFacProviderException, GFacException{
+ jobExecutionContext.getNotifier().publish(new StartExecutionEvent());
+ GlobusHostType host = (GlobusHostType) jobExecutionContext.getApplicationContext().
+ getHostDescription().getType();
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().
+ getApplicationDeploymentDescription().getType();
+
+ StringBuilder stringBuilder = new StringBuilder();
+ try {
+
+ GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.
+ getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentials();
+ job.setCredentials(gssCred);
+ // We do not support multiple gatekeepers in XBaya GUI, so we simply pick the 0th element in the array
+ String gateKeeper = host.getGlobusGateKeeperEndPointArray(0);
+ log.info("Request to contact:" + gateKeeper);
+
+ stringBuilder.append("Finished launching job, Host = ").append(host.getHostAddress()).append(" RSL = ")
+ .append(job.getRSL()).append(" working directory = ").append(app.getStaticWorkingDirectory())
+ .append(" temp directory = ").append(app.getScratchWorkingDirectory())
+ .append(" Globus GateKeeper Endpoint = ").append(gateKeeper);
+
+ log.info(stringBuilder.toString());
+
+ submitJobs(gateKeeper, jobExecutionContext, host);
+
+ } catch (ApplicationSettingsException e) {
+ throw new GFacException(e.getMessage(), e);
+ } finally {
+ if (job != null) {
+ try {
+ /*
+ * Remove listener
+ */
+ job.removeListener(listener);
+ } catch (Exception e) {
+ log.error(e.getMessage());
+ }
+ }
+ }
+ }
+
+ private void submitJobs(String gateKeeper,
+ JobExecutionContext jobExecutionContext,
+ GlobusHostType globusHostType) throws GFacException, GFacProviderException {
+ boolean applicationSaved=false;
+ String taskID = jobExecutionContext.getTaskData().getTaskID();
+
+ if (twoPhase) {
+ try {
+ /*
+ * The first boolean is to force communication through SSLv3
+ * The second boolean is to specify the job is a batch job - use true for interactive and false for
+ * batch.
+ * The third boolean is to specify to use the full proxy and not delegate a limited proxy.
+ */
+ job.request(true, gateKeeper, false, false);
+
+ // Single boolean to track all authentication failures, therefore we need to re-initialize
+ // this here
+ renewCredentialsAttempt = false;
+
+ } catch (WaitingForCommitException e) {
+ String jobID = job.getIDAsString();
+
+ details.setJobID(jobID);
+ details.setJobDescription(job.getRSL());
+ jobExecutionContext.setJobDetails(details);
+ GFacUtils.saveJobStatus(jobExecutionContext, details, JobState.UN_SUBMITTED);
+
+ applicationSaved=true;
+ String jobStatusMessage = "Un-submitted JobID= " + jobID;
+ log.info(jobStatusMessage);
+ jobExecutionContext.getNotifier().publish(new JobIDEvent(jobStatusMessage));
+
+ log.info("Two phase commit: sending COMMIT_REQUEST signal; Job id - " + jobID);
+
+ try {
+ job.signal(GramJob.SIGNAL_COMMIT_REQUEST);
+
+ } catch (GramException gramException) {
+ throw new GFacException("Error while sending commit request. Job Id - "
+ + job.getIDAsString(), gramException);
+ } catch (GSSException gssException) {
+
+ // User credentials are invalid
+ log.error("Error while submitting commit request - Credentials provided are invalid. Job Id - "
+ + job.getIDAsString(), e);
+ log.info("Attempting to renew credentials and re-submit commit signal...");
+ GFacUtils.saveErrorDetails(jobExecutionContext, gssException.getLocalizedMessage(), CorrectiveAction.RETRY_SUBMISSION, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
+ renewCredentials(jobExecutionContext);
+
+ try {
+ job.signal(GramJob.SIGNAL_COMMIT_REQUEST);
+ } catch (GramException e1) {
+ GFacUtils.saveErrorDetails(jobExecutionContext, gssException.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
+ throw new GFacException("Error while sending commit request. Job Id - "
+ + job.getIDAsString(), e1);
+ } catch (GSSException e1) {
+ GFacUtils.saveErrorDetails(jobExecutionContext, gssException.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
+ throw new GFacException("Error while sending commit request. Job Id - "
+ + job.getIDAsString() + ". Credentials provided invalid", e1);
+ }
+ }
+ GFacUtils.updateJobStatus(jobExecutionContext, details, JobState.SUBMITTED);
+ jobStatusMessage = "Submitted JobID= " + job.getIDAsString();
+ log.info(jobStatusMessage);
+ jobExecutionContext.getNotifier().publish(new JobIDEvent(jobStatusMessage));
+
+ } catch (GSSException e) {
+ // Renew credentials and re-submit
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.RETRY_SUBMISSION, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
+
+ reSubmitJob(gateKeeper, jobExecutionContext, globusHostType, e);
+
+ } catch (GramException e) {
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
+
+ throw new GFacException("An error occurred while submitting a job, job id = " + job.getIDAsString(), e);
+ }
+ } else {
+
+ /*
+ * The first boolean is to force communication through SSLv3
+ * The second boolean is to specify the job is a batch job - use true for interactive and false for
+ * batch.
+ * The third boolean is to specify to use the full proxy and not delegate a limited proxy.
+ */
+ try {
+
+ job.request(true, gateKeeper, false, false);
+ renewCredentialsAttempt = false;
+
+ } catch (GramException e) {
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
+ throw new GFacException("An error occurred while submitting a job, job id = " + job.getIDAsString(), e);
+ } catch (GSSException e) {
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.RETRY_SUBMISSION, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
+ // Renew credentials and re-submit
+ reSubmitJob(gateKeeper, jobExecutionContext, globusHostType, e);
+ }
+
+ String jobStatusMessage = "Un-submitted JobID= " + job.getIDAsString();
+ log.info(jobStatusMessage);
+ jobExecutionContext.getNotifier().publish(new JobIDEvent(jobStatusMessage));
+
+ }
+
+ currentlyExecutingJobCache.put(job.getIDAsString(), job);
+ /*
+ * Wait until job is done
+ */
+ listener.waitFor();
+
+ checkJobStatus(jobExecutionContext, globusHostType, gateKeeper);
+
+ }
+
+ private void renewCredentials(JobExecutionContext jobExecutionContext) throws GFacException {
+
+ renewCredentials(this.job, jobExecutionContext);
+ }
+
+ private void renewCredentials(GramJob gramJob, JobExecutionContext jobExecutionContext) throws GFacException {
+
+ try {
+ GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.
+ getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).renewCredentials();
+ gramJob.renew(gssCred);
+ } catch (GramException e1) {
+ throw new GFacException("Unable to renew credentials. Job Id - "
+ + gramJob.getIDAsString(), e1);
+ } catch (GSSException e1) {
+ throw new GFacException("Unable to renew credentials. Job Id - "
+ + gramJob.getIDAsString(), e1);
+ } catch (ApplicationSettingsException e) {
+ throw new GFacException(e.getLocalizedMessage(), e);
+ }
+ }
+
+ private void reSubmitJob(String gateKeeper,
+ JobExecutionContext jobExecutionContext,
+ GlobusHostType globusHostType, Exception e) throws GFacException, GFacProviderException {
+
+ if (!renewCredentialsAttempt) {
+
+ renewCredentialsAttempt = true;
+
+ // User credentials are invalid
+ log.error("Error while submitting job - Credentials provided are invalid. Job Id - "
+ + job.getIDAsString(), e);
+ log.info("Attempting to renew credentials and re-submit jobs...");
+
+ // Remove existing listener and register a new listener
+ job.removeListener(listener);
+ listener = new GramJobSubmissionListener(job, jobExecutionContext);
+
+ job.addListener(listener);
+
+ renewCredentials(jobExecutionContext);
+
+ submitJobs(gateKeeper, jobExecutionContext, globusHostType);
+
+ } else {
+ throw new GFacException("Error while submitting job - Credentials provided are invalid. Job Id - "
+ + job.getIDAsString(), e);
+ }
+
+ }
+
+ private void reSubmitJob(String gateKeeper,
+ JobExecutionContext jobExecutionContext,
+ GlobusHostType globusHostType) throws GFacException, GFacProviderException {
+
+ // User credentials are invalid
+ log.info("Attempting to renew credentials and re-submit jobs...");
+
+ // Remove existing listener and register a new listener
+ job.removeListener(listener);
+ listener = new GramJobSubmissionListener(job, jobExecutionContext);
+
+ job.addListener(listener);
+
+ renewCredentials(jobExecutionContext);
+
+ submitJobs(gateKeeper, jobExecutionContext, globusHostType);
+
+ }
+
+
+
+ public void dispose(JobExecutionContext jobExecutionContext) throws GFacProviderException {
+ }
+
+ public void cancelJob(String jobId, JobExecutionContext jobExecutionContext) throws GFacException {
+ cancelSingleJob(jobId, jobExecutionContext);
+ }
+
+
+ private void cancelSingleJob(String jobId, JobExecutionContext context) throws GFacException {
+ // First check whether job id is in the cache
+ if (currentlyExecutingJobCache.containsKey(jobId)) {
+
+ synchronized (this) {
+ GramJob gramJob = currentlyExecutingJobCache.get(jobId);
+
+ // Even though we check using containsKey, at this point job could be null
+ if (gramJob != null && (gramJob.getStatus() != GRAMConstants.STATUS_DONE ||
+ gramJob.getStatus() != GRAMConstants.STATUS_FAILED)) {
+ cancelJob(gramJob, context);
+ }
+ }
+
+ } else {
+
+ try {
+ GSSCredential gssCred = ((GSISecurityContext)context.
+ getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentials();
+
+ GramJob gramJob = new GramJob(null);
+ try {
+ gramJob.setID(jobId);
+ } catch (MalformedURLException e) {
+ throw new GFacException("Invalid job id - " + jobId, e);
+ }
+ gramJob.setCredentials(gssCred);
+
+ synchronized (this) {
+ if (gramJob.getStatus() != GRAMConstants.STATUS_DONE ||
+ gramJob.getStatus() != GRAMConstants.STATUS_FAILED) {
+ cancelJob(gramJob, context);
+ }
+ }
+ } catch (ApplicationSettingsException e) {
+ throw new GFacException(e);
+ }
+ }
+ }
+
+ private void cancelJob(GramJob gramJob, JobExecutionContext context) throws GFacException{
+
+ try {
+ gramJob.cancel();
+ } catch (GramException e) {
+ throw new GFacException("Error cancelling job, id - " + gramJob.getIDAsString(), e);
+ } catch (GSSException e) {
+
+ log.warn("Credentials invalid to cancel job. Attempting to renew credentials and re-try. " +
+ "Job id - " + gramJob.getIDAsString());
+ renewCredentials(gramJob, context);
+
+ try {
+ gramJob.cancel();
+ gramJob.signal(GramJob.SIGNAL_COMMIT_END);
+ } catch (GramException e1) {
+ throw new GFacException("Error cancelling job, id - " + gramJob.getIDAsString(), e1);
+ } catch (GSSException e1) {
+ throw new GFacException("Error cancelling job, invalid credentials. Job id - "
+ + gramJob.getIDAsString(), e);
+ }
+ }
+
+ }
+
+ public void initProperties(Map<String, String> properties) throws GFacException {
+
+ }
+
+ private void checkJobStatus(JobExecutionContext jobExecutionContext, GlobusHostType host, String gateKeeper)
+ throws GFacProviderException {
+ int jobStatus = listener.getCurrentStatus();
+
+ if (jobStatus == GramJob.STATUS_FAILED) {
+
+ String errorMsg = "Job " + job.getIDAsString() + " on host " + host.getHostAddress() + " Job Exit Code = "
+ + listener.getError() + " Error Description = " + getGramErrorString(listener.getError());
+
+ if (listener.getError() == GRAMProtocolErrorConstants.INVALID_SCRIPT_REPLY) {
+
+ // re-submitting without renewing
+ // TODO verify why we re-submit jobs when we get a invalid script reply
+ if (!reSubmissionInProgress) {
+ reSubmissionInProgress = true;
+
+ log.info("Invalid script reply received. Re-submitting job, id - " + job.getIDAsString());
+ try {
+ reSubmitJob(gateKeeper, jobExecutionContext, host);
+ } catch (GFacException e) {
+ throw new GFacProviderException
+ ("Error during re-submission. Original job submission data - " + errorMsg, e);
+ }
+ return;
+ }
+
+ } else if (listener.getError() == GRAMProtocolErrorConstants.ERROR_AUTHORIZATION) {
+
+ // re-submit with renewed credentials
+ if (!authorisationFailedAttempt) {
+ authorisationFailedAttempt = true;
+ log.info("Authorisation error contacting provider. Re-submitting job with renewed credentials.");
+
+ try {
+ renewCredentials(jobExecutionContext);
+ reSubmitJob(gateKeeper, jobExecutionContext, host);
+ } catch (GFacException e) {
+ throw new GFacProviderException
+ ("Error during re-submission. Original job submission data - " + errorMsg, e);
+ }
+
+ return;
+ }
+
+ } else if (listener.getError() == GRAMProtocolErrorConstants.USER_CANCELLED) {
+
+ log.info("User successfully cancelled job id " + job.getIDAsString());
+ return;
+ }
+
+
+
+ log.error(errorMsg);
+
+ synchronized (this) {
+ currentlyExecutingJobCache.remove(job.getIDAsString());
+ }
+
+ throw new JobSubmissionFault(new Exception(errorMsg), host.getHostAddress(), gateKeeper,
+ job.getRSL(), jobExecutionContext, getGramErrorString(listener.getError()),
+ listener.getError());
+
+ } else if (jobStatus == GramJob.STATUS_DONE) {
+ log.info("Job " + job.getIDAsString() + " on host " + host.getHostAddress() + " is successfully executed.");
+
+ synchronized (this) {
+ currentlyExecutingJobCache.remove(job.getIDAsString());
+ }
+ }
+ }
+
+ public String getGramErrorString(int errorCode) {
+
+ if (resources != null) {
+ try {
+ return resources.getProperty(String.valueOf(errorCode));
+ } catch (MissingResourceException mre) {
+ log.warn("Error reading globus error descriptions.", mre);
+ return "Error code: " + errorCode;
+ }
+ } else {
+ return "Error code: " + errorCode;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/persistence/DBJobPersistenceManager.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/persistence/DBJobPersistenceManager.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/persistence/DBJobPersistenceManager.java
new file mode 100644
index 0000000..68e7f48
--- /dev/null
+++ b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/persistence/DBJobPersistenceManager.java
@@ -0,0 +1,225 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.gram.persistence;
+
+import org.apache.airavata.common.utils.DBUtil;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.persistence.JobData;
+import org.apache.airavata.gfac.persistence.JobPersistenceManager;
+import org.apache.log4j.Logger;
+import org.globus.gram.internal.GRAMConstants;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * User: AmilaJ (amilaj@apache.org)
+ * Date: 6/18/13
+ * Time: 4:16 PM
+ * Database based job persistence manager. Current default implementation.
+ */
+
+public class DBJobPersistenceManager implements JobPersistenceManager {
+
+ private DBUtil dbUtil;
+
+ private static final Logger log = Logger.getLogger(DBJobPersistenceManager.class);
+
+
+ public DBJobPersistenceManager(DBUtil db) {
+ this.dbUtil = db;
+ }
+
+ public synchronized void updateJobStatus(JobData jobData) throws GFacException {
+
+ if (jobData.getState() == GRAMConstants.STATUS_UNSUBMITTED) {
+ insertJob(jobData);
+ } else {
+
+ String sql = "update gram_job set status = ? where job_id = ?";
+
+ Connection connection = null;
+ PreparedStatement stmt = null;
+
+ try {
+ connection = getConnection();
+ stmt = connection.prepareStatement(sql);
+ stmt.setInt(1, jobData.getState());
+ stmt.setString(2, jobData.getJobId());
+
+ stmt.executeUpdate();
+ connection.commit();
+
+ } catch (SQLException e) {
+ throw new GFacException(e);
+ } finally {
+ try {
+ if (stmt != null) {
+ stmt.close();
+ }
+
+ if (connection != null) {
+ connection.close();
+ }
+
+ } catch (SQLException e) {
+ log.error("Error closing streams", e);
+ }
+ }
+ }
+ }
+
+ private void insertJob(JobData jobData) throws GFacException {
+
+ String sql = "insert into gram_job values (?, ?)";
+
+ PreparedStatement stmt = null;
+ Connection connection = null;
+
+ try {
+ connection = getConnection();
+ stmt = connection.prepareStatement(sql);
+ stmt.setString(1, jobData.getJobId());
+ stmt.setInt(2, jobData.getState());
+
+ stmt.executeUpdate();
+ } catch (SQLException e) {
+ throw new GFacException(e);
+ } finally {
+ try {
+ if (stmt != null) {
+ stmt.close();
+ }
+
+ if (connection != null) {
+ connection.close();
+ }
+
+ } catch (SQLException e) {
+ log.error("Error closing streams", e);
+ }
+ }
+
+ }
+
+ public List<JobData> getRunningJobs() throws GFacException {
+
+ String sql = "select * from gram_job where status not in (?, ?, ?)";
+
+ int[] statuses = new int[3];
+ statuses[0] = GRAMConstants.STATUS_UNSUBMITTED;
+ statuses[1] = GRAMConstants.STATUS_DONE;
+ statuses[2] = GRAMConstants.STATUS_FAILED;
+
+ return getJobs(sql, statuses);
+ }
+
+ public List<JobData> getFailedJobs() throws GFacException {
+
+ String sql = "select * from gram_job where status in (?)";
+
+ int[] statuses = new int[1];
+ statuses[0] = GRAMConstants.STATUS_FAILED;
+
+ return getJobs(sql, statuses);
+ }
+
+ public List<JobData> getUnSubmittedJobs() throws GFacException {
+
+ String sql = "select * from gram_job where status in (?)";
+
+ int[] statuses = new int[1];
+ statuses[0] = GRAMConstants.STATUS_UNSUBMITTED;
+
+ return getJobs(sql, statuses);
+ }
+
+ public List<JobData> getSuccessfullyCompletedJobs() throws GFacException {
+
+ String sql = "select * from gram_job where status in (?)";
+
+ int[] statuses = new int[1];
+ statuses[0] = GRAMConstants.STATUS_DONE;
+
+ return getJobs(sql, statuses);
+
+ }
+
+
+ protected List<JobData> getJobs(String sql, int[] statuses) throws GFacException {
+
+ List<JobData> jobs = new ArrayList<JobData>();
+
+ PreparedStatement preparedStatement = null;
+ Connection connection = null;
+
+ try {
+ connection = getConnection();
+ preparedStatement = connection.prepareStatement(sql);
+
+ int index = 1;
+ for (int status : statuses) {
+ preparedStatement.setInt(index, status);
+ ++index;
+ }
+
+ ResultSet resultSet = preparedStatement.executeQuery();
+
+ while (resultSet.next()) {
+
+ String jobId = resultSet.getString("job_id");
+ int state = resultSet.getInt("status");
+
+ jobs.add(new JobData(jobId, state));
+ }
+
+ } catch (SQLException e) {
+ throw new GFacException(e);
+ } finally {
+ try {
+ if (preparedStatement != null) {
+ preparedStatement.close();
+ }
+
+ if (connection != null) {
+ connection.close();
+ }
+
+ } catch (SQLException e) {
+ log.error("Error closing connection", e);
+ }
+ }
+
+ return jobs;
+ }
+
+ private synchronized Connection getConnection() throws SQLException {
+ Connection connection = dbUtil.getConnection();
+ connection.setAutoCommit(true);
+
+ return connection;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/security/GSISecurityContext.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/security/GSISecurityContext.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/security/GSISecurityContext.java
new file mode 100644
index 0000000..395b797
--- /dev/null
+++ b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/security/GSISecurityContext.java
@@ -0,0 +1,275 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.gram.security;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.credential.store.credential.Credential;
+import org.apache.airavata.credential.store.credential.impl.certificate.CertificateCredential;
+import org.apache.airavata.credential.store.store.CredentialReader;
+import org.apache.airavata.gfac.AbstractSecurityContext;
+import org.apache.airavata.gfac.Constants;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.RequestData;
+import org.globus.gsi.X509Credential;
+import org.globus.gsi.gssapi.GlobusGSSCredentialImpl;
+import org.globus.gsi.provider.GlobusProvider;
+import org.globus.myproxy.GetParams;
+import org.globus.myproxy.MyProxy;
+import org.globus.myproxy.MyProxyException;
+import org.gridforum.jgss.ExtendedGSSCredential;
+import org.ietf.jgss.GSSCredential;
+import org.ietf.jgss.GSSException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.security.Security;
+import java.security.cert.X509Certificate;
+
+/**
+ * Handles GRID related security.
+ */
+public class GSISecurityContext extends AbstractSecurityContext {
+
+ protected static final Logger log = LoggerFactory.getLogger(GSISecurityContext.class);
+ /*
+ * context name
+ */
+ public static final String GSI_SECURITY_CONTEXT = "gsi";
+
+ public static int CREDENTIAL_RENEWING_THRESH_HOLD = 10 * 90;
+
+ private GSSCredential gssCredentials = null;
+
+ // Set trusted cert path and add provider
+ static {
+ Security.addProvider(new GlobusProvider());
+ try {
+ setUpTrustedCertificatePath();
+ } catch (ApplicationSettingsException e) {
+ log.error(e.getLocalizedMessage(), e);
+ }
+ }
+
+ public static void setUpTrustedCertificatePath(String trustedCertificatePath) {
+
+ File file = new File(trustedCertificatePath);
+
+ if (!file.exists() || !file.canRead()) {
+ File f = new File(".");
+ log.info("Current directory " + f.getAbsolutePath());
+ throw new RuntimeException("Cannot read trusted certificate path " + trustedCertificatePath);
+ } else {
+ System.setProperty(Constants.TRUSTED_CERTIFICATE_SYSTEM_PROPERTY, file.getAbsolutePath());
+ }
+ }
+
+ private static void setUpTrustedCertificatePath() throws ApplicationSettingsException {
+
+ String trustedCertificatePath = ServerSettings.getSetting(Constants.TRUSTED_CERT_LOCATION);
+
+ setUpTrustedCertificatePath(trustedCertificatePath);
+ }
+
+ /**
+ * Gets the trusted certificate path. Trusted certificate path is stored in "X509_CERT_DIR"
+ * system property.
+ * @return The trusted certificate path as a string.
+ */
+ public static String getTrustedCertificatePath() {
+ return System.getProperty(Constants.TRUSTED_CERTIFICATE_SYSTEM_PROPERTY);
+ }
+
+
+ public GSISecurityContext(CredentialReader credentialReader, RequestData requestData) {
+ super(credentialReader, requestData);
+ }
+
+
+
+ /**
+ * Gets GSSCredentials. The process is as follows;
+ * If credentials were queried for the first time create credentials.
+ * 1. Try creating credentials using certificates stored in the credential store
+ * 2. If 1 fails use user name and password to create credentials
+ * If credentials are already created check the remaining life time of the credential. If
+ * remaining life time is less than CREDENTIAL_RENEWING_THRESH_HOLD, then renew credentials.
+ * @return GSSCredentials to be used.
+ * @throws org.apache.airavata.gfac.GFacException If an error occurred while creating credentials.
+ * @throws org.apache.airavata.common.exception.ApplicationSettingsException
+ */
+ public GSSCredential getGssCredentials() throws GFacException, ApplicationSettingsException {
+
+ if (gssCredentials == null) {
+
+ try {
+ gssCredentials = getCredentialsFromStore();
+ } catch (Exception e) {
+ log.error("An exception occurred while retrieving credentials from the credential store. " +
+ "Will continue with my proxy user name and password.", e);
+ }
+
+ // If store does not have credentials try to get from user name and password
+ if (gssCredentials == null) {
+ gssCredentials = getDefaultCredentials();
+ }
+
+ // if still null, throw an exception
+ if (gssCredentials == null) {
+ throw new GFacException("Unable to retrieve my proxy credentials to continue operation.");
+ }
+ } else {
+ try {
+ if (gssCredentials.getRemainingLifetime() < CREDENTIAL_RENEWING_THRESH_HOLD) {
+ return renewCredentials();
+ }
+ } catch (GSSException e) {
+ throw new GFacException("Unable to retrieve remaining life time from credentials.", e);
+ }
+ }
+
+ return gssCredentials;
+ }
+
+ /**
+ * Renews credentials. First try to renew credentials as a trusted renewer. If that failed
+ * use user name and password to renew credentials.
+ * @return Renewed credentials.
+ * @throws org.apache.airavata.gfac.GFacException If an error occurred while renewing credentials.
+ * @throws org.apache.airavata.common.exception.ApplicationSettingsException
+ */
+ public GSSCredential renewCredentials() throws GFacException, ApplicationSettingsException {
+
+ // First try to renew credentials as a trusted renewer
+ try {
+ gssCredentials = renewCredentialsAsATrustedHost();
+ } catch (Exception e) {
+ log.warn("Renewing credentials as a trusted renewer failed", e);
+ gssCredentials = getProxyCredentials();
+ }
+
+ return gssCredentials;
+ }
+
+ /**
+ * Reads the credentials from credential store.
+ * @return If token is found in the credential store, will return a valid credential. Else returns null.
+ * @throws Exception If an error occurred while retrieving credentials.
+ */
+ public GSSCredential getCredentialsFromStore() throws Exception {
+
+ if (getCredentialReader() == null) {
+ return null;
+ }
+
+ Credential credential = getCredentialReader().getCredential(getRequestData().getGatewayId(),
+ getRequestData().getTokenId());
+
+ if (credential != null) {
+ if (credential instanceof CertificateCredential) {
+
+ log.info("Successfully found credentials for token id - " + getRequestData().getTokenId() +
+ " gateway id - " + getRequestData().getGatewayId());
+
+ CertificateCredential certificateCredential = (CertificateCredential) credential;
+
+ X509Certificate[] certificates = certificateCredential.getCertificates();
+ X509Credential newCredential = new X509Credential(certificateCredential.getPrivateKey(), certificates);
+
+ GlobusGSSCredentialImpl cred = new GlobusGSSCredentialImpl(newCredential, GSSCredential.INITIATE_AND_ACCEPT);
+ System.out.print(cred.export(ExtendedGSSCredential.IMPEXP_OPAQUE));
+ return cred;
+ //return new GlobusGSSCredentialImpl(newCredential,
+ // GSSCredential.INITIATE_AND_ACCEPT);
+ } else {
+ log.info("Credential type is not CertificateCredential. Cannot create mapping globus credentials. " +
+ "Credential type - " + credential.getClass().getName());
+ }
+ } else {
+ log.info("Could not find credentials for token - " + getRequestData().getTokenId() + " and "
+ + "gateway id - " + getRequestData().getGatewayId());
+ }
+
+ return null;
+ }
+
+ /**
+ * Gets the default proxy certificate.
+ * @return Default my proxy credentials.
+ * @throws org.apache.airavata.gfac.GFacException If an error occurred while retrieving credentials.
+ * @throws org.apache.airavata.common.exception.ApplicationSettingsException
+ */
+ public GSSCredential getDefaultCredentials() throws GFacException, ApplicationSettingsException{
+ MyProxy myproxy = new MyProxy(getRequestData().getMyProxyServerUrl(), getRequestData().getMyProxyPort());
+ try {
+ return myproxy.get(getRequestData().getMyProxyUserName(), getRequestData().getMyProxyPassword(),
+ getRequestData().getMyProxyLifeTime());
+ } catch (MyProxyException e) {
+ throw new GFacException("An error occurred while retrieving default security credentials.", e);
+ }
+ }
+
+ /**
+ * Gets a new proxy certificate given current credentials.
+ * @return The short lived GSSCredentials
+ * @throws org.apache.airavata.gfac.GFacException If an error is occurred while retrieving credentials.
+ * @throws org.apache.airavata.common.exception.ApplicationSettingsException
+ */
+ public GSSCredential getProxyCredentials() throws GFacException, ApplicationSettingsException {
+
+ MyProxy myproxy = new MyProxy(getRequestData().getMyProxyServerUrl(), getRequestData().getMyProxyPort());
+ try {
+ return myproxy.get(gssCredentials, getRequestData().getMyProxyUserName(), getRequestData().getMyProxyPassword(),
+ getRequestData().getMyProxyLifeTime());
+ } catch (MyProxyException e) {
+ throw new GFacException("An error occurred while renewing security credentials using user/password.", e);
+ }
+ }
+
+ /**
+ * Renew GSSCredentials.
+ * Before executing we need to add current host as a trusted renewer. Note to renew credentials
+ * we dont need user name and password.
+ * To do that execute following command
+ * > myproxy-logon -t <LIFETIME></LIFETIME> -s <MY PROXY SERVER> -l <USER NAME>
+ * E.g :- > myproxy-logon -t 264 -s myproxy.teragrid.org -l us3
+ * Enter MyProxy pass phrase:
+ * A credential has been received for user us3 in /tmp/x509up_u501.
+ * > myproxy-init -A --cert /tmp/x509up_u501 --key /tmp/x509up_u501 -l ogce -s myproxy.teragrid.org
+ * @return Renewed credentials.
+ * @throws org.apache.airavata.gfac.GFacException If an error occurred while renewing credentials.
+ * @throws org.apache.airavata.common.exception.ApplicationSettingsException
+ */
+ public GSSCredential renewCredentialsAsATrustedHost() throws GFacException, ApplicationSettingsException {
+ MyProxy myproxy = new MyProxy(getRequestData().getMyProxyServerUrl(), getRequestData().getMyProxyPort());
+ GetParams getParams = new GetParams();
+ getParams.setAuthzCreds(gssCredentials);
+ getParams.setUserName(getRequestData().getMyProxyUserName());
+ getParams.setLifetime(getRequestData().getMyProxyLifeTime());
+ try {
+ return myproxy.get(gssCredentials, getParams);
+ } catch (MyProxyException e) {
+ throw new GFacException("An error occurred while renewing security credentials.", e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/util/GramJobSubmissionListener.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/util/GramJobSubmissionListener.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/util/GramJobSubmissionListener.java
new file mode 100644
index 0000000..d91ae08
--- /dev/null
+++ b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/util/GramJobSubmissionListener.java
@@ -0,0 +1,141 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.gram.util;
+
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.gram.security.GSISecurityContext;
+import org.apache.airavata.gfac.notification.events.StatusChangeEvent;
+import org.apache.airavata.gfac.utils.GFacUtils;
+import org.globus.gram.GramJob;
+import org.globus.gram.GramJobListener;
+import org.ietf.jgss.GSSCredential;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class GramJobSubmissionListener implements GramJobListener{
+ private final Logger log = LoggerFactory.getLogger(GramJobSubmissionListener.class);
+
+ public static final int NO_ERROR = -42;
+ public static final int INITIAL_STATUS = -43;
+
+ private volatile boolean jobDone = false;
+ private volatile int error = NO_ERROR;
+ private int currentStatus = INITIAL_STATUS;
+
+ private JobExecutionContext context;
+ private GramJob job;
+
+ public GramJobSubmissionListener(GramJob job, JobExecutionContext context) {
+ this.job = job;
+ this.context = context;
+ }
+
+ /**
+ * This method is used to block the process until the currentStatus of the job is DONE or FAILED
+ */
+ public void waitFor() {
+ while (!isJobDone()) {
+ synchronized (this) {
+ try {
+ wait();
+ } catch (InterruptedException e) {}
+ }
+ }
+ }
+
+
+
+ private synchronized boolean isJobDone() {
+ return this.jobDone;
+ }
+
+ private void setStatus(int status, int error) {
+ try {
+ GFacUtils.updateJobStatus(context, context.getJobDetails(), GramProviderUtils.getApplicationJobStatus(status));
+ } catch (GFacException e) {
+ log.error("Error persisting status" + e.getLocalizedMessage(), e);
+ }
+ this.currentStatus = status;
+ this.error = error;
+
+ switch (this.currentStatus) {
+ case GramJob.STATUS_FAILED:
+ log.info("Job Error Code: " + error);
+ this.jobDone = true;
+ notifyAll();
+ case GramJob.STATUS_DONE:
+ this.jobDone = true;
+ notifyAll();
+ }
+
+ }
+
+ public synchronized void statusChanged(GramJob job) {
+
+ int jobStatus = job.getStatus();
+ String jobStatusMessage = "Status of job " + job.getIDAsString() + "is " + job.getStatusAsString();
+ /*
+ * Notify currentStatus change
+ */
+ this.context.getNotifier().publish(new StatusChangeEvent(jobStatusMessage));
+
+ /*
+ * Set new currentStatus if it is jobDone, notify all wait object
+ */
+ if (currentStatus != jobStatus) {
+ currentStatus = jobStatus;
+
+ setStatus(job.getStatus(), job.getError());
+
+ // Test to see whether we need to renew credentials
+ renewCredentials(job);
+ }
+ }
+
+ private void renewCredentials(GramJob job) {
+
+ try {
+
+ int proxyExpTime = job.getCredentials().getRemainingLifetime();
+ if (proxyExpTime < GSISecurityContext.CREDENTIAL_RENEWING_THRESH_HOLD) {
+ log.info("Job proxy expired. Trying to renew proxy");
+ GSSCredential gssCred = ((GSISecurityContext)context.
+ getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).renewCredentials();
+ job.renew(gssCred);
+ log.info("MyProxy credentials are renewed .");
+ }
+
+ } catch (Exception e) {
+ log.error("An error occurred while trying to renew credentials. Job id " + job.getIDAsString());
+ }
+
+
+ }
+
+ public synchronized int getError() {
+ return error;
+ }
+
+ public synchronized int getCurrentStatus() {
+ return currentStatus;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/util/GramProviderUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/util/GramProviderUtils.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/util/GramProviderUtils.java
new file mode 100644
index 0000000..74bb674
--- /dev/null
+++ b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/util/GramProviderUtils.java
@@ -0,0 +1,113 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.gram.util;
+
+import org.apache.airavata.gfac.ToolsException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.provider.GFacProviderException;
+import org.apache.airavata.model.workspace.experiment.JobState;
+import org.globus.gram.GramAttributes;
+import org.globus.gram.GramJob;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+
+public class GramProviderUtils {
+ private static final Logger log = LoggerFactory.getLogger(GramProviderUtils.class);
+
+ public static GramJob setupEnvironment(JobExecutionContext jobExecutionContext, boolean enableTwoPhase) throws GFacProviderException {
+ log.debug("Searching for Gate Keeper");
+ try {
+ GramAttributes jobAttr = GramRSLGenerator.configureRemoteJob(jobExecutionContext);
+ String rsl = jobAttr.toRSL();
+
+ if (enableTwoPhase) {
+ rsl = rsl + "(twoPhase=yes)";
+ }
+
+ log.debug("RSL = " + rsl);
+ GramJob job = new GramJob(rsl);
+ return job;
+ } catch (ToolsException te) {
+ throw new GFacProviderException(te.getMessage(), te);
+ }
+ }
+
+ public static JobState getApplicationJobStatus(int gramStatus) {
+ switch (gramStatus) {
+ case GramJob.STATUS_UNSUBMITTED:
+ return JobState.HELD;
+ case GramJob.STATUS_ACTIVE:
+ return JobState.ACTIVE;
+ case GramJob.STATUS_DONE:
+ return JobState.COMPLETE;
+ case GramJob.STATUS_FAILED:
+ return JobState.FAILED;
+ case GramJob.STATUS_PENDING:
+ return JobState.QUEUED;
+ case GramJob.STATUS_STAGE_IN:
+ return JobState.QUEUED;
+ case GramJob.STATUS_STAGE_OUT:
+ return JobState.COMPLETE;
+ case GramJob.STATUS_SUSPENDED:
+ return JobState.SUSPENDED;
+ default:
+ return JobState.UNKNOWN;
+ }
+ }
+
+ public static URI createGsiftpURI(String host, String localPath) throws URISyntaxException {
+ StringBuffer buf = new StringBuffer();
+ if (!host.startsWith("gsiftp://"))
+ buf.append("gsiftp://");
+ buf.append(host);
+ if (!host.endsWith("/"))
+ buf.append("/");
+ buf.append(localPath);
+ return new URI(buf.toString());
+ }
+
+ public static URI createGsiftpURI(GridFTPContactInfo host, String localPath) throws URISyntaxException {
+ StringBuffer buf = new StringBuffer();
+
+ if (!host.hostName.startsWith("gsiftp://"))
+ buf.append("gsiftp://");
+ buf.append(host).append(":").append(host.port);
+ if (!host.hostName.endsWith("/"))
+ buf.append("/");
+ buf.append(localPath);
+ return new URI(buf.toString());
+ }
+
+ public static String createGsiftpURIAsString(String host, String localPath) throws URISyntaxException {
+ StringBuffer buf = new StringBuffer();
+ if (!host.startsWith("gsiftp://"))
+ buf.append("gsiftp://");
+ buf.append(host);
+ if (!host.endsWith("/"))
+ buf.append("/");
+ buf.append(localPath);
+ return buf.toString();
+ }
+
+}
[8/8] git commit: changing package names of gfac implementations
Posted by la...@apache.org.
changing package names of gfac implementations
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/9bb8c2be
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/9bb8c2be
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/9bb8c2be
Branch: refs/heads/master
Commit: 9bb8c2befdfb242706c6bfe28e28b40e91227ff5
Parents: 0da5982
Author: lahiru <la...@apache.org>
Authored: Mon May 5 10:44:07 2014 -0400
Committer: lahiru <la...@apache.org>
Committed: Mon May 5 10:44:07 2014 -0400
----------------------------------------------------------------------
.../server/src/main/resources/gfac-config.xml | 40 +-
.../gfac-bes/src/test/resources/gfac-config.xml | 9 +-
.../context/security/GSISecurityContext.java | 275 ---------
.../apache/airavata/gfac/external/GridFtp.java | 558 -------------------
.../airavata/gfac/gram/external/GridFtp.java | 558 +++++++++++++++++++
.../gram/handler/GramDirectorySetupHandler.java | 139 +++++
.../gfac/gram/handler/GridFTPInputHandler.java | 207 +++++++
.../gfac/gram/handler/GridFTPOutputHandler.java | 349 ++++++++++++
.../airavata/gfac/gram/impl/GramProvider.java | 527 ++++++++++++++++++
.../persistence/DBJobPersistenceManager.java | 225 ++++++++
.../gfac/gram/security/GSISecurityContext.java | 275 +++++++++
.../gram/util/GramJobSubmissionListener.java | 141 +++++
.../gfac/gram/util/GramProviderUtils.java | 113 ++++
.../gfac/gram/util/GramRSLGenerator.java | 211 +++++++
.../gfac/gram/util/GridFTPContactInfo.java | 61 ++
.../gfac/handler/GramDirectorySetupHandler.java | 137 -----
.../gfac/handler/GridFTPInputHandler.java | 204 -------
.../gfac/handler/GridFTPOutputHandler.java | 347 ------------
.../persistence/DBJobPersistenceManager.java | 223 --------
.../gfac/provider/impl/GramProvider.java | 527 ------------------
.../gfac/util/GramJobSubmissionListener.java | 141 -----
.../airavata/gfac/util/GramProviderUtils.java | 114 ----
.../airavata/gfac/util/GramRSLGenerator.java | 211 -------
.../airavata/gfac/util/GridFTPContactInfo.java | 61 --
.../impl/GFacBaseTestWithMyProxyAuth.java | 2 +-
.../impl/GramProviderTestWithMyProxyAuth.java | 6 +-
.../src/test/resources/gfac-config.xml | 13 +-
.../context/security/GSISecurityContext.java | 288 ----------
.../handler/GSISSHDirectorySetupHandler.java | 103 ++++
.../gfac/gsissh/handler/GSISSHInputHandler.java | 138 +++++
.../gsissh/handler/GSISSHOutputHandler.java | 221 ++++++++
.../gfac/gsissh/impl/GSISSHProvider.java | 138 +++++
.../gsissh/security/GSISecurityContext.java | 288 ++++++++++
.../gfac/gsissh/util/GFACGSISSHUtils.java | 192 +++++++
.../handler/GSISSHDirectorySetupHandler.java | 101 ----
.../gfac/handler/GSISSHInputHandler.java | 136 -----
.../gfac/handler/GSISSHOutputHandler.java | 219 --------
.../gfac/provider/impl/GSISSHProvider.java | 138 -----
.../airavata/gfac/util/GFACGSISSHUtils.java | 192 -------
.../impl/GSISSHProviderTestWithMyProxyAuth.java | 2 +-
.../GSISecurityContextTestWithMyProxyAuth.java | 2 +-
.../hadoop/handler/HDFSDataMovementHandler.java | 103 ++++
.../hadoop/handler/HadoopDeploymentHandler.java | 276 +++++++++
.../hadoop/provider/impl/HadoopProvider.java | 154 +++++
.../gfac/hadoop/provider/utils/HadoopUtils.java | 60 ++
.../gfac/handler/HDFSDataMovementHandler.java | 101 ----
.../gfac/handler/HadoopDeploymentHandler.java | 274 ---------
.../gfac/provider/impl/HadoopProvider.java | 154 -----
.../gfac/provider/utils/HadoopUtils.java | 60 --
.../src/test/resources/gfac-config.xml | 16 +-
.../handler/LocalDirectorySetupHandler.java | 61 --
.../handler/LocalDirectorySetupHandler.java | 63 +++
.../airavata/gfac/local/impl/LocalProvider.java | 239 ++++++++
.../local/utils/InputStreamToFileWriter.java | 68 +++
.../airavata/gfac/local/utils/InputUtils.java | 39 ++
.../gfac/local/utils/LocalProviderUtil.java | 54 ++
.../gfac/provider/impl/LocalProvider.java | 239 --------
.../gfac/utils/InputStreamToFileWriter.java | 68 ---
.../apache/airavata/gfac/utils/InputUtils.java | 39 --
.../airavata/gfac/utils/LocalProviderUtil.java | 54 --
.../gfac/services/impl/LocalProviderTest.java | 4 +-
.../src/test/resources/gfac-config.xml | 4 +-
.../impl/pull/qstat/ResourceConnection.java | 8 +-
.../context/security/SSHSecurityContext.java | 120 ----
.../gfac/handler/AdvancedSCPInputHandler.java | 150 -----
.../gfac/handler/AdvancedSCPOutputHandler.java | 128 -----
.../gfac/handler/SSHDirectorySetupHandler.java | 105 ----
.../airavata/gfac/handler/SSHInputHandler.java | 141 -----
.../airavata/gfac/handler/SSHOutputHandler.java | 218 --------
.../gfac/provider/impl/SSHProvider.java | 310 -----------
.../ssh/handler/AdvancedSCPInputHandler.java | 152 +++++
.../ssh/handler/AdvancedSCPOutputHandler.java | 130 +++++
.../ssh/handler/SSHDirectorySetupHandler.java | 107 ++++
.../gfac/ssh/handler/SSHInputHandler.java | 143 +++++
.../gfac/ssh/handler/SSHOutputHandler.java | 220 ++++++++
.../airavata/gfac/ssh/impl/SSHProvider.java | 310 +++++++++++
.../gfac/ssh/security/SSHSecurityContext.java | 120 ++++
.../airavata/gfac/ssh/util/GFACSSHUtils.java | 179 ++++++
.../apache/airavata/gfac/util/GFACSSHUtils.java | 180 ------
.../services/impl/BigRed2TestWithSSHAuth.java | 3 +-
.../impl/SSHProviderTestWithSSHAuth.java | 2 +-
.../gfac-ssh/src/test/resources/gfac-config.xml | 36 +-
.../src/test/resources/gfac-config.xml | 82 +--
.../src/main/resources/gfac-config.xml | 82 +--
.../src/test/resources/gfac-config.xml | 82 +--
85 files changed, 6522 insertions(+), 6448 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/configuration/server/src/main/resources/gfac-config.xml
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/gfac-config.xml b/modules/configuration/server/src/main/resources/gfac-config.xml
index 5b80367..0e95bc4 100644
--- a/modules/configuration/server/src/main/resources/gfac-config.xml
+++ b/modules/configuration/server/src/main/resources/gfac-config.xml
@@ -24,24 +24,24 @@
</InHandlers>
<OutHandlers></OutHandlers>
</GlobalHandlers>
- <Provider class="org.apache.airavata.gfac.provider.impl.LocalProvider" host="org.apache.airavata.schemas.gfac.impl.HostDescriptionTypeImpl">
+ <Provider class="org.apache.airavata.gfac.local.impl.LocalProvider" host="org.apache.airavata.schemas.gfac.impl.HostDescriptionTypeImpl">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.LocalDirectorySetupHandler"/>
+ <Handler class="org.apache.airavata.gfac.local.handler.LocalDirectorySetupHandler"/>
</InHandlers>
</Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.GramProvider" host="org.apache.airavata.schemas.gfac.impl.GlobusHostTypeImpl">
+ <Provider class="org.apache.airavata.gfac.gram.provider.impl.GramProvider" host="org.apache.airavata.schemas.gfac.impl.GlobusHostTypeImpl">
<property name="name" value="value"/>
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler">
+ <Handler class="org.apache.airavata.gfac..gram.handler.GramDirectorySetupHandler">
<property name="name" value="value"/>
</Handler>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
+ <Handler class="org.apache.airavata.gfac..gram.handler.GridFTPInputHandler"/>
</InHandlers>
<OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
+ <Handler class="org.apache.airavata.gfac..gram.handler.GridFTPOutputHandler"/>
</OutHandlers>
</Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.BESProvider" host="org.apache.airavata.schemas.gfac.impl.UnicoreHostTypeImpl">
+ <Provider class="org.apache.airavata.gfac.bes.provider.impl.BESProvider" host="org.apache.airavata.schemas.gfac.impl.UnicoreHostTypeImpl">
</Provider>
<Provider class="org.apache.airavata.gfac.ec2.EC2Provider" host="org.apache.airavata.schemas.gfac.impl.Ec2HostTypeImpl">
@@ -49,10 +49,10 @@
<OutHandlers/>
</Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.HadoopProvider" host="org.apache.airavata.schemas.gfac.impl.HadoopHostTypeImpl">
+ <Provider class="org.apache.airavata.gfac.hadoop.provider.impl.HadoopProvider" host="org.apache.airavata.schemas.gfac.impl.HadoopHostTypeImpl">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.HadoopDeploymentHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.HDFSDataMovementHandler"/>
+ <Handler class="org.apache.airavata.gfac.hadoop.handler.HadoopDeploymentHandler"/>
+ <Handler class="org.apache.airavata.gfac.hadoop.handler.HDFSDataMovementHandler"/>
</InHandlers>
<OutHandlers/>
</Provider>
@@ -60,25 +60,25 @@
<Application name="UltraScan">
<InHandlers>
<Handler class="org.apache.airavata.gfac.handler.GfacDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
+ <Handler class="org.apache.airavata.gfac.gram.handler.GridFTPInputHandler"/>
</InHandlers>
<OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
+ <Handler class="org.apache.airavata.gfac.gram.handler.GridFTPOutputHandler"/>
</OutHandlers>
</Application>
- <Provider class="org.apache.airavata.gfac.provider.impl.SSHProvider" host="org.apache.airavata.schemas.gfac.impl.SSHHostTypeImpl">
+ <Provider class="org.apache.airavata.gfac.ssh.provider.impl.SSHProvider" host="org.apache.airavata.schemas.gfac.impl.SSHHostTypeImpl">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SSHDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.SSHInputHandler"/>
+ <Handler class="org.apache.airavata.gfac.ssh.handler.SSHDirectorySetupHandler"/>
+ <Handler class="org.apache.airavata.gfac.ssh.handler.SSHInputHandler"/>
</InHandlers>
<OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SSHOutputHandler"/>
+ <Handler class="org.apache.airavata.gfac.ssh.handler.SSHOutputHandler"/>
</OutHandlers>
</Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.GSISSHProvider" host="org.apache.airavata.schemas.gfac.impl.GsisshHostTypeImpl" executionMode="async">
+ <Provider class="org.apache.airavata.gfac.gsissh.provider.impl.GSISSHProvider" host="org.apache.airavata.schemas.gfac.impl.GsisshHostTypeImpl" executionMode="async">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GSISSHDirectorySetupHandler"/>
+ <Handler class="org.apache.airavata.gfac.gsissh.handler.GSISSHDirectorySetupHandler"/>
<!--Handler class="org.apache.airavata.gfac.handler.AdvancedSCPInputHandler">
<property name="privateKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa"/>
<property name="publicKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa.pub"/>
@@ -88,10 +88,10 @@
<property name="passPhrase" value="/home/airavata/outputData"/>
<property name="password" value="/home/airavata/outputData"/> <either we have to set password or keys, password has higher preference>
</Handler-->
- <Handler class="org.apache.airavata.gfac.handler.GSISSHInputHandler"/>
+ <Handler class="org.apache.airavata.gfac.gsissh.handler.GSISSHInputHandler"/>
</InHandlers>
<OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GSISSHOutputHandler"/>
+ <Handler class="org.apache.airavata.gfac.gsissh.handler.GSISSHOutputHandler"/>
<!--Handler class="org.apache.airavata.gfac.handler.AdvancedSCPOutputHandler">
<property name="privateKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa"/>
<property name="publicKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa.pub"/>
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-bes/src/test/resources/gfac-config.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/test/resources/gfac-config.xml b/modules/gfac/gfac-bes/src/test/resources/gfac-config.xml
index 85d148d..0598356 100644
--- a/modules/gfac/gfac-bes/src/test/resources/gfac-config.xml
+++ b/modules/gfac/gfac-bes/src/test/resources/gfac-config.xml
@@ -21,13 +21,6 @@
</GlobalHandlers>
- <Provider class="org.apache.airavata.gfac.provider.impl.SSHProvider" host="org.apache.airavata.schemas.gfac.impl.SSHHostTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SSHDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.SSHInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SSHOutputHandler"/>
- </OutHandlers>
+ <Provider class="org.apache.airavata.gfac.bes.provider.impl.BESProvider" host="org.apache.airavata.schemas.gfac.impl.UnicoreHostTypeImpl">
</Provider>
</GFac>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/context/security/GSISecurityContext.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/context/security/GSISecurityContext.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/context/security/GSISecurityContext.java
deleted file mode 100644
index d8f58a1..0000000
--- a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/context/security/GSISecurityContext.java
+++ /dev/null
@@ -1,275 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.context.security;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.credential.store.credential.Credential;
-import org.apache.airavata.credential.store.credential.impl.certificate.CertificateCredential;
-import org.apache.airavata.credential.store.store.CredentialReader;
-import org.apache.airavata.gfac.AbstractSecurityContext;
-import org.apache.airavata.gfac.Constants;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.RequestData;
-import org.globus.gsi.X509Credential;
-import org.globus.gsi.gssapi.GlobusGSSCredentialImpl;
-import org.globus.gsi.provider.GlobusProvider;
-import org.globus.myproxy.GetParams;
-import org.globus.myproxy.MyProxy;
-import org.globus.myproxy.MyProxyException;
-import org.gridforum.jgss.ExtendedGSSCredential;
-import org.ietf.jgss.GSSCredential;
-import org.ietf.jgss.GSSException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.security.Security;
-import java.security.cert.X509Certificate;
-
-/**
- * Handles GRID related security.
- */
-public class GSISecurityContext extends AbstractSecurityContext {
-
- protected static final Logger log = LoggerFactory.getLogger(GSISecurityContext.class);
- /*
- * context name
- */
- public static final String GSI_SECURITY_CONTEXT = "gsi";
-
- public static int CREDENTIAL_RENEWING_THRESH_HOLD = 10 * 90;
-
- private GSSCredential gssCredentials = null;
-
- // Set trusted cert path and add provider
- static {
- Security.addProvider(new GlobusProvider());
- try {
- setUpTrustedCertificatePath();
- } catch (ApplicationSettingsException e) {
- log.error(e.getLocalizedMessage(), e);
- }
- }
-
- public static void setUpTrustedCertificatePath(String trustedCertificatePath) {
-
- File file = new File(trustedCertificatePath);
-
- if (!file.exists() || !file.canRead()) {
- File f = new File(".");
- log.info("Current directory " + f.getAbsolutePath());
- throw new RuntimeException("Cannot read trusted certificate path " + trustedCertificatePath);
- } else {
- System.setProperty(Constants.TRUSTED_CERTIFICATE_SYSTEM_PROPERTY, file.getAbsolutePath());
- }
- }
-
- private static void setUpTrustedCertificatePath() throws ApplicationSettingsException {
-
- String trustedCertificatePath = ServerSettings.getSetting(Constants.TRUSTED_CERT_LOCATION);
-
- setUpTrustedCertificatePath(trustedCertificatePath);
- }
-
- /**
- * Gets the trusted certificate path. Trusted certificate path is stored in "X509_CERT_DIR"
- * system property.
- * @return The trusted certificate path as a string.
- */
- public static String getTrustedCertificatePath() {
- return System.getProperty(Constants.TRUSTED_CERTIFICATE_SYSTEM_PROPERTY);
- }
-
-
- public GSISecurityContext(CredentialReader credentialReader, RequestData requestData) {
- super(credentialReader, requestData);
- }
-
-
-
- /**
- * Gets GSSCredentials. The process is as follows;
- * If credentials were queried for the first time create credentials.
- * 1. Try creating credentials using certificates stored in the credential store
- * 2. If 1 fails use user name and password to create credentials
- * If credentials are already created check the remaining life time of the credential. If
- * remaining life time is less than CREDENTIAL_RENEWING_THRESH_HOLD, then renew credentials.
- * @return GSSCredentials to be used.
- * @throws org.apache.airavata.gfac.GFacException If an error occurred while creating credentials.
- * @throws org.apache.airavata.common.exception.ApplicationSettingsException
- */
- public GSSCredential getGssCredentials() throws GFacException, ApplicationSettingsException {
-
- if (gssCredentials == null) {
-
- try {
- gssCredentials = getCredentialsFromStore();
- } catch (Exception e) {
- log.error("An exception occurred while retrieving credentials from the credential store. " +
- "Will continue with my proxy user name and password.", e);
- }
-
- // If store does not have credentials try to get from user name and password
- if (gssCredentials == null) {
- gssCredentials = getDefaultCredentials();
- }
-
- // if still null, throw an exception
- if (gssCredentials == null) {
- throw new GFacException("Unable to retrieve my proxy credentials to continue operation.");
- }
- } else {
- try {
- if (gssCredentials.getRemainingLifetime() < CREDENTIAL_RENEWING_THRESH_HOLD) {
- return renewCredentials();
- }
- } catch (GSSException e) {
- throw new GFacException("Unable to retrieve remaining life time from credentials.", e);
- }
- }
-
- return gssCredentials;
- }
-
- /**
- * Renews credentials. First try to renew credentials as a trusted renewer. If that failed
- * use user name and password to renew credentials.
- * @return Renewed credentials.
- * @throws org.apache.airavata.gfac.GFacException If an error occurred while renewing credentials.
- * @throws org.apache.airavata.common.exception.ApplicationSettingsException
- */
- public GSSCredential renewCredentials() throws GFacException, ApplicationSettingsException {
-
- // First try to renew credentials as a trusted renewer
- try {
- gssCredentials = renewCredentialsAsATrustedHost();
- } catch (Exception e) {
- log.warn("Renewing credentials as a trusted renewer failed", e);
- gssCredentials = getProxyCredentials();
- }
-
- return gssCredentials;
- }
-
- /**
- * Reads the credentials from credential store.
- * @return If token is found in the credential store, will return a valid credential. Else returns null.
- * @throws Exception If an error occurred while retrieving credentials.
- */
- public GSSCredential getCredentialsFromStore() throws Exception {
-
- if (getCredentialReader() == null) {
- return null;
- }
-
- Credential credential = getCredentialReader().getCredential(getRequestData().getGatewayId(),
- getRequestData().getTokenId());
-
- if (credential != null) {
- if (credential instanceof CertificateCredential) {
-
- log.info("Successfully found credentials for token id - " + getRequestData().getTokenId() +
- " gateway id - " + getRequestData().getGatewayId());
-
- CertificateCredential certificateCredential = (CertificateCredential) credential;
-
- X509Certificate[] certificates = certificateCredential.getCertificates();
- X509Credential newCredential = new X509Credential(certificateCredential.getPrivateKey(), certificates);
-
- GlobusGSSCredentialImpl cred = new GlobusGSSCredentialImpl(newCredential, GSSCredential.INITIATE_AND_ACCEPT);
- System.out.print(cred.export(ExtendedGSSCredential.IMPEXP_OPAQUE));
- return cred;
- //return new GlobusGSSCredentialImpl(newCredential,
- // GSSCredential.INITIATE_AND_ACCEPT);
- } else {
- log.info("Credential type is not CertificateCredential. Cannot create mapping globus credentials. " +
- "Credential type - " + credential.getClass().getName());
- }
- } else {
- log.info("Could not find credentials for token - " + getRequestData().getTokenId() + " and "
- + "gateway id - " + getRequestData().getGatewayId());
- }
-
- return null;
- }
-
- /**
- * Gets the default proxy certificate.
- * @return Default my proxy credentials.
- * @throws org.apache.airavata.gfac.GFacException If an error occurred while retrieving credentials.
- * @throws org.apache.airavata.common.exception.ApplicationSettingsException
- */
- public GSSCredential getDefaultCredentials() throws GFacException, ApplicationSettingsException{
- MyProxy myproxy = new MyProxy(getRequestData().getMyProxyServerUrl(), getRequestData().getMyProxyPort());
- try {
- return myproxy.get(getRequestData().getMyProxyUserName(), getRequestData().getMyProxyPassword(),
- getRequestData().getMyProxyLifeTime());
- } catch (MyProxyException e) {
- throw new GFacException("An error occurred while retrieving default security credentials.", e);
- }
- }
-
- /**
- * Gets a new proxy certificate given current credentials.
- * @return The short lived GSSCredentials
- * @throws org.apache.airavata.gfac.GFacException If an error is occurred while retrieving credentials.
- * @throws org.apache.airavata.common.exception.ApplicationSettingsException
- */
- public GSSCredential getProxyCredentials() throws GFacException, ApplicationSettingsException {
-
- MyProxy myproxy = new MyProxy(getRequestData().getMyProxyServerUrl(), getRequestData().getMyProxyPort());
- try {
- return myproxy.get(gssCredentials, getRequestData().getMyProxyUserName(), getRequestData().getMyProxyPassword(),
- getRequestData().getMyProxyLifeTime());
- } catch (MyProxyException e) {
- throw new GFacException("An error occurred while renewing security credentials using user/password.", e);
- }
- }
-
- /**
- * Renew GSSCredentials.
- * Before executing we need to add current host as a trusted renewer. Note to renew credentials
- * we dont need user name and password.
- * To do that execute following command
- * > myproxy-logon -t <LIFETIME></LIFETIME> -s <MY PROXY SERVER> -l <USER NAME>
- * E.g :- > myproxy-logon -t 264 -s myproxy.teragrid.org -l us3
- * Enter MyProxy pass phrase:
- * A credential has been received for user us3 in /tmp/x509up_u501.
- * > myproxy-init -A --cert /tmp/x509up_u501 --key /tmp/x509up_u501 -l ogce -s myproxy.teragrid.org
- * @return Renewed credentials.
- * @throws org.apache.airavata.gfac.GFacException If an error occurred while renewing credentials.
- * @throws org.apache.airavata.common.exception.ApplicationSettingsException
- */
- public GSSCredential renewCredentialsAsATrustedHost() throws GFacException, ApplicationSettingsException {
- MyProxy myproxy = new MyProxy(getRequestData().getMyProxyServerUrl(), getRequestData().getMyProxyPort());
- GetParams getParams = new GetParams();
- getParams.setAuthzCreds(gssCredentials);
- getParams.setUserName(getRequestData().getMyProxyUserName());
- getParams.setLifetime(getRequestData().getMyProxyLifeTime());
- try {
- return myproxy.get(gssCredentials, getParams);
- } catch (MyProxyException e) {
- throw new GFacException("An error occurred while renewing security credentials.", e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/external/GridFtp.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/external/GridFtp.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/external/GridFtp.java
deleted file mode 100644
index 5be087e..0000000
--- a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/external/GridFtp.java
+++ /dev/null
@@ -1,558 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.external;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.Vector;
-
-import org.apache.airavata.gfac.Constants;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.ToolsException;
-import org.apache.airavata.gfac.util.GramProviderUtils;
-import org.apache.airavata.gfac.util.GridFTPContactInfo;
-import org.globus.ftp.DataChannelAuthentication;
-import org.globus.ftp.DataSourceStream;
-import org.globus.ftp.FileInfo;
-import org.globus.ftp.GridFTPClient;
-import org.globus.ftp.HostPort;
-import org.globus.ftp.Marker;
-import org.globus.ftp.MarkerListener;
-import org.globus.ftp.MlsxEntry;
-import org.globus.ftp.Session;
-import org.globus.ftp.exception.ClientException;
-import org.globus.ftp.exception.ServerException;
-import org.globus.gsi.gssapi.auth.HostAuthorization;
-import org.ietf.jgss.GSSCredential;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * GridFTP tools
- */
-public class GridFtp {
- public static final Logger log = LoggerFactory.getLogger(GridFtp.class);
-
- public static final String GSIFTP_SCHEME = "gsiftp";
- public static final String HOST = "host";
-
- /**
- * Make directory at remote location
- *
- * @param destURI
- * @param gssCred
- * @throws ServerException
- * @throws IOException
- */
- public void makeDir(URI destURI, GSSCredential gssCred) throws ToolsException {
- GridFTPClient destClient = null;
- GridFTPContactInfo destHost = new GridFTPContactInfo(destURI.getHost(), destURI.getPort());
- try {
-
- String destPath = destURI.getPath();
- log.info(("Creating Directory = " + destHost + "=" + destPath));
-
- destClient = new GridFTPClient(destHost.hostName, destHost.port);
-
- int tryCount = 0;
- while (true) {
- try {
- destClient.setAuthorization(new HostAuthorization(GridFtp.HOST));
- destClient.authenticate(gssCred);
- destClient.setDataChannelAuthentication(DataChannelAuthentication.SELF);
-
- if (!destClient.exists(destPath)) {
- destClient.makeDir(destPath);
- }
- break;
- } catch (ServerException e) {
- tryCount++;
- if (tryCount >= 3) {
- throw new ToolsException(e.getMessage(), e);
- }
- Thread.sleep(10000);
- } catch (IOException e) {
- tryCount++;
- if (tryCount >= 3) {
- throw new ToolsException(e.getMessage(), e);
- }
- Thread.sleep(10000);
- }
- }
- } catch (ServerException e) {
- throw new ToolsException("Cannot Create GridFTP Client to:" + destHost.toString(), e);
- } catch (IOException e) {
- throw new ToolsException("Cannot Create GridFTP Client to:" + destHost.toString(), e);
- } catch (InterruptedException e) {
- throw new ToolsException("Internal Error cannot sleep", e);
- } finally {
- if (destClient != null) {
- try {
- destClient.close();
- } catch (Exception e) {
- log.warn("Cannot close GridFTP client connection",e);
- }
- }
- }
- }
-
- /**
- * Upload file from stream
- *
- * @param destURI
- * @param gsCredential
- * @param io
- * @throws GFacException
- */
- public void uploadFile(URI destURI, GSSCredential gsCredential, InputStream io) throws ToolsException {
- GridFTPClient ftpClient = null;
- GridFTPContactInfo contactInfo = new GridFTPContactInfo(destURI.getHost(), destURI.getPort());
-
- try {
-
- String remoteFile = destURI.getPath();
- log.info("The remote file is " + remoteFile);
-
- log.debug("Setup GridFTP Client");
-
- ftpClient = new GridFTPClient(contactInfo.hostName, contactInfo.port);
- ftpClient.setAuthorization(new HostAuthorization(GridFtp.HOST));
- ftpClient.authenticate(gsCredential);
- ftpClient.setDataChannelAuthentication(DataChannelAuthentication.SELF);
-
- log.info("Uploading file");
- if (checkBinaryExtensions(remoteFile)) {
- log.debug("Transfer mode is set to Binary for a file upload");
- ftpClient.setType(Session.TYPE_IMAGE);
- }
-
- ftpClient.put(remoteFile, new DataSourceStream(io), new MarkerListener() {
- public void markerArrived(Marker marker) {
- }
- });
-
- log.info("Upload file to:" + remoteFile + " is done");
-
- } catch (ServerException e) {
- throw new ToolsException("Cannot upload file to GridFTP:" + contactInfo.toString(), e);
- } catch (IOException e) {
- throw new ToolsException("Cannot upload file to GridFTP:" + contactInfo.toString(), e);
- } catch (ClientException e) {
- throw new ToolsException("Cannot upload file to GridFTP:" + contactInfo.toString(), e);
- } finally {
- if (ftpClient != null) {
- try {
- ftpClient.close();
- } catch (Exception e) {
- log.warn("Cannot close GridFTP client connection",e);
- }
- }
- }
- }
-
- public void uploadFile(URI srcURI, URI destURI, GSSCredential gsCredential) throws ToolsException {
- GridFTPClient srcClient = null;
- GridFTPContactInfo destContactInfo = new GridFTPContactInfo(destURI.getHost(), destURI.getPort());
- GridFTPContactInfo srcContactInfo = new GridFTPContactInfo(srcURI.getHost(),srcURI.getPort());
- try {
- String remoteFile = destURI.getPath();
- log.info("The remote file is " + remoteFile);
- log.debug("Setup GridFTP Client");
- srcClient = new GridFTPClient(srcContactInfo.hostName, srcContactInfo.port);
- srcClient.setAuthorization(new HostAuthorization(GridFtp.HOST));
- srcClient.authenticate(gsCredential);
- srcClient.setDataChannelAuthentication(DataChannelAuthentication.SELF);
-
- GridFTPClient destClient = new GridFTPClient(destContactInfo.hostName, destContactInfo.port);
- destClient.setAuthorization(new HostAuthorization(GridFtp.HOST));
- destClient.authenticate(gsCredential);
- destClient.setDataChannelAuthentication(DataChannelAuthentication.SELF);
- log.debug("Uploading file");
- if (checkBinaryExtensions(remoteFile)) {
- log.debug("Transfer mode is set to Binary for a file upload");
- srcClient.setType(Session.TYPE_IMAGE);
- }
-
- srcClient.transfer(srcURI.getPath(),destClient, remoteFile, false, null);
-
- log.info("Upload file to:" + remoteFile + " is done");
-
- } catch (ServerException e) {
- throw new ToolsException("Cannot upload file to GridFTP:" + destContactInfo.toString(), e);
- } catch (IOException e) {
- throw new ToolsException("Cannot upload file to GridFTP:" + destContactInfo.toString(), e);
- } catch (ClientException e) {
- throw new ToolsException("Cannot upload file to GridFTP:" + destContactInfo.toString(), e);
- } finally {
- if (srcClient != null) {
- try {
- srcClient.close();
- } catch (Exception e) {
- log.warn("Cannot close GridFTP client connection",e);
- }
- }
- }
- }
-
- /**
- * Upload file to remote location
- *
- * @param destURI
- * @param gsCredential
- * @param localFile
- * @throws GFacException
- */
- public void uploadFile(URI destURI, GSSCredential gsCredential, File localFile) throws ToolsException {
- GridFTPClient ftpClient = null;
- GridFTPContactInfo contactInfo = new GridFTPContactInfo(destURI.getHost(), destURI.getPort());
- try {
-
- String remoteFile = destURI.getPath();
-
- log.info("The local temp file is " + localFile);
- log.info("the remote file is " + remoteFile);
-
- log.debug("Setup GridFTP Client");
-
- ftpClient = new GridFTPClient(contactInfo.hostName, contactInfo.port);
- ftpClient.setAuthorization(new HostAuthorization(GridFtp.HOST));
- ftpClient.authenticate(gsCredential);
- ftpClient.setDataChannelAuthentication(DataChannelAuthentication.SELF);
-
- log.debug("Uploading file");
- if (checkBinaryExtensions(remoteFile)) {
- log.debug("Transfer mode is set to Binary for a file upload");
- ftpClient.setType(Session.TYPE_IMAGE);
- }
-
-
- ftpClient.put(localFile, remoteFile, false);
-
- log.info("Upload file to:" + remoteFile + " is done");
-
- } catch (ServerException e) {
- throw new ToolsException("Cannot upload file to GridFTP:" + contactInfo.toString(), e);
- } catch (IOException e) {
- throw new ToolsException("Cannot upload file to GridFTP:" + contactInfo.toString(), e);
- } catch (ClientException e) {
- throw new ToolsException("Cannot upload file to GridFTP:" + contactInfo.toString(), e);
- } finally {
- if (ftpClient != null) {
- try {
- ftpClient.close();
- } catch (Exception e) {
- log.warn("Cannot close GridFTP client connection",e);
- }
- }
- }
- }
-
- /**
- * Download File from remote location
- *
- * @param destURI
- * @param gsCredential
- * @param localFile
- * @throws GFacException
- */
- public void downloadFile(URI destURI, GSSCredential gsCredential, File localFile) throws ToolsException {
- GridFTPClient ftpClient = null;
- GridFTPContactInfo contactInfo = new GridFTPContactInfo(destURI.getHost(), destURI.getPort());
- try {
- String remoteFile = destURI.getPath();
-
- log.info("The local temp file is " + localFile);
- log.info("the remote file is " + remoteFile);
-
- log.debug("Setup GridFTP Client");
-
- ftpClient = new GridFTPClient(contactInfo.hostName, contactInfo.port);
- ftpClient.setAuthorization(new HostAuthorization(GridFtp.HOST));
- ftpClient.authenticate(gsCredential);
- ftpClient.setDataChannelAuthentication(DataChannelAuthentication.SELF);
-
- log.debug("Downloading file");
- if (checkBinaryExtensions(remoteFile)) {
- log.debug("Transfer mode is set to Binary to download a file");
- ftpClient.setType(Session.TYPE_IMAGE);
- }
-
- ftpClient.get(remoteFile, localFile);
-
- log.info("Download file to:" + localFile + " is done");
-
- } catch (ServerException e) {
- throw new ToolsException("Cannot download file from GridFTP:" + contactInfo.toString(), e);
- } catch (IOException e) {
- throw new ToolsException("Cannot download file from GridFTP:" + contactInfo.toString(), e);
- } catch (ClientException e) {
- throw new ToolsException("Cannot download file from GridFTP:" + contactInfo.toString(), e);
- } finally {
- if (ftpClient != null) {
- try {
- //ftpClient.close();
- ftpClient.close(false);
- } catch (Exception e) {
- log.warn("Cannot close GridFTP client connection",e);
- }
- }
- }
- }
-
- /**
- * Stream remote file
- *
- * @param destURI
- * @param gsCredential
- * @param localFile
- * @return
- * @throws GFacException
- */
- public String readRemoteFile(URI destURI, GSSCredential gsCredential, File localFile) throws ToolsException {
- BufferedReader instream = null;
- File localTempfile = null;
- try {
-
- if (localFile == null) {
- localTempfile = File.createTempFile("stderr", "err");
- } else {
- localTempfile = localFile;
- }
-
- log.info("Local temporary file:" + localTempfile);
-
- downloadFile(destURI, gsCredential, localTempfile);
-
- instream = new BufferedReader(new FileReader(localTempfile));
- StringBuffer buff = new StringBuffer();
- String temp = null;
- while ((temp = instream.readLine()) != null) {
- buff.append(temp);
- buff.append(Constants.NEWLINE);
- }
-
- log.info("finish read file:" + localTempfile);
-
- return buff.toString();
- } catch (FileNotFoundException e) {
- throw new ToolsException("Cannot read localfile file:" + localTempfile, e);
- } catch (IOException e) {
- throw new ToolsException("Cannot read localfile file:" + localTempfile, e);
- } finally {
- if (instream != null) {
- try {
- instream.close();
- } catch (Exception e) {
- log.warn("Cannot close GridFTP client connection",e);
- }
- }
- }
- }
-
- /**
- * Transfer data from one GridFTp Endpoint to another GridFTP Endpoint
- *
- * @param srchost
- * @param desthost
- * @param gssCred
- * @param srcActive
- * @throws ServerException
- * @throws ClientException
- * @throws IOException
- */
- public void transfer(URI srchost, URI desthost, GSSCredential gssCred, boolean srcActive) throws ToolsException {
- GridFTPClient destClient = null;
- GridFTPClient srcClient = null;
-
- try {
- destClient = new GridFTPClient(desthost.getHost(), desthost.getPort());
- destClient.setAuthorization(new HostAuthorization(GridFtp.HOST));
- destClient.authenticate(gssCred);
-
- if (checkBinaryExtensions(desthost.getPath())) {
- log.debug("Transfer mode is set to Binary");
- destClient.setType(Session.TYPE_IMAGE);
- }
-
- srcClient = new GridFTPClient(srchost.getHost(), srchost.getPort());
- srcClient.setAuthorization(new HostAuthorization(GridFtp.HOST));
- srcClient.authenticate(gssCred);
-
- if (checkBinaryExtensions(srchost.getPath())) {
- log.debug("Transfer mode is set to Binary");
- srcClient.setType(Session.TYPE_IMAGE);
- }
-
- if (srcActive) {
- log.debug("Set src active");
- HostPort hp = destClient.setPassive();
- srcClient.setActive(hp);
- } else {
- log.debug("Set dst active");
- HostPort hp = srcClient.setPassive();
- destClient.setActive(hp);
- }
-
- log.debug("Start transfer file from GridFTP:" + srchost.toString() + " to " + desthost.toString());
-
- /**
- * Transfer a file. The transfer() function blocks until the transfer is complete.
- */
- srcClient.transfer(srchost.getPath(), destClient, desthost.getPath(), false, null);
- if (srcClient.getSize(srchost.getPath()) == destClient.getSize(desthost.getPath())) {
- log.debug("CHECK SUM OK");
- } else {
- log.debug("****CHECK SUM FAILED****");
- }
-
- } catch (ServerException e) {
- throw new ToolsException("Cannot transfer file from GridFTP:" + srchost.toString() + " to "
- + desthost.toString(), e);
- } catch (IOException e) {
- throw new ToolsException("Cannot transfer file from GridFTP:" + srchost.toString() + " to "
- + desthost.toString(), e);
- } catch (ClientException e) {
- throw new ToolsException("Cannot transfer file from GridFTP:" + srchost.toString() + " to "
- + desthost.toString(), e);
- } finally {
- if (destClient != null) {
- try {
- destClient.close();
- } catch (Exception e) {
- log.warn("Cannot close GridFTP client connection at Desitnation:" + desthost.toString());
- }
- }
- if (srcClient != null) {
- try {
- srcClient.close();
- } catch (Exception e) {
- log.warn("Cannot close GridFTP client connection at Source:" + srchost.toString(),e);
- }
- }
- }
- }
-
- /**
- * List files in a GridFTP directory
- * @param dirURI
- * @param gssCred
- * @return
- * @throws ToolsException
- */
- @SuppressWarnings("unchecked")
- public List<String> listDir(URI dirURI, GSSCredential gssCred) throws ToolsException {
- List<String> files = new ArrayList<String>();
- GridFTPClient srcClient = null;
- try {
- GridFTPContactInfo contactInfo = new GridFTPContactInfo(dirURI.getHost(), dirURI.getPort());
-
- srcClient = new GridFTPClient(contactInfo.hostName, contactInfo.port);
- srcClient.setAuthorization(new HostAuthorization(GridFtp.HOST));
- srcClient.authenticate(gssCred);
- srcClient.setDataChannelAuthentication(DataChannelAuthentication.SELF);
- srcClient.setType(Session.TYPE_ASCII);
- srcClient.changeDir(dirURI.getPath());
-
- Vector<Object> fileInfo = null;
- try {
- fileInfo = srcClient.mlsd();
- } catch (Throwable e) {
- fileInfo = srcClient.list();
- }
-
- if (!fileInfo.isEmpty()) {
- for (int j = 0; j < fileInfo.size(); ++j) {
- String name = null;
- if (fileInfo.get(j) instanceof MlsxEntry) {
- name = ((MlsxEntry) fileInfo.get(j)).getFileName();
- } else if (fileInfo.get(j) instanceof FileInfo) {
- name = ((FileInfo) fileInfo.get(j)).getName();
- } else {
- throw new ToolsException("Unsupported type returned by gridftp " + fileInfo.get(j));
- }
-
- if (!name.equals(".") && !name.equals("..")) {
- URI uri = GramProviderUtils.createGsiftpURI(contactInfo.hostName, dirURI.getPath() + File.separator + name);
- files.add(uri.getPath());
- }
- }
- }
- return files;
- } catch (IOException e) {
- throw new ToolsException("Could not list directory: " + dirURI.toString() ,e);
- } catch (ServerException e) {
- throw new ToolsException("Could not list directory: " + dirURI.toString() ,e);
- } catch (ClientException e) {
- throw new ToolsException("Could not list directory: " + dirURI.toString() ,e);
- } catch (URISyntaxException e) {
- throw new ToolsException("Error creating URL of listed files: " + dirURI.toString() ,e);
- } finally {
- if (srcClient != null) {
- try {
- srcClient.close();
- } catch (Exception e) {
- log.warn("Cannot close GridFTP client connection", e);
- }
- }
- }
- }
- /**
- * Method to check file extension as binary to set transfer type
- * @param filePath
- * @return
- */
- private static boolean checkBinaryExtensions(String filePath){
- String extension = filePath.substring(filePath.lastIndexOf(".")+1,filePath.length());
- Set<String> extensions = new HashSet<String>(Arrays.asList(new String[] {"tar","zip","gz","tgz"}));
- if(extensions.contains(extension)){
- return true;
- }else{
- return false;
- }
-
- }
-
-
-
-
- public String gridFTPFileExist(URI inputDirectory,String fileName,GSSCredential gssCred) throws ToolsException {
- List<String> strings = listDir(inputDirectory, gssCred);
- for(String fileExist:strings){
- if(fileName.equals(fileExist)) {
- fileName = "duplicate_" + fileName;
- return fileName;
- }
- }
- return fileName;
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/external/GridFtp.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/external/GridFtp.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/external/GridFtp.java
new file mode 100644
index 0000000..fef9fad
--- /dev/null
+++ b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/external/GridFtp.java
@@ -0,0 +1,558 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.gram.external;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.Vector;
+
+import org.apache.airavata.gfac.Constants;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.ToolsException;
+import org.apache.airavata.gfac.gram.util.GramProviderUtils;
+import org.apache.airavata.gfac.gram.util.GridFTPContactInfo;
+import org.globus.ftp.DataChannelAuthentication;
+import org.globus.ftp.DataSourceStream;
+import org.globus.ftp.FileInfo;
+import org.globus.ftp.GridFTPClient;
+import org.globus.ftp.HostPort;
+import org.globus.ftp.Marker;
+import org.globus.ftp.MarkerListener;
+import org.globus.ftp.MlsxEntry;
+import org.globus.ftp.Session;
+import org.globus.ftp.exception.ClientException;
+import org.globus.ftp.exception.ServerException;
+import org.globus.gsi.gssapi.auth.HostAuthorization;
+import org.ietf.jgss.GSSCredential;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * GridFTP tools
+ */
+public class GridFtp {
+ public static final Logger log = LoggerFactory.getLogger(GridFtp.class);
+
+ public static final String GSIFTP_SCHEME = "gsiftp";
+ public static final String HOST = "host";
+
+ /**
+ * Make directory at remote location
+ *
+ * @param destURI
+ * @param gssCred
+ * @throws ServerException
+ * @throws IOException
+ */
+ public void makeDir(URI destURI, GSSCredential gssCred) throws ToolsException {
+ GridFTPClient destClient = null;
+ GridFTPContactInfo destHost = new GridFTPContactInfo(destURI.getHost(), destURI.getPort());
+ try {
+
+ String destPath = destURI.getPath();
+ log.info(("Creating Directory = " + destHost + "=" + destPath));
+
+ destClient = new GridFTPClient(destHost.hostName, destHost.port);
+
+ int tryCount = 0;
+ while (true) {
+ try {
+ destClient.setAuthorization(new HostAuthorization(GridFtp.HOST));
+ destClient.authenticate(gssCred);
+ destClient.setDataChannelAuthentication(DataChannelAuthentication.SELF);
+
+ if (!destClient.exists(destPath)) {
+ destClient.makeDir(destPath);
+ }
+ break;
+ } catch (ServerException e) {
+ tryCount++;
+ if (tryCount >= 3) {
+ throw new ToolsException(e.getMessage(), e);
+ }
+ Thread.sleep(10000);
+ } catch (IOException e) {
+ tryCount++;
+ if (tryCount >= 3) {
+ throw new ToolsException(e.getMessage(), e);
+ }
+ Thread.sleep(10000);
+ }
+ }
+ } catch (ServerException e) {
+ throw new ToolsException("Cannot Create GridFTP Client to:" + destHost.toString(), e);
+ } catch (IOException e) {
+ throw new ToolsException("Cannot Create GridFTP Client to:" + destHost.toString(), e);
+ } catch (InterruptedException e) {
+ throw new ToolsException("Internal Error cannot sleep", e);
+ } finally {
+ if (destClient != null) {
+ try {
+ destClient.close();
+ } catch (Exception e) {
+ log.warn("Cannot close GridFTP client connection",e);
+ }
+ }
+ }
+ }
+
+ /**
+ * Upload file from stream
+ *
+ * @param destURI
+ * @param gsCredential
+ * @param io
+ * @throws GFacException
+ */
+ public void uploadFile(URI destURI, GSSCredential gsCredential, InputStream io) throws ToolsException {
+ GridFTPClient ftpClient = null;
+ GridFTPContactInfo contactInfo = new GridFTPContactInfo(destURI.getHost(), destURI.getPort());
+
+ try {
+
+ String remoteFile = destURI.getPath();
+ log.info("The remote file is " + remoteFile);
+
+ log.debug("Setup GridFTP Client");
+
+ ftpClient = new GridFTPClient(contactInfo.hostName, contactInfo.port);
+ ftpClient.setAuthorization(new HostAuthorization(GridFtp.HOST));
+ ftpClient.authenticate(gsCredential);
+ ftpClient.setDataChannelAuthentication(DataChannelAuthentication.SELF);
+
+ log.info("Uploading file");
+ if (checkBinaryExtensions(remoteFile)) {
+ log.debug("Transfer mode is set to Binary for a file upload");
+ ftpClient.setType(Session.TYPE_IMAGE);
+ }
+
+ ftpClient.put(remoteFile, new DataSourceStream(io), new MarkerListener() {
+ public void markerArrived(Marker marker) {
+ }
+ });
+
+ log.info("Upload file to:" + remoteFile + " is done");
+
+ } catch (ServerException e) {
+ throw new ToolsException("Cannot upload file to GridFTP:" + contactInfo.toString(), e);
+ } catch (IOException e) {
+ throw new ToolsException("Cannot upload file to GridFTP:" + contactInfo.toString(), e);
+ } catch (ClientException e) {
+ throw new ToolsException("Cannot upload file to GridFTP:" + contactInfo.toString(), e);
+ } finally {
+ if (ftpClient != null) {
+ try {
+ ftpClient.close();
+ } catch (Exception e) {
+ log.warn("Cannot close GridFTP client connection",e);
+ }
+ }
+ }
+ }
+
+ public void uploadFile(URI srcURI, URI destURI, GSSCredential gsCredential) throws ToolsException {
+ GridFTPClient srcClient = null;
+ GridFTPContactInfo destContactInfo = new GridFTPContactInfo(destURI.getHost(), destURI.getPort());
+ GridFTPContactInfo srcContactInfo = new GridFTPContactInfo(srcURI.getHost(),srcURI.getPort());
+ try {
+ String remoteFile = destURI.getPath();
+ log.info("The remote file is " + remoteFile);
+ log.debug("Setup GridFTP Client");
+ srcClient = new GridFTPClient(srcContactInfo.hostName, srcContactInfo.port);
+ srcClient.setAuthorization(new HostAuthorization(GridFtp.HOST));
+ srcClient.authenticate(gsCredential);
+ srcClient.setDataChannelAuthentication(DataChannelAuthentication.SELF);
+
+ GridFTPClient destClient = new GridFTPClient(destContactInfo.hostName, destContactInfo.port);
+ destClient.setAuthorization(new HostAuthorization(GridFtp.HOST));
+ destClient.authenticate(gsCredential);
+ destClient.setDataChannelAuthentication(DataChannelAuthentication.SELF);
+ log.debug("Uploading file");
+ if (checkBinaryExtensions(remoteFile)) {
+ log.debug("Transfer mode is set to Binary for a file upload");
+ srcClient.setType(Session.TYPE_IMAGE);
+ }
+
+ srcClient.transfer(srcURI.getPath(),destClient, remoteFile, false, null);
+
+ log.info("Upload file to:" + remoteFile + " is done");
+
+ } catch (ServerException e) {
+ throw new ToolsException("Cannot upload file to GridFTP:" + destContactInfo.toString(), e);
+ } catch (IOException e) {
+ throw new ToolsException("Cannot upload file to GridFTP:" + destContactInfo.toString(), e);
+ } catch (ClientException e) {
+ throw new ToolsException("Cannot upload file to GridFTP:" + destContactInfo.toString(), e);
+ } finally {
+ if (srcClient != null) {
+ try {
+ srcClient.close();
+ } catch (Exception e) {
+ log.warn("Cannot close GridFTP client connection",e);
+ }
+ }
+ }
+ }
+
+ /**
+ * Upload file to remote location
+ *
+ * @param destURI
+ * @param gsCredential
+ * @param localFile
+ * @throws GFacException
+ */
+ public void uploadFile(URI destURI, GSSCredential gsCredential, File localFile) throws ToolsException {
+ GridFTPClient ftpClient = null;
+ GridFTPContactInfo contactInfo = new GridFTPContactInfo(destURI.getHost(), destURI.getPort());
+ try {
+
+ String remoteFile = destURI.getPath();
+
+ log.info("The local temp file is " + localFile);
+ log.info("the remote file is " + remoteFile);
+
+ log.debug("Setup GridFTP Client");
+
+ ftpClient = new GridFTPClient(contactInfo.hostName, contactInfo.port);
+ ftpClient.setAuthorization(new HostAuthorization(GridFtp.HOST));
+ ftpClient.authenticate(gsCredential);
+ ftpClient.setDataChannelAuthentication(DataChannelAuthentication.SELF);
+
+ log.debug("Uploading file");
+ if (checkBinaryExtensions(remoteFile)) {
+ log.debug("Transfer mode is set to Binary for a file upload");
+ ftpClient.setType(Session.TYPE_IMAGE);
+ }
+
+
+ ftpClient.put(localFile, remoteFile, false);
+
+ log.info("Upload file to:" + remoteFile + " is done");
+
+ } catch (ServerException e) {
+ throw new ToolsException("Cannot upload file to GridFTP:" + contactInfo.toString(), e);
+ } catch (IOException e) {
+ throw new ToolsException("Cannot upload file to GridFTP:" + contactInfo.toString(), e);
+ } catch (ClientException e) {
+ throw new ToolsException("Cannot upload file to GridFTP:" + contactInfo.toString(), e);
+ } finally {
+ if (ftpClient != null) {
+ try {
+ ftpClient.close();
+ } catch (Exception e) {
+ log.warn("Cannot close GridFTP client connection",e);
+ }
+ }
+ }
+ }
+
+ /**
+ * Download File from remote location
+ *
+ * @param destURI
+ * @param gsCredential
+ * @param localFile
+ * @throws GFacException
+ */
+ public void downloadFile(URI destURI, GSSCredential gsCredential, File localFile) throws ToolsException {
+ GridFTPClient ftpClient = null;
+ GridFTPContactInfo contactInfo = new GridFTPContactInfo(destURI.getHost(), destURI.getPort());
+ try {
+ String remoteFile = destURI.getPath();
+
+ log.info("The local temp file is " + localFile);
+ log.info("the remote file is " + remoteFile);
+
+ log.debug("Setup GridFTP Client");
+
+ ftpClient = new GridFTPClient(contactInfo.hostName, contactInfo.port);
+ ftpClient.setAuthorization(new HostAuthorization(GridFtp.HOST));
+ ftpClient.authenticate(gsCredential);
+ ftpClient.setDataChannelAuthentication(DataChannelAuthentication.SELF);
+
+ log.debug("Downloading file");
+ if (checkBinaryExtensions(remoteFile)) {
+ log.debug("Transfer mode is set to Binary to download a file");
+ ftpClient.setType(Session.TYPE_IMAGE);
+ }
+
+ ftpClient.get(remoteFile, localFile);
+
+ log.info("Download file to:" + localFile + " is done");
+
+ } catch (ServerException e) {
+ throw new ToolsException("Cannot download file from GridFTP:" + contactInfo.toString(), e);
+ } catch (IOException e) {
+ throw new ToolsException("Cannot download file from GridFTP:" + contactInfo.toString(), e);
+ } catch (ClientException e) {
+ throw new ToolsException("Cannot download file from GridFTP:" + contactInfo.toString(), e);
+ } finally {
+ if (ftpClient != null) {
+ try {
+ //ftpClient.close();
+ ftpClient.close(false);
+ } catch (Exception e) {
+ log.warn("Cannot close GridFTP client connection",e);
+ }
+ }
+ }
+ }
+
+ /**
+ * Stream remote file
+ *
+ * @param destURI
+ * @param gsCredential
+ * @param localFile
+ * @return
+ * @throws GFacException
+ */
+ public String readRemoteFile(URI destURI, GSSCredential gsCredential, File localFile) throws ToolsException {
+ BufferedReader instream = null;
+ File localTempfile = null;
+ try {
+
+ if (localFile == null) {
+ localTempfile = File.createTempFile("stderr", "err");
+ } else {
+ localTempfile = localFile;
+ }
+
+ log.info("Local temporary file:" + localTempfile);
+
+ downloadFile(destURI, gsCredential, localTempfile);
+
+ instream = new BufferedReader(new FileReader(localTempfile));
+ StringBuffer buff = new StringBuffer();
+ String temp = null;
+ while ((temp = instream.readLine()) != null) {
+ buff.append(temp);
+ buff.append(Constants.NEWLINE);
+ }
+
+ log.info("finish read file:" + localTempfile);
+
+ return buff.toString();
+ } catch (FileNotFoundException e) {
+ throw new ToolsException("Cannot read localfile file:" + localTempfile, e);
+ } catch (IOException e) {
+ throw new ToolsException("Cannot read localfile file:" + localTempfile, e);
+ } finally {
+ if (instream != null) {
+ try {
+ instream.close();
+ } catch (Exception e) {
+ log.warn("Cannot close GridFTP client connection",e);
+ }
+ }
+ }
+ }
+
+ /**
+ * Transfer data from one GridFTp Endpoint to another GridFTP Endpoint
+ *
+ * @param srchost
+ * @param desthost
+ * @param gssCred
+ * @param srcActive
+ * @throws ServerException
+ * @throws ClientException
+ * @throws IOException
+ */
+ public void transfer(URI srchost, URI desthost, GSSCredential gssCred, boolean srcActive) throws ToolsException {
+ GridFTPClient destClient = null;
+ GridFTPClient srcClient = null;
+
+ try {
+ destClient = new GridFTPClient(desthost.getHost(), desthost.getPort());
+ destClient.setAuthorization(new HostAuthorization(GridFtp.HOST));
+ destClient.authenticate(gssCred);
+
+ if (checkBinaryExtensions(desthost.getPath())) {
+ log.debug("Transfer mode is set to Binary");
+ destClient.setType(Session.TYPE_IMAGE);
+ }
+
+ srcClient = new GridFTPClient(srchost.getHost(), srchost.getPort());
+ srcClient.setAuthorization(new HostAuthorization(GridFtp.HOST));
+ srcClient.authenticate(gssCred);
+
+ if (checkBinaryExtensions(srchost.getPath())) {
+ log.debug("Transfer mode is set to Binary");
+ srcClient.setType(Session.TYPE_IMAGE);
+ }
+
+ if (srcActive) {
+ log.debug("Set src active");
+ HostPort hp = destClient.setPassive();
+ srcClient.setActive(hp);
+ } else {
+ log.debug("Set dst active");
+ HostPort hp = srcClient.setPassive();
+ destClient.setActive(hp);
+ }
+
+ log.debug("Start transfer file from GridFTP:" + srchost.toString() + " to " + desthost.toString());
+
+ /**
+ * Transfer a file. The transfer() function blocks until the transfer is complete.
+ */
+ srcClient.transfer(srchost.getPath(), destClient, desthost.getPath(), false, null);
+ if (srcClient.getSize(srchost.getPath()) == destClient.getSize(desthost.getPath())) {
+ log.debug("CHECK SUM OK");
+ } else {
+ log.debug("****CHECK SUM FAILED****");
+ }
+
+ } catch (ServerException e) {
+ throw new ToolsException("Cannot transfer file from GridFTP:" + srchost.toString() + " to "
+ + desthost.toString(), e);
+ } catch (IOException e) {
+ throw new ToolsException("Cannot transfer file from GridFTP:" + srchost.toString() + " to "
+ + desthost.toString(), e);
+ } catch (ClientException e) {
+ throw new ToolsException("Cannot transfer file from GridFTP:" + srchost.toString() + " to "
+ + desthost.toString(), e);
+ } finally {
+ if (destClient != null) {
+ try {
+ destClient.close();
+ } catch (Exception e) {
+ log.warn("Cannot close GridFTP client connection at Desitnation:" + desthost.toString());
+ }
+ }
+ if (srcClient != null) {
+ try {
+ srcClient.close();
+ } catch (Exception e) {
+ log.warn("Cannot close GridFTP client connection at Source:" + srchost.toString(),e);
+ }
+ }
+ }
+ }
+
+ /**
+ * List files in a GridFTP directory
+ * @param dirURI
+ * @param gssCred
+ * @return
+ * @throws ToolsException
+ */
+ @SuppressWarnings("unchecked")
+ public List<String> listDir(URI dirURI, GSSCredential gssCred) throws ToolsException {
+ List<String> files = new ArrayList<String>();
+ GridFTPClient srcClient = null;
+ try {
+ GridFTPContactInfo contactInfo = new GridFTPContactInfo(dirURI.getHost(), dirURI.getPort());
+
+ srcClient = new GridFTPClient(contactInfo.hostName, contactInfo.port);
+ srcClient.setAuthorization(new HostAuthorization(GridFtp.HOST));
+ srcClient.authenticate(gssCred);
+ srcClient.setDataChannelAuthentication(DataChannelAuthentication.SELF);
+ srcClient.setType(Session.TYPE_ASCII);
+ srcClient.changeDir(dirURI.getPath());
+
+ Vector<Object> fileInfo = null;
+ try {
+ fileInfo = srcClient.mlsd();
+ } catch (Throwable e) {
+ fileInfo = srcClient.list();
+ }
+
+ if (!fileInfo.isEmpty()) {
+ for (int j = 0; j < fileInfo.size(); ++j) {
+ String name = null;
+ if (fileInfo.get(j) instanceof MlsxEntry) {
+ name = ((MlsxEntry) fileInfo.get(j)).getFileName();
+ } else if (fileInfo.get(j) instanceof FileInfo) {
+ name = ((FileInfo) fileInfo.get(j)).getName();
+ } else {
+ throw new ToolsException("Unsupported type returned by gridftp " + fileInfo.get(j));
+ }
+
+ if (!name.equals(".") && !name.equals("..")) {
+ URI uri = GramProviderUtils.createGsiftpURI(contactInfo.hostName, dirURI.getPath() + File.separator + name);
+ files.add(uri.getPath());
+ }
+ }
+ }
+ return files;
+ } catch (IOException e) {
+ throw new ToolsException("Could not list directory: " + dirURI.toString() ,e);
+ } catch (ServerException e) {
+ throw new ToolsException("Could not list directory: " + dirURI.toString() ,e);
+ } catch (ClientException e) {
+ throw new ToolsException("Could not list directory: " + dirURI.toString() ,e);
+ } catch (URISyntaxException e) {
+ throw new ToolsException("Error creating URL of listed files: " + dirURI.toString() ,e);
+ } finally {
+ if (srcClient != null) {
+ try {
+ srcClient.close();
+ } catch (Exception e) {
+ log.warn("Cannot close GridFTP client connection", e);
+ }
+ }
+ }
+ }
+ /**
+ * Method to check file extension as binary to set transfer type
+ * @param filePath
+ * @return
+ */
+ private static boolean checkBinaryExtensions(String filePath){
+ String extension = filePath.substring(filePath.lastIndexOf(".")+1,filePath.length());
+ Set<String> extensions = new HashSet<String>(Arrays.asList(new String[] {"tar","zip","gz","tgz"}));
+ if(extensions.contains(extension)){
+ return true;
+ }else{
+ return false;
+ }
+
+ }
+
+
+
+
+ public String gridFTPFileExist(URI inputDirectory,String fileName,GSSCredential gssCred) throws ToolsException {
+ List<String> strings = listDir(inputDirectory, gssCred);
+ for(String fileExist:strings){
+ if(fileName.equals(fileExist)) {
+ fileName = "duplicate_" + fileName;
+ return fileName;
+ }
+ }
+ return fileName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/handler/GramDirectorySetupHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/handler/GramDirectorySetupHandler.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/handler/GramDirectorySetupHandler.java
new file mode 100644
index 0000000..76fa2db
--- /dev/null
+++ b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/handler/GramDirectorySetupHandler.java
@@ -0,0 +1,139 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.gram.handler;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Map;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.commons.gfac.type.ApplicationDescription;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.gram.security.GSISecurityContext;
+import org.apache.airavata.gfac.gram.external.GridFtp;
+import org.apache.airavata.gfac.gram.util.GramProviderUtils;
+import org.apache.airavata.gfac.handler.AbstractHandler;
+import org.apache.airavata.gfac.handler.GFacHandlerException;
+import org.apache.airavata.gfac.utils.GFacUtils;
+import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
+import org.apache.airavata.model.workspace.experiment.DataTransferDetails;
+import org.apache.airavata.model.workspace.experiment.ErrorCategory;
+import org.apache.airavata.model.workspace.experiment.TransferState;
+import org.apache.airavata.model.workspace.experiment.TransferStatus;
+import org.apache.airavata.registry.cpi.ChildDataType;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.GlobusHostType;
+import org.apache.airavata.schemas.gfac.HostDescriptionType;
+import org.apache.airavata.schemas.gfac.UnicoreHostType;
+import org.ietf.jgss.GSSCredential;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class GramDirectorySetupHandler extends AbstractHandler {
+ private static final Logger log = LoggerFactory.getLogger(GramDirectorySetupHandler.class);
+
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ log.info("Invoking GramDirectorySetupHandler ...");
+ super.invoke(jobExecutionContext);
+ String[] gridFTPEndpointArray = null;
+
+ //TODO: why it is tightly coupled with gridftp
+// GlobusHostType host = (GlobusHostType) jobExecutionContext.getApplicationContext().getHostDescription().getType();
+
+ //TODO: make it more reusable
+ HostDescriptionType hostType = jobExecutionContext.getApplicationContext().getHostDescription().getType();
+
+
+
+ if(hostType instanceof GlobusHostType){
+ gridFTPEndpointArray = ((GlobusHostType) hostType).getGridFTPEndPointArray();
+ }
+ else if (hostType instanceof UnicoreHostType){
+ gridFTPEndpointArray = ((UnicoreHostType) hostType).getGridFTPEndPointArray();
+ }
+
+
+
+ ApplicationDescription applicationDeploymentDescription = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription();
+ ApplicationDeploymentDescriptionType app = applicationDeploymentDescription.getType();
+ GridFtp ftp = new GridFtp();
+
+ try {
+
+ GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.
+ getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentials();
+
+ if (gridFTPEndpointArray == null || gridFTPEndpointArray.length == 0) {
+ gridFTPEndpointArray = new String[]{hostType.getHostAddress()};
+ }
+ boolean success = false;
+ GFacHandlerException pe = null;// = new ProviderException("");
+ for (String endpoint : gridFTPEndpointArray) {
+ try {
+
+ URI tmpdirURI = GramProviderUtils.createGsiftpURI(endpoint, app.getScratchWorkingDirectory());
+ URI workingDirURI = GramProviderUtils.createGsiftpURI(endpoint, app.getStaticWorkingDirectory());
+ URI inputURI = GramProviderUtils.createGsiftpURI(endpoint, app.getInputDataDirectory());
+ URI outputURI = GramProviderUtils.createGsiftpURI(endpoint, app.getOutputDataDirectory());
+
+ log.info("Host FTP = " + gridFTPEndpointArray[0]);
+ log.info("temp directory = " + tmpdirURI);
+ log.info("Working directory = " + workingDirURI);
+ log.info("Input directory = " + inputURI);
+ log.info("Output directory = " + outputURI);
+ ftp.makeDir(tmpdirURI, gssCred);
+ ftp.makeDir(workingDirURI, gssCred);
+ ftp.makeDir(inputURI, gssCred);
+ ftp.makeDir(outputURI, gssCred);
+ success = true;
+ DataTransferDetails detail = new DataTransferDetails();
+ TransferStatus status = new TransferStatus();
+ status.setTransferState(TransferState.DIRECTORY_SETUP);
+ detail.setTransferStatus(status);
+ detail.setTransferDescription("Working directory = " + workingDirURI);
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL,detail, jobExecutionContext.getTaskData().getTaskID());
+
+ break;
+ } catch (URISyntaxException e) {
+ pe = new GFacHandlerException("URI is malformatted:" + e.getMessage(), e);
+
+ } catch (Exception e) {
+ pe = new GFacHandlerException(e.getMessage(), e);
+ }
+ }
+ if (success == false) {
+ GFacUtils.saveErrorDetails(jobExecutionContext, pe.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
+ throw pe;
+ }
+ } catch (SecurityException e) {
+ throw new GFacHandlerException(e.getMessage(), e);
+ } catch (ApplicationSettingsException e1) {
+ throw new GFacHandlerException(e1.getMessage(), e1);
+ } catch (GFacException e) {
+ throw new GFacHandlerException(e);
+ }
+ }
+
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
+
+ }
+}
[5/8] changing package names of gfac implementations
Posted by la...@apache.org.
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GramProviderUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GramProviderUtils.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GramProviderUtils.java
deleted file mode 100644
index d76b067..0000000
--- a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GramProviderUtils.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.util;
-
-import org.apache.airavata.gfac.ToolsException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.provider.GFacProviderException;
-import org.apache.airavata.gfac.utils.*;
-import org.apache.airavata.model.workspace.experiment.JobState;
-import org.globus.gram.GramAttributes;
-import org.globus.gram.GramJob;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-
-public class GramProviderUtils {
- private static final Logger log = LoggerFactory.getLogger(GramProviderUtils.class);
-
- public static GramJob setupEnvironment(JobExecutionContext jobExecutionContext, boolean enableTwoPhase) throws GFacProviderException {
- log.debug("Searching for Gate Keeper");
- try {
- GramAttributes jobAttr = GramRSLGenerator.configureRemoteJob(jobExecutionContext);
- String rsl = jobAttr.toRSL();
-
- if (enableTwoPhase) {
- rsl = rsl + "(twoPhase=yes)";
- }
-
- log.debug("RSL = " + rsl);
- GramJob job = new GramJob(rsl);
- return job;
- } catch (ToolsException te) {
- throw new GFacProviderException(te.getMessage(), te);
- }
- }
-
- public static JobState getApplicationJobStatus(int gramStatus) {
- switch (gramStatus) {
- case GramJob.STATUS_UNSUBMITTED:
- return JobState.HELD;
- case GramJob.STATUS_ACTIVE:
- return JobState.ACTIVE;
- case GramJob.STATUS_DONE:
- return JobState.COMPLETE;
- case GramJob.STATUS_FAILED:
- return JobState.FAILED;
- case GramJob.STATUS_PENDING:
- return JobState.QUEUED;
- case GramJob.STATUS_STAGE_IN:
- return JobState.QUEUED;
- case GramJob.STATUS_STAGE_OUT:
- return JobState.COMPLETE;
- case GramJob.STATUS_SUSPENDED:
- return JobState.SUSPENDED;
- default:
- return JobState.UNKNOWN;
- }
- }
-
- public static URI createGsiftpURI(String host, String localPath) throws URISyntaxException {
- StringBuffer buf = new StringBuffer();
- if (!host.startsWith("gsiftp://"))
- buf.append("gsiftp://");
- buf.append(host);
- if (!host.endsWith("/"))
- buf.append("/");
- buf.append(localPath);
- return new URI(buf.toString());
- }
-
- public static URI createGsiftpURI(GridFTPContactInfo host, String localPath) throws URISyntaxException {
- StringBuffer buf = new StringBuffer();
-
- if (!host.hostName.startsWith("gsiftp://"))
- buf.append("gsiftp://");
- buf.append(host).append(":").append(host.port);
- if (!host.hostName.endsWith("/"))
- buf.append("/");
- buf.append(localPath);
- return new URI(buf.toString());
- }
-
- public static String createGsiftpURIAsString(String host, String localPath) throws URISyntaxException {
- StringBuffer buf = new StringBuffer();
- if (!host.startsWith("gsiftp://"))
- buf.append("gsiftp://");
- buf.append(host);
- if (!host.endsWith("/"))
- buf.append("/");
- buf.append(localPath);
- return buf.toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GramRSLGenerator.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GramRSLGenerator.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GramRSLGenerator.java
deleted file mode 100644
index 81782ff..0000000
--- a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GramRSLGenerator.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.util;
-
-import org.apache.airavata.common.utils.StringUtil;
-import org.apache.airavata.commons.gfac.type.ActualParameter;
-import org.apache.airavata.commons.gfac.type.MappingFactory;
-import org.apache.airavata.gfac.Constants;
-import org.apache.airavata.gfac.ToolsException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.gfac.provider.GFacProviderException;
-import org.apache.airavata.model.workspace.experiment.ComputationalResourceScheduling;
-import org.apache.airavata.model.workspace.experiment.TaskDetails;
-import org.apache.airavata.schemas.gfac.*;
-import org.globus.gram.GramAttributes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-public class GramRSLGenerator {
- protected static final Logger log = LoggerFactory.getLogger(GramRSLGenerator.class);
-
- private enum JobType {
- SERIAL, SINGLE, MPI, MULTIPLE, CONDOR
- }
-
- ;
-
- public static GramAttributes configureRemoteJob(JobExecutionContext context) throws ToolsException {
- HpcApplicationDeploymentType app = (HpcApplicationDeploymentType) context.getApplicationContext().getApplicationDeploymentDescription().getType();
- GramAttributes jobAttr = new GramAttributes();
- jobAttr.setExecutable(app.getExecutableLocation());
- jobAttr.setDirectory(app.getStaticWorkingDirectory());
- jobAttr.setStdout(app.getStandardOutput());
- jobAttr.setStderr(app.getStandardError());
- /*
- * The env here contains the env of the host and the application. i.e the env specified in the host description
- * and application description documents
- */
- NameValuePairType[] env = app.getApplicationEnvironmentArray();
- if (env.length != 0) {
- Map<String, String> nv = new HashMap<String, String>();
- for (int i = 0; i < env.length; i++) {
- String key = env[i].getName();
- String value = env[i].getValue();
- nv.put(key, value);
- }
-
- for (Map.Entry<String, String> entry : nv.entrySet()) {
- jobAttr.addEnvVariable(entry.getKey(), entry.getValue());
- }
- }
- jobAttr.addEnvVariable(Constants.INPUT_DATA_DIR_VAR_NAME, app.getInputDataDirectory());
- jobAttr.addEnvVariable(Constants.OUTPUT_DATA_DIR_VAR_NAME, app.getOutputDataDirectory());
-
-
-
- if (app.getStandardInput() != null && !"".equals(app.getStandardInput())) {
- jobAttr.setStdin(app.getStandardInput());
- } else {
- MessageContext input = context.getInMessageContext();;
- Map<String,Object> inputs = input.getParameters();
- Set<String> keys = inputs.keySet();
- for (String paramName : keys ) {
- ActualParameter actualParameter = (ActualParameter) inputs.get(paramName);
- if ("URIArray".equals(actualParameter.getType().getType().toString()) || "StringArray".equals(actualParameter.getType().getType().toString())
- || "FileArray".equals(actualParameter.getType().getType().toString())) {
- String[] values = null;
- if (actualParameter.getType() instanceof URIArrayType) {
- values = ((URIArrayType) actualParameter.getType()).getValueArray();
- } else if (actualParameter.getType() instanceof StringArrayType) {
- values = ((StringArrayType) actualParameter.getType()).getValueArray();
- } else if (actualParameter.getType() instanceof FileArrayType) {
- values = ((FileArrayType) actualParameter.getType()).getValueArray();
- }
- String value = StringUtil.createDelimiteredString(values, " ");
- jobAttr.addArgument(value);
- } else {
- String paramValue = MappingFactory.toString(actualParameter);
- jobAttr.addArgument(paramValue);
- }
- }
- }
- // Using the workflowContext Header values if user provided them in the request and overwrite the default values in DD
- //todo finish the scheduling based on workflow execution context
- TaskDetails taskData = context.getTaskData();
- if(taskData != null && taskData.isSetTaskScheduling()){
- ComputationalResourceScheduling computionnalResource = taskData.getTaskScheduling();
- try {
- int cpuCount = computionnalResource.getTotalCPUCount();
- if(cpuCount>0){
- app.setCpuCount(cpuCount);
- }
- } catch (NullPointerException e) {
- log.debug("No Value sent in WorkflowContextHeader for CPU Count, value in the Deployment Descriptor will be used");
- new GFacProviderException("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used",e);
- }
- try {
- int nodeCount = computionnalResource.getNodeCount();
- if(nodeCount>0){
- app.setNodeCount(nodeCount);
- }
- } catch (NullPointerException e) {
- log.debug("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used");
- new GFacProviderException("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used",e);
- }
- try {
- String queueName = computionnalResource.getQueueName();
- if (queueName != null) {
- if(app.getQueue() == null){
- QueueType queueType = app.addNewQueue();
- queueType.setQueueName(queueName);
- }else{
- app.getQueue().setQueueName(queueName);
- }
- }
- } catch (NullPointerException e) {
- log.debug("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used");
- new GFacProviderException("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used",e);
- }
- try {
- int maxwallTime = computionnalResource.getWallTimeLimit();
- if(maxwallTime>0){
- app.setMaxWallTime(maxwallTime);
- }
- } catch (NullPointerException e) {
- log.debug("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used");
- new GFacProviderException("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used",e);
- }
- }
- if (app.getNodeCount() > 0) {
- jobAttr.set("hostCount", String.valueOf(app.getNodeCount()));
- log.debug("Setting number of Nodes to " + app.getCpuCount());
- }
- if (app.getCpuCount() > 0) {
- log.debug("Setting number of procs to " + app.getCpuCount());
- jobAttr.setNumProcs(app.getCpuCount());
- }
- if (app.getMinMemory() > 0) {
- log.debug("Setting minimum memory to " + app.getMinMemory());
- jobAttr.setMinMemory(app.getMinMemory());
- }
- if (app.getMaxMemory() > 0) {
- log.debug("Setting maximum memory to " + app.getMaxMemory());
- jobAttr.setMaxMemory(app.getMaxMemory());
- }
- if (app.getProjectAccount() != null) {
- if (app.getProjectAccount().getProjectAccountNumber() != null) {
- log.debug("Setting project to " + app.getProjectAccount().getProjectAccountNumber());
- jobAttr.setProject(app.getProjectAccount().getProjectAccountNumber());
- }
- }
- if (app.getQueue() != null) {
- if (app.getQueue().getQueueName() != null) {
- log.debug("Setting job queue to " + app.getQueue().getQueueName());
- jobAttr.setQueue(app.getQueue().getQueueName());
- }
- }
- if (app.getMaxWallTime() > 0) {
- log.debug("Setting max wall clock time to " + app.getMaxWallTime());
-
- jobAttr.setMaxWallTime(app.getMaxWallTime());
- jobAttr.set("proxy_timeout", "1");
- } else {
- jobAttr.setMaxWallTime(30);
- }
- String jobType = JobType.SINGLE.toString();
- if (app.getJobType() != null) {
- jobType = app.getJobType().toString();
- }
- if (jobType.equalsIgnoreCase(JobType.SINGLE.toString())) {
- log.debug("Setting job type to single");
- jobAttr.setJobType(GramAttributes.JOBTYPE_SINGLE);
- } if (jobType.equalsIgnoreCase(JobType.SERIAL.toString())) {
- log.debug("Setting job type to single");
- jobAttr.setJobType(GramAttributes.JOBTYPE_SINGLE);
- } else if (jobType.equalsIgnoreCase(JobType.MPI.toString())) {
- log.debug("Setting job type to mpi");
- jobAttr.setJobType(GramAttributes.JOBTYPE_MPI);
- } else if (jobType.equalsIgnoreCase(JobType.MULTIPLE.toString())) {
- log.debug("Setting job type to multiple");
- jobAttr.setJobType(GramAttributes.JOBTYPE_MULTIPLE);
- } else if (jobType.equalsIgnoreCase(JobType.CONDOR.toString())) {
- jobAttr.setJobType(GramAttributes.JOBTYPE_CONDOR);
- }
-
- return jobAttr;
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GridFTPContactInfo.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GridFTPContactInfo.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GridFTPContactInfo.java
deleted file mode 100644
index d37beba..0000000
--- a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GridFTPContactInfo.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.util;
-
-import org.apache.airavata.gfac.Constants;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class GridFTPContactInfo {
- protected final static Logger log = LoggerFactory.getLogger(GridFTPContactInfo.class);
- public String hostName;
- public int port;
-
- public GridFTPContactInfo(String hostName, int port) {
- if (port <= 0 || port == 80) {
- log.debug(hostName + "port recived " + port + " setting it to " + Constants.DEFAULT_GSI_FTP_PORT);
- port = Constants.DEFAULT_GSI_FTP_PORT;
- }
- this.hostName = hostName;
- this.port = port;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof GridFTPContactInfo) {
- return hostName.equals(((GridFTPContactInfo) obj).hostName) && port == ((GridFTPContactInfo) obj).port;
- } else {
- return false;
- }
- }
-
- @Override
- public int hashCode() {
- return hostName.hashCode();
- }
-
- @Override
- public String toString() {
- StringBuffer buf = new StringBuffer();
- buf.append(hostName).append(":").append(port);
- return buf.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/test/java/org/apache/airavata/core/gfac/services/impl/GFacBaseTestWithMyProxyAuth.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/test/java/org/apache/airavata/core/gfac/services/impl/GFacBaseTestWithMyProxyAuth.java b/modules/gfac/gfac-gram/src/test/java/org/apache/airavata/core/gfac/services/impl/GFacBaseTestWithMyProxyAuth.java
index 24c364c..a5960d4 100644
--- a/modules/gfac/gfac-gram/src/test/java/org/apache/airavata/core/gfac/services/impl/GFacBaseTestWithMyProxyAuth.java
+++ b/modules/gfac/gfac-gram/src/test/java/org/apache/airavata/core/gfac/services/impl/GFacBaseTestWithMyProxyAuth.java
@@ -28,7 +28,7 @@ import org.apache.airavata.common.utils.DerbyUtil;
import org.apache.airavata.credential.store.store.CredentialReader;
import org.apache.airavata.credential.store.store.impl.CredentialReaderImpl;
import org.apache.airavata.gfac.RequestData;
-import org.apache.airavata.gfac.context.security.GSISecurityContext;
+import org.apache.airavata.gfac.gram.security.GSISecurityContext;
import org.apache.log4j.Logger;
import org.junit.BeforeClass;
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/test/java/org/apache/airavata/core/gfac/services/impl/GramProviderTestWithMyProxyAuth.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/test/java/org/apache/airavata/core/gfac/services/impl/GramProviderTestWithMyProxyAuth.java b/modules/gfac/gfac-gram/src/test/java/org/apache/airavata/core/gfac/services/impl/GramProviderTestWithMyProxyAuth.java
index ffaaaae..4fa945e 100644
--- a/modules/gfac/gfac-gram/src/test/java/org/apache/airavata/core/gfac/services/impl/GramProviderTestWithMyProxyAuth.java
+++ b/modules/gfac/gfac-gram/src/test/java/org/apache/airavata/core/gfac/services/impl/GramProviderTestWithMyProxyAuth.java
@@ -37,7 +37,7 @@ import org.apache.airavata.gfac.GFacException;
import org.apache.airavata.gfac.context.ApplicationContext;
import org.apache.airavata.gfac.context.JobExecutionContext;
import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.gfac.context.security.GSISecurityContext;
+import org.apache.airavata.gfac.gram.security.GSISecurityContext;
import org.apache.airavata.gfac.cpi.GFacImpl;
import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
import org.apache.airavata.schemas.gfac.GlobusHostType;
@@ -80,8 +80,8 @@ public class GramProviderTestWithMyProxyAuth extends GFacBaseTestWithMyProxyAuth
// gFacConfiguration.setMyProxyPassphrase("*****");
// gFacConfiguration.setTrustedCertLocation("./certificates");
// //have to set InFlwo Handlers and outFlowHandlers
-// gFacConfiguration.setInHandlers(Arrays.asList(new String[] {"org.apache.airavata.gfac.handler.GramDirectorySetupHandler","org.apache.airavata.gfac.handler.GridFTPInputHandler"}));
-// gFacConfiguration.setOutHandlers(Arrays.asList(new String[] {"org.apache.airavata.gfac.handler.GridFTPOutputHandler"}));
+// gFacConfiguration.setInHandlers(Arrays.asList(new String[] {"GramDirectorySetupHandler","GridFTPInputHandler"}));
+// gFacConfiguration.setOutHandlers(Arrays.asList(new String[] {"GridFTPOutputHandler"}));
/*
* Host
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/test/resources/gfac-config.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/test/resources/gfac-config.xml b/modules/gfac/gfac-gram/src/test/resources/gfac-config.xml
index 85d148d..ad1305e 100644
--- a/modules/gfac/gfac-gram/src/test/resources/gfac-config.xml
+++ b/modules/gfac/gfac-gram/src/test/resources/gfac-config.xml
@@ -21,13 +21,16 @@
</GlobalHandlers>
- <Provider class="org.apache.airavata.gfac.provider.impl.SSHProvider" host="org.apache.airavata.schemas.gfac.impl.SSHHostTypeImpl">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SSHDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.SSHInputHandler"/>
+ <Provider class="org.apache.airavata.gfac.gram.provider.impl.GramProvider" host="org.apache.airavata.schemas.gfac.impl.GlobusHostTypeImpl">
+ <property name="name" value="value"/>
+ <InHandlers>
+ <Handler class="org.apache.airavata.gfac..gram.handler.GramDirectorySetupHandler">
+ <property name="name" value="value"/>
+ </Handler>
+ <Handler class="org.apache.airavata.gfac..gram.handler.GridFTPInputHandler"/>
</InHandlers>
<OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.SSHOutputHandler"/>
+ <Handler class="org.apache.airavata.gfac..gram.handler.GridFTPOutputHandler"/>
</OutHandlers>
</Provider>
</GFac>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/context/security/GSISecurityContext.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/context/security/GSISecurityContext.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/context/security/GSISecurityContext.java
deleted file mode 100644
index 3eb020f..0000000
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/context/security/GSISecurityContext.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.context.security;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.credential.store.credential.Credential;
-import org.apache.airavata.credential.store.credential.impl.certificate.CertificateCredential;
-import org.apache.airavata.credential.store.store.CredentialReader;
-import org.apache.airavata.gfac.AbstractSecurityContext;
-import org.apache.airavata.gfac.Constants;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.RequestData;
-import org.apache.airavata.gsi.ssh.api.Cluster;
-import org.globus.gsi.X509Credential;
-import org.globus.gsi.gssapi.GlobusGSSCredentialImpl;
-import org.globus.gsi.provider.GlobusProvider;
-import org.globus.myproxy.GetParams;
-import org.globus.myproxy.MyProxy;
-import org.globus.myproxy.MyProxyException;
-import org.gridforum.jgss.ExtendedGSSCredential;
-import org.ietf.jgss.GSSCredential;
-import org.ietf.jgss.GSSException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.security.Security;
-import java.security.cert.X509Certificate;
-
-/**
- * Handles GRID related security.
- */
-public class GSISecurityContext extends AbstractSecurityContext {
-
- protected static final Logger log = LoggerFactory.getLogger(GSISecurityContext.class);
- /*
- * context name
- */
- public static final String GSI_SECURITY_CONTEXT = "gsi";
-
- public static int CREDENTIAL_RENEWING_THRESH_HOLD = 10 * 90;
-
- private GSSCredential gssCredentials = null;
-
- private Cluster pbsCluster = null;
-
- // Set trusted cert path and add provider
- static {
- Security.addProvider(new GlobusProvider());
- try {
- setUpTrustedCertificatePath();
- } catch (ApplicationSettingsException e) {
- log.error(e.getLocalizedMessage(), e);
- }
- }
-
- public static void setUpTrustedCertificatePath(String trustedCertificatePath) {
-
- File file = new File(trustedCertificatePath);
-
- if (!file.exists() || !file.canRead()) {
- File f = new File(".");
- log.info("Current directory " + f.getAbsolutePath());
- throw new RuntimeException("Cannot read trusted certificate path " + trustedCertificatePath);
- } else {
- System.setProperty(Constants.TRUSTED_CERTIFICATE_SYSTEM_PROPERTY, file.getAbsolutePath());
- }
- }
-
- private static void setUpTrustedCertificatePath() throws ApplicationSettingsException {
-
- String trustedCertificatePath = ServerSettings.getSetting(Constants.TRUSTED_CERT_LOCATION);
-
- setUpTrustedCertificatePath(trustedCertificatePath);
- }
-
- /**
- * Gets the trusted certificate path. Trusted certificate path is stored in "X509_CERT_DIR"
- * system property.
- * @return The trusted certificate path as a string.
- */
- public static String getTrustedCertificatePath() {
- return System.getProperty(Constants.TRUSTED_CERTIFICATE_SYSTEM_PROPERTY);
- }
-
-
- public GSISecurityContext(CredentialReader credentialReader, RequestData requestData) {
- super(credentialReader, requestData);
- }
-
-
- public GSISecurityContext(Cluster pbsCluster) {
- this.setPbsCluster(pbsCluster);
- }
-
- /**
- * Gets GSSCredentials. The process is as follows;
- * If credentials were queried for the first time create credentials.
- * 1. Try creating credentials using certificates stored in the credential store
- * 2. If 1 fails use user name and password to create credentials
- * If credentials are already created check the remaining life time of the credential. If
- * remaining life time is less than CREDENTIAL_RENEWING_THRESH_HOLD, then renew credentials.
- * @return GSSCredentials to be used.
- * @throws org.apache.airavata.gfac.GFacException If an error occurred while creating credentials.
- * @throws org.apache.airavata.common.exception.ApplicationSettingsException
- */
- public GSSCredential getGssCredentials() throws GFacException, ApplicationSettingsException {
-
- if (gssCredentials == null) {
-
- try {
- gssCredentials = getCredentialsFromStore();
- } catch (Exception e) {
- log.error("An exception occurred while retrieving credentials from the credential store. " +
- "Will continue with my proxy user name and password.", e);
- }
-
- // If store does not have credentials try to get from user name and password
- if (gssCredentials == null) {
- gssCredentials = getDefaultCredentials();
- }
-
- // if still null, throw an exception
- if (gssCredentials == null) {
- throw new GFacException("Unable to retrieve my proxy credentials to continue operation.");
- }
- } else {
- try {
- if (gssCredentials.getRemainingLifetime() < CREDENTIAL_RENEWING_THRESH_HOLD) {
- return renewCredentials();
- }
- } catch (GSSException e) {
- throw new GFacException("Unable to retrieve remaining life time from credentials.", e);
- }
- }
-
- return gssCredentials;
- }
-
- /**
- * Renews credentials. First try to renew credentials as a trusted renewer. If that failed
- * use user name and password to renew credentials.
- * @return Renewed credentials.
- * @throws org.apache.airavata.gfac.GFacException If an error occurred while renewing credentials.
- * @throws org.apache.airavata.common.exception.ApplicationSettingsException
- */
- public GSSCredential renewCredentials() throws GFacException, ApplicationSettingsException {
-
- // First try to renew credentials as a trusted renewer
- try {
- gssCredentials = renewCredentialsAsATrustedHost();
- } catch (Exception e) {
- log.warn("Renewing credentials as a trusted renewer failed", e);
- gssCredentials = getProxyCredentials();
- }
-
- return gssCredentials;
- }
-
- /**
- * Reads the credentials from credential store.
- * @return If token is found in the credential store, will return a valid credential. Else returns null.
- * @throws Exception If an error occurred while retrieving credentials.
- */
- public GSSCredential getCredentialsFromStore() throws Exception {
-
- if (getCredentialReader() == null) {
- return null;
- }
-
- Credential credential = getCredentialReader().getCredential(getRequestData().getGatewayId(),
- getRequestData().getTokenId());
-
- if (credential != null) {
- if (credential instanceof CertificateCredential) {
-
- log.info("Successfully found credentials for token id - " + getRequestData().getTokenId() +
- " gateway id - " + getRequestData().getGatewayId());
-
- CertificateCredential certificateCredential = (CertificateCredential) credential;
-
- X509Certificate[] certificates = certificateCredential.getCertificates();
- X509Credential newCredential = new X509Credential(certificateCredential.getPrivateKey(), certificates);
-
- GlobusGSSCredentialImpl cred = new GlobusGSSCredentialImpl(newCredential, GSSCredential.INITIATE_AND_ACCEPT);
- System.out.print(cred.export(ExtendedGSSCredential.IMPEXP_OPAQUE));
- return cred;
- //return new GlobusGSSCredentialImpl(newCredential,
- // GSSCredential.INITIATE_AND_ACCEPT);
- } else {
- log.info("Credential type is not CertificateCredential. Cannot create mapping globus credentials. " +
- "Credential type - " + credential.getClass().getName());
- }
- } else {
- log.info("Could not find credentials for token - " + getRequestData().getTokenId() + " and "
- + "gateway id - " + getRequestData().getGatewayId());
- }
-
- return null;
- }
-
- /**
- * Gets the default proxy certificate.
- * @return Default my proxy credentials.
- * @throws org.apache.airavata.gfac.GFacException If an error occurred while retrieving credentials.
- * @throws org.apache.airavata.common.exception.ApplicationSettingsException
- */
- public GSSCredential getDefaultCredentials() throws GFacException, ApplicationSettingsException{
- MyProxy myproxy = new MyProxy(getRequestData().getMyProxyServerUrl(), getRequestData().getMyProxyPort());
- try {
- return myproxy.get(getRequestData().getMyProxyUserName(), getRequestData().getMyProxyPassword(),
- getRequestData().getMyProxyLifeTime());
- } catch (MyProxyException e) {
- throw new GFacException("An error occurred while retrieving default security credentials.", e);
- }
- }
-
- /**
- * Gets a new proxy certificate given current credentials.
- * @return The short lived GSSCredentials
- * @throws org.apache.airavata.gfac.GFacException If an error is occurred while retrieving credentials.
- * @throws org.apache.airavata.common.exception.ApplicationSettingsException
- */
- public GSSCredential getProxyCredentials() throws GFacException, ApplicationSettingsException {
-
- MyProxy myproxy = new MyProxy(getRequestData().getMyProxyServerUrl(), getRequestData().getMyProxyPort());
- try {
- return myproxy.get(gssCredentials, getRequestData().getMyProxyUserName(), getRequestData().getMyProxyPassword(),
- getRequestData().getMyProxyLifeTime());
- } catch (MyProxyException e) {
- throw new GFacException("An error occurred while renewing security credentials using user/password.", e);
- }
- }
-
- /**
- * Renew GSSCredentials.
- * Before executing we need to add current host as a trusted renewer. Note to renew credentials
- * we dont need user name and password.
- * To do that execute following command
- * > myproxy-logon -t <LIFETIME></LIFETIME> -s <MY PROXY SERVER> -l <USER NAME>
- * E.g :- > myproxy-logon -t 264 -s myproxy.teragrid.org -l us3
- * Enter MyProxy pass phrase:
- * A credential has been received for user us3 in /tmp/x509up_u501.
- * > myproxy-init -A --cert /tmp/x509up_u501 --key /tmp/x509up_u501 -l ogce -s myproxy.teragrid.org
- * @return Renewed credentials.
- * @throws org.apache.airavata.gfac.GFacException If an error occurred while renewing credentials.
- * @throws org.apache.airavata.common.exception.ApplicationSettingsException
- */
- public GSSCredential renewCredentialsAsATrustedHost() throws GFacException, ApplicationSettingsException {
- MyProxy myproxy = new MyProxy(getRequestData().getMyProxyServerUrl(), getRequestData().getMyProxyPort());
- GetParams getParams = new GetParams();
- getParams.setAuthzCreds(gssCredentials);
- getParams.setUserName(getRequestData().getMyProxyUserName());
- getParams.setLifetime(getRequestData().getMyProxyLifeTime());
- try {
- return myproxy.get(gssCredentials, getParams);
- } catch (MyProxyException e) {
- throw new GFacException("An error occurred while renewing security credentials.", e);
- }
- }
-
- public Cluster getPbsCluster() {
- return pbsCluster;
- }
-
- public void setPbsCluster(Cluster pbsCluster) {
- this.pbsCluster = pbsCluster;
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHDirectorySetupHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHDirectorySetupHandler.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHDirectorySetupHandler.java
new file mode 100644
index 0000000..0f9e7fc
--- /dev/null
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHDirectorySetupHandler.java
@@ -0,0 +1,103 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.gsissh.handler;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.gsissh.security.GSISecurityContext;
+import org.apache.airavata.gfac.gsissh.util.GFACGSISSHUtils;
+import org.apache.airavata.gfac.handler.AbstractHandler;
+import org.apache.airavata.gfac.handler.GFacHandlerException;
+import org.apache.airavata.gfac.utils.GFacUtils;
+import org.apache.airavata.gsi.ssh.api.Cluster;
+import org.apache.airavata.gsi.ssh.api.SSHApiException;
+import org.apache.airavata.model.workspace.experiment.*;
+import org.apache.airavata.registry.cpi.ChildDataType;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+public class GSISSHDirectorySetupHandler extends AbstractHandler {
+ private static final Logger log = LoggerFactory.getLogger(GSISSHDirectorySetupHandler.class);
+
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ try {
+ if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) == null) {
+ GFACGSISSHUtils.addSecurityContext(jobExecutionContext);
+ }
+ } catch (ApplicationSettingsException e) {
+ log.error(e.getMessage());
+ throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
+ } catch (GFacException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+
+ log.info("Setup SSH job directorties");
+ super.invoke(jobExecutionContext);
+ makeDirectory(jobExecutionContext);
+ }
+ private void makeDirectory(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ try {
+ Cluster cluster = ((GSISecurityContext) jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getPbsCluster();
+ if (cluster == null) {
+ throw new GFacHandlerException("Security context is not set properly");
+ } else {
+ log.info("Successfully retrieved the Security Context");
+ }
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
+
+ String workingDirectory = app.getScratchWorkingDirectory();
+ cluster.makeDirectory(workingDirectory);
+ cluster.makeDirectory(app.getScratchWorkingDirectory());
+ cluster.makeDirectory(app.getInputDataDirectory());
+ cluster.makeDirectory(app.getOutputDataDirectory());
+ DataTransferDetails detail = new DataTransferDetails();
+ TransferStatus status = new TransferStatus();
+ status.setTransferState(TransferState.DIRECTORY_SETUP);
+ detail.setTransferStatus(status);
+ detail.setTransferDescription("Working directory = " + workingDirectory);
+
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
+
+ } catch (SSHApiException e) {
+ throw new GFacHandlerException("Error executing the Handler: " + GSISSHDirectorySetupHandler.class, e);
+ } catch (Exception e) {
+ DataTransferDetails detail = new DataTransferDetails();
+ TransferStatus status = new TransferStatus();
+ status.setTransferState(TransferState.FAILED);
+ detail.setTransferStatus(status);
+ try {
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
+ } catch (Exception e1) {
+ throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
+ }
+ throw new GFacHandlerException("Error executing the Handler: " + GSISSHDirectorySetupHandler.class, e);
+ }
+ }
+
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHInputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHInputHandler.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHInputHandler.java
new file mode 100644
index 0000000..ce9102a
--- /dev/null
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHInputHandler.java
@@ -0,0 +1,138 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.gsissh.handler;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.StringUtil;
+import org.apache.airavata.commons.gfac.type.ActualParameter;
+import org.apache.airavata.commons.gfac.type.MappingFactory;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.context.MessageContext;
+import org.apache.airavata.gfac.gsissh.security.GSISecurityContext;
+import org.apache.airavata.gfac.gsissh.util.GFACGSISSHUtils;
+import org.apache.airavata.gfac.handler.AbstractHandler;
+import org.apache.airavata.gfac.handler.GFacHandlerException;
+import org.apache.airavata.gfac.utils.GFacUtils;
+import org.apache.airavata.gsi.ssh.api.Cluster;
+import org.apache.airavata.gsi.ssh.api.SSHApiException;
+import org.apache.airavata.model.workspace.experiment.*;
+import org.apache.airavata.registry.cpi.ChildDataType;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.URIArrayType;
+import org.apache.airavata.schemas.gfac.URIParameterType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+
+public class GSISSHInputHandler extends AbstractHandler {
+ private static final Logger log = LoggerFactory.getLogger(GSISSHInputHandler.class);
+
+
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ MessageContext inputNew = new MessageContext();
+ DataTransferDetails detail = new DataTransferDetails();
+ TransferStatus status = new TransferStatus();
+ try {
+ if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) == null) {
+ try {
+ GFACGSISSHUtils.addSecurityContext(jobExecutionContext);
+ } catch (ApplicationSettingsException e) {
+ log.error(e.getMessage());
+ throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
+ }
+ }
+ log.info("Invoking SCPInputHandler");
+ super.invoke(jobExecutionContext);
+
+
+ MessageContext input = jobExecutionContext.getInMessageContext();
+ Set<String> parameters = input.getParameters().keySet();
+ for (String paramName : parameters) {
+ ActualParameter actualParameter = (ActualParameter) input.getParameters().get(paramName);
+ String paramValue = MappingFactory.toString(actualParameter);
+ //TODO: Review this with type
+ if ("URI".equals(actualParameter.getType().getType().toString())) {
+ ((URIParameterType) actualParameter.getType()).setValue(stageInputFiles(jobExecutionContext, paramValue));
+ } else if ("URIArray".equals(actualParameter.getType().getType().toString())) {
+ List<String> split = Arrays.asList(StringUtil.getElementsFromString(paramValue));
+ List<String> newFiles = new ArrayList<String>();
+ for (String paramValueEach : split) {
+ String stageInputFiles = stageInputFiles(jobExecutionContext, paramValueEach);
+ status.setTransferState(TransferState.UPLOAD);
+ detail.setTransferStatus(status);
+ detail.setTransferDescription("Input Data Staged: " + stageInputFiles);
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
+ newFiles.add(stageInputFiles);
+ }
+ ((URIArrayType) actualParameter.getType()).setValueArray(newFiles.toArray(new String[newFiles.size()]));
+ }
+ inputNew.getParameters().put(paramName, actualParameter);
+ }
+ } catch (Exception e) {
+ log.error(e.getMessage());
+ status.setTransferState(TransferState.FAILED);
+ detail.setTransferStatus(status);
+ try {
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
+ } catch (Exception e1) {
+ throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
+ }
+ throw new GFacHandlerException("Error while input File Staging", e, e.getLocalizedMessage());
+ }
+ jobExecutionContext.setInMessageContext(inputNew);
+ }
+
+ private static String stageInputFiles(JobExecutionContext jobExecutionContext, String paramValue) throws IOException, GFacException {
+ Cluster cluster = null;
+ if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) != null) {
+ cluster = ((GSISecurityContext) jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getPbsCluster();
+ } else {
+ cluster = ((GSISecurityContext) jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getPbsCluster();
+ }
+ if (cluster == null) {
+ throw new GFacException("Security context is not set properly");
+ } else {
+ log.info("Successfully retrieved the Security Context");
+ }
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
+ int i = paramValue.lastIndexOf(File.separator);
+ String substring = paramValue.substring(i + 1);
+ try {
+ String targetFile = app.getInputDataDirectory() + File.separator + substring;
+ if (paramValue.startsWith("file")) {
+ paramValue = paramValue.substring(paramValue.indexOf(":") + 1, paramValue.length());
+ }
+ cluster.scpTo(targetFile, paramValue);
+ return targetFile;
+ } catch (SSHApiException e) {
+ throw new GFacHandlerException("Error while input File Staging", e, e.getLocalizedMessage());
+ }
+ }
+
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java
new file mode 100644
index 0000000..41932d9
--- /dev/null
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java
@@ -0,0 +1,221 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.gsissh.handler;
+
+import net.schmizz.sshj.connection.ConnectionException;
+import net.schmizz.sshj.transport.TransportException;
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.Constants;
+import org.apache.airavata.commons.gfac.type.ActualParameter;
+import org.apache.airavata.commons.gfac.type.ApplicationDescription;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.gsissh.security.GSISecurityContext;
+import org.apache.airavata.gfac.gsissh.util.GFACGSISSHUtils;
+import org.apache.airavata.gfac.handler.AbstractHandler;
+import org.apache.airavata.gfac.handler.GFacHandlerException;
+import org.apache.airavata.gfac.provider.GFacProviderException;
+import org.apache.airavata.gfac.utils.GFacUtils;
+import org.apache.airavata.gfac.utils.OutputUtils;
+import org.apache.airavata.gsi.ssh.api.Cluster;
+import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
+import org.apache.airavata.model.workspace.experiment.*;
+import org.apache.airavata.registry.cpi.ChildDataType;
+import org.apache.airavata.registry.cpi.DataType;
+import org.apache.airavata.registry.cpi.RegistryException;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.GsisshHostType;
+import org.apache.airavata.schemas.gfac.URIParameterType;
+import org.apache.xmlbeans.XmlException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class GSISSHOutputHandler extends AbstractHandler {
+ private static final Logger log = LoggerFactory.getLogger(GSISSHOutputHandler.class);
+
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ if(jobExecutionContext.getApplicationContext().getHostDescription().getType() instanceof GsisshHostType) { // this is because we don't have the right jobexecution context
+ // so attempting to get it from the registry
+ if (Constants.PUSH.equals(((GsisshHostType) jobExecutionContext.getApplicationContext().getHostDescription().getType()).getMonitorMode())) {
+ log.warn("During the out handler chain jobExecution context came null, so trying to handler");
+ ApplicationDescription applicationDeploymentDescription = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription();
+ TaskDetails taskData = null;
+ try {
+ taskData = (TaskDetails) jobExecutionContext.getRegistry().get(DataType.TASK_DETAIL, jobExecutionContext.getTaskData().getTaskID());
+ } catch (RegistryException e) {
+ log.error("Error retrieving job details from Registry");
+ throw new GFacHandlerException("Error retrieving job details from Registry", e);
+ }
+ JobDetails jobDetails = taskData.getJobDetailsList().get(0);
+ String jobDescription = jobDetails.getJobDescription();
+ if (jobDescription != null) {
+ JobDescriptor jobDescriptor = null;
+ try {
+ jobDescriptor = JobDescriptor.fromXML(jobDescription);
+ } catch (XmlException e1) {
+ e1.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+ applicationDeploymentDescription.getType().setScratchWorkingDirectory(
+ jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getWorkingDirectory());
+ applicationDeploymentDescription.getType().setInputDataDirectory(jobDescriptor.getInputDirectory());
+ applicationDeploymentDescription.getType().setOutputDataDirectory(jobDescriptor.getOutputDirectory());
+ applicationDeploymentDescription.getType().setStandardError(jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getStandardErrorFile());
+ applicationDeploymentDescription.getType().setStandardOutput(jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getStandardOutFile());
+ }
+ }
+ }
+ try {
+ if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) == null) {
+
+ GFACGSISSHUtils.addSecurityContext(jobExecutionContext);
+ }
+ } catch (ApplicationSettingsException e) {
+ log.error(e.getMessage());
+ throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
+ } catch (GFacException e) {
+ log.error(e.getMessage());
+ throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
+ }
+ super.invoke(jobExecutionContext);
+ DataTransferDetails detail = new DataTransferDetails();
+ TransferStatus status = new TransferStatus();
+
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext()
+ .getApplicationDeploymentDescription().getType();
+ try {
+ Cluster cluster = null;
+ if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) != null) {
+ cluster = ((GSISecurityContext) jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getPbsCluster();
+ } else {
+ cluster = ((GSISecurityContext) jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getPbsCluster();
+ }
+ if (cluster == null) {
+ throw new GFacProviderException("Security context is not set properly");
+ } else {
+ log.info("Successfully retrieved the Security Context");
+ }
+
+ // Get the Stdouts and StdErrs
+ String timeStampedServiceName = GFacUtils.createUniqueNameForService(jobExecutionContext.getServiceName());
+
+ TaskDetails taskData = jobExecutionContext.getTaskData();
+ String outputDataDir = null;
+ File localStdOutFile;
+ File localStdErrFile;
+
+ if (taskData.getAdvancedOutputDataHandling() != null) {
+ outputDataDir = taskData.getAdvancedOutputDataHandling().getOutputDataDir();
+ }
+ if (outputDataDir == null) {
+ outputDataDir = File.separator + "tmp";
+ }
+ outputDataDir = outputDataDir + File.separator + jobExecutionContext.getExperimentID() + "-" + jobExecutionContext.getTaskData().getTaskID();
+ (new File(outputDataDir)).mkdirs();
+
+
+ localStdOutFile = new File(outputDataDir + File.separator + timeStampedServiceName + "stdout");
+ localStdErrFile = new File(outputDataDir + File.separator + timeStampedServiceName + "stderr");
+// cluster.makeDirectory(outputDataDir);
+ cluster.scpFrom(app.getStandardOutput(), localStdOutFile.getAbsolutePath());
+ Thread.sleep(1000);
+ cluster.scpFrom(app.getStandardError(), localStdErrFile.getAbsolutePath());
+ Thread.sleep(1000);
+
+ String stdOutStr = GFacUtils.readFileToString(localStdOutFile.getAbsolutePath());
+ String stdErrStr = GFacUtils.readFileToString(localStdErrFile.getAbsolutePath());
+ status.setTransferState(TransferState.COMPLETE);
+ detail.setTransferStatus(status);
+ detail.setTransferDescription("STDOUT:" + stdOutStr);
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
+
+ status.setTransferState(TransferState.COMPLETE);
+ detail.setTransferStatus(status);
+ detail.setTransferDescription("STDERR:" + stdErrStr);
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
+
+
+ Map<String, ActualParameter> stringMap = new HashMap<String, ActualParameter>();
+ Map<String, Object> output = jobExecutionContext.getOutMessageContext().getParameters();
+ Set<String> keys = output.keySet();
+ for (String paramName : keys) {
+ ActualParameter actualParameter = (ActualParameter) output.get(paramName);
+ if ("URI".equals(actualParameter.getType().getType().toString())) {
+
+ List<String> outputList = cluster.listDirectory(app.getOutputDataDirectory());
+ if (outputList.size() == 0 || outputList.get(0).isEmpty()) {
+ stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr);
+ } else {
+ String valueList = outputList.get(0);
+ cluster.scpFrom(app.getOutputDataDirectory() + File.separator + valueList, outputDataDir);
+ jobExecutionContext.addOutputFile(outputDataDir + File.separator + valueList);
+ ((URIParameterType) actualParameter.getType()).setValue(valueList);
+ stringMap = new HashMap<String, ActualParameter>();
+ stringMap.put(paramName, actualParameter);
+ }
+ } else {
+ stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr);
+ }
+ }
+ if (stringMap == null || stringMap.isEmpty()) {
+ throw new GFacHandlerException(
+ "Empty Output returned from the Application, Double check the application"
+ + "and ApplicationDescriptor output Parameter Names");
+ }
+ status.setTransferState(TransferState.DOWNLOAD);
+ detail.setTransferStatus(status);
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
+
+ app.setStandardError(localStdErrFile.getAbsolutePath());
+ app.setStandardOutput(localStdOutFile.getAbsolutePath());
+ app.setOutputDataDirectory(outputDataDir);
+ } catch (XmlException e) {
+ throw new GFacHandlerException("Cannot read output:" + e.getMessage(), e);
+ } catch (ConnectionException e) {
+ throw new GFacHandlerException(e.getMessage(), e);
+ } catch (TransportException e) {
+ throw new GFacHandlerException(e.getMessage(), e);
+ } catch (IOException e) {
+ throw new GFacHandlerException(e.getMessage(), e);
+ } catch (Exception e) {
+ try {
+ status.setTransferState(TransferState.FAILED);
+ detail.setTransferStatus(status);
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
+ } catch (Exception e1) {
+ throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
+ }
+ throw new GFacHandlerException("Error in retrieving results", e);
+ }
+
+ }
+
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/impl/GSISSHProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/impl/GSISSHProvider.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/impl/GSISSHProvider.java
new file mode 100644
index 0000000..5ba5ebf
--- /dev/null
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/impl/GSISSHProvider.java
@@ -0,0 +1,138 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.gsissh.impl;
+
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.gsissh.security.GSISecurityContext;
+import org.apache.airavata.gfac.cpi.GFacImpl;
+import org.apache.airavata.gfac.gsissh.util.GFACGSISSHUtils;
+import org.apache.airavata.gfac.handler.ThreadedHandler;
+import org.apache.airavata.gfac.notification.events.StartExecutionEvent;
+import org.apache.airavata.gfac.provider.AbstractProvider;
+import org.apache.airavata.gfac.provider.GFacProviderException;
+import org.apache.airavata.gfac.utils.GFacUtils;
+import org.apache.airavata.gsi.ssh.api.Cluster;
+import org.apache.airavata.gsi.ssh.api.SSHApiException;
+import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
+import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
+import org.apache.airavata.model.workspace.experiment.ErrorCategory;
+import org.apache.airavata.model.workspace.experiment.JobDetails;
+import org.apache.airavata.model.workspace.experiment.JobState;
+import org.apache.airavata.schemas.gfac.GsisshHostType;
+import org.apache.airavata.schemas.gfac.HostDescriptionType;
+import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+
+public class GSISSHProvider extends AbstractProvider {
+ private static final Logger log = LoggerFactory.getLogger(GSISSHProvider.class);
+
+ public void initProperties(Map<String, String> properties) throws GFacProviderException, GFacException {
+
+ }
+
+ public void initialize(JobExecutionContext jobExecutionContext) throws GFacProviderException, GFacException {
+ super.initialize(jobExecutionContext);
+ }
+
+ public void execute(JobExecutionContext jobExecutionContext) throws GFacProviderException, GFacException {
+ log.info("Invoking GSISSH Provider Invoke ...");
+ jobExecutionContext.getNotifier().publish(new StartExecutionEvent());
+ HostDescriptionType host = jobExecutionContext.getApplicationContext().
+ getHostDescription().getType();
+ HpcApplicationDeploymentType app = (HpcApplicationDeploymentType) jobExecutionContext.getApplicationContext().
+ getApplicationDeploymentDescription().getType();
+ JobDetails jobDetails = new JobDetails();
+ String taskID = jobExecutionContext.getTaskData().getTaskID();
+ try {
+ Cluster cluster = null;
+ if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) != null) {
+ cluster = ((GSISecurityContext) jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getPbsCluster();
+ }
+ if (cluster == null) {
+ throw new GFacProviderException("Security context is not set properly");
+ } else {
+ log.info("Successfully retrieved the Security Context");
+ }
+ // This installed path is a mandetory field, because this could change based on the computing resource
+ JobDescriptor jobDescriptor = GFACGSISSHUtils.createJobDescriptor(jobExecutionContext, app, cluster);
+
+ log.info(jobDescriptor.toXML());
+
+ jobDetails.setJobDescription(jobDescriptor.toXML());
+
+ String jobID = cluster.submitBatchJob(jobDescriptor);
+ jobExecutionContext.setJobDetails(jobDetails);
+ if(jobID == null){
+ jobDetails.setJobID("none");
+ GFacUtils.saveJobStatus(jobExecutionContext, jobDetails, JobState.FAILED);
+ }else{
+ jobDetails.setJobID(jobID);
+ GFacUtils.saveJobStatus(jobExecutionContext, jobDetails, JobState.SUBMITTED);
+ }
+
+
+ // Now job has submitted to the resource, its up to the Provider to parse the information to daemon handler
+ // to perform monitoring, daemon handlers can be accessed from anywhere
+ List<ThreadedHandler> daemonHandlers = GFacImpl.getDaemonHandlers();
+ ThreadedHandler pullMonitorHandler = null;
+ for(ThreadedHandler threadedHandler:daemonHandlers){
+ if("org.apache.airavata.gfac.monitor.handlers.GridPullMonitorHandler".equals(threadedHandler.getClass().getName())){
+ pullMonitorHandler = threadedHandler;
+ }
+ }
+ // we know this hos is type GsiSSHHostType
+ String monitorMode = ((GsisshHostType) host).getMonitorMode();
+ if("".equals(monitorMode) || monitorMode == null || org.apache.airavata.common.utils.Constants.PULL.equals(monitorMode)){
+ log.info("Job is launched successfully now parsing it to monitoring in pull mode, JobID Returned: " + jobID);
+ pullMonitorHandler.invoke(jobExecutionContext);
+ }else{
+ log.error("Currently we only support Pull monitoring");
+ }
+ } catch (SSHApiException e) {
+ String error = "Error submitting the job to host " + host.getHostAddress() + " message: " + e.getMessage();
+ log.error(error);
+ jobDetails.setJobID("none");
+ GFacUtils.saveJobStatus(jobExecutionContext, jobDetails,JobState.FAILED);
+ GFacUtils.saveErrorDetails(jobExecutionContext, error, CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
+ throw new GFacProviderException(error, e);
+ } catch (Exception e) {
+ String error = "Error submitting the job to host " + host.getHostAddress() + " message: " + e.getMessage();
+ log.error(error);
+ jobDetails.setJobID("none");
+ GFacUtils.saveJobStatus(jobExecutionContext, jobDetails,JobState.FAILED);
+ GFacUtils.saveErrorDetails(jobExecutionContext, error, CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
+ throw new GFacProviderException(error, e);
+ }
+ }
+
+ public void dispose(JobExecutionContext jobExecutionContext) throws GFacProviderException, GFacException {
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+ public void cancelJob(String jobId, JobExecutionContext jobExecutionContext) throws GFacException {
+ //To change body of implemented methods use File | Settings | File Templates.
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/security/GSISecurityContext.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/security/GSISecurityContext.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/security/GSISecurityContext.java
new file mode 100644
index 0000000..a09b72e
--- /dev/null
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/security/GSISecurityContext.java
@@ -0,0 +1,288 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.gsissh.security;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.credential.store.credential.Credential;
+import org.apache.airavata.credential.store.credential.impl.certificate.CertificateCredential;
+import org.apache.airavata.credential.store.store.CredentialReader;
+import org.apache.airavata.gfac.AbstractSecurityContext;
+import org.apache.airavata.gfac.Constants;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.RequestData;
+import org.apache.airavata.gsi.ssh.api.Cluster;
+import org.globus.gsi.X509Credential;
+import org.globus.gsi.gssapi.GlobusGSSCredentialImpl;
+import org.globus.gsi.provider.GlobusProvider;
+import org.globus.myproxy.GetParams;
+import org.globus.myproxy.MyProxy;
+import org.globus.myproxy.MyProxyException;
+import org.gridforum.jgss.ExtendedGSSCredential;
+import org.ietf.jgss.GSSCredential;
+import org.ietf.jgss.GSSException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.security.Security;
+import java.security.cert.X509Certificate;
+
+/**
+ * Handles GRID related security.
+ */
+public class GSISecurityContext extends AbstractSecurityContext {
+
+ protected static final Logger log = LoggerFactory.getLogger(GSISecurityContext.class);
+ /*
+ * context name
+ */
+ public static final String GSI_SECURITY_CONTEXT = "gsi";
+
+ public static int CREDENTIAL_RENEWING_THRESH_HOLD = 10 * 90;
+
+ private GSSCredential gssCredentials = null;
+
+ private Cluster pbsCluster = null;
+
+ // Set trusted cert path and add provider
+ static {
+ Security.addProvider(new GlobusProvider());
+ try {
+ setUpTrustedCertificatePath();
+ } catch (ApplicationSettingsException e) {
+ log.error(e.getLocalizedMessage(), e);
+ }
+ }
+
+ public static void setUpTrustedCertificatePath(String trustedCertificatePath) {
+
+ File file = new File(trustedCertificatePath);
+
+ if (!file.exists() || !file.canRead()) {
+ File f = new File(".");
+ log.info("Current directory " + f.getAbsolutePath());
+ throw new RuntimeException("Cannot read trusted certificate path " + trustedCertificatePath);
+ } else {
+ System.setProperty(Constants.TRUSTED_CERTIFICATE_SYSTEM_PROPERTY, file.getAbsolutePath());
+ }
+ }
+
+ private static void setUpTrustedCertificatePath() throws ApplicationSettingsException {
+
+ String trustedCertificatePath = ServerSettings.getSetting(Constants.TRUSTED_CERT_LOCATION);
+
+ setUpTrustedCertificatePath(trustedCertificatePath);
+ }
+
+ /**
+ * Gets the trusted certificate path. Trusted certificate path is stored in "X509_CERT_DIR"
+ * system property.
+ * @return The trusted certificate path as a string.
+ */
+ public static String getTrustedCertificatePath() {
+ return System.getProperty(Constants.TRUSTED_CERTIFICATE_SYSTEM_PROPERTY);
+ }
+
+
+ public GSISecurityContext(CredentialReader credentialReader, RequestData requestData) {
+ super(credentialReader, requestData);
+ }
+
+
+ public GSISecurityContext(Cluster pbsCluster) {
+ this.setPbsCluster(pbsCluster);
+ }
+
+ /**
+ * Gets GSSCredentials. The process is as follows;
+ * If credentials were queried for the first time create credentials.
+ * 1. Try creating credentials using certificates stored in the credential store
+ * 2. If 1 fails use user name and password to create credentials
+ * If credentials are already created check the remaining life time of the credential. If
+ * remaining life time is less than CREDENTIAL_RENEWING_THRESH_HOLD, then renew credentials.
+ * @return GSSCredentials to be used.
+ * @throws org.apache.airavata.gfac.GFacException If an error occurred while creating credentials.
+ * @throws org.apache.airavata.common.exception.ApplicationSettingsException
+ */
+ public GSSCredential getGssCredentials() throws GFacException, ApplicationSettingsException {
+
+ if (gssCredentials == null) {
+
+ try {
+ gssCredentials = getCredentialsFromStore();
+ } catch (Exception e) {
+ log.error("An exception occurred while retrieving credentials from the credential store. " +
+ "Will continue with my proxy user name and password.", e);
+ }
+
+ // If store does not have credentials try to get from user name and password
+ if (gssCredentials == null) {
+ gssCredentials = getDefaultCredentials();
+ }
+
+ // if still null, throw an exception
+ if (gssCredentials == null) {
+ throw new GFacException("Unable to retrieve my proxy credentials to continue operation.");
+ }
+ } else {
+ try {
+ if (gssCredentials.getRemainingLifetime() < CREDENTIAL_RENEWING_THRESH_HOLD) {
+ return renewCredentials();
+ }
+ } catch (GSSException e) {
+ throw new GFacException("Unable to retrieve remaining life time from credentials.", e);
+ }
+ }
+
+ return gssCredentials;
+ }
+
+ /**
+ * Renews credentials. First try to renew credentials as a trusted renewer. If that failed
+ * use user name and password to renew credentials.
+ * @return Renewed credentials.
+ * @throws org.apache.airavata.gfac.GFacException If an error occurred while renewing credentials.
+ * @throws org.apache.airavata.common.exception.ApplicationSettingsException
+ */
+ public GSSCredential renewCredentials() throws GFacException, ApplicationSettingsException {
+
+ // First try to renew credentials as a trusted renewer
+ try {
+ gssCredentials = renewCredentialsAsATrustedHost();
+ } catch (Exception e) {
+ log.warn("Renewing credentials as a trusted renewer failed", e);
+ gssCredentials = getProxyCredentials();
+ }
+
+ return gssCredentials;
+ }
+
+ /**
+ * Reads the credentials from credential store.
+ * @return If token is found in the credential store, will return a valid credential. Else returns null.
+ * @throws Exception If an error occurred while retrieving credentials.
+ */
+ public GSSCredential getCredentialsFromStore() throws Exception {
+
+ if (getCredentialReader() == null) {
+ return null;
+ }
+
+ Credential credential = getCredentialReader().getCredential(getRequestData().getGatewayId(),
+ getRequestData().getTokenId());
+
+ if (credential != null) {
+ if (credential instanceof CertificateCredential) {
+
+ log.info("Successfully found credentials for token id - " + getRequestData().getTokenId() +
+ " gateway id - " + getRequestData().getGatewayId());
+
+ CertificateCredential certificateCredential = (CertificateCredential) credential;
+
+ X509Certificate[] certificates = certificateCredential.getCertificates();
+ X509Credential newCredential = new X509Credential(certificateCredential.getPrivateKey(), certificates);
+
+ GlobusGSSCredentialImpl cred = new GlobusGSSCredentialImpl(newCredential, GSSCredential.INITIATE_AND_ACCEPT);
+ System.out.print(cred.export(ExtendedGSSCredential.IMPEXP_OPAQUE));
+ return cred;
+ //return new GlobusGSSCredentialImpl(newCredential,
+ // GSSCredential.INITIATE_AND_ACCEPT);
+ } else {
+ log.info("Credential type is not CertificateCredential. Cannot create mapping globus credentials. " +
+ "Credential type - " + credential.getClass().getName());
+ }
+ } else {
+ log.info("Could not find credentials for token - " + getRequestData().getTokenId() + " and "
+ + "gateway id - " + getRequestData().getGatewayId());
+ }
+
+ return null;
+ }
+
+ /**
+ * Gets the default proxy certificate.
+ * @return Default my proxy credentials.
+ * @throws org.apache.airavata.gfac.GFacException If an error occurred while retrieving credentials.
+ * @throws org.apache.airavata.common.exception.ApplicationSettingsException
+ */
+ public GSSCredential getDefaultCredentials() throws GFacException, ApplicationSettingsException{
+ MyProxy myproxy = new MyProxy(getRequestData().getMyProxyServerUrl(), getRequestData().getMyProxyPort());
+ try {
+ return myproxy.get(getRequestData().getMyProxyUserName(), getRequestData().getMyProxyPassword(),
+ getRequestData().getMyProxyLifeTime());
+ } catch (MyProxyException e) {
+ throw new GFacException("An error occurred while retrieving default security credentials.", e);
+ }
+ }
+
+ /**
+ * Gets a new proxy certificate given current credentials.
+ * @return The short lived GSSCredentials
+ * @throws org.apache.airavata.gfac.GFacException If an error is occurred while retrieving credentials.
+ * @throws org.apache.airavata.common.exception.ApplicationSettingsException
+ */
+ public GSSCredential getProxyCredentials() throws GFacException, ApplicationSettingsException {
+
+ MyProxy myproxy = new MyProxy(getRequestData().getMyProxyServerUrl(), getRequestData().getMyProxyPort());
+ try {
+ return myproxy.get(gssCredentials, getRequestData().getMyProxyUserName(), getRequestData().getMyProxyPassword(),
+ getRequestData().getMyProxyLifeTime());
+ } catch (MyProxyException e) {
+ throw new GFacException("An error occurred while renewing security credentials using user/password.", e);
+ }
+ }
+
+ /**
+ * Renew GSSCredentials.
+ * Before executing we need to add current host as a trusted renewer. Note to renew credentials
+ * we dont need user name and password.
+ * To do that execute following command
+ * > myproxy-logon -t <LIFETIME></LIFETIME> -s <MY PROXY SERVER> -l <USER NAME>
+ * E.g :- > myproxy-logon -t 264 -s myproxy.teragrid.org -l us3
+ * Enter MyProxy pass phrase:
+ * A credential has been received for user us3 in /tmp/x509up_u501.
+ * > myproxy-init -A --cert /tmp/x509up_u501 --key /tmp/x509up_u501 -l ogce -s myproxy.teragrid.org
+ * @return Renewed credentials.
+ * @throws org.apache.airavata.gfac.GFacException If an error occurred while renewing credentials.
+ * @throws org.apache.airavata.common.exception.ApplicationSettingsException
+ */
+ public GSSCredential renewCredentialsAsATrustedHost() throws GFacException, ApplicationSettingsException {
+ MyProxy myproxy = new MyProxy(getRequestData().getMyProxyServerUrl(), getRequestData().getMyProxyPort());
+ GetParams getParams = new GetParams();
+ getParams.setAuthzCreds(gssCredentials);
+ getParams.setUserName(getRequestData().getMyProxyUserName());
+ getParams.setLifetime(getRequestData().getMyProxyLifeTime());
+ try {
+ return myproxy.get(gssCredentials, getParams);
+ } catch (MyProxyException e) {
+ throw new GFacException("An error occurred while renewing security credentials.", e);
+ }
+ }
+
+ public Cluster getPbsCluster() {
+ return pbsCluster;
+ }
+
+ public void setPbsCluster(Cluster pbsCluster) {
+ this.pbsCluster = pbsCluster;
+ }
+}
[6/8] changing package names of gfac implementations
Posted by la...@apache.org.
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/util/GramRSLGenerator.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/util/GramRSLGenerator.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/util/GramRSLGenerator.java
new file mode 100644
index 0000000..f8a7502
--- /dev/null
+++ b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/util/GramRSLGenerator.java
@@ -0,0 +1,211 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.gram.util;
+
+import org.apache.airavata.common.utils.StringUtil;
+import org.apache.airavata.commons.gfac.type.ActualParameter;
+import org.apache.airavata.commons.gfac.type.MappingFactory;
+import org.apache.airavata.gfac.Constants;
+import org.apache.airavata.gfac.ToolsException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.context.MessageContext;
+import org.apache.airavata.gfac.provider.GFacProviderException;
+import org.apache.airavata.model.workspace.experiment.ComputationalResourceScheduling;
+import org.apache.airavata.model.workspace.experiment.TaskDetails;
+import org.apache.airavata.schemas.gfac.*;
+import org.globus.gram.GramAttributes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+public class GramRSLGenerator {
+ protected static final Logger log = LoggerFactory.getLogger(GramRSLGenerator.class);
+
+ private enum JobType {
+ SERIAL, SINGLE, MPI, MULTIPLE, CONDOR
+ }
+
+ ;
+
+ public static GramAttributes configureRemoteJob(JobExecutionContext context) throws ToolsException {
+ HpcApplicationDeploymentType app = (HpcApplicationDeploymentType) context.getApplicationContext().getApplicationDeploymentDescription().getType();
+ GramAttributes jobAttr = new GramAttributes();
+ jobAttr.setExecutable(app.getExecutableLocation());
+ jobAttr.setDirectory(app.getStaticWorkingDirectory());
+ jobAttr.setStdout(app.getStandardOutput());
+ jobAttr.setStderr(app.getStandardError());
+ /*
+ * The env here contains the env of the host and the application. i.e the env specified in the host description
+ * and application description documents
+ */
+ NameValuePairType[] env = app.getApplicationEnvironmentArray();
+ if (env.length != 0) {
+ Map<String, String> nv = new HashMap<String, String>();
+ for (int i = 0; i < env.length; i++) {
+ String key = env[i].getName();
+ String value = env[i].getValue();
+ nv.put(key, value);
+ }
+
+ for (Map.Entry<String, String> entry : nv.entrySet()) {
+ jobAttr.addEnvVariable(entry.getKey(), entry.getValue());
+ }
+ }
+ jobAttr.addEnvVariable(Constants.INPUT_DATA_DIR_VAR_NAME, app.getInputDataDirectory());
+ jobAttr.addEnvVariable(Constants.OUTPUT_DATA_DIR_VAR_NAME, app.getOutputDataDirectory());
+
+
+
+ if (app.getStandardInput() != null && !"".equals(app.getStandardInput())) {
+ jobAttr.setStdin(app.getStandardInput());
+ } else {
+ MessageContext input = context.getInMessageContext();;
+ Map<String,Object> inputs = input.getParameters();
+ Set<String> keys = inputs.keySet();
+ for (String paramName : keys ) {
+ ActualParameter actualParameter = (ActualParameter) inputs.get(paramName);
+ if ("URIArray".equals(actualParameter.getType().getType().toString()) || "StringArray".equals(actualParameter.getType().getType().toString())
+ || "FileArray".equals(actualParameter.getType().getType().toString())) {
+ String[] values = null;
+ if (actualParameter.getType() instanceof URIArrayType) {
+ values = ((URIArrayType) actualParameter.getType()).getValueArray();
+ } else if (actualParameter.getType() instanceof StringArrayType) {
+ values = ((StringArrayType) actualParameter.getType()).getValueArray();
+ } else if (actualParameter.getType() instanceof FileArrayType) {
+ values = ((FileArrayType) actualParameter.getType()).getValueArray();
+ }
+ String value = StringUtil.createDelimiteredString(values, " ");
+ jobAttr.addArgument(value);
+ } else {
+ String paramValue = MappingFactory.toString(actualParameter);
+ jobAttr.addArgument(paramValue);
+ }
+ }
+ }
+ // Using the workflowContext Header values if user provided them in the request and overwrite the default values in DD
+ //todo finish the scheduling based on workflow execution context
+ TaskDetails taskData = context.getTaskData();
+ if(taskData != null && taskData.isSetTaskScheduling()){
+ ComputationalResourceScheduling computionnalResource = taskData.getTaskScheduling();
+ try {
+ int cpuCount = computionnalResource.getTotalCPUCount();
+ if(cpuCount>0){
+ app.setCpuCount(cpuCount);
+ }
+ } catch (NullPointerException e) {
+ log.debug("No Value sent in WorkflowContextHeader for CPU Count, value in the Deployment Descriptor will be used");
+ new GFacProviderException("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used",e);
+ }
+ try {
+ int nodeCount = computionnalResource.getNodeCount();
+ if(nodeCount>0){
+ app.setNodeCount(nodeCount);
+ }
+ } catch (NullPointerException e) {
+ log.debug("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used");
+ new GFacProviderException("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used",e);
+ }
+ try {
+ String queueName = computionnalResource.getQueueName();
+ if (queueName != null) {
+ if(app.getQueue() == null){
+ QueueType queueType = app.addNewQueue();
+ queueType.setQueueName(queueName);
+ }else{
+ app.getQueue().setQueueName(queueName);
+ }
+ }
+ } catch (NullPointerException e) {
+ log.debug("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used");
+ new GFacProviderException("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used",e);
+ }
+ try {
+ int maxwallTime = computionnalResource.getWallTimeLimit();
+ if(maxwallTime>0){
+ app.setMaxWallTime(maxwallTime);
+ }
+ } catch (NullPointerException e) {
+ log.debug("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used");
+ new GFacProviderException("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used",e);
+ }
+ }
+ if (app.getNodeCount() > 0) {
+ jobAttr.set("hostCount", String.valueOf(app.getNodeCount()));
+ log.debug("Setting number of Nodes to " + app.getCpuCount());
+ }
+ if (app.getCpuCount() > 0) {
+ log.debug("Setting number of procs to " + app.getCpuCount());
+ jobAttr.setNumProcs(app.getCpuCount());
+ }
+ if (app.getMinMemory() > 0) {
+ log.debug("Setting minimum memory to " + app.getMinMemory());
+ jobAttr.setMinMemory(app.getMinMemory());
+ }
+ if (app.getMaxMemory() > 0) {
+ log.debug("Setting maximum memory to " + app.getMaxMemory());
+ jobAttr.setMaxMemory(app.getMaxMemory());
+ }
+ if (app.getProjectAccount() != null) {
+ if (app.getProjectAccount().getProjectAccountNumber() != null) {
+ log.debug("Setting project to " + app.getProjectAccount().getProjectAccountNumber());
+ jobAttr.setProject(app.getProjectAccount().getProjectAccountNumber());
+ }
+ }
+ if (app.getQueue() != null) {
+ if (app.getQueue().getQueueName() != null) {
+ log.debug("Setting job queue to " + app.getQueue().getQueueName());
+ jobAttr.setQueue(app.getQueue().getQueueName());
+ }
+ }
+ if (app.getMaxWallTime() > 0) {
+ log.debug("Setting max wall clock time to " + app.getMaxWallTime());
+
+ jobAttr.setMaxWallTime(app.getMaxWallTime());
+ jobAttr.set("proxy_timeout", "1");
+ } else {
+ jobAttr.setMaxWallTime(30);
+ }
+ String jobType = JobType.SINGLE.toString();
+ if (app.getJobType() != null) {
+ jobType = app.getJobType().toString();
+ }
+ if (jobType.equalsIgnoreCase(JobType.SINGLE.toString())) {
+ log.debug("Setting job type to single");
+ jobAttr.setJobType(GramAttributes.JOBTYPE_SINGLE);
+ } if (jobType.equalsIgnoreCase(JobType.SERIAL.toString())) {
+ log.debug("Setting job type to single");
+ jobAttr.setJobType(GramAttributes.JOBTYPE_SINGLE);
+ } else if (jobType.equalsIgnoreCase(JobType.MPI.toString())) {
+ log.debug("Setting job type to mpi");
+ jobAttr.setJobType(GramAttributes.JOBTYPE_MPI);
+ } else if (jobType.equalsIgnoreCase(JobType.MULTIPLE.toString())) {
+ log.debug("Setting job type to multiple");
+ jobAttr.setJobType(GramAttributes.JOBTYPE_MULTIPLE);
+ } else if (jobType.equalsIgnoreCase(JobType.CONDOR.toString())) {
+ jobAttr.setJobType(GramAttributes.JOBTYPE_CONDOR);
+ }
+
+ return jobAttr;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/util/GridFTPContactInfo.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/util/GridFTPContactInfo.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/util/GridFTPContactInfo.java
new file mode 100644
index 0000000..f331b0e
--- /dev/null
+++ b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/gram/util/GridFTPContactInfo.java
@@ -0,0 +1,61 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.gram.util;
+
+import org.apache.airavata.gfac.Constants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class GridFTPContactInfo {
+ protected final static Logger log = LoggerFactory.getLogger(GridFTPContactInfo.class);
+ public String hostName;
+ public int port;
+
+ public GridFTPContactInfo(String hostName, int port) {
+ if (port <= 0 || port == 80) {
+ log.debug(hostName + "port recived " + port + " setting it to " + Constants.DEFAULT_GSI_FTP_PORT);
+ port = Constants.DEFAULT_GSI_FTP_PORT;
+ }
+ this.hostName = hostName;
+ this.port = port;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof GridFTPContactInfo) {
+ return hostName.equals(((GridFTPContactInfo) obj).hostName) && port == ((GridFTPContactInfo) obj).port;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return hostName.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ StringBuffer buf = new StringBuffer();
+ buf.append(hostName).append(":").append(port);
+ return buf.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GramDirectorySetupHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GramDirectorySetupHandler.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GramDirectorySetupHandler.java
deleted file mode 100644
index 2a91afd..0000000
--- a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GramDirectorySetupHandler.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.handler;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.Map;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.commons.gfac.type.ApplicationDescription;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.security.GSISecurityContext;
-import org.apache.airavata.gfac.external.GridFtp;
-import org.apache.airavata.gfac.util.GramProviderUtils;
-import org.apache.airavata.gfac.utils.GFacUtils;
-import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
-import org.apache.airavata.model.workspace.experiment.DataTransferDetails;
-import org.apache.airavata.model.workspace.experiment.ErrorCategory;
-import org.apache.airavata.model.workspace.experiment.TransferState;
-import org.apache.airavata.model.workspace.experiment.TransferStatus;
-import org.apache.airavata.registry.cpi.ChildDataType;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.apache.airavata.schemas.gfac.GlobusHostType;
-import org.apache.airavata.schemas.gfac.HostDescriptionType;
-import org.apache.airavata.schemas.gfac.UnicoreHostType;
-import org.ietf.jgss.GSSCredential;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class GramDirectorySetupHandler extends AbstractHandler {
- private static final Logger log = LoggerFactory.getLogger(GramDirectorySetupHandler.class);
-
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- log.info("Invoking GramDirectorySetupHandler ...");
- super.invoke(jobExecutionContext);
- String[] gridFTPEndpointArray = null;
-
- //TODO: why it is tightly coupled with gridftp
-// GlobusHostType host = (GlobusHostType) jobExecutionContext.getApplicationContext().getHostDescription().getType();
-
- //TODO: make it more reusable
- HostDescriptionType hostType = jobExecutionContext.getApplicationContext().getHostDescription().getType();
-
-
-
- if(hostType instanceof GlobusHostType){
- gridFTPEndpointArray = ((GlobusHostType) hostType).getGridFTPEndPointArray();
- }
- else if (hostType instanceof UnicoreHostType){
- gridFTPEndpointArray = ((UnicoreHostType) hostType).getGridFTPEndPointArray();
- }
-
-
-
- ApplicationDescription applicationDeploymentDescription = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription();
- ApplicationDeploymentDescriptionType app = applicationDeploymentDescription.getType();
- GridFtp ftp = new GridFtp();
-
- try {
-
- GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.
- getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentials();
-
- if (gridFTPEndpointArray == null || gridFTPEndpointArray.length == 0) {
- gridFTPEndpointArray = new String[]{hostType.getHostAddress()};
- }
- boolean success = false;
- GFacHandlerException pe = null;// = new ProviderException("");
- for (String endpoint : gridFTPEndpointArray) {
- try {
-
- URI tmpdirURI = GramProviderUtils.createGsiftpURI(endpoint, app.getScratchWorkingDirectory());
- URI workingDirURI = GramProviderUtils.createGsiftpURI(endpoint, app.getStaticWorkingDirectory());
- URI inputURI = GramProviderUtils.createGsiftpURI(endpoint, app.getInputDataDirectory());
- URI outputURI = GramProviderUtils.createGsiftpURI(endpoint, app.getOutputDataDirectory());
-
- log.info("Host FTP = " + gridFTPEndpointArray[0]);
- log.info("temp directory = " + tmpdirURI);
- log.info("Working directory = " + workingDirURI);
- log.info("Input directory = " + inputURI);
- log.info("Output directory = " + outputURI);
- ftp.makeDir(tmpdirURI, gssCred);
- ftp.makeDir(workingDirURI, gssCred);
- ftp.makeDir(inputURI, gssCred);
- ftp.makeDir(outputURI, gssCred);
- success = true;
- DataTransferDetails detail = new DataTransferDetails();
- TransferStatus status = new TransferStatus();
- status.setTransferState(TransferState.DIRECTORY_SETUP);
- detail.setTransferStatus(status);
- detail.setTransferDescription("Working directory = " + workingDirURI);
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL,detail, jobExecutionContext.getTaskData().getTaskID());
-
- break;
- } catch (URISyntaxException e) {
- pe = new GFacHandlerException("URI is malformatted:" + e.getMessage(), e);
-
- } catch (Exception e) {
- pe = new GFacHandlerException(e.getMessage(), e);
- }
- }
- if (success == false) {
- GFacUtils.saveErrorDetails(jobExecutionContext, pe.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
- throw pe;
- }
- } catch (SecurityException e) {
- throw new GFacHandlerException(e.getMessage(), e);
- } catch (ApplicationSettingsException e1) {
- throw new GFacHandlerException(e1.getMessage(), e1);
- } catch (GFacException e) {
- throw new GFacHandlerException(e);
- }
- }
-
- public void initProperties(Map<String, String> properties) throws GFacHandlerException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GridFTPInputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GridFTPInputHandler.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GridFTPInputHandler.java
deleted file mode 100644
index ffbacdd..0000000
--- a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GridFTPInputHandler.java
+++ /dev/null
@@ -1,204 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.handler;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.StringUtil;
-import org.apache.airavata.commons.gfac.type.ActualParameter;
-import org.apache.airavata.commons.gfac.type.MappingFactory;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.ToolsException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.gfac.context.security.GSISecurityContext;
-import org.apache.airavata.gfac.external.GridFtp;
-import org.apache.airavata.gfac.util.GramProviderUtils;
-import org.apache.airavata.gfac.utils.GFacUtils;
-import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
-import org.apache.airavata.model.workspace.experiment.DataTransferDetails;
-import org.apache.airavata.model.workspace.experiment.ErrorCategory;
-import org.apache.airavata.model.workspace.experiment.TransferState;
-import org.apache.airavata.model.workspace.experiment.TransferStatus;
-import org.apache.airavata.registry.cpi.ChildDataType;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.apache.airavata.schemas.gfac.GlobusHostType;
-import org.apache.airavata.schemas.gfac.HostDescriptionType;
-import org.apache.airavata.schemas.gfac.URIArrayType;
-import org.apache.airavata.schemas.gfac.URIParameterType;
-import org.apache.airavata.schemas.gfac.UnicoreHostType;
-import org.ietf.jgss.GSSCredential;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class GridFTPInputHandler extends AbstractHandler {
- private static final Logger log = LoggerFactory.getLogger(AppDescriptorCheckHandler.class);
-
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- log.info("Invoking GridFTPInputHandler ...");
- super.invoke(jobExecutionContext);
- DataTransferDetails detail = new DataTransferDetails();
- TransferStatus status = new TransferStatus();
-
- MessageContext inputNew = new MessageContext();
- try {
- MessageContext input = jobExecutionContext.getInMessageContext();
- Set<String> parameters = input.getParameters().keySet();
- for (String paramName : parameters) {
- ActualParameter actualParameter = (ActualParameter) input.getParameters().get(paramName);
- String paramValue = MappingFactory.toString(actualParameter);
- //TODO: Review this with type
- if ("URI".equals(actualParameter.getType().getType().toString())) {
- ((URIParameterType) actualParameter.getType()).setValue(stageInputFiles(jobExecutionContext, paramValue));
- } else if ("URIArray".equals(actualParameter.getType().getType().toString())) {
- List<String> split = Arrays.asList(StringUtil.getElementsFromString(paramValue));
- List<String> newFiles = new ArrayList<String>();
- for (String paramValueEach : split) {
- String stageInputFiles = stageInputFiles(jobExecutionContext, paramValueEach);
- detail.setTransferDescription("Input Data Staged: " + stageInputFiles);
- status.setTransferState(TransferState.UPLOAD);
- detail.setTransferStatus(status);
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL,detail, jobExecutionContext.getTaskData().getTaskID());
-
- newFiles.add(stageInputFiles);
- }
- ((URIArrayType) actualParameter.getType()).setValueArray(newFiles.toArray(new String[newFiles.size()]));
- }
- inputNew.getParameters().put(paramName, actualParameter);
-
- }
- } catch (Exception e) {
- try {
- status.setTransferState(TransferState.FAILED);
- detail.setTransferStatus(status);
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL,detail, jobExecutionContext.getTaskData().getTaskID());
- GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
- } catch (Exception e1) {
- throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
- }
- log.error(e.getMessage());
- throw new GFacHandlerException("Error while input File Staging", e, e.getLocalizedMessage());
- }
- jobExecutionContext.setInMessageContext(inputNew);
- }
-
- private static String stageInputFiles(JobExecutionContext jobExecutionContext, String paramValue) throws URISyntaxException, SecurityException, ToolsException, IOException,GFacException, ApplicationSettingsException {
- URI gridftpURL = new URI(paramValue);
-
- String[] gridFTPEndpointArray = null;
-
- // not to download input files to the input dir if its http / gsiftp
- // but if local then yes
- boolean isInputNonLocal = true;
-
- //TODO: why it is tightly coupled with gridftp
-// GlobusHostType host = (GlobusHostType) jobExecutionContext.getApplicationContext().getHostDescription().getType();
-
- //TODO: make it more reusable
- HostDescriptionType hostType = jobExecutionContext.getApplicationContext().getHostDescription().getType();
-
- if(jobExecutionContext.getApplicationContext().getHostDescription().getType() instanceof GlobusHostType){
- gridFTPEndpointArray = ((GlobusHostType) hostType).getGridFTPEndPointArray();
- }
- else if (jobExecutionContext.getApplicationContext().getHostDescription().getType() instanceof UnicoreHostType){
- gridFTPEndpointArray = ((UnicoreHostType) hostType).getGridFTPEndPointArray();
- isInputNonLocal = false;
- }
- else {
- //TODO
- }
-
-
- ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
- GridFtp ftp = new GridFtp();
- URI destURI = null;
- GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentials();
-
- for (String endpoint : gridFTPEndpointArray) {
- URI inputURI = GramProviderUtils.createGsiftpURI(endpoint, app.getInputDataDirectory());
- String fileName = new File(gridftpURL.getPath()).getName();
- fileName = ftp.gridFTPFileExist(inputURI, fileName,gssCred);
-
- String destLocalPath = inputURI.getPath() + File.separator + fileName;
- //if user give a url just to refer an endpoint, not a web resource we are not doing any transfer
- if (fileName != null && !"".equals(fileName)) {
- destURI = GramProviderUtils.createGsiftpURI(endpoint, destLocalPath);
- if (paramValue.startsWith("gsiftp")) {
- // no need to do if it is unicore, as unicore will download this on user's behalf to the job space dir
- if(isInputNonLocal) ftp.uploadFile(gridftpURL, destURI, gssCred);
- else return paramValue;
- } else if (paramValue.startsWith("file")) {
- String localFile = paramValue.substring(paramValue.indexOf(":") + 1, paramValue.length());
- FileInputStream fis = null;
- try {
- fis = new FileInputStream(localFile);
- ftp.uploadFile(destURI, gssCred, fis);
- } catch (IOException e) {
- throw new GFacException("Unable to create file : " + localFile ,e);
- } finally {
- if (fis != null) {
- fis.close();
- }
- }
- } else if (paramValue.startsWith("http")) {
- // no need to do if it is unicore
- if(isInputNonLocal) {
- InputStream is = null;
- try {
- is = gridftpURL.toURL().openStream();
- ftp.uploadFile(destURI, gssCred, (is));
- }finally {
- is.close();
- }
- } else {
- // don't return destUri
- return paramValue;
- }
-
- } else {
- //todo throw exception telling unsupported protocol
- return paramValue;
- }
- } else {
- // When the given input is not a web resource but a URI type input, then we don't do any transfer just keep the same value as it isin the input
- return paramValue;
- }
- }
- return destURI.getPath();
- }
-
- public void initProperties(Map<String, String> properties) throws GFacHandlerException {
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GridFTPOutputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GridFTPOutputHandler.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GridFTPOutputHandler.java
deleted file mode 100644
index 33e549f..0000000
--- a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/handler/GridFTPOutputHandler.java
+++ /dev/null
@@ -1,347 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.handler;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.StringUtil;
-import org.apache.airavata.commons.gfac.type.ActualParameter;
-import org.apache.airavata.commons.gfac.type.ApplicationDescription;
-import org.apache.airavata.commons.gfac.type.MappingFactory;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.ToolsException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.gfac.context.security.GSISecurityContext;
-import org.apache.airavata.gfac.external.GridFtp;
-import org.apache.airavata.gfac.provider.GFacProviderException;
-import org.apache.airavata.gfac.util.GramProviderUtils;
-import org.apache.airavata.gfac.utils.GFacUtils;
-import org.apache.airavata.gfac.utils.OutputUtils;
-import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
-import org.apache.airavata.model.workspace.experiment.DataTransferDetails;
-import org.apache.airavata.model.workspace.experiment.ErrorCategory;
-import org.apache.airavata.model.workspace.experiment.TaskDetails;
-import org.apache.airavata.model.workspace.experiment.TransferState;
-import org.apache.airavata.model.workspace.experiment.TransferStatus;
-import org.apache.airavata.registry.cpi.ChildDataType;
-import org.apache.airavata.registry.cpi.Registry;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.apache.airavata.schemas.gfac.GlobusHostType;
-import org.apache.airavata.schemas.gfac.HostDescriptionType;
-import org.apache.airavata.schemas.gfac.StringArrayType;
-import org.apache.airavata.schemas.gfac.URIArrayType;
-import org.apache.airavata.schemas.gfac.URIParameterType;
-import org.apache.airavata.schemas.gfac.UnicoreHostType;
-import org.ietf.jgss.GSSCredential;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-public class GridFTPOutputHandler extends AbstractHandler {
- private static final Logger log = LoggerFactory.getLogger(GridFTPOutputHandler.class);
- private Registry registry;
-
-
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- log.info("Invoking GridFTPOutputHandler ...");
- super.invoke(jobExecutionContext);
-
- ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
-
- HostDescriptionType hostType = jobExecutionContext.getApplicationContext().getHostDescription().getType();
- String[] gridFTPEndpointArray = null;
- String hostName = null;
-
- if(jobExecutionContext.getApplicationContext().getHostDescription().getType() instanceof GlobusHostType){
- gridFTPEndpointArray = ((GlobusHostType) hostType).getGridFTPEndPointArray();
- hostName = ((GlobusHostType) hostType).getHostName();
-
- }
- else if (jobExecutionContext.getApplicationContext().getHostDescription().getType() instanceof UnicoreHostType){
- gridFTPEndpointArray = ((UnicoreHostType) hostType).getGridFTPEndPointArray();
- hostName = ((UnicoreHostType) hostType).getHostName();
- }
- else {
- //TODO
- }
-
- GridFtp ftp = new GridFtp();
- File localStdErrFile = null;
- Map<String, ActualParameter> stringMap = new HashMap<String, ActualParameter>();
- DataTransferDetails detail = new DataTransferDetails();
- TransferStatus status = new TransferStatus();
-
- try {
- GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentials();
- String[] hostgridFTP = gridFTPEndpointArray;
- if (hostgridFTP == null || hostgridFTP.length == 0) {
- hostgridFTP = new String[]{hostName};
- }
- for (String endpoint : gridFTPEndpointArray) {
- try {
- /*
- * Read Stdout and Stderror
- */
- URI stdoutURI = GramProviderUtils.createGsiftpURI(endpoint, app.getStandardOutput());
- URI stderrURI = GramProviderUtils.createGsiftpURI(endpoint, app.getStandardError());
- status.setTransferState(TransferState.COMPLETE);
- detail.setTransferStatus(status);
- detail.setTransferDescription("STDOUT:" + stdoutURI.toString());
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL,detail, jobExecutionContext.getTaskData().getTaskID());
- status.setTransferState(TransferState.COMPLETE);
- detail.setTransferStatus(status);
- detail.setTransferDescription("STDERR:" + stderrURI.toString());
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL,detail, jobExecutionContext.getTaskData().getTaskID());
-
- log.info("STDOUT:" + stdoutURI.toString());
- log.info("STDERR:" + stderrURI.toString());
-
- File logDir = new File("./service_logs");
- if (!logDir.exists()) {
- logDir.mkdir();
- }
-
- String timeStampedServiceName = GFacUtils.createUniqueNameForService(jobExecutionContext
- .getServiceName());
- File localStdOutFile = File.createTempFile(timeStampedServiceName, "stdout");
- localStdErrFile = File.createTempFile(timeStampedServiceName, "stderr");
-
-
- String stdout = null;
- String stderr = null;
-
- // TODO: what if job is failed
- // and this handler is not able to find std* files?
- try {
- stdout = ftp.readRemoteFile(stdoutURI, gssCred, localStdOutFile);
- stderr = ftp.readRemoteFile(stderrURI, gssCred, localStdErrFile);
- //TODO: do we also need to set them as output parameters for another job
- ApplicationDescription application = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription();
- ApplicationDeploymentDescriptionType appDesc = application.getType();
- appDesc.setStandardOutput(stdout);
- appDesc.setStandardError(stderr);
- jobExecutionContext.getApplicationContext().setApplicationDeploymentDescription(application);
- }
- catch(ToolsException e) {
- log.error("Cannot download stdout/err files. One reason could be the job is not successfully finished: "+e.getMessage());
- }
-
-
- Map<String, Object> output = jobExecutionContext.getOutMessageContext().getParameters();
- Set<String> keys = output.keySet();
- for (String paramName : keys) {
- ActualParameter actualParameter = (ActualParameter) output.get(paramName);
- if ("URIArray".equals(actualParameter.getType().getType().toString())) {
- URI outputURI = GramProviderUtils.createGsiftpURI(endpoint, app.getOutputDataDirectory());
- List<String> outputList = ftp.listDir(outputURI, gssCred);
- String[] valueList = outputList.toArray(new String[outputList.size()]);
- ((URIArrayType) actualParameter.getType()).setValueArray(valueList);
- // why to instantiate new instance?
-// stringMap = new HashMap<String, ActualParameter>();
- stringMap.put(paramName, actualParameter);
- }else if ("StringArray".equals(actualParameter.getType().getType().toString())) {
- String[] valueList = OutputUtils.parseStdoutArray(stdout, paramName);
- ((StringArrayType) actualParameter.getType()).setValueArray(valueList);
-// stringMap = new HashMap<String, ActualParameter>();
- stringMap.put(paramName, actualParameter);
- } else if ("URI".equals(actualParameter.getType().getType().toString())) {
- URI outputURI = GramProviderUtils.createGsiftpURI(endpoint, app.getOutputDataDirectory());
- List<String> outputList = ftp.listDir(outputURI, gssCred);
- if (outputList.size() == 0 || outputList.get(0).isEmpty()) {
- stringMap = OutputUtils.fillOutputFromStdout(output, stdout, stderr);
- } else {
- String valueList = outputList.get(0);
- ((URIParameterType) actualParameter.getType()).setValue(valueList);
- stringMap = new HashMap<String, ActualParameter>();
- stringMap.put(paramName, actualParameter);
- }
- }
- else {
- // This is to handle exception during the output parsing.
- stringMap = OutputUtils.fillOutputFromStdout(output, stdout, stderr);
- }
- status.setTransferState(TransferState.DOWNLOAD);
- detail.setTransferStatus(status);
- detail.setTransferDescription("Output: " + stringMap.get(paramName).toString());
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL,detail, jobExecutionContext.getTaskData().getTaskID());
-
- }
- if (stringMap == null || stringMap.isEmpty()) {
- throw new GFacHandlerException("Empty Output returned from the Application, Double check the application" +
- "and ApplicationDescriptor output Parameter Names");
- }
- // If users has given an output Data path to download the output files this will download the file on machine where GFac is installed
- TaskDetails taskData = jobExecutionContext.getTaskData();
- if(taskData != null && taskData.getAdvancedOutputDataHandling() != null){
- String outputDataDirectory = taskData.getAdvancedOutputDataHandling().getOutputDataDir();
- if(outputDataDirectory != null && !"".equals(outputDataDirectory)){
- stageOutputFiles(jobExecutionContext,outputDataDirectory);
- }
- }
- } catch (ToolsException e) {
- log.error(e.getMessage());
- throw new GFacHandlerException(e.getMessage() + "\n StdError Data: \n" +readLastLinesofStdOut(localStdErrFile.getPath(), 20),e);
- } catch (URISyntaxException e) {
- log.error(e.getMessage());
- throw new GFacHandlerException("URI is malformatted:" + e.getMessage(), e, readLastLinesofStdOut(localStdErrFile.getPath(), 20));
- }
- }
- } catch (Exception e) {
- try {
- status.setTransferState(TransferState.FAILED);
- detail.setTransferStatus(status);
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL,detail, jobExecutionContext.getTaskData().getTaskID());
- GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
- } catch (Exception e1) {
- throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
- }
- log.error(e.getMessage());
- throw new GFacHandlerException(e.getMessage(), e, readLastLinesofStdOut(localStdErrFile.getPath(), 20));
- }
-
- }
-
- private static String readLastLinesofStdOut(String path, int count) {
- StringBuffer buffer = new StringBuffer();
- FileInputStream in = null;
- try {
- in = new FileInputStream(path);
- } catch (FileNotFoundException e) {
- e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
- }
- BufferedReader br = new BufferedReader(new InputStreamReader(in));
- List<String> strLine = new ArrayList<String>();
- String tmp = null;
- int numberofLines = 0;
- try {
- while ((tmp = br.readLine()) != null) {
- strLine.add(tmp);
- numberofLines++;
- }
- } catch (IOException e) {
- e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
- }
- if (numberofLines > count) {
- for (int i = numberofLines - count; i < numberofLines; i++) {
- buffer.append(strLine.get(i));
- buffer.append("\n");
- }
- } else {
- for (int i = 0; i < numberofLines; i++) {
- buffer.append(strLine.get(i));
- buffer.append("\n");
- }
- }
- try {
- in.close();
- } catch (IOException e) {
- e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
- }
- return buffer.toString();
- }
-
- private static void stageOutputFiles(JobExecutionContext jobExecutionContext, String outputFileStagingPath) throws GFacProviderException,GFacException, ApplicationSettingsException {
-
-
- HostDescriptionType hostType = jobExecutionContext.getApplicationContext().getHostDescription().getType();
- String[] gridFTPEndpointArray = null;
-
- if(jobExecutionContext.getApplicationContext().getHostDescription().getType() instanceof GlobusHostType){
- gridFTPEndpointArray = ((GlobusHostType) hostType).getGridFTPEndPointArray();
- }
- else if (jobExecutionContext.getApplicationContext().getHostDescription().getType() instanceof UnicoreHostType){
- gridFTPEndpointArray = ((UnicoreHostType) hostType).getGridFTPEndPointArray();
- }
- else {
- //TODO
- }
-
-
- MessageContext outputNew = new MessageContext();
- MessageContext output = jobExecutionContext.getOutMessageContext();
- Map<String, Object> parameters = output.getParameters();
- for (String paramName : parameters.keySet()) {
- ActualParameter actualParameter = (ActualParameter) parameters
- .get(paramName);
-
- GridFtp ftp = new GridFtp();
- GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentials();
- try {
- if ("URI".equals(actualParameter.getType().getType().toString())) {
- for (String endpoint : gridFTPEndpointArray) {
- ((URIParameterType) actualParameter.getType()).setValue(doStaging(outputFileStagingPath,
- MappingFactory.toString(actualParameter), ftp, gssCred, endpoint));
- }
- } else if ("URIArray".equals(actualParameter.getType().getType().toString())) {
- List<String> split = Arrays.asList(StringUtil.getElementsFromString(MappingFactory.toString(actualParameter)));
- List<String> newFiles = new ArrayList<String>();
- for (String endpoint : gridFTPEndpointArray) {
- for (String paramValueEach : split) {
- newFiles.add(doStaging(outputFileStagingPath, paramValueEach, ftp, gssCred, endpoint));
- }
- ((URIArrayType) actualParameter.getType()).setValueArray(newFiles.toArray(new String[newFiles.size()]));
- }
-
- }
- } catch (URISyntaxException e) {
- log.error(e.getMessage());
- throw new GFacProviderException(e.getMessage(), e);
- } catch (ToolsException e) {
- log.error(e.getMessage());
- throw new GFacProviderException(e.getMessage(), e);
- }
- outputNew.getParameters().put(paramName, actualParameter);
- }
- jobExecutionContext.setOutMessageContext(outputNew);
- }
-
- private static String doStaging(String outputFileStagingPath, String paramValue, GridFtp ftp, GSSCredential gssCred, String endpoint) throws URISyntaxException, ToolsException {
- URI srcURI = GramProviderUtils.createGsiftpURI(endpoint, paramValue);
- String fileName = new File(srcURI.getPath()).getName();
- File outputpath = new File(outputFileStagingPath);
- if(!outputpath.exists()){
- outputpath.mkdirs();
- }
- File outputFile = new File(outputpath.getAbsolutePath() + File.separator + fileName);
- ftp.readRemoteFile(srcURI,
- gssCred, outputFile);
- return outputFileStagingPath + File.separator + fileName;
- }
-
- public void initProperties(Map<String, String> properties) throws GFacHandlerException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/persistence/DBJobPersistenceManager.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/persistence/DBJobPersistenceManager.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/persistence/DBJobPersistenceManager.java
deleted file mode 100644
index 3086b95..0000000
--- a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/persistence/DBJobPersistenceManager.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.persistence;
-
-import org.apache.airavata.common.utils.DBUtil;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.log4j.Logger;
-import org.globus.gram.internal.GRAMConstants;
-
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * User: AmilaJ (amilaj@apache.org)
- * Date: 6/18/13
- * Time: 4:16 PM
- * Database based job persistence manager. Current default implementation.
- */
-
-public class DBJobPersistenceManager implements JobPersistenceManager {
-
- private DBUtil dbUtil;
-
- private static final Logger log = Logger.getLogger(DBJobPersistenceManager.class);
-
-
- public DBJobPersistenceManager(DBUtil db) {
- this.dbUtil = db;
- }
-
- public synchronized void updateJobStatus(JobData jobData) throws GFacException {
-
- if (jobData.getState() == GRAMConstants.STATUS_UNSUBMITTED) {
- insertJob(jobData);
- } else {
-
- String sql = "update gram_job set status = ? where job_id = ?";
-
- Connection connection = null;
- PreparedStatement stmt = null;
-
- try {
- connection = getConnection();
- stmt = connection.prepareStatement(sql);
- stmt.setInt(1, jobData.getState());
- stmt.setString(2, jobData.getJobId());
-
- stmt.executeUpdate();
- connection.commit();
-
- } catch (SQLException e) {
- throw new GFacException(e);
- } finally {
- try {
- if (stmt != null) {
- stmt.close();
- }
-
- if (connection != null) {
- connection.close();
- }
-
- } catch (SQLException e) {
- log.error("Error closing streams", e);
- }
- }
- }
- }
-
- private void insertJob(JobData jobData) throws GFacException {
-
- String sql = "insert into gram_job values (?, ?)";
-
- PreparedStatement stmt = null;
- Connection connection = null;
-
- try {
- connection = getConnection();
- stmt = connection.prepareStatement(sql);
- stmt.setString(1, jobData.getJobId());
- stmt.setInt(2, jobData.getState());
-
- stmt.executeUpdate();
- } catch (SQLException e) {
- throw new GFacException(e);
- } finally {
- try {
- if (stmt != null) {
- stmt.close();
- }
-
- if (connection != null) {
- connection.close();
- }
-
- } catch (SQLException e) {
- log.error("Error closing streams", e);
- }
- }
-
- }
-
- public List<JobData> getRunningJobs() throws GFacException {
-
- String sql = "select * from gram_job where status not in (?, ?, ?)";
-
- int[] statuses = new int[3];
- statuses[0] = GRAMConstants.STATUS_UNSUBMITTED;
- statuses[1] = GRAMConstants.STATUS_DONE;
- statuses[2] = GRAMConstants.STATUS_FAILED;
-
- return getJobs(sql, statuses);
- }
-
- public List<JobData> getFailedJobs() throws GFacException {
-
- String sql = "select * from gram_job where status in (?)";
-
- int[] statuses = new int[1];
- statuses[0] = GRAMConstants.STATUS_FAILED;
-
- return getJobs(sql, statuses);
- }
-
- public List<JobData> getUnSubmittedJobs() throws GFacException {
-
- String sql = "select * from gram_job where status in (?)";
-
- int[] statuses = new int[1];
- statuses[0] = GRAMConstants.STATUS_UNSUBMITTED;
-
- return getJobs(sql, statuses);
- }
-
- public List<JobData> getSuccessfullyCompletedJobs() throws GFacException {
-
- String sql = "select * from gram_job where status in (?)";
-
- int[] statuses = new int[1];
- statuses[0] = GRAMConstants.STATUS_DONE;
-
- return getJobs(sql, statuses);
-
- }
-
-
- protected List<JobData> getJobs(String sql, int[] statuses) throws GFacException {
-
- List<JobData> jobs = new ArrayList<JobData>();
-
- PreparedStatement preparedStatement = null;
- Connection connection = null;
-
- try {
- connection = getConnection();
- preparedStatement = connection.prepareStatement(sql);
-
- int index = 1;
- for (int status : statuses) {
- preparedStatement.setInt(index, status);
- ++index;
- }
-
- ResultSet resultSet = preparedStatement.executeQuery();
-
- while (resultSet.next()) {
-
- String jobId = resultSet.getString("job_id");
- int state = resultSet.getInt("status");
-
- jobs.add(new JobData(jobId, state));
- }
-
- } catch (SQLException e) {
- throw new GFacException(e);
- } finally {
- try {
- if (preparedStatement != null) {
- preparedStatement.close();
- }
-
- if (connection != null) {
- connection.close();
- }
-
- } catch (SQLException e) {
- log.error("Error closing connection", e);
- }
- }
-
- return jobs;
- }
-
- private synchronized Connection getConnection() throws SQLException {
- Connection connection = dbUtil.getConnection();
- connection.setAutoCommit(true);
-
- return connection;
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/provider/impl/GramProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/provider/impl/GramProvider.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/provider/impl/GramProvider.java
deleted file mode 100644
index 2ba4b4b..0000000
--- a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/provider/impl/GramProvider.java
+++ /dev/null
@@ -1,527 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.provider.impl;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.MalformedURLException;
-import java.util.Map;
-import java.util.MissingResourceException;
-import java.util.Properties;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.JobSubmissionFault;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.security.GSISecurityContext;
-import org.apache.airavata.gfac.notification.events.JobIDEvent;
-import org.apache.airavata.gfac.notification.events.StartExecutionEvent;
-import org.apache.airavata.gfac.provider.AbstractProvider;
-import org.apache.airavata.gfac.provider.GFacProviderException;
-import org.apache.airavata.gfac.util.GramProviderUtils;
-import org.apache.airavata.gfac.utils.GFacUtils;
-import org.apache.airavata.gfac.util.GramJobSubmissionListener;
-import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
-import org.apache.airavata.model.workspace.experiment.ErrorCategory;
-import org.apache.airavata.model.workspace.experiment.JobState;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.apache.airavata.schemas.gfac.GlobusHostType;
-import org.globus.gram.GramException;
-import org.globus.gram.GramJob;
-import org.globus.gram.WaitingForCommitException;
-import org.globus.gram.internal.GRAMConstants;
-import org.globus.gram.internal.GRAMProtocolErrorConstants;
-import org.ietf.jgss.GSSCredential;
-import org.ietf.jgss.GSSException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class GramProvider extends AbstractProvider {
- private static final Logger log = LoggerFactory.getLogger(GramJobSubmissionListener.class);
-
- private GramJob job;
- private GramJobSubmissionListener listener;
- private boolean twoPhase = true;
-
- /**
- * If normal job submission fail due to an authorisation failure or script failure we
- * will re-attempt to submit the job. In-order to avoid any recursive loop during a continuous
- * failure we track whether failure paths are tried or not. Following booleans keeps track whether
- * we already tried a failure path or not.
- */
- /**
- * To track job submissions during a authorisation failure while requesting job.
- */
- private boolean renewCredentialsAttempt = false;
- /**
- * To track job submission during a script error situation.
- */
- private boolean reSubmissionInProgress = false;
- /**
- * To track authorisation failures during status monitoring.
- */
- private boolean authorisationFailedAttempt = false;
-
- private static final Map<String, GramJob> currentlyExecutingJobCache
- = new ConcurrentHashMap<String, GramJob>();
-
- private static Properties resources;
-
- static {
- try {
-
- String propFileName = "errors.properties";
- resources = new Properties();
- InputStream inputStream = GramProvider.class.getClassLoader()
- .getResourceAsStream(propFileName);
-
- if (inputStream == null) {
- throw new FileNotFoundException("property file '" + propFileName
- + "' not found in the classpath");
- }
-
- resources.load(inputStream);
-
- } catch (FileNotFoundException mre) {
- log.error("errors.properties not found", mre);
- } catch (IOException e) {
- log.error("Error reading errors.properties file", e);
- }
- }
-
-
- // This method prepare the environment before the application invocation.
- public void initialize(JobExecutionContext jobExecutionContext) throws GFacProviderException, GFacException {
-
- try {
- super.initialize(jobExecutionContext);
- String strTwoPhase = ServerSettings.getSetting("TwoPhase");
- if (strTwoPhase != null) {
- twoPhase = Boolean.parseBoolean(strTwoPhase);
- log.info("Two phase commit is set to " + twoPhase);
- }
- } catch (ApplicationSettingsException e) {
- log.warn("Error reading TwoPhase property from configurations.", e);
- }
-
- job = GramProviderUtils.setupEnvironment(jobExecutionContext, twoPhase);
- listener = new GramJobSubmissionListener(job, jobExecutionContext);
- job.addListener(listener);
- }
-
- public void execute(JobExecutionContext jobExecutionContext) throws GFacProviderException, GFacException{
- jobExecutionContext.getNotifier().publish(new StartExecutionEvent());
- GlobusHostType host = (GlobusHostType) jobExecutionContext.getApplicationContext().
- getHostDescription().getType();
- ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().
- getApplicationDeploymentDescription().getType();
-
- StringBuilder stringBuilder = new StringBuilder();
- try {
-
- GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.
- getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentials();
- job.setCredentials(gssCred);
- // We do not support multiple gatekeepers in XBaya GUI, so we simply pick the 0th element in the array
- String gateKeeper = host.getGlobusGateKeeperEndPointArray(0);
- log.info("Request to contact:" + gateKeeper);
-
- stringBuilder.append("Finished launching job, Host = ").append(host.getHostAddress()).append(" RSL = ")
- .append(job.getRSL()).append(" working directory = ").append(app.getStaticWorkingDirectory())
- .append(" temp directory = ").append(app.getScratchWorkingDirectory())
- .append(" Globus GateKeeper Endpoint = ").append(gateKeeper);
-
- log.info(stringBuilder.toString());
-
- submitJobs(gateKeeper, jobExecutionContext, host);
-
- } catch (ApplicationSettingsException e) {
- throw new GFacException(e.getMessage(), e);
- } finally {
- if (job != null) {
- try {
- /*
- * Remove listener
- */
- job.removeListener(listener);
- } catch (Exception e) {
- log.error(e.getMessage());
- }
- }
- }
- }
-
- private void submitJobs(String gateKeeper,
- JobExecutionContext jobExecutionContext,
- GlobusHostType globusHostType) throws GFacException, GFacProviderException {
- boolean applicationSaved=false;
- String taskID = jobExecutionContext.getTaskData().getTaskID();
-
- if (twoPhase) {
- try {
- /*
- * The first boolean is to force communication through SSLv3
- * The second boolean is to specify the job is a batch job - use true for interactive and false for
- * batch.
- * The third boolean is to specify to use the full proxy and not delegate a limited proxy.
- */
- job.request(true, gateKeeper, false, false);
-
- // Single boolean to track all authentication failures, therefore we need to re-initialize
- // this here
- renewCredentialsAttempt = false;
-
- } catch (WaitingForCommitException e) {
- String jobID = job.getIDAsString();
-
- details.setJobID(jobID);
- details.setJobDescription(job.getRSL());
- jobExecutionContext.setJobDetails(details);
- GFacUtils.saveJobStatus(jobExecutionContext, details, JobState.UN_SUBMITTED);
-
- applicationSaved=true;
- String jobStatusMessage = "Un-submitted JobID= " + jobID;
- log.info(jobStatusMessage);
- jobExecutionContext.getNotifier().publish(new JobIDEvent(jobStatusMessage));
-
- log.info("Two phase commit: sending COMMIT_REQUEST signal; Job id - " + jobID);
-
- try {
- job.signal(GramJob.SIGNAL_COMMIT_REQUEST);
-
- } catch (GramException gramException) {
- throw new GFacException("Error while sending commit request. Job Id - "
- + job.getIDAsString(), gramException);
- } catch (GSSException gssException) {
-
- // User credentials are invalid
- log.error("Error while submitting commit request - Credentials provided are invalid. Job Id - "
- + job.getIDAsString(), e);
- log.info("Attempting to renew credentials and re-submit commit signal...");
- GFacUtils.saveErrorDetails(jobExecutionContext, gssException.getLocalizedMessage(), CorrectiveAction.RETRY_SUBMISSION, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
- renewCredentials(jobExecutionContext);
-
- try {
- job.signal(GramJob.SIGNAL_COMMIT_REQUEST);
- } catch (GramException e1) {
- GFacUtils.saveErrorDetails(jobExecutionContext, gssException.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
- throw new GFacException("Error while sending commit request. Job Id - "
- + job.getIDAsString(), e1);
- } catch (GSSException e1) {
- GFacUtils.saveErrorDetails(jobExecutionContext, gssException.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
- throw new GFacException("Error while sending commit request. Job Id - "
- + job.getIDAsString() + ". Credentials provided invalid", e1);
- }
- }
- GFacUtils.updateJobStatus(jobExecutionContext, details, JobState.SUBMITTED);
- jobStatusMessage = "Submitted JobID= " + job.getIDAsString();
- log.info(jobStatusMessage);
- jobExecutionContext.getNotifier().publish(new JobIDEvent(jobStatusMessage));
-
- } catch (GSSException e) {
- // Renew credentials and re-submit
- GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.RETRY_SUBMISSION, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
-
- reSubmitJob(gateKeeper, jobExecutionContext, globusHostType, e);
-
- } catch (GramException e) {
- GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
-
- throw new GFacException("An error occurred while submitting a job, job id = " + job.getIDAsString(), e);
- }
- } else {
-
- /*
- * The first boolean is to force communication through SSLv3
- * The second boolean is to specify the job is a batch job - use true for interactive and false for
- * batch.
- * The third boolean is to specify to use the full proxy and not delegate a limited proxy.
- */
- try {
-
- job.request(true, gateKeeper, false, false);
- renewCredentialsAttempt = false;
-
- } catch (GramException e) {
- GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
- throw new GFacException("An error occurred while submitting a job, job id = " + job.getIDAsString(), e);
- } catch (GSSException e) {
- GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.RETRY_SUBMISSION, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
- // Renew credentials and re-submit
- reSubmitJob(gateKeeper, jobExecutionContext, globusHostType, e);
- }
-
- String jobStatusMessage = "Un-submitted JobID= " + job.getIDAsString();
- log.info(jobStatusMessage);
- jobExecutionContext.getNotifier().publish(new JobIDEvent(jobStatusMessage));
-
- }
-
- currentlyExecutingJobCache.put(job.getIDAsString(), job);
- /*
- * Wait until job is done
- */
- listener.waitFor();
-
- checkJobStatus(jobExecutionContext, globusHostType, gateKeeper);
-
- }
-
- private void renewCredentials(JobExecutionContext jobExecutionContext) throws GFacException {
-
- renewCredentials(this.job, jobExecutionContext);
- }
-
- private void renewCredentials(GramJob gramJob, JobExecutionContext jobExecutionContext) throws GFacException {
-
- try {
- GSSCredential gssCred = ((GSISecurityContext)jobExecutionContext.
- getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).renewCredentials();
- gramJob.renew(gssCred);
- } catch (GramException e1) {
- throw new GFacException("Unable to renew credentials. Job Id - "
- + gramJob.getIDAsString(), e1);
- } catch (GSSException e1) {
- throw new GFacException("Unable to renew credentials. Job Id - "
- + gramJob.getIDAsString(), e1);
- } catch (ApplicationSettingsException e) {
- throw new GFacException(e.getLocalizedMessage(), e);
- }
- }
-
- private void reSubmitJob(String gateKeeper,
- JobExecutionContext jobExecutionContext,
- GlobusHostType globusHostType, Exception e) throws GFacException, GFacProviderException {
-
- if (!renewCredentialsAttempt) {
-
- renewCredentialsAttempt = true;
-
- // User credentials are invalid
- log.error("Error while submitting job - Credentials provided are invalid. Job Id - "
- + job.getIDAsString(), e);
- log.info("Attempting to renew credentials and re-submit jobs...");
-
- // Remove existing listener and register a new listener
- job.removeListener(listener);
- listener = new GramJobSubmissionListener(job, jobExecutionContext);
-
- job.addListener(listener);
-
- renewCredentials(jobExecutionContext);
-
- submitJobs(gateKeeper, jobExecutionContext, globusHostType);
-
- } else {
- throw new GFacException("Error while submitting job - Credentials provided are invalid. Job Id - "
- + job.getIDAsString(), e);
- }
-
- }
-
- private void reSubmitJob(String gateKeeper,
- JobExecutionContext jobExecutionContext,
- GlobusHostType globusHostType) throws GFacException, GFacProviderException {
-
- // User credentials are invalid
- log.info("Attempting to renew credentials and re-submit jobs...");
-
- // Remove existing listener and register a new listener
- job.removeListener(listener);
- listener = new GramJobSubmissionListener(job, jobExecutionContext);
-
- job.addListener(listener);
-
- renewCredentials(jobExecutionContext);
-
- submitJobs(gateKeeper, jobExecutionContext, globusHostType);
-
- }
-
-
-
- public void dispose(JobExecutionContext jobExecutionContext) throws GFacProviderException {
- }
-
- public void cancelJob(String jobId, JobExecutionContext jobExecutionContext) throws GFacException {
- cancelSingleJob(jobId, jobExecutionContext);
- }
-
-
- private void cancelSingleJob(String jobId, JobExecutionContext context) throws GFacException {
- // First check whether job id is in the cache
- if (currentlyExecutingJobCache.containsKey(jobId)) {
-
- synchronized (this) {
- GramJob gramJob = currentlyExecutingJobCache.get(jobId);
-
- // Even though we check using containsKey, at this point job could be null
- if (gramJob != null && (gramJob.getStatus() != GRAMConstants.STATUS_DONE ||
- gramJob.getStatus() != GRAMConstants.STATUS_FAILED)) {
- cancelJob(gramJob, context);
- }
- }
-
- } else {
-
- try {
- GSSCredential gssCred = ((GSISecurityContext)context.
- getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getGssCredentials();
-
- GramJob gramJob = new GramJob(null);
- try {
- gramJob.setID(jobId);
- } catch (MalformedURLException e) {
- throw new GFacException("Invalid job id - " + jobId, e);
- }
- gramJob.setCredentials(gssCred);
-
- synchronized (this) {
- if (gramJob.getStatus() != GRAMConstants.STATUS_DONE ||
- gramJob.getStatus() != GRAMConstants.STATUS_FAILED) {
- cancelJob(gramJob, context);
- }
- }
- } catch (ApplicationSettingsException e) {
- throw new GFacException(e);
- }
- }
- }
-
- private void cancelJob(GramJob gramJob, JobExecutionContext context) throws GFacException{
-
- try {
- gramJob.cancel();
- } catch (GramException e) {
- throw new GFacException("Error cancelling job, id - " + gramJob.getIDAsString(), e);
- } catch (GSSException e) {
-
- log.warn("Credentials invalid to cancel job. Attempting to renew credentials and re-try. " +
- "Job id - " + gramJob.getIDAsString());
- renewCredentials(gramJob, context);
-
- try {
- gramJob.cancel();
- gramJob.signal(GramJob.SIGNAL_COMMIT_END);
- } catch (GramException e1) {
- throw new GFacException("Error cancelling job, id - " + gramJob.getIDAsString(), e1);
- } catch (GSSException e1) {
- throw new GFacException("Error cancelling job, invalid credentials. Job id - "
- + gramJob.getIDAsString(), e);
- }
- }
-
- }
-
- public void initProperties(Map<String, String> properties) throws GFacException {
-
- }
-
- private void checkJobStatus(JobExecutionContext jobExecutionContext, GlobusHostType host, String gateKeeper)
- throws GFacProviderException {
- int jobStatus = listener.getCurrentStatus();
-
- if (jobStatus == GramJob.STATUS_FAILED) {
-
- String errorMsg = "Job " + job.getIDAsString() + " on host " + host.getHostAddress() + " Job Exit Code = "
- + listener.getError() + " Error Description = " + getGramErrorString(listener.getError());
-
- if (listener.getError() == GRAMProtocolErrorConstants.INVALID_SCRIPT_REPLY) {
-
- // re-submitting without renewing
- // TODO verify why we re-submit jobs when we get a invalid script reply
- if (!reSubmissionInProgress) {
- reSubmissionInProgress = true;
-
- log.info("Invalid script reply received. Re-submitting job, id - " + job.getIDAsString());
- try {
- reSubmitJob(gateKeeper, jobExecutionContext, host);
- } catch (GFacException e) {
- throw new GFacProviderException
- ("Error during re-submission. Original job submission data - " + errorMsg, e);
- }
- return;
- }
-
- } else if (listener.getError() == GRAMProtocolErrorConstants.ERROR_AUTHORIZATION) {
-
- // re-submit with renewed credentials
- if (!authorisationFailedAttempt) {
- authorisationFailedAttempt = true;
- log.info("Authorisation error contacting provider. Re-submitting job with renewed credentials.");
-
- try {
- renewCredentials(jobExecutionContext);
- reSubmitJob(gateKeeper, jobExecutionContext, host);
- } catch (GFacException e) {
- throw new GFacProviderException
- ("Error during re-submission. Original job submission data - " + errorMsg, e);
- }
-
- return;
- }
-
- } else if (listener.getError() == GRAMProtocolErrorConstants.USER_CANCELLED) {
-
- log.info("User successfully cancelled job id " + job.getIDAsString());
- return;
- }
-
-
-
- log.error(errorMsg);
-
- synchronized (this) {
- currentlyExecutingJobCache.remove(job.getIDAsString());
- }
-
- throw new JobSubmissionFault(new Exception(errorMsg), host.getHostAddress(), gateKeeper,
- job.getRSL(), jobExecutionContext, getGramErrorString(listener.getError()),
- listener.getError());
-
- } else if (jobStatus == GramJob.STATUS_DONE) {
- log.info("Job " + job.getIDAsString() + " on host " + host.getHostAddress() + " is successfully executed.");
-
- synchronized (this) {
- currentlyExecutingJobCache.remove(job.getIDAsString());
- }
- }
- }
-
- public String getGramErrorString(int errorCode) {
-
- if (resources != null) {
- try {
- return resources.getProperty(String.valueOf(errorCode));
- } catch (MissingResourceException mre) {
- log.warn("Error reading globus error descriptions.", mre);
- return "Error code: " + errorCode;
- }
- } else {
- return "Error code: " + errorCode;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GramJobSubmissionListener.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GramJobSubmissionListener.java b/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GramJobSubmissionListener.java
deleted file mode 100644
index a77864d..0000000
--- a/modules/gfac/gfac-gram/src/main/java/org/apache/airavata/gfac/util/GramJobSubmissionListener.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.util;
-
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.security.GSISecurityContext;
-import org.apache.airavata.gfac.notification.events.StatusChangeEvent;
-import org.apache.airavata.gfac.utils.GFacUtils;
-import org.globus.gram.GramJob;
-import org.globus.gram.GramJobListener;
-import org.ietf.jgss.GSSCredential;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class GramJobSubmissionListener implements GramJobListener{
- private final Logger log = LoggerFactory.getLogger(GramJobSubmissionListener.class);
-
- public static final int NO_ERROR = -42;
- public static final int INITIAL_STATUS = -43;
-
- private volatile boolean jobDone = false;
- private volatile int error = NO_ERROR;
- private int currentStatus = INITIAL_STATUS;
-
- private JobExecutionContext context;
- private GramJob job;
-
- public GramJobSubmissionListener(GramJob job, JobExecutionContext context) {
- this.job = job;
- this.context = context;
- }
-
- /**
- * This method is used to block the process until the currentStatus of the job is DONE or FAILED
- */
- public void waitFor() {
- while (!isJobDone()) {
- synchronized (this) {
- try {
- wait();
- } catch (InterruptedException e) {}
- }
- }
- }
-
-
-
- private synchronized boolean isJobDone() {
- return this.jobDone;
- }
-
- private void setStatus(int status, int error) {
- try {
- GFacUtils.updateJobStatus(context, context.getJobDetails(), GramProviderUtils.getApplicationJobStatus(status));
- } catch (GFacException e) {
- log.error("Error persisting status" + e.getLocalizedMessage(), e);
- }
- this.currentStatus = status;
- this.error = error;
-
- switch (this.currentStatus) {
- case GramJob.STATUS_FAILED:
- log.info("Job Error Code: " + error);
- this.jobDone = true;
- notifyAll();
- case GramJob.STATUS_DONE:
- this.jobDone = true;
- notifyAll();
- }
-
- }
-
- public synchronized void statusChanged(GramJob job) {
-
- int jobStatus = job.getStatus();
- String jobStatusMessage = "Status of job " + job.getIDAsString() + "is " + job.getStatusAsString();
- /*
- * Notify currentStatus change
- */
- this.context.getNotifier().publish(new StatusChangeEvent(jobStatusMessage));
-
- /*
- * Set new currentStatus if it is jobDone, notify all wait object
- */
- if (currentStatus != jobStatus) {
- currentStatus = jobStatus;
-
- setStatus(job.getStatus(), job.getError());
-
- // Test to see whether we need to renew credentials
- renewCredentials(job);
- }
- }
-
- private void renewCredentials(GramJob job) {
-
- try {
-
- int proxyExpTime = job.getCredentials().getRemainingLifetime();
- if (proxyExpTime < GSISecurityContext.CREDENTIAL_RENEWING_THRESH_HOLD) {
- log.info("Job proxy expired. Trying to renew proxy");
- GSSCredential gssCred = ((GSISecurityContext)context.
- getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).renewCredentials();
- job.renew(gssCred);
- log.info("MyProxy credentials are renewed .");
- }
-
- } catch (Exception e) {
- log.error("An error occurred while trying to renew credentials. Job id " + job.getIDAsString());
- }
-
-
- }
-
- public synchronized int getError() {
- return error;
- }
-
- public synchronized int getCurrentStatus() {
- return currentStatus;
- }
-}
[4/8] changing package names of gfac implementations
Posted by la...@apache.org.
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java
new file mode 100644
index 0000000..2c4c956
--- /dev/null
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java
@@ -0,0 +1,192 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.gsissh.util;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.StringUtil;
+import org.apache.airavata.commons.gfac.type.ActualParameter;
+import org.apache.airavata.commons.gfac.type.HostDescription;
+import org.apache.airavata.commons.gfac.type.MappingFactory;
+import org.apache.airavata.gfac.Constants;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.RequestData;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.context.MessageContext;
+import org.apache.airavata.gfac.gsissh.security.GSISecurityContext;
+import org.apache.airavata.gsi.ssh.api.Cluster;
+import org.apache.airavata.gsi.ssh.api.SSHApiException;
+import org.apache.airavata.gsi.ssh.api.ServerInfo;
+import org.apache.airavata.gsi.ssh.api.authentication.GSIAuthenticationInfo;
+import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
+import org.apache.airavata.gsi.ssh.api.job.JobManagerConfiguration;
+import org.apache.airavata.gsi.ssh.impl.PBSCluster;
+import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo;
+import org.apache.airavata.gsi.ssh.util.CommonUtils;
+import org.apache.airavata.model.workspace.experiment.ComputationalResourceScheduling;
+import org.apache.airavata.model.workspace.experiment.TaskDetails;
+import org.apache.airavata.schemas.gfac.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+
+public class GFACGSISSHUtils {
+ private final static Logger logger = LoggerFactory.getLogger(GFACGSISSHUtils.class);
+
+ public static final String PBS_JOB_MANAGER = "pbs";
+ public static final String SLURM_JOB_MANAGER = "slurm";
+ public static final String SUN_GRID_ENGINE_JOB_MANAGER = "sge";
+
+ public static void addSecurityContext(JobExecutionContext jobExecutionContext) throws GFacException, ApplicationSettingsException {
+ RequestData requestData = new RequestData("default");
+ GSISecurityContext context = null;
+ try {
+ //todo fix this
+ context = new GSISecurityContext(null, requestData);
+ } catch (Exception e) {
+ throw new GFacException("An error occurred while creating GSI security context", e);
+ }
+ HostDescription registeredHost = jobExecutionContext.getApplicationContext().getHostDescription();
+ if (registeredHost.getType() instanceof GlobusHostType || registeredHost.getType() instanceof UnicoreHostType
+ || registeredHost.getType() instanceof SSHHostType) {
+ logger.error("This is a wrong method to invoke to non ssh host types,please check your gfac-config.xml");
+ } else if (registeredHost.getType() instanceof GsisshHostType) {
+ GSIAuthenticationInfo authenticationInfo
+ = new MyProxyAuthenticationInfo(requestData.getMyProxyUserName(), requestData.getMyProxyPassword(), requestData.getMyProxyServerUrl(),
+ requestData.getMyProxyPort(), requestData.getMyProxyLifeTime(), System.getProperty(Constants.TRUSTED_CERTIFICATE_SYSTEM_PROPERTY));
+ GsisshHostType gsisshHostType = (GsisshHostType) registeredHost.getType();
+ ServerInfo serverInfo = new ServerInfo(requestData.getMyProxyUserName(), registeredHost.getType().getHostAddress(),
+ gsisshHostType.getPort());
+
+ Cluster pbsCluster = null;
+ try {
+ JobManagerConfiguration jConfig = null;
+ String installedParentPath = ((HpcApplicationDeploymentType)
+ jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType()).getInstalledParentPath();
+ String jobManager = ((GsisshHostType) registeredHost.getType()).getJobManager();
+ if (jobManager == null) {
+ logger.error("No Job Manager is configured, so we are picking pbs as the default job manager");
+ jConfig = CommonUtils.getPBSJobManager(installedParentPath);
+ } else {
+ if (PBS_JOB_MANAGER.equalsIgnoreCase(jobManager)) {
+ jConfig = CommonUtils.getPBSJobManager(installedParentPath);
+ } else if (SLURM_JOB_MANAGER.equalsIgnoreCase(jobManager)) {
+ jConfig = CommonUtils.getSLURMJobManager(installedParentPath);
+ } else if (SUN_GRID_ENGINE_JOB_MANAGER.equalsIgnoreCase(jobManager)) {
+ jConfig = CommonUtils.getSGEJobManager(installedParentPath);
+ }
+ }
+ pbsCluster = new PBSCluster(serverInfo, authenticationInfo, jConfig);
+ } catch (SSHApiException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+
+ context.setPbsCluster(pbsCluster);
+ }
+ jobExecutionContext.addSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT,context);
+ }
+ public static JobDescriptor createJobDescriptor(JobExecutionContext jobExecutionContext,
+ ApplicationDeploymentDescriptionType app, Cluster cluster) {
+ JobDescriptor jobDescriptor = new JobDescriptor();
+ // this is common for any application descriptor
+ jobDescriptor.setInputDirectory(app.getInputDataDirectory());
+ jobDescriptor.setOutputDirectory(app.getOutputDataDirectory());
+ jobDescriptor.setExecutablePath(app.getExecutableLocation());
+ jobDescriptor.setStandardOutFile(app.getStandardOutput());
+ jobDescriptor.setStandardErrorFile(app.getStandardError());
+ Random random = new Random();
+ int i = random.nextInt();
+ jobDescriptor.setJobName(app.getApplicationName().getStringValue() + String.valueOf(i));
+ jobDescriptor.setWorkingDirectory(app.getStaticWorkingDirectory());
+
+
+ List<String> inputValues = new ArrayList<String>();
+ MessageContext input = jobExecutionContext.getInMessageContext();
+ Map<String, Object> inputs = input.getParameters();
+ Set<String> keys = inputs.keySet();
+ for (String paramName : keys) {
+ ActualParameter actualParameter = (ActualParameter) inputs.get(paramName);
+ if ("URIArray".equals(actualParameter.getType().getType().toString()) || "StringArray".equals(actualParameter.getType().getType().toString())
+ || "FileArray".equals(actualParameter.getType().getType().toString())) {
+ String[] values = null;
+ if (actualParameter.getType() instanceof URIArrayType) {
+ values = ((URIArrayType) actualParameter.getType()).getValueArray();
+ } else if (actualParameter.getType() instanceof StringArrayType) {
+ values = ((StringArrayType) actualParameter.getType()).getValueArray();
+ } else if (actualParameter.getType() instanceof FileArrayType) {
+ values = ((FileArrayType) actualParameter.getType()).getValueArray();
+ }
+ String value = StringUtil.createDelimiteredString(values, " ");
+ inputValues.add(value);
+ } else {
+ String paramValue = MappingFactory.toString(actualParameter);
+ inputValues.add(paramValue);
+ }
+ }
+ jobDescriptor.setInputValues(inputValues);
+
+ // this part will fill out the hpcApplicationDescriptor
+ if (app instanceof HpcApplicationDeploymentType) {
+ HpcApplicationDeploymentType applicationDeploymentType
+ = (HpcApplicationDeploymentType) app;
+ jobDescriptor.setShellName("/bin/bash");
+ jobDescriptor.setAllEnvExport(true);
+ jobDescriptor.setMailOptions("n");
+ jobDescriptor.setNodes(applicationDeploymentType.getNodeCount());
+ jobDescriptor.setProcessesPerNode(applicationDeploymentType.getProcessorsPerNode());
+ jobDescriptor.setMaxWallTime(String.valueOf(applicationDeploymentType.getMaxWallTime()));
+ jobDescriptor.setJobSubmitter(applicationDeploymentType.getJobSubmitterCommand());
+ if (applicationDeploymentType.getProjectAccount() != null) {
+ if (applicationDeploymentType.getProjectAccount().getProjectAccountNumber() != null) {
+ jobDescriptor.setAcountString(applicationDeploymentType.getProjectAccount().getProjectAccountNumber());
+ }
+ }
+ if (applicationDeploymentType.getQueue() != null) {
+ if (applicationDeploymentType.getQueue().getQueueName() != null) {
+ jobDescriptor.setQueueName(applicationDeploymentType.getQueue().getQueueName());
+ }
+ }
+ jobDescriptor.setOwner(((PBSCluster) cluster).getServerInfo().getUserName());
+ TaskDetails taskData = jobExecutionContext.getTaskData();
+ if (taskData != null && taskData.isSetTaskScheduling()) {
+ ComputationalResourceScheduling computionnalResource = taskData.getTaskScheduling();
+ if (computionnalResource.getNodeCount() > 0) {
+ jobDescriptor.setNodes(computionnalResource.getNodeCount());
+ }
+ if (computionnalResource.getComputationalProjectAccount() != null) {
+ jobDescriptor.setAcountString(computionnalResource.getComputationalProjectAccount());
+ }
+ if (computionnalResource.getQueueName() != null) {
+ jobDescriptor.setQueueName(computionnalResource.getQueueName());
+ }
+ if (computionnalResource.getTotalCPUCount() > 0) {
+ jobDescriptor.setProcessesPerNode(computionnalResource.getTotalCPUCount());
+ }
+ if (computionnalResource.getWallTimeLimit() > 0) {
+ jobDescriptor.setMaxWallTime(String.valueOf(computionnalResource.getWallTimeLimit()));
+ }
+ }
+
+ }
+ return jobDescriptor;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHDirectorySetupHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHDirectorySetupHandler.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHDirectorySetupHandler.java
deleted file mode 100644
index 1b9741d..0000000
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHDirectorySetupHandler.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.handler;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.security.GSISecurityContext;
-import org.apache.airavata.gfac.util.GFACGSISSHUtils;
-import org.apache.airavata.gfac.utils.GFacUtils;
-import org.apache.airavata.gsi.ssh.api.Cluster;
-import org.apache.airavata.gsi.ssh.api.SSHApiException;
-import org.apache.airavata.model.workspace.experiment.*;
-import org.apache.airavata.registry.cpi.ChildDataType;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-
-public class GSISSHDirectorySetupHandler extends AbstractHandler{
- private static final Logger log = LoggerFactory.getLogger(GSISSHDirectorySetupHandler.class);
-
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- try {
- if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) == null) {
- GFACGSISSHUtils.addSecurityContext(jobExecutionContext);
- }
- } catch (ApplicationSettingsException e) {
- log.error(e.getMessage());
- throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
- } catch (GFacException e) {
- e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
- }
-
- log.info("Setup SSH job directorties");
- super.invoke(jobExecutionContext);
- makeDirectory(jobExecutionContext);
- }
- private void makeDirectory(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- try {
- Cluster cluster = ((GSISecurityContext) jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getPbsCluster();
- if (cluster == null) {
- throw new GFacHandlerException("Security context is not set properly");
- } else {
- log.info("Successfully retrieved the Security Context");
- }
- ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
-
- String workingDirectory = app.getScratchWorkingDirectory();
- cluster.makeDirectory(workingDirectory);
- cluster.makeDirectory(app.getScratchWorkingDirectory());
- cluster.makeDirectory(app.getInputDataDirectory());
- cluster.makeDirectory(app.getOutputDataDirectory());
- DataTransferDetails detail = new DataTransferDetails();
- TransferStatus status = new TransferStatus();
- status.setTransferState(TransferState.DIRECTORY_SETUP);
- detail.setTransferStatus(status);
- detail.setTransferDescription("Working directory = " + workingDirectory);
-
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
-
- } catch (SSHApiException e) {
- throw new GFacHandlerException("Error executing the Handler: " + GSISSHDirectorySetupHandler.class, e);
- } catch (Exception e) {
- DataTransferDetails detail = new DataTransferDetails();
- TransferStatus status = new TransferStatus();
- status.setTransferState(TransferState.FAILED);
- detail.setTransferStatus(status);
- try {
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
- GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
- } catch (Exception e1) {
- throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
- }
- throw new GFacHandlerException("Error executing the Handler: " + GSISSHDirectorySetupHandler.class, e);
- }
- }
-
- public void initProperties(Map<String, String> properties) throws GFacHandlerException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHInputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHInputHandler.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHInputHandler.java
deleted file mode 100644
index 6f4fafd..0000000
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHInputHandler.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.handler;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.StringUtil;
-import org.apache.airavata.commons.gfac.type.ActualParameter;
-import org.apache.airavata.commons.gfac.type.MappingFactory;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.gfac.context.security.GSISecurityContext;
-import org.apache.airavata.gfac.util.GFACGSISSHUtils;
-import org.apache.airavata.gfac.utils.GFacUtils;
-import org.apache.airavata.gsi.ssh.api.Cluster;
-import org.apache.airavata.gsi.ssh.api.SSHApiException;
-import org.apache.airavata.model.workspace.experiment.*;
-import org.apache.airavata.registry.cpi.ChildDataType;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.apache.airavata.schemas.gfac.URIArrayType;
-import org.apache.airavata.schemas.gfac.URIParameterType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.*;
-
-public class GSISSHInputHandler extends AbstractHandler {
- private static final Logger log = LoggerFactory.getLogger(GSISSHInputHandler.class);
-
-
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- MessageContext inputNew = new MessageContext();
- DataTransferDetails detail = new DataTransferDetails();
- TransferStatus status = new TransferStatus();
- try {
- if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) == null) {
- try {
- GFACGSISSHUtils.addSecurityContext(jobExecutionContext);
- } catch (ApplicationSettingsException e) {
- log.error(e.getMessage());
- throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
- }
- }
- log.info("Invoking SCPInputHandler");
- super.invoke(jobExecutionContext);
-
-
- MessageContext input = jobExecutionContext.getInMessageContext();
- Set<String> parameters = input.getParameters().keySet();
- for (String paramName : parameters) {
- ActualParameter actualParameter = (ActualParameter) input.getParameters().get(paramName);
- String paramValue = MappingFactory.toString(actualParameter);
- //TODO: Review this with type
- if ("URI".equals(actualParameter.getType().getType().toString())) {
- ((URIParameterType) actualParameter.getType()).setValue(stageInputFiles(jobExecutionContext, paramValue));
- } else if ("URIArray".equals(actualParameter.getType().getType().toString())) {
- List<String> split = Arrays.asList(StringUtil.getElementsFromString(paramValue));
- List<String> newFiles = new ArrayList<String>();
- for (String paramValueEach : split) {
- String stageInputFiles = stageInputFiles(jobExecutionContext, paramValueEach);
- status.setTransferState(TransferState.UPLOAD);
- detail.setTransferStatus(status);
- detail.setTransferDescription("Input Data Staged: " + stageInputFiles);
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
- newFiles.add(stageInputFiles);
- }
- ((URIArrayType) actualParameter.getType()).setValueArray(newFiles.toArray(new String[newFiles.size()]));
- }
- inputNew.getParameters().put(paramName, actualParameter);
- }
- } catch (Exception e) {
- log.error(e.getMessage());
- status.setTransferState(TransferState.FAILED);
- detail.setTransferStatus(status);
- try {
- GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
- } catch (Exception e1) {
- throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
- }
- throw new GFacHandlerException("Error while input File Staging", e, e.getLocalizedMessage());
- }
- jobExecutionContext.setInMessageContext(inputNew);
- }
-
- private static String stageInputFiles(JobExecutionContext jobExecutionContext, String paramValue) throws IOException, GFacException {
- Cluster cluster = null;
- if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) != null) {
- cluster = ((GSISecurityContext) jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getPbsCluster();
- } else {
- cluster = ((GSISecurityContext) jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getPbsCluster();
- }
- if (cluster == null) {
- throw new GFacException("Security context is not set properly");
- } else {
- log.info("Successfully retrieved the Security Context");
- }
- ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
- int i = paramValue.lastIndexOf(File.separator);
- String substring = paramValue.substring(i + 1);
- try {
- String targetFile = app.getInputDataDirectory() + File.separator + substring;
- if (paramValue.startsWith("file")) {
- paramValue = paramValue.substring(paramValue.indexOf(":") + 1, paramValue.length());
- }
- cluster.scpTo(targetFile, paramValue);
- return targetFile;
- } catch (SSHApiException e) {
- throw new GFacHandlerException("Error while input File Staging", e, e.getLocalizedMessage());
- }
- }
-
- public void initProperties(Map<String, String> properties) throws GFacHandlerException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHOutputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHOutputHandler.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHOutputHandler.java
deleted file mode 100644
index 3855248..0000000
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHOutputHandler.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.handler;
-
-import net.schmizz.sshj.connection.ConnectionException;
-import net.schmizz.sshj.transport.TransportException;
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.Constants;
-import org.apache.airavata.commons.gfac.type.ActualParameter;
-import org.apache.airavata.commons.gfac.type.ApplicationDescription;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.security.GSISecurityContext;
-import org.apache.airavata.gfac.provider.GFacProviderException;
-import org.apache.airavata.gfac.util.GFACGSISSHUtils;
-import org.apache.airavata.gfac.utils.GFacUtils;
-import org.apache.airavata.gfac.utils.OutputUtils;
-import org.apache.airavata.gsi.ssh.api.Cluster;
-import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
-import org.apache.airavata.model.workspace.experiment.*;
-import org.apache.airavata.registry.cpi.ChildDataType;
-import org.apache.airavata.registry.cpi.DataType;
-import org.apache.airavata.registry.cpi.RegistryException;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.apache.airavata.schemas.gfac.GsisshHostType;
-import org.apache.airavata.schemas.gfac.URIParameterType;
-import org.apache.xmlbeans.XmlException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-public class GSISSHOutputHandler extends AbstractHandler{
- private static final Logger log = LoggerFactory.getLogger(GSISSHOutputHandler.class);
-
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- if(jobExecutionContext.getApplicationContext().getHostDescription().getType() instanceof GsisshHostType) { // this is because we don't have the right jobexecution context
- // so attempting to get it from the registry
- if (Constants.PUSH.equals(((GsisshHostType) jobExecutionContext.getApplicationContext().getHostDescription().getType()).getMonitorMode())) {
- log.warn("During the out handler chain jobExecution context came null, so trying to handler");
- ApplicationDescription applicationDeploymentDescription = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription();
- TaskDetails taskData = null;
- try {
- taskData = (TaskDetails) jobExecutionContext.getRegistry().get(DataType.TASK_DETAIL, jobExecutionContext.getTaskData().getTaskID());
- } catch (RegistryException e) {
- log.error("Error retrieving job details from Registry");
- throw new GFacHandlerException("Error retrieving job details from Registry", e);
- }
- JobDetails jobDetails = taskData.getJobDetailsList().get(0);
- String jobDescription = jobDetails.getJobDescription();
- if (jobDescription != null) {
- JobDescriptor jobDescriptor = null;
- try {
- jobDescriptor = JobDescriptor.fromXML(jobDescription);
- } catch (XmlException e1) {
- e1.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
- }
- applicationDeploymentDescription.getType().setScratchWorkingDirectory(
- jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getWorkingDirectory());
- applicationDeploymentDescription.getType().setInputDataDirectory(jobDescriptor.getInputDirectory());
- applicationDeploymentDescription.getType().setOutputDataDirectory(jobDescriptor.getOutputDirectory());
- applicationDeploymentDescription.getType().setStandardError(jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getStandardErrorFile());
- applicationDeploymentDescription.getType().setStandardOutput(jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getStandardOutFile());
- }
- }
- }
- try {
- if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) == null) {
-
- GFACGSISSHUtils.addSecurityContext(jobExecutionContext);
- }
- } catch (ApplicationSettingsException e) {
- log.error(e.getMessage());
- throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
- } catch (GFacException e) {
- log.error(e.getMessage());
- throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
- }
- super.invoke(jobExecutionContext);
- DataTransferDetails detail = new DataTransferDetails();
- TransferStatus status = new TransferStatus();
-
- ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext()
- .getApplicationDeploymentDescription().getType();
- try {
- Cluster cluster = null;
- if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) != null) {
- cluster = ((GSISecurityContext) jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getPbsCluster();
- } else {
- cluster = ((GSISecurityContext) jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getPbsCluster();
- }
- if (cluster == null) {
- throw new GFacProviderException("Security context is not set properly");
- } else {
- log.info("Successfully retrieved the Security Context");
- }
-
- // Get the Stdouts and StdErrs
- String timeStampedServiceName = GFacUtils.createUniqueNameForService(jobExecutionContext.getServiceName());
-
- TaskDetails taskData = jobExecutionContext.getTaskData();
- String outputDataDir = null;
- File localStdOutFile;
- File localStdErrFile;
-
- if (taskData.getAdvancedOutputDataHandling() != null) {
- outputDataDir = taskData.getAdvancedOutputDataHandling().getOutputDataDir();
- }
- if (outputDataDir == null) {
- outputDataDir = File.separator + "tmp";
- }
- outputDataDir = outputDataDir + File.separator + jobExecutionContext.getExperimentID() + "-" + jobExecutionContext.getTaskData().getTaskID();
- (new File(outputDataDir)).mkdirs();
-
-
- localStdOutFile = new File(outputDataDir + File.separator + timeStampedServiceName + "stdout");
- localStdErrFile = new File(outputDataDir + File.separator + timeStampedServiceName + "stderr");
-// cluster.makeDirectory(outputDataDir);
- cluster.scpFrom(app.getStandardOutput(), localStdOutFile.getAbsolutePath());
- Thread.sleep(1000);
- cluster.scpFrom(app.getStandardError(), localStdErrFile.getAbsolutePath());
- Thread.sleep(1000);
-
- String stdOutStr = GFacUtils.readFileToString(localStdOutFile.getAbsolutePath());
- String stdErrStr = GFacUtils.readFileToString(localStdErrFile.getAbsolutePath());
- status.setTransferState(TransferState.COMPLETE);
- detail.setTransferStatus(status);
- detail.setTransferDescription("STDOUT:" + stdOutStr);
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
-
- status.setTransferState(TransferState.COMPLETE);
- detail.setTransferStatus(status);
- detail.setTransferDescription("STDERR:" + stdErrStr);
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
-
-
- Map<String, ActualParameter> stringMap = new HashMap<String, ActualParameter>();
- Map<String, Object> output = jobExecutionContext.getOutMessageContext().getParameters();
- Set<String> keys = output.keySet();
- for (String paramName : keys) {
- ActualParameter actualParameter = (ActualParameter) output.get(paramName);
- if ("URI".equals(actualParameter.getType().getType().toString())) {
-
- List<String> outputList = cluster.listDirectory(app.getOutputDataDirectory());
- if (outputList.size() == 0 || outputList.get(0).isEmpty()) {
- stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr);
- } else {
- String valueList = outputList.get(0);
- cluster.scpFrom(app.getOutputDataDirectory() + File.separator + valueList, outputDataDir);
- jobExecutionContext.addOutputFile(outputDataDir + File.separator + valueList);
- ((URIParameterType) actualParameter.getType()).setValue(valueList);
- stringMap = new HashMap<String, ActualParameter>();
- stringMap.put(paramName, actualParameter);
- }
- } else {
- stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr);
- }
- }
- if (stringMap == null || stringMap.isEmpty()) {
- throw new GFacHandlerException(
- "Empty Output returned from the Application, Double check the application"
- + "and ApplicationDescriptor output Parameter Names");
- }
- status.setTransferState(TransferState.DOWNLOAD);
- detail.setTransferStatus(status);
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
-
- app.setStandardError(localStdErrFile.getAbsolutePath());
- app.setStandardOutput(localStdOutFile.getAbsolutePath());
- app.setOutputDataDirectory(outputDataDir);
- } catch (XmlException e) {
- throw new GFacHandlerException("Cannot read output:" + e.getMessage(), e);
- } catch (ConnectionException e) {
- throw new GFacHandlerException(e.getMessage(), e);
- } catch (TransportException e) {
- throw new GFacHandlerException(e.getMessage(), e);
- } catch (IOException e) {
- throw new GFacHandlerException(e.getMessage(), e);
- } catch (Exception e) {
- try {
- status.setTransferState(TransferState.FAILED);
- detail.setTransferStatus(status);
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
- GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
- } catch (Exception e1) {
- throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
- }
- throw new GFacHandlerException("Error in retrieving results", e);
- }
-
- }
-
- public void initProperties(Map<String, String> properties) throws GFacHandlerException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/provider/impl/GSISSHProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/provider/impl/GSISSHProvider.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/provider/impl/GSISSHProvider.java
deleted file mode 100644
index da5b330..0000000
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/provider/impl/GSISSHProvider.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.provider.impl;
-
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.security.GSISecurityContext;
-import org.apache.airavata.gfac.cpi.GFacImpl;
-import org.apache.airavata.gfac.handler.ThreadedHandler;
-import org.apache.airavata.gfac.notification.events.StartExecutionEvent;
-import org.apache.airavata.gfac.provider.AbstractProvider;
-import org.apache.airavata.gfac.provider.GFacProviderException;
-import org.apache.airavata.gfac.util.GFACGSISSHUtils;
-import org.apache.airavata.gfac.utils.GFacUtils;
-import org.apache.airavata.gsi.ssh.api.Cluster;
-import org.apache.airavata.gsi.ssh.api.SSHApiException;
-import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
-import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
-import org.apache.airavata.model.workspace.experiment.ErrorCategory;
-import org.apache.airavata.model.workspace.experiment.JobDetails;
-import org.apache.airavata.model.workspace.experiment.JobState;
-import org.apache.airavata.schemas.gfac.GsisshHostType;
-import org.apache.airavata.schemas.gfac.HostDescriptionType;
-import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-
-public class GSISSHProvider extends AbstractProvider {
- private static final Logger log = LoggerFactory.getLogger(GSISSHProvider.class);
-
- public void initProperties(Map<String, String> properties) throws GFacProviderException, GFacException {
-
- }
-
- public void initialize(JobExecutionContext jobExecutionContext) throws GFacProviderException, GFacException {
- super.initialize(jobExecutionContext);
- }
-
- public void execute(JobExecutionContext jobExecutionContext) throws GFacProviderException, GFacException {
- log.info("Invoking GSISSH Provider Invoke ...");
- jobExecutionContext.getNotifier().publish(new StartExecutionEvent());
- HostDescriptionType host = jobExecutionContext.getApplicationContext().
- getHostDescription().getType();
- HpcApplicationDeploymentType app = (HpcApplicationDeploymentType) jobExecutionContext.getApplicationContext().
- getApplicationDeploymentDescription().getType();
- JobDetails jobDetails = new JobDetails();
- String taskID = jobExecutionContext.getTaskData().getTaskID();
- try {
- Cluster cluster = null;
- if (jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT) != null) {
- cluster = ((GSISecurityContext) jobExecutionContext.getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT)).getPbsCluster();
- }
- if (cluster == null) {
- throw new GFacProviderException("Security context is not set properly");
- } else {
- log.info("Successfully retrieved the Security Context");
- }
- // This installed path is a mandetory field, because this could change based on the computing resource
- JobDescriptor jobDescriptor = GFACGSISSHUtils.createJobDescriptor(jobExecutionContext, app, cluster);
-
- log.info(jobDescriptor.toXML());
-
- jobDetails.setJobDescription(jobDescriptor.toXML());
-
- String jobID = cluster.submitBatchJob(jobDescriptor);
- jobExecutionContext.setJobDetails(jobDetails);
- if(jobID == null){
- jobDetails.setJobID("none");
- GFacUtils.saveJobStatus(jobExecutionContext, jobDetails, JobState.FAILED);
- }else{
- jobDetails.setJobID(jobID);
- GFacUtils.saveJobStatus(jobExecutionContext, jobDetails, JobState.SUBMITTED);
- }
-
-
- // Now job has submitted to the resource, its up to the Provider to parse the information to daemon handler
- // to perform monitoring, daemon handlers can be accessed from anywhere
- List<ThreadedHandler> daemonHandlers = GFacImpl.getDaemonHandlers();
- ThreadedHandler pullMonitorHandler = null;
- for(ThreadedHandler threadedHandler:daemonHandlers){
- if("org.apache.airavata.gfac.monitor.handlers.GridPullMonitorHandler".equals(threadedHandler.getClass().getName())){
- pullMonitorHandler = threadedHandler;
- }
- }
- // we know this hos is type GsiSSHHostType
- String monitorMode = ((GsisshHostType) host).getMonitorMode();
- if("".equals(monitorMode) || monitorMode == null || org.apache.airavata.common.utils.Constants.PULL.equals(monitorMode)){
- log.info("Job is launched successfully now parsing it to monitoring in pull mode, JobID Returned: " + jobID);
- pullMonitorHandler.invoke(jobExecutionContext);
- }else{
- log.error("Currently we only support Pull monitoring");
- }
- } catch (SSHApiException e) {
- String error = "Error submitting the job to host " + host.getHostAddress() + " message: " + e.getMessage();
- log.error(error);
- jobDetails.setJobID("none");
- GFacUtils.saveJobStatus(jobExecutionContext, jobDetails,JobState.FAILED);
- GFacUtils.saveErrorDetails(jobExecutionContext, error, CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
- throw new GFacProviderException(error, e);
- } catch (Exception e) {
- String error = "Error submitting the job to host " + host.getHostAddress() + " message: " + e.getMessage();
- log.error(error);
- jobDetails.setJobID("none");
- GFacUtils.saveJobStatus(jobExecutionContext, jobDetails,JobState.FAILED);
- GFacUtils.saveErrorDetails(jobExecutionContext, error, CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
- throw new GFacProviderException(error, e);
- }
- }
-
- public void dispose(JobExecutionContext jobExecutionContext) throws GFacProviderException, GFacException {
- //To change body of implemented methods use File | Settings | File Templates.
- }
-
- public void cancelJob(String jobId, JobExecutionContext jobExecutionContext) throws GFacException {
- //To change body of implemented methods use File | Settings | File Templates.
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/util/GFACGSISSHUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/util/GFACGSISSHUtils.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/util/GFACGSISSHUtils.java
deleted file mode 100644
index 0351d00..0000000
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/util/GFACGSISSHUtils.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.util;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.StringUtil;
-import org.apache.airavata.commons.gfac.type.ActualParameter;
-import org.apache.airavata.commons.gfac.type.HostDescription;
-import org.apache.airavata.commons.gfac.type.MappingFactory;
-import org.apache.airavata.gfac.Constants;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.RequestData;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.gfac.context.security.GSISecurityContext;
-import org.apache.airavata.gsi.ssh.api.Cluster;
-import org.apache.airavata.gsi.ssh.api.SSHApiException;
-import org.apache.airavata.gsi.ssh.api.ServerInfo;
-import org.apache.airavata.gsi.ssh.api.authentication.GSIAuthenticationInfo;
-import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
-import org.apache.airavata.gsi.ssh.api.job.JobManagerConfiguration;
-import org.apache.airavata.gsi.ssh.impl.PBSCluster;
-import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo;
-import org.apache.airavata.gsi.ssh.util.CommonUtils;
-import org.apache.airavata.model.workspace.experiment.ComputationalResourceScheduling;
-import org.apache.airavata.model.workspace.experiment.TaskDetails;
-import org.apache.airavata.schemas.gfac.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.*;
-
-
-public class GFACGSISSHUtils {
- private final static Logger logger = LoggerFactory.getLogger(GFACGSISSHUtils.class);
-
- public static final String PBS_JOB_MANAGER = "pbs";
- public static final String SLURM_JOB_MANAGER = "slurm";
- public static final String SUN_GRID_ENGINE_JOB_MANAGER = "sge";
-
- public static void addSecurityContext(JobExecutionContext jobExecutionContext) throws GFacException, ApplicationSettingsException {
- RequestData requestData = new RequestData("default");
- GSISecurityContext context = null;
- try {
- //todo fix this
- context = new GSISecurityContext(null, requestData);
- } catch (Exception e) {
- throw new GFacException("An error occurred while creating GSI security context", e);
- }
- HostDescription registeredHost = jobExecutionContext.getApplicationContext().getHostDescription();
- if (registeredHost.getType() instanceof GlobusHostType || registeredHost.getType() instanceof UnicoreHostType
- || registeredHost.getType() instanceof SSHHostType) {
- logger.error("This is a wrong method to invoke to non ssh host types,please check your gfac-config.xml");
- } else if (registeredHost.getType() instanceof GsisshHostType) {
- GSIAuthenticationInfo authenticationInfo
- = new MyProxyAuthenticationInfo(requestData.getMyProxyUserName(), requestData.getMyProxyPassword(), requestData.getMyProxyServerUrl(),
- requestData.getMyProxyPort(), requestData.getMyProxyLifeTime(), System.getProperty(Constants.TRUSTED_CERTIFICATE_SYSTEM_PROPERTY));
- GsisshHostType gsisshHostType = (GsisshHostType) registeredHost.getType();
- ServerInfo serverInfo = new ServerInfo(requestData.getMyProxyUserName(), registeredHost.getType().getHostAddress(),
- gsisshHostType.getPort());
-
- Cluster pbsCluster = null;
- try {
- JobManagerConfiguration jConfig = null;
- String installedParentPath = ((HpcApplicationDeploymentType)
- jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType()).getInstalledParentPath();
- String jobManager = ((GsisshHostType) registeredHost.getType()).getJobManager();
- if (jobManager == null) {
- logger.error("No Job Manager is configured, so we are picking pbs as the default job manager");
- jConfig = CommonUtils.getPBSJobManager(installedParentPath);
- } else {
- if (PBS_JOB_MANAGER.equalsIgnoreCase(jobManager)) {
- jConfig = CommonUtils.getPBSJobManager(installedParentPath);
- } else if (SLURM_JOB_MANAGER.equalsIgnoreCase(jobManager)) {
- jConfig = CommonUtils.getSLURMJobManager(installedParentPath);
- } else if (SUN_GRID_ENGINE_JOB_MANAGER.equalsIgnoreCase(jobManager)) {
- jConfig = CommonUtils.getSGEJobManager(installedParentPath);
- }
- }
- pbsCluster = new PBSCluster(serverInfo, authenticationInfo, jConfig);
- } catch (SSHApiException e) {
- e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
- }
-
- context.setPbsCluster(pbsCluster);
- }
- jobExecutionContext.addSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT,context);
- }
- public static JobDescriptor createJobDescriptor(JobExecutionContext jobExecutionContext,
- ApplicationDeploymentDescriptionType app, Cluster cluster) {
- JobDescriptor jobDescriptor = new JobDescriptor();
- // this is common for any application descriptor
- jobDescriptor.setInputDirectory(app.getInputDataDirectory());
- jobDescriptor.setOutputDirectory(app.getOutputDataDirectory());
- jobDescriptor.setExecutablePath(app.getExecutableLocation());
- jobDescriptor.setStandardOutFile(app.getStandardOutput());
- jobDescriptor.setStandardErrorFile(app.getStandardError());
- Random random = new Random();
- int i = random.nextInt();
- jobDescriptor.setJobName(app.getApplicationName().getStringValue() + String.valueOf(i));
- jobDescriptor.setWorkingDirectory(app.getStaticWorkingDirectory());
-
-
- List<String> inputValues = new ArrayList<String>();
- MessageContext input = jobExecutionContext.getInMessageContext();
- Map<String, Object> inputs = input.getParameters();
- Set<String> keys = inputs.keySet();
- for (String paramName : keys) {
- ActualParameter actualParameter = (ActualParameter) inputs.get(paramName);
- if ("URIArray".equals(actualParameter.getType().getType().toString()) || "StringArray".equals(actualParameter.getType().getType().toString())
- || "FileArray".equals(actualParameter.getType().getType().toString())) {
- String[] values = null;
- if (actualParameter.getType() instanceof URIArrayType) {
- values = ((URIArrayType) actualParameter.getType()).getValueArray();
- } else if (actualParameter.getType() instanceof StringArrayType) {
- values = ((StringArrayType) actualParameter.getType()).getValueArray();
- } else if (actualParameter.getType() instanceof FileArrayType) {
- values = ((FileArrayType) actualParameter.getType()).getValueArray();
- }
- String value = StringUtil.createDelimiteredString(values, " ");
- inputValues.add(value);
- } else {
- String paramValue = MappingFactory.toString(actualParameter);
- inputValues.add(paramValue);
- }
- }
- jobDescriptor.setInputValues(inputValues);
-
- // this part will fill out the hpcApplicationDescriptor
- if (app instanceof HpcApplicationDeploymentType) {
- HpcApplicationDeploymentType applicationDeploymentType
- = (HpcApplicationDeploymentType) app;
- jobDescriptor.setShellName("/bin/bash");
- jobDescriptor.setAllEnvExport(true);
- jobDescriptor.setMailOptions("n");
- jobDescriptor.setNodes(applicationDeploymentType.getNodeCount());
- jobDescriptor.setProcessesPerNode(applicationDeploymentType.getProcessorsPerNode());
- jobDescriptor.setMaxWallTime(String.valueOf(applicationDeploymentType.getMaxWallTime()));
- jobDescriptor.setJobSubmitter(applicationDeploymentType.getJobSubmitterCommand());
- if (applicationDeploymentType.getProjectAccount() != null) {
- if (applicationDeploymentType.getProjectAccount().getProjectAccountNumber() != null) {
- jobDescriptor.setAcountString(applicationDeploymentType.getProjectAccount().getProjectAccountNumber());
- }
- }
- if (applicationDeploymentType.getQueue() != null) {
- if (applicationDeploymentType.getQueue().getQueueName() != null) {
- jobDescriptor.setQueueName(applicationDeploymentType.getQueue().getQueueName());
- }
- }
- jobDescriptor.setOwner(((PBSCluster) cluster).getServerInfo().getUserName());
- TaskDetails taskData = jobExecutionContext.getTaskData();
- if (taskData != null && taskData.isSetTaskScheduling()) {
- ComputationalResourceScheduling computionnalResource = taskData.getTaskScheduling();
- if (computionnalResource.getNodeCount() > 0) {
- jobDescriptor.setNodes(computionnalResource.getNodeCount());
- }
- if (computionnalResource.getComputationalProjectAccount() != null) {
- jobDescriptor.setAcountString(computionnalResource.getComputationalProjectAccount());
- }
- if (computionnalResource.getQueueName() != null) {
- jobDescriptor.setQueueName(computionnalResource.getQueueName());
- }
- if (computionnalResource.getTotalCPUCount() > 0) {
- jobDescriptor.setProcessesPerNode(computionnalResource.getTotalCPUCount());
- }
- if (computionnalResource.getWallTimeLimit() > 0) {
- jobDescriptor.setMaxWallTime(String.valueOf(computionnalResource.getWallTimeLimit()));
- }
- }
-
- }
- return jobDescriptor;
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gsissh/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTestWithMyProxyAuth.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTestWithMyProxyAuth.java b/modules/gfac/gfac-gsissh/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTestWithMyProxyAuth.java
index be78570..162c2d2 100644
--- a/modules/gfac/gfac-gsissh/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTestWithMyProxyAuth.java
+++ b/modules/gfac/gfac-gsissh/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTestWithMyProxyAuth.java
@@ -37,7 +37,7 @@ import org.apache.airavata.gfac.SecurityContext;
import org.apache.airavata.gfac.context.ApplicationContext;
import org.apache.airavata.gfac.context.JobExecutionContext;
import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.gfac.context.security.GSISecurityContext;
+import org.apache.airavata.gfac.gsissh.security.GSISecurityContext;
import org.apache.airavata.gfac.cpi.GFacImpl;
import org.apache.airavata.gsi.ssh.api.Cluster;
import org.apache.airavata.gsi.ssh.api.SSHApiException;
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-gsissh/src/test/java/org/apache/airavata/core/gfac/services/impl/security/GSISecurityContextTestWithMyProxyAuth.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/test/java/org/apache/airavata/core/gfac/services/impl/security/GSISecurityContextTestWithMyProxyAuth.java b/modules/gfac/gfac-gsissh/src/test/java/org/apache/airavata/core/gfac/services/impl/security/GSISecurityContextTestWithMyProxyAuth.java
index 6306e3c..86bd4d3 100644
--- a/modules/gfac/gfac-gsissh/src/test/java/org/apache/airavata/core/gfac/services/impl/security/GSISecurityContextTestWithMyProxyAuth.java
+++ b/modules/gfac/gfac-gsissh/src/test/java/org/apache/airavata/core/gfac/services/impl/security/GSISecurityContextTestWithMyProxyAuth.java
@@ -29,7 +29,7 @@ import org.apache.airavata.common.utils.ServerSettings;
import org.apache.airavata.credential.store.store.CredentialReader;
import org.apache.airavata.credential.store.store.CredentialReaderFactory;
import org.apache.airavata.gfac.RequestData;
-import org.apache.airavata.gfac.context.security.GSISecurityContext;
+import org.apache.airavata.gfac.gsissh.security.GSISecurityContext;
import org.apache.log4j.Logger;
import org.ietf.jgss.GSSCredential;
import org.testng.annotations.AfterClass;
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/handler/HDFSDataMovementHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/handler/HDFSDataMovementHandler.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/handler/HDFSDataMovementHandler.java
new file mode 100644
index 0000000..f54b7a5
--- /dev/null
+++ b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/handler/HDFSDataMovementHandler.java
@@ -0,0 +1,103 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+
+package org.apache.airavata.gfac.hadoop.handler;
+
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.context.MessageContext;
+import org.apache.airavata.gfac.hadoop.provider.utils.HadoopUtils;
+import org.apache.airavata.gfac.handler.GFacHandler;
+import org.apache.airavata.gfac.handler.GFacHandlerException;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.HadoopApplicationDeploymentDescriptionType;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Map;
+
+public class HDFSDataMovementHandler implements GFacHandler {
+ private static final Logger logger = LoggerFactory.getLogger(HDFSDataMovementHandler.class);
+
+ private boolean isWhirrBasedDeployment = false;
+ private File hadoopConfigDir;
+
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ MessageContext inMessageContext = jobExecutionContext.getInMessageContext();
+ if(inMessageContext.getParameter("HADOOP_DEPLOYMENT_TYPE").equals("WHIRR")){
+ isWhirrBasedDeployment = true;
+ } else {
+ String hadoopConfigDirPath = (String)inMessageContext.getParameter("HADOOP_CONFIG_DIR");
+ File hadoopConfigDir = new File(hadoopConfigDirPath);
+ if (!hadoopConfigDir.exists()){
+ throw new GFacHandlerException("Specified hadoop configuration directory doesn't exist.");
+ } else if (FileUtils.listFiles(hadoopConfigDir, null, null).size() <= 0){
+ throw new GFacHandlerException("Cannot find any hadoop configuration files inside specified directory.");
+ }
+
+ this.hadoopConfigDir = hadoopConfigDir;
+ }
+
+ if(jobExecutionContext.isInPath()){
+ try {
+ handleInPath(jobExecutionContext);
+ } catch (IOException e) {
+ throw new GFacHandlerException("Error while copying input data from local file system to HDFS.",e);
+ }
+ } else {
+ handleOutPath(jobExecutionContext);
+ }
+ }
+
+ private void handleInPath(JobExecutionContext jobExecutionContext) throws GFacHandlerException, IOException {
+ ApplicationDeploymentDescriptionType appDepDesc =
+ jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
+ HadoopApplicationDeploymentDescriptionType hadoopAppDesc =
+ (HadoopApplicationDeploymentDescriptionType)appDepDesc;
+ if(appDepDesc.isSetInputDataDirectory() && isInputDataDirectoryLocal(appDepDesc)){
+ Configuration hadoopConf = HadoopUtils.createHadoopConfiguration(jobExecutionContext, isWhirrBasedDeployment, hadoopConfigDir);
+ FileSystem hdfs = FileSystem.get(hadoopConf);
+ hdfs.copyFromLocalFile(new Path(appDepDesc.getInputDataDirectory()),
+ new Path(hadoopAppDesc.getHadoopJobConfiguration().getHdfsInputDirectory()));
+ }
+ }
+
+ private boolean isInputDataDirectoryLocal(ApplicationDeploymentDescriptionType appDepDesc){
+ String inputDataDirectoryPath = appDepDesc.getInputDataDirectory();
+ File inputDataDirectory = new File(inputDataDirectoryPath);
+ if(inputDataDirectory.exists() && FileUtils.listFiles(inputDataDirectory, null, null).size() > 0){
+ return true;
+ }
+
+ return false;
+ }
+
+ private void handleOutPath(JobExecutionContext jobExecutionContext){}
+
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/handler/HadoopDeploymentHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/handler/HadoopDeploymentHandler.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/handler/HadoopDeploymentHandler.java
new file mode 100644
index 0000000..24e0e9a
--- /dev/null
+++ b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/handler/HadoopDeploymentHandler.java
@@ -0,0 +1,276 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+
+package org.apache.airavata.gfac.hadoop.handler;
+
+import com.google.common.io.Files;
+import org.apache.airavata.commons.gfac.type.HostDescription;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.context.MessageContext;
+import org.apache.airavata.gfac.handler.GFacHandler;
+import org.apache.airavata.gfac.handler.GFacHandlerException;
+import org.apache.airavata.schemas.gfac.HadoopHostType;
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.io.FileUtils;
+import org.apache.whirr.Cluster;
+import org.apache.whirr.ClusterController;
+import org.apache.whirr.ClusterControllerFactory;
+import org.apache.whirr.ClusterSpec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.*;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Properties;
+
+import static org.apache.whirr.ClusterSpec.Property.*;
+import static org.apache.whirr.ClusterSpec.Property.INSTANCE_TEMPLATES;
+import static org.apache.whirr.ClusterSpec.Property.PRIVATE_KEY_FILE;
+
+/**
+ * This handler takes care of deploying hadoop in cloud(in cloud bursting scenarios) and
+ * deploying hadoop in local cluster. In case of existing hadoop cluster this will ignore
+ * cluster setup just use the hadoop configuration provided by user.
+ */
+public class HadoopDeploymentHandler implements GFacHandler {
+ private static final Logger logger = LoggerFactory.getLogger("hadoop-dep-handler");
+
+ /**
+ * Once invoked this method will deploy Hadoop in a local cluster or cloud based on the
+ * configuration provided. If there is a already deployed hadoop cluster this will skip
+ * deployment.
+ *
+ *
+ * @param jobExecutionContext job execution context containing all the required configurations
+ * and runtime information.
+ * @throws org.apache.airavata.gfac.handler.GFacHandlerException
+ */
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ if(jobExecutionContext.isInPath()){
+ handleInPath(jobExecutionContext);
+ } else {
+ handleOutPath(jobExecutionContext);
+ }
+ }
+
+ private void handleInPath(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ HostDescription hostDescription =
+ jobExecutionContext.getApplicationContext().getHostDescription();
+ if (!isHadoopDeploymentAvailable(hostDescription)) {
+ // Temp directory to keep generated configuration files.
+ File tempDirectory = Files.createTempDir();
+ try {
+ File hadoopSiteXML = launchHadoopCluster(hostDescription, tempDirectory);
+ jobExecutionContext.getInMessageContext().addParameter("HADOOP_SITE_XML", hadoopSiteXML.getAbsolutePath());
+ jobExecutionContext.getInMessageContext().addParameter("HADOOP_DEPLOYMENT_TYPE", "WHIRR");
+ // TODO: Add hadoop-site.xml to job execution context.
+ } catch (IOException e) {
+ throw new GFacHandlerException("IO Error while processing configurations.",e);
+ } catch (ConfigurationException e) {
+ throw new GFacHandlerException("Whirr configuration error.", e);
+ } catch (InterruptedException e) {
+ throw new GFacHandlerException("Hadoop cluster launch interrupted.", e);
+ } catch (TransformerException e) {
+ throw new GFacHandlerException("Error while creating hadoop-site.xml", e);
+ } catch (ParserConfigurationException e) {
+ throw new GFacHandlerException("Error while creating hadoop-site.xml", e);
+ }
+ } else {
+ jobExecutionContext.getInMessageContext().addParameter("HADOOP_DEPLOYMENT_TYPE",
+ "MANUAL");
+ jobExecutionContext.getInMessageContext().addParameter("HADOOP_CONFIG_DIR",
+ ((HadoopHostType)hostDescription.getType()).getHadoopConfigurationDirectory());
+ logger.info("Hadoop configuration is available. Skipping hadoop deployment.");
+ if(logger.isDebugEnabled()){
+ logger.debug("Hadoop configuration directory: " +
+ getHadoopConfigDirectory(hostDescription));
+ }
+ }
+ }
+
+ private void handleOutPath(JobExecutionContext jobExecutionContext){
+ MessageContext inMessageContext = jobExecutionContext.getInMessageContext();
+ if(((String)inMessageContext.getParameter("HADOOP_DEPLOYMENT_TYPE")).equals("WHIRR")){
+ // TODO: Shutdown hadoop cluster.
+ logger.info("Shutdown hadoop cluster.");
+ }
+ }
+
+ private File launchHadoopCluster(HostDescription hostDescription, File workingDirectory)
+ throws IOException, GFacHandlerException, ConfigurationException, InterruptedException, TransformerException, ParserConfigurationException {
+ ClusterSpec hadoopClusterSpec =
+ whirrConfigurationToClusterSpec(hostDescription, workingDirectory);
+ ClusterController hadoopClusterController =
+ createClusterController(hadoopClusterSpec.getServiceName());
+ Cluster hadoopCluster = hadoopClusterController.launchCluster(hadoopClusterSpec);
+
+ logger.info(String.format("Started cluster of %s instances.\n",
+ hadoopCluster.getInstances().size()));
+
+ File siteXML = new File(workingDirectory, "hadoop-site.xml");
+ clusterPropertiesToHadoopSiteXml(hadoopCluster.getConfiguration(), siteXML);
+
+ return siteXML;
+ }
+
+ private ClusterController createClusterController(String serviceName){
+ ClusterControllerFactory factory = new ClusterControllerFactory();
+ ClusterController controller = factory.create(serviceName);
+
+ if(controller == null){
+ logger.warn("Unable to find the service {0}, using default.", serviceName);
+ controller = factory.create(null);
+ }
+
+ return controller;
+ }
+
+ private ClusterSpec whirrConfigurationToClusterSpec(HostDescription hostDescription,
+ File workingDirectory) throws IOException, GFacHandlerException, ConfigurationException {
+ File whirrConfig = getWhirrConfigurationFile(hostDescription, workingDirectory);
+ CompositeConfiguration compositeConfiguration = new CompositeConfiguration();
+ Configuration configuration = new PropertiesConfiguration(whirrConfig);
+ compositeConfiguration.addConfiguration(configuration);
+
+ ClusterSpec hadoopClusterSpec = new ClusterSpec(compositeConfiguration);
+
+ for (ClusterSpec.Property required : EnumSet.of(CLUSTER_NAME, PROVIDER, IDENTITY, CREDENTIAL,
+ INSTANCE_TEMPLATES, PRIVATE_KEY_FILE)) {
+ if (hadoopClusterSpec.getConfiguration().getString(required.getConfigName()) == null) {
+ throw new IllegalArgumentException(String.format("Option '%s' not set.",
+ required.getSimpleName()));
+ }
+ }
+
+ return hadoopClusterSpec;
+ }
+
+ private File getWhirrConfigurationFile(HostDescription hostDescription, File workingDirectory)
+ throws GFacHandlerException, IOException {
+ HadoopHostType hadoopHostDesc = (HadoopHostType)hostDescription;
+ if(hadoopHostDesc.isSetWhirrConfiguration()){
+ HadoopHostType.WhirrConfiguration whirrConfig = hadoopHostDesc.getWhirrConfiguration();
+ if(whirrConfig.isSetConfigurationFile()){
+ File whirrConfigFile = new File(whirrConfig.getConfigurationFile());
+ if(!whirrConfigFile.exists()){
+ throw new GFacHandlerException(
+ "Specified whirr configuration file doesn't exists.");
+ }
+
+ FileUtils.copyFileToDirectory(whirrConfigFile, workingDirectory);
+
+ return new File(workingDirectory, whirrConfigFile.getName());
+ } else if(whirrConfig.isSetConfiguration()){
+ Properties whirrConfigProps =
+ whirrConfigurationsToProperties(whirrConfig.getConfiguration());
+ File whirrConfigFile = new File(workingDirectory, "whirr-hadoop.config");
+ whirrConfigProps.store(
+ new FileOutputStream(whirrConfigFile), null);
+
+ return whirrConfigFile;
+ }
+ }
+
+ throw new GFacHandlerException("Cannot find Whirr configurations. Whirr configuration "
+ + "is required if you don't have already running Hadoop deployment.");
+ }
+
+ private Properties whirrConfigurationsToProperties(
+ HadoopHostType.WhirrConfiguration.Configuration configuration){
+ Properties whirrConfigProps = new Properties();
+
+ for(HadoopHostType.WhirrConfiguration.Configuration.Property property:
+ configuration.getPropertyArray()) {
+ whirrConfigProps.put(property.getName(), property.getValue());
+ }
+
+ return whirrConfigProps;
+ }
+
+ private void clusterPropertiesToHadoopSiteXml(Properties props, File hadoopSiteXml) throws ParserConfigurationException, TransformerException {
+ DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance();
+ DocumentBuilder documentBuilder = domFactory.newDocumentBuilder();
+
+ Document hadoopSiteXmlDoc = documentBuilder.newDocument();
+
+ hadoopSiteXmlDoc.setXmlVersion("1.0");
+ hadoopSiteXmlDoc.setXmlStandalone(true);
+ hadoopSiteXmlDoc.createProcessingInstruction("xml-stylesheet", "type=\"text/xsl\" href=\"configuration.xsl\"");
+
+ Element configEle = hadoopSiteXmlDoc.createElement("configuration");
+
+ hadoopSiteXmlDoc.appendChild(configEle);
+
+ for(Map.Entry<Object, Object> entry : props.entrySet()){
+ addPropertyToConfiguration(entry, configEle, hadoopSiteXmlDoc);
+ }
+
+ saveDomToFile(hadoopSiteXmlDoc, hadoopSiteXml);
+ }
+
+ private void saveDomToFile(Document dom, File destFile) throws TransformerException {
+ Source source = new DOMSource(dom);
+
+ Result result = new StreamResult(destFile);
+
+ Transformer transformer = TransformerFactory.newInstance().newTransformer();
+ transformer.transform(source, result);
+ }
+
+ private void addPropertyToConfiguration(Map.Entry<Object, Object> entry, Element configElement, Document doc){
+ Element property = doc.createElement("property");
+ configElement.appendChild(property);
+
+ Element nameEle = doc.createElement("name");
+ nameEle.setTextContent(entry.getKey().toString());
+ property.appendChild(nameEle);
+
+ Element valueEle = doc.createElement("value");
+ valueEle.setTextContent(entry.getValue().toString());
+ property.appendChild(valueEle);
+ }
+
+ private boolean isHadoopDeploymentAvailable(HostDescription hostDescription) {
+ return ((HadoopHostType) hostDescription.getType()).isSetHadoopConfigurationDirectory();
+ }
+
+ private String getHadoopConfigDirectory(HostDescription hostDescription){
+ return ((HadoopHostType)hostDescription.getType()).getHadoopConfigurationDirectory();
+ }
+
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/provider/impl/HadoopProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/provider/impl/HadoopProvider.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/provider/impl/HadoopProvider.java
new file mode 100644
index 0000000..f0eb06b
--- /dev/null
+++ b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/provider/impl/HadoopProvider.java
@@ -0,0 +1,154 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+
+package org.apache.airavata.gfac.hadoop.provider.impl;
+
+import java.io.File;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Map;
+
+import org.apache.airavata.commons.gfac.type.ActualParameter;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.context.MessageContext;
+import org.apache.airavata.gfac.hadoop.provider.utils.HadoopUtils;
+import org.apache.airavata.gfac.provider.AbstractProvider;
+import org.apache.airavata.gfac.provider.GFacProviderException;
+import org.apache.airavata.schemas.gfac.HadoopApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.OutputParameterType;
+import org.apache.airavata.schemas.gfac.StringParameterType;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import sun.reflect.generics.reflectiveObjects.NotImplementedException;
+
+/**
+ * Executes hadoop job using the cluster configuration provided by handlers in
+ * in-flow.
+ */
+public class HadoopProvider extends AbstractProvider {
+ private static final Logger logger = LoggerFactory.getLogger(HadoopProvider.class);
+
+ private boolean isWhirrBasedDeployment = false;
+ private File hadoopConfigDir;
+
+ public void initialize(JobExecutionContext jobExecutionContext) throws GFacProviderException {
+ MessageContext inMessageContext = jobExecutionContext.getInMessageContext();
+ if(inMessageContext.getParameter("HADOOP_DEPLOYMENT_TYPE").equals("WHIRR")){
+ isWhirrBasedDeployment = true;
+ } else {
+ String hadoopConfigDirPath = (String)inMessageContext.getParameter("HADOOP_CONFIG_DIR");
+ File hadoopConfigDir = new File(hadoopConfigDirPath);
+ if (!hadoopConfigDir.exists()){
+ throw new GFacProviderException("Specified hadoop configuration directory doesn't exist.");
+ } else if (FileUtils.listFiles(hadoopConfigDir, null, null).size() <= 0){
+ throw new GFacProviderException("Cannot find any hadoop configuration files inside specified directory.");
+ }
+
+ this.hadoopConfigDir = hadoopConfigDir;
+ }
+ }
+
+ public void execute(JobExecutionContext jobExecutionContext) throws GFacProviderException {
+ HadoopApplicationDeploymentDescriptionType hadoopAppDesc =
+ (HadoopApplicationDeploymentDescriptionType)jobExecutionContext
+ .getApplicationContext().getApplicationDeploymentDescription().getType();
+ MessageContext inMessageContext = jobExecutionContext.getInMessageContext();
+ HadoopApplicationDeploymentDescriptionType.HadoopJobConfiguration jobConf = hadoopAppDesc.getHadoopJobConfiguration();
+
+ try{
+ // Preparing Hadoop configuration
+ Configuration hadoopConf = HadoopUtils.createHadoopConfiguration(
+ jobExecutionContext, isWhirrBasedDeployment, hadoopConfigDir);
+
+ // Load jar containing map-reduce job implementation
+ ArrayList<URL> mapRedJars = new ArrayList<URL>();
+ mapRedJars.add(new File(jobConf.getJarLocation()).toURL());
+ URLClassLoader childClassLoader = new URLClassLoader(mapRedJars.toArray(new URL[mapRedJars.size()]),
+ this.getClass().getClassLoader());
+
+ Job job = new Job(hadoopConf);
+
+ job.setJobName(jobConf.getJobName());
+
+ job.setOutputKeyClass(Class.forName(jobConf.getOutputKeyClass(), true, childClassLoader));
+ job.setOutputValueClass(Class.forName(jobConf.getOutputValueClass(), true, childClassLoader));
+
+ job.setMapperClass((Class<? extends Mapper>)Class.forName(jobConf.getMapperClass(), true, childClassLoader));
+ job.setCombinerClass((Class<? extends Reducer>) Class.forName(jobConf.getCombinerClass(), true, childClassLoader));
+ job.setReducerClass((Class<? extends Reducer>) Class.forName(jobConf.getCombinerClass(), true, childClassLoader));
+
+ job.setInputFormatClass((Class<? extends InputFormat>)Class.forName(jobConf.getInputFormatClass(), true, childClassLoader));
+ job.setOutputFormatClass((Class<? extends OutputFormat>) Class.forName(jobConf.getOutputFormatClass(), true, childClassLoader));
+
+ FileInputFormat.setInputPaths(job, new Path(hadoopAppDesc.getInputDataDirectory()));
+ FileOutputFormat.setOutputPath(job, new Path(hadoopAppDesc.getOutputDataDirectory()));
+
+ job.waitForCompletion(true);
+ System.out.println(job.getTrackingURL());
+ if(jobExecutionContext.getOutMessageContext() == null){
+ jobExecutionContext.setOutMessageContext(new MessageContext());
+ }
+
+ OutputParameterType[] outputParametersArray = jobExecutionContext.getApplicationContext().
+ getServiceDescription().getType().getOutputParametersArray();
+ for(OutputParameterType outparamType : outputParametersArray){
+ String paramName = outparamType.getParameterName();
+ if(paramName.equals("test-hadoop")){
+ ActualParameter outParam = new ActualParameter();
+ outParam.getType().changeType(StringParameterType.type);
+ ((StringParameterType) outParam.getType()).setValue(job.getTrackingURL());
+ jobExecutionContext.getOutMessageContext().addParameter("test-hadoop", outParam);
+ }
+ }
+ } catch (Exception e) {
+ String errMessage = "Error occurred during Map-Reduce job execution.";
+ logger.error(errMessage, e);
+ throw new GFacProviderException(errMessage, e);
+ }
+ }
+
+ public void dispose(JobExecutionContext jobExecutionContext) throws GFacProviderException {
+ // TODO: How to handle cluster shutdown. Best way is to introduce inPath/outPath to handler.
+ }
+
+ @Override
+ public void cancelJob(String jobId, JobExecutionContext jobExecutionContext) throws GFacException {
+ throw new NotImplementedException();
+ }
+
+
+ public void initProperties(Map<String, String> properties) throws GFacProviderException, GFacException {
+
+ }
+}
[3/8] changing package names of gfac implementations
Posted by la...@apache.org.
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/provider/utils/HadoopUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/provider/utils/HadoopUtils.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/provider/utils/HadoopUtils.java
new file mode 100644
index 0000000..2947fc3
--- /dev/null
+++ b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/hadoop/provider/utils/HadoopUtils.java
@@ -0,0 +1,60 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+
+package org.apache.airavata.gfac.hadoop.provider.utils;
+
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.context.MessageContext;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.util.Collection;
+
+public class HadoopUtils {
+ public static Configuration createHadoopConfiguration(
+ JobExecutionContext jobExecutionContext,
+ boolean isWhirrBasedDeployment,
+ File hadoopConfigDir) throws FileNotFoundException {
+ MessageContext inMessageContext = jobExecutionContext.getInMessageContext();
+ Configuration hadoopConf = new Configuration();
+
+ if(isWhirrBasedDeployment){
+ hadoopConf.addResource(new FileInputStream(
+ new File((String)inMessageContext.getParameter("HADOOP_SITE_XML"))));
+ } else {
+ readHadoopClusterConfigurationFromDirectory(hadoopConfigDir, hadoopConf);
+ }
+
+ return hadoopConf;
+ }
+
+ private static void readHadoopClusterConfigurationFromDirectory(File localHadoopConfigurationDirectory, Configuration hadoopConf)
+ throws FileNotFoundException {
+ Collection hadoopConfigurationFiles =
+ FileUtils.listFiles(localHadoopConfigurationDirectory, null, false);
+ for (Object f : hadoopConfigurationFiles) {
+ hadoopConf.addResource(new FileInputStream((File)f));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HDFSDataMovementHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HDFSDataMovementHandler.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HDFSDataMovementHandler.java
deleted file mode 100644
index 49eadf4..0000000
--- a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HDFSDataMovementHandler.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-
-package org.apache.airavata.gfac.handler;
-
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.gfac.provider.utils.HadoopUtils;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.apache.airavata.schemas.gfac.HadoopApplicationDeploymentDescriptionType;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Map;
-
-public class HDFSDataMovementHandler implements GFacHandler {
- private static final Logger logger = LoggerFactory.getLogger(HDFSDataMovementHandler.class);
-
- private boolean isWhirrBasedDeployment = false;
- private File hadoopConfigDir;
-
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- MessageContext inMessageContext = jobExecutionContext.getInMessageContext();
- if(inMessageContext.getParameter("HADOOP_DEPLOYMENT_TYPE").equals("WHIRR")){
- isWhirrBasedDeployment = true;
- } else {
- String hadoopConfigDirPath = (String)inMessageContext.getParameter("HADOOP_CONFIG_DIR");
- File hadoopConfigDir = new File(hadoopConfigDirPath);
- if (!hadoopConfigDir.exists()){
- throw new GFacHandlerException("Specified hadoop configuration directory doesn't exist.");
- } else if (FileUtils.listFiles(hadoopConfigDir, null, null).size() <= 0){
- throw new GFacHandlerException("Cannot find any hadoop configuration files inside specified directory.");
- }
-
- this.hadoopConfigDir = hadoopConfigDir;
- }
-
- if(jobExecutionContext.isInPath()){
- try {
- handleInPath(jobExecutionContext);
- } catch (IOException e) {
- throw new GFacHandlerException("Error while copying input data from local file system to HDFS.",e);
- }
- } else {
- handleOutPath(jobExecutionContext);
- }
- }
-
- private void handleInPath(JobExecutionContext jobExecutionContext) throws GFacHandlerException, IOException {
- ApplicationDeploymentDescriptionType appDepDesc =
- jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
- HadoopApplicationDeploymentDescriptionType hadoopAppDesc =
- (HadoopApplicationDeploymentDescriptionType)appDepDesc;
- if(appDepDesc.isSetInputDataDirectory() && isInputDataDirectoryLocal(appDepDesc)){
- Configuration hadoopConf = HadoopUtils.createHadoopConfiguration(jobExecutionContext, isWhirrBasedDeployment, hadoopConfigDir);
- FileSystem hdfs = FileSystem.get(hadoopConf);
- hdfs.copyFromLocalFile(new Path(appDepDesc.getInputDataDirectory()),
- new Path(hadoopAppDesc.getHadoopJobConfiguration().getHdfsInputDirectory()));
- }
- }
-
- private boolean isInputDataDirectoryLocal(ApplicationDeploymentDescriptionType appDepDesc){
- String inputDataDirectoryPath = appDepDesc.getInputDataDirectory();
- File inputDataDirectory = new File(inputDataDirectoryPath);
- if(inputDataDirectory.exists() && FileUtils.listFiles(inputDataDirectory, null, null).size() > 0){
- return true;
- }
-
- return false;
- }
-
- private void handleOutPath(JobExecutionContext jobExecutionContext){}
-
- public void initProperties(Map<String, String> properties) throws GFacHandlerException {
-
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HadoopDeploymentHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HadoopDeploymentHandler.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HadoopDeploymentHandler.java
deleted file mode 100644
index c7de3f9..0000000
--- a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/handler/HadoopDeploymentHandler.java
+++ /dev/null
@@ -1,274 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-
-package org.apache.airavata.gfac.handler;
-
-import com.google.common.io.Files;
-import org.apache.airavata.commons.gfac.type.HostDescription;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.schemas.gfac.HadoopHostType;
-import org.apache.commons.configuration.CompositeConfiguration;
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.apache.commons.io.FileUtils;
-import org.apache.whirr.Cluster;
-import org.apache.whirr.ClusterController;
-import org.apache.whirr.ClusterControllerFactory;
-import org.apache.whirr.ClusterSpec;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.transform.*;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.Map;
-import java.util.Properties;
-
-import static org.apache.whirr.ClusterSpec.Property.*;
-import static org.apache.whirr.ClusterSpec.Property.INSTANCE_TEMPLATES;
-import static org.apache.whirr.ClusterSpec.Property.PRIVATE_KEY_FILE;
-
-/**
- * This handler takes care of deploying hadoop in cloud(in cloud bursting scenarios) and
- * deploying hadoop in local cluster. In case of existing hadoop cluster this will ignore
- * cluster setup just use the hadoop configuration provided by user.
- */
-public class HadoopDeploymentHandler implements GFacHandler {
- private static final Logger logger = LoggerFactory.getLogger("hadoop-dep-handler");
-
- /**
- * Once invoked this method will deploy Hadoop in a local cluster or cloud based on the
- * configuration provided. If there is a already deployed hadoop cluster this will skip
- * deployment.
- *
- *
- * @param jobExecutionContext job execution context containing all the required configurations
- * and runtime information.
- * @throws GFacHandlerException
- */
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- if(jobExecutionContext.isInPath()){
- handleInPath(jobExecutionContext);
- } else {
- handleOutPath(jobExecutionContext);
- }
- }
-
- private void handleInPath(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- HostDescription hostDescription =
- jobExecutionContext.getApplicationContext().getHostDescription();
- if (!isHadoopDeploymentAvailable(hostDescription)) {
- // Temp directory to keep generated configuration files.
- File tempDirectory = Files.createTempDir();
- try {
- File hadoopSiteXML = launchHadoopCluster(hostDescription, tempDirectory);
- jobExecutionContext.getInMessageContext().addParameter("HADOOP_SITE_XML", hadoopSiteXML.getAbsolutePath());
- jobExecutionContext.getInMessageContext().addParameter("HADOOP_DEPLOYMENT_TYPE", "WHIRR");
- // TODO: Add hadoop-site.xml to job execution context.
- } catch (IOException e) {
- throw new GFacHandlerException("IO Error while processing configurations.",e);
- } catch (ConfigurationException e) {
- throw new GFacHandlerException("Whirr configuration error.", e);
- } catch (InterruptedException e) {
- throw new GFacHandlerException("Hadoop cluster launch interrupted.", e);
- } catch (TransformerException e) {
- throw new GFacHandlerException("Error while creating hadoop-site.xml", e);
- } catch (ParserConfigurationException e) {
- throw new GFacHandlerException("Error while creating hadoop-site.xml", e);
- }
- } else {
- jobExecutionContext.getInMessageContext().addParameter("HADOOP_DEPLOYMENT_TYPE",
- "MANUAL");
- jobExecutionContext.getInMessageContext().addParameter("HADOOP_CONFIG_DIR",
- ((HadoopHostType)hostDescription.getType()).getHadoopConfigurationDirectory());
- logger.info("Hadoop configuration is available. Skipping hadoop deployment.");
- if(logger.isDebugEnabled()){
- logger.debug("Hadoop configuration directory: " +
- getHadoopConfigDirectory(hostDescription));
- }
- }
- }
-
- private void handleOutPath(JobExecutionContext jobExecutionContext){
- MessageContext inMessageContext = jobExecutionContext.getInMessageContext();
- if(((String)inMessageContext.getParameter("HADOOP_DEPLOYMENT_TYPE")).equals("WHIRR")){
- // TODO: Shutdown hadoop cluster.
- logger.info("Shutdown hadoop cluster.");
- }
- }
-
- private File launchHadoopCluster(HostDescription hostDescription, File workingDirectory)
- throws IOException, GFacHandlerException, ConfigurationException, InterruptedException, TransformerException, ParserConfigurationException {
- ClusterSpec hadoopClusterSpec =
- whirrConfigurationToClusterSpec(hostDescription, workingDirectory);
- ClusterController hadoopClusterController =
- createClusterController(hadoopClusterSpec.getServiceName());
- Cluster hadoopCluster = hadoopClusterController.launchCluster(hadoopClusterSpec);
-
- logger.info(String.format("Started cluster of %s instances.\n",
- hadoopCluster.getInstances().size()));
-
- File siteXML = new File(workingDirectory, "hadoop-site.xml");
- clusterPropertiesToHadoopSiteXml(hadoopCluster.getConfiguration(), siteXML);
-
- return siteXML;
- }
-
- private ClusterController createClusterController(String serviceName){
- ClusterControllerFactory factory = new ClusterControllerFactory();
- ClusterController controller = factory.create(serviceName);
-
- if(controller == null){
- logger.warn("Unable to find the service {0}, using default.", serviceName);
- controller = factory.create(null);
- }
-
- return controller;
- }
-
- private ClusterSpec whirrConfigurationToClusterSpec(HostDescription hostDescription,
- File workingDirectory) throws IOException, GFacHandlerException, ConfigurationException {
- File whirrConfig = getWhirrConfigurationFile(hostDescription, workingDirectory);
- CompositeConfiguration compositeConfiguration = new CompositeConfiguration();
- Configuration configuration = new PropertiesConfiguration(whirrConfig);
- compositeConfiguration.addConfiguration(configuration);
-
- ClusterSpec hadoopClusterSpec = new ClusterSpec(compositeConfiguration);
-
- for (ClusterSpec.Property required : EnumSet.of(CLUSTER_NAME, PROVIDER, IDENTITY, CREDENTIAL,
- INSTANCE_TEMPLATES, PRIVATE_KEY_FILE)) {
- if (hadoopClusterSpec.getConfiguration().getString(required.getConfigName()) == null) {
- throw new IllegalArgumentException(String.format("Option '%s' not set.",
- required.getSimpleName()));
- }
- }
-
- return hadoopClusterSpec;
- }
-
- private File getWhirrConfigurationFile(HostDescription hostDescription, File workingDirectory)
- throws GFacHandlerException, IOException {
- HadoopHostType hadoopHostDesc = (HadoopHostType)hostDescription;
- if(hadoopHostDesc.isSetWhirrConfiguration()){
- HadoopHostType.WhirrConfiguration whirrConfig = hadoopHostDesc.getWhirrConfiguration();
- if(whirrConfig.isSetConfigurationFile()){
- File whirrConfigFile = new File(whirrConfig.getConfigurationFile());
- if(!whirrConfigFile.exists()){
- throw new GFacHandlerException(
- "Specified whirr configuration file doesn't exists.");
- }
-
- FileUtils.copyFileToDirectory(whirrConfigFile, workingDirectory);
-
- return new File(workingDirectory, whirrConfigFile.getName());
- } else if(whirrConfig.isSetConfiguration()){
- Properties whirrConfigProps =
- whirrConfigurationsToProperties(whirrConfig.getConfiguration());
- File whirrConfigFile = new File(workingDirectory, "whirr-hadoop.config");
- whirrConfigProps.store(
- new FileOutputStream(whirrConfigFile), null);
-
- return whirrConfigFile;
- }
- }
-
- throw new GFacHandlerException("Cannot find Whirr configurations. Whirr configuration "
- + "is required if you don't have already running Hadoop deployment.");
- }
-
- private Properties whirrConfigurationsToProperties(
- HadoopHostType.WhirrConfiguration.Configuration configuration){
- Properties whirrConfigProps = new Properties();
-
- for(HadoopHostType.WhirrConfiguration.Configuration.Property property:
- configuration.getPropertyArray()) {
- whirrConfigProps.put(property.getName(), property.getValue());
- }
-
- return whirrConfigProps;
- }
-
- private void clusterPropertiesToHadoopSiteXml(Properties props, File hadoopSiteXml) throws ParserConfigurationException, TransformerException {
- DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance();
- DocumentBuilder documentBuilder = domFactory.newDocumentBuilder();
-
- Document hadoopSiteXmlDoc = documentBuilder.newDocument();
-
- hadoopSiteXmlDoc.setXmlVersion("1.0");
- hadoopSiteXmlDoc.setXmlStandalone(true);
- hadoopSiteXmlDoc.createProcessingInstruction("xml-stylesheet", "type=\"text/xsl\" href=\"configuration.xsl\"");
-
- Element configEle = hadoopSiteXmlDoc.createElement("configuration");
-
- hadoopSiteXmlDoc.appendChild(configEle);
-
- for(Map.Entry<Object, Object> entry : props.entrySet()){
- addPropertyToConfiguration(entry, configEle, hadoopSiteXmlDoc);
- }
-
- saveDomToFile(hadoopSiteXmlDoc, hadoopSiteXml);
- }
-
- private void saveDomToFile(Document dom, File destFile) throws TransformerException {
- Source source = new DOMSource(dom);
-
- Result result = new StreamResult(destFile);
-
- Transformer transformer = TransformerFactory.newInstance().newTransformer();
- transformer.transform(source, result);
- }
-
- private void addPropertyToConfiguration(Map.Entry<Object, Object> entry, Element configElement, Document doc){
- Element property = doc.createElement("property");
- configElement.appendChild(property);
-
- Element nameEle = doc.createElement("name");
- nameEle.setTextContent(entry.getKey().toString());
- property.appendChild(nameEle);
-
- Element valueEle = doc.createElement("value");
- valueEle.setTextContent(entry.getValue().toString());
- property.appendChild(valueEle);
- }
-
- private boolean isHadoopDeploymentAvailable(HostDescription hostDescription) {
- return ((HadoopHostType) hostDescription.getType()).isSetHadoopConfigurationDirectory();
- }
-
- private String getHadoopConfigDirectory(HostDescription hostDescription){
- return ((HadoopHostType)hostDescription.getType()).getHadoopConfigurationDirectory();
- }
-
- public void initProperties(Map<String, String> properties) throws GFacHandlerException {
-
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/impl/HadoopProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/impl/HadoopProvider.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/impl/HadoopProvider.java
deleted file mode 100644
index 0fc8b47..0000000
--- a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/impl/HadoopProvider.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-
-package org.apache.airavata.gfac.provider.impl;
-
-import java.io.File;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.ArrayList;
-import java.util.Map;
-
-import org.apache.airavata.commons.gfac.type.ActualParameter;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.gfac.provider.AbstractProvider;
-import org.apache.airavata.gfac.provider.GFacProviderException;
-import org.apache.airavata.gfac.provider.utils.HadoopUtils;
-import org.apache.airavata.schemas.gfac.HadoopApplicationDeploymentDescriptionType;
-import org.apache.airavata.schemas.gfac.OutputParameterType;
-import org.apache.airavata.schemas.gfac.StringParameterType;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.OutputFormat;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import sun.reflect.generics.reflectiveObjects.NotImplementedException;
-
-/**
- * Executes hadoop job using the cluster configuration provided by handlers in
- * in-flow.
- */
-public class HadoopProvider extends AbstractProvider {
- private static final Logger logger = LoggerFactory.getLogger(HadoopProvider.class);
-
- private boolean isWhirrBasedDeployment = false;
- private File hadoopConfigDir;
-
- public void initialize(JobExecutionContext jobExecutionContext) throws GFacProviderException {
- MessageContext inMessageContext = jobExecutionContext.getInMessageContext();
- if(inMessageContext.getParameter("HADOOP_DEPLOYMENT_TYPE").equals("WHIRR")){
- isWhirrBasedDeployment = true;
- } else {
- String hadoopConfigDirPath = (String)inMessageContext.getParameter("HADOOP_CONFIG_DIR");
- File hadoopConfigDir = new File(hadoopConfigDirPath);
- if (!hadoopConfigDir.exists()){
- throw new GFacProviderException("Specified hadoop configuration directory doesn't exist.");
- } else if (FileUtils.listFiles(hadoopConfigDir, null, null).size() <= 0){
- throw new GFacProviderException("Cannot find any hadoop configuration files inside specified directory.");
- }
-
- this.hadoopConfigDir = hadoopConfigDir;
- }
- }
-
- public void execute(JobExecutionContext jobExecutionContext) throws GFacProviderException {
- HadoopApplicationDeploymentDescriptionType hadoopAppDesc =
- (HadoopApplicationDeploymentDescriptionType)jobExecutionContext
- .getApplicationContext().getApplicationDeploymentDescription().getType();
- MessageContext inMessageContext = jobExecutionContext.getInMessageContext();
- HadoopApplicationDeploymentDescriptionType.HadoopJobConfiguration jobConf = hadoopAppDesc.getHadoopJobConfiguration();
-
- try{
- // Preparing Hadoop configuration
- Configuration hadoopConf = HadoopUtils.createHadoopConfiguration(
- jobExecutionContext, isWhirrBasedDeployment, hadoopConfigDir);
-
- // Load jar containing map-reduce job implementation
- ArrayList<URL> mapRedJars = new ArrayList<URL>();
- mapRedJars.add(new File(jobConf.getJarLocation()).toURL());
- URLClassLoader childClassLoader = new URLClassLoader(mapRedJars.toArray(new URL[mapRedJars.size()]),
- this.getClass().getClassLoader());
-
- Job job = new Job(hadoopConf);
-
- job.setJobName(jobConf.getJobName());
-
- job.setOutputKeyClass(Class.forName(jobConf.getOutputKeyClass(), true, childClassLoader));
- job.setOutputValueClass(Class.forName(jobConf.getOutputValueClass(), true, childClassLoader));
-
- job.setMapperClass((Class<? extends Mapper>)Class.forName(jobConf.getMapperClass(), true, childClassLoader));
- job.setCombinerClass((Class<? extends Reducer>) Class.forName(jobConf.getCombinerClass(), true, childClassLoader));
- job.setReducerClass((Class<? extends Reducer>) Class.forName(jobConf.getCombinerClass(), true, childClassLoader));
-
- job.setInputFormatClass((Class<? extends InputFormat>)Class.forName(jobConf.getInputFormatClass(), true, childClassLoader));
- job.setOutputFormatClass((Class<? extends OutputFormat>) Class.forName(jobConf.getOutputFormatClass(), true, childClassLoader));
-
- FileInputFormat.setInputPaths(job, new Path(hadoopAppDesc.getInputDataDirectory()));
- FileOutputFormat.setOutputPath(job, new Path(hadoopAppDesc.getOutputDataDirectory()));
-
- job.waitForCompletion(true);
- System.out.println(job.getTrackingURL());
- if(jobExecutionContext.getOutMessageContext() == null){
- jobExecutionContext.setOutMessageContext(new MessageContext());
- }
-
- OutputParameterType[] outputParametersArray = jobExecutionContext.getApplicationContext().
- getServiceDescription().getType().getOutputParametersArray();
- for(OutputParameterType outparamType : outputParametersArray){
- String paramName = outparamType.getParameterName();
- if(paramName.equals("test-hadoop")){
- ActualParameter outParam = new ActualParameter();
- outParam.getType().changeType(StringParameterType.type);
- ((StringParameterType) outParam.getType()).setValue(job.getTrackingURL());
- jobExecutionContext.getOutMessageContext().addParameter("test-hadoop", outParam);
- }
- }
- } catch (Exception e) {
- String errMessage = "Error occurred during Map-Reduce job execution.";
- logger.error(errMessage, e);
- throw new GFacProviderException(errMessage, e);
- }
- }
-
- public void dispose(JobExecutionContext jobExecutionContext) throws GFacProviderException {
- // TODO: How to handle cluster shutdown. Best way is to introduce inPath/outPath to handler.
- }
-
- @Override
- public void cancelJob(String jobId, JobExecutionContext jobExecutionContext) throws GFacException {
- throw new NotImplementedException();
- }
-
-
- public void initProperties(Map<String, String> properties) throws GFacProviderException, GFacException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/utils/HadoopUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/utils/HadoopUtils.java b/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/utils/HadoopUtils.java
deleted file mode 100644
index c3053d1..0000000
--- a/modules/gfac/gfac-hadoop/src/main/java/org/apache/airavata/gfac/provider/utils/HadoopUtils.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-
-package org.apache.airavata.gfac.provider.utils;
-
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.util.Collection;
-
-public class HadoopUtils {
- public static Configuration createHadoopConfiguration(
- JobExecutionContext jobExecutionContext,
- boolean isWhirrBasedDeployment,
- File hadoopConfigDir) throws FileNotFoundException {
- MessageContext inMessageContext = jobExecutionContext.getInMessageContext();
- Configuration hadoopConf = new Configuration();
-
- if(isWhirrBasedDeployment){
- hadoopConf.addResource(new FileInputStream(
- new File((String)inMessageContext.getParameter("HADOOP_SITE_XML"))));
- } else {
- readHadoopClusterConfigurationFromDirectory(hadoopConfigDir, hadoopConf);
- }
-
- return hadoopConf;
- }
-
- private static void readHadoopClusterConfigurationFromDirectory(File localHadoopConfigurationDirectory, Configuration hadoopConf)
- throws FileNotFoundException {
- Collection hadoopConfigurationFiles =
- FileUtils.listFiles(localHadoopConfigurationDirectory, null, false);
- for (Object f : hadoopConfigurationFiles) {
- hadoopConf.addResource(new FileInputStream((File)f));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-hadoop/src/test/resources/gfac-config.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-hadoop/src/test/resources/gfac-config.xml b/modules/gfac/gfac-hadoop/src/test/resources/gfac-config.xml
index f43a5fb..9c4774c 100644
--- a/modules/gfac/gfac-hadoop/src/test/resources/gfac-config.xml
+++ b/modules/gfac/gfac-hadoop/src/test/resources/gfac-config.xml
@@ -20,21 +20,11 @@
<OutHandlers></OutHandlers>
</GlobalHandlers>
- <Provider class="org.apache.airavata.gfac.provider.impl.HadoopProvider" host="org.apache.airavata.schemas.gfac.impl.HadoopHostTypeImpl">
+ <Provider class="org.apache.airavata.gfac.hadoop.provider.impl.HadoopProvider" host="org.apache.airavata.schemas.gfac.impl.HadoopHostTypeImpl">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.HadoopDeploymentHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.HDFSDataMovementHandler"/>
+ <Handler class="org.apache.airavata.gfac.hadoop.handler.HadoopDeploymentHandler"/>
+ <Handler class="org.apache.airavata.gfac.hadoop.handler.HDFSDataMovementHandler"/>
</InHandlers>
<OutHandlers/>
</Provider>
-
- <Application name="UltraScan">
- <InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler"/>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
- </InHandlers>
- <OutHandlers>
- <Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
- </OutHandlers>
- </Application>
</GFac>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/handler/LocalDirectorySetupHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/handler/LocalDirectorySetupHandler.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/handler/LocalDirectorySetupHandler.java
deleted file mode 100644
index 8a11c2a..0000000
--- a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/handler/LocalDirectorySetupHandler.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.handler;
-
-import org.apache.airavata.commons.gfac.type.ApplicationDescription;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.apache.airavata.schemas.gfac.HostDescriptionType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.Map;
-
-public class LocalDirectorySetupHandler implements GFacHandler{
- private static final Logger log = LoggerFactory.getLogger(LocalDirectorySetupHandler.class);
-
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- log.info("Invoking GramDirectorySetupHandler ...");
- HostDescriptionType type = jobExecutionContext.getApplicationContext().getHostDescription().getType();
- ApplicationDescription applicationDeploymentDescription = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription();
- ApplicationDeploymentDescriptionType app = applicationDeploymentDescription.getType();
- log.debug("working diectroy = " + app.getStaticWorkingDirectory());
- log.debug("temp directory = " + app.getScratchWorkingDirectory());
-
- makeFileSystemDir(app.getStaticWorkingDirectory(),jobExecutionContext);
- makeFileSystemDir(app.getScratchWorkingDirectory(),jobExecutionContext);
- makeFileSystemDir(app.getInputDataDirectory(),jobExecutionContext);
- makeFileSystemDir(app.getOutputDataDirectory(),jobExecutionContext);
- }
- private void makeFileSystemDir(String dir, JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- File f = new File(dir);
- if (f.isDirectory() && f.exists()) {
- return;
- } else if (!new File(dir).mkdir()) {
- throw new GFacHandlerException("Cannot make directory "+dir);
- }
- }
-
- public void initProperties(Map<String, String> properties) throws GFacHandlerException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/handler/LocalDirectorySetupHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/handler/LocalDirectorySetupHandler.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/handler/LocalDirectorySetupHandler.java
new file mode 100644
index 0000000..81493fb
--- /dev/null
+++ b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/handler/LocalDirectorySetupHandler.java
@@ -0,0 +1,63 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.local.handler;
+
+import org.apache.airavata.commons.gfac.type.ApplicationDescription;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.handler.GFacHandler;
+import org.apache.airavata.gfac.handler.GFacHandlerException;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.HostDescriptionType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.Map;
+
+public class LocalDirectorySetupHandler implements GFacHandler {
+ private static final Logger log = LoggerFactory.getLogger(LocalDirectorySetupHandler.class);
+
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ log.info("Invoking GramDirectorySetupHandler ...");
+ HostDescriptionType type = jobExecutionContext.getApplicationContext().getHostDescription().getType();
+ ApplicationDescription applicationDeploymentDescription = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription();
+ ApplicationDeploymentDescriptionType app = applicationDeploymentDescription.getType();
+ log.debug("working diectroy = " + app.getStaticWorkingDirectory());
+ log.debug("temp directory = " + app.getScratchWorkingDirectory());
+
+ makeFileSystemDir(app.getStaticWorkingDirectory(),jobExecutionContext);
+ makeFileSystemDir(app.getScratchWorkingDirectory(),jobExecutionContext);
+ makeFileSystemDir(app.getInputDataDirectory(),jobExecutionContext);
+ makeFileSystemDir(app.getOutputDataDirectory(),jobExecutionContext);
+ }
+ private void makeFileSystemDir(String dir, JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ File f = new File(dir);
+ if (f.isDirectory() && f.exists()) {
+ return;
+ } else if (!new File(dir).mkdir()) {
+ throw new GFacHandlerException("Cannot make directory "+dir);
+ }
+ }
+
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/impl/LocalProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/impl/LocalProvider.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/impl/LocalProvider.java
new file mode 100644
index 0000000..f0a0bf9
--- /dev/null
+++ b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/impl/LocalProvider.java
@@ -0,0 +1,239 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.local.impl;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.airavata.gfac.Constants;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.local.utils.InputStreamToFileWriter;
+import org.apache.airavata.gfac.local.utils.InputUtils;
+import org.apache.airavata.gfac.notification.events.StartExecutionEvent;
+import org.apache.airavata.gfac.provider.AbstractProvider;
+import org.apache.airavata.gfac.provider.GFacProviderException;
+import org.apache.airavata.gfac.provider.utils.ProviderUtils;
+import org.apache.airavata.gfac.utils.GFacUtils;
+import org.apache.airavata.gfac.utils.OutputUtils;
+import org.apache.airavata.model.workspace.experiment.JobDetails;
+import org.apache.airavata.model.workspace.experiment.JobState;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.NameValuePairType;
+import org.apache.xmlbeans.XmlException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import sun.reflect.generics.reflectiveObjects.NotImplementedException;
+
+public class LocalProvider extends AbstractProvider {
+ private static final Logger log = LoggerFactory.getLogger(LocalProvider.class);
+ private ProcessBuilder builder;
+ private List<String> cmdList;
+ private String jobId;
+
+ public static class LocalProviderJobData{
+ private String applicationName;
+ private List<String> inputParameters;
+ private String workingDir;
+ private String inputDir;
+ private String outputDir;
+ public String getApplicationName() {
+ return applicationName;
+ }
+ public void setApplicationName(String applicationName) {
+ this.applicationName = applicationName;
+ }
+ public List<String> getInputParameters() {
+ return inputParameters;
+ }
+ public void setInputParameters(List<String> inputParameters) {
+ this.inputParameters = inputParameters;
+ }
+ public String getWorkingDir() {
+ return workingDir;
+ }
+ public void setWorkingDir(String workingDir) {
+ this.workingDir = workingDir;
+ }
+ public String getInputDir() {
+ return inputDir;
+ }
+ public void setInputDir(String inputDir) {
+ this.inputDir = inputDir;
+ }
+ public String getOutputDir() {
+ return outputDir;
+ }
+ public void setOutputDir(String outputDir) {
+ this.outputDir = outputDir;
+ }
+ }
+ public LocalProvider(){
+ cmdList = new ArrayList<String>();
+ }
+
+ public void initialize(JobExecutionContext jobExecutionContext) throws GFacProviderException,GFacException {
+ super.initialize(jobExecutionContext);
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().
+ getApplicationDeploymentDescription().getType();
+
+ buildCommand(app.getExecutableLocation(), ProviderUtils.getInputParameters(jobExecutionContext));
+ initProcessBuilder(app);
+
+ // extra environment variables
+ builder.environment().put(Constants.INPUT_DATA_DIR_VAR_NAME, app.getInputDataDirectory());
+ builder.environment().put(Constants.OUTPUT_DATA_DIR_VAR_NAME, app.getOutputDataDirectory());
+
+ // set working directory
+ builder.directory(new File(app.getStaticWorkingDirectory()));
+
+ // log info
+ log.info("Command = " + InputUtils.buildCommand(cmdList));
+ log.info("Working dir = " + builder.directory());
+ for (String key : builder.environment().keySet()) {
+ log.info("Env[" + key + "] = " + builder.environment().get(key));
+ }
+ }
+
+ public void execute(JobExecutionContext jobExecutionContext) throws GFacProviderException {
+ jobExecutionContext.getNotifier().publish(new StartExecutionEvent());
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.
+ getApplicationContext().getApplicationDeploymentDescription().getType();
+ JobDetails jobDetails = new JobDetails();
+ try {
+ jobId = jobExecutionContext.getTaskData().getTaskID();
+ jobDetails.setJobID(jobId);
+ jobDetails.setJobDescription(app.toString());
+ jobExecutionContext.setJobDetails(jobDetails);
+ jobDetails.setJobDescription(app.toString());
+ GFacUtils.saveJobStatus(jobExecutionContext,jobDetails, JobState.SETUP);
+ // running cmd
+ Process process = builder.start();
+
+ Thread standardOutWriter = new InputStreamToFileWriter(process.getInputStream(), app.getStandardOutput());
+ Thread standardErrorWriter = new InputStreamToFileWriter(process.getErrorStream(), app.getStandardError());
+
+ // start output threads
+ standardOutWriter.setDaemon(true);
+ standardErrorWriter.setDaemon(true);
+ standardOutWriter.start();
+ standardErrorWriter.start();
+
+ int returnValue = process.waitFor();
+
+ // make sure other two threads are done
+ standardOutWriter.join();
+ standardErrorWriter.join();
+
+ /*
+ * check return value. usually not very helpful to draw conclusions based on return values so don't bother.
+ * just provide warning in the log messages
+ */
+ if (returnValue != 0) {
+ log.error("Process finished with non zero return value. Process may have failed");
+ } else {
+ log.info("Process finished with return value of zero.");
+ }
+
+ StringBuffer buf = new StringBuffer();
+ buf.append("Executed ").append(InputUtils.buildCommand(cmdList))
+ .append(" on the localHost, working directory = ").append(app.getStaticWorkingDirectory())
+ .append(" tempDirectory = ").append(app.getScratchWorkingDirectory()).append(" With the status ")
+ .append(String.valueOf(returnValue));
+ log.info(buf.toString());
+ } catch (IOException io) {
+ throw new GFacProviderException(io.getMessage(), io);
+ } catch (InterruptedException e) {
+ throw new GFacProviderException(e.getMessage(), e);
+ }catch (GFacException e) {
+ throw new GFacProviderException(e.getMessage(), e);
+ }
+ }
+
+// private void saveApplicationJob(JobExecutionContext jobExecutionContext)
+// throws GFacProviderException {
+// ApplicationDeploymentDescriptionType app = jobExecutionContext.
+// getApplicationContext().getApplicationDeploymentDescription().getType();
+// ApplicationJob appJob = GFacUtils.createApplicationJob(jobExecutionContext);
+// appJob.setJobId(jobId);
+// LocalProviderJobData data = new LocalProviderJobData();
+// data.setApplicationName(app.getExecutableLocation());
+// data.setInputDir(app.getInputDataDirectory());
+// data.setOutputDir(app.getOutputDataDirectory());
+// data.setWorkingDir(builder.directory().toString());
+// data.setInputParameters(ProviderUtils.getInputParameters(jobExecutionContext));
+// ByteArrayOutputStream stream = new ByteArrayOutputStream();
+// JAXB.marshal(data, stream);
+// appJob.setJobData(stream.toString());
+// appJob.setSubmittedTime(Calendar.getInstance().getTime());
+// appJob.setStatus(ApplicationJobStatus.SUBMITTED);
+// appJob.setStatusUpdateTime(appJob.getSubmittedTime());
+// GFacUtils.recordApplicationJob(jobExecutionContext, appJob);
+// }
+
+ public void dispose(JobExecutionContext jobExecutionContext) throws GFacProviderException {
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
+
+ try {
+ String stdOutStr = GFacUtils.readFileToString(app.getStandardOutput());
+ String stdErrStr = GFacUtils.readFileToString(app.getStandardError());
+ Map<String, Object> output = jobExecutionContext.getOutMessageContext().getParameters();
+ OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr);
+ } catch (XmlException e) {
+ throw new GFacProviderException("Cannot read output:" + e.getMessage(), e);
+ } catch (IOException io) {
+ throw new GFacProviderException(io.getMessage(), io);
+ } catch (Exception e){
+ throw new GFacProviderException("Error in retrieving results",e);
+ }
+ }
+
+ public void cancelJob(String jobId, JobExecutionContext jobExecutionContext) throws GFacException {
+ throw new NotImplementedException();
+ }
+
+
+ private void buildCommand(String executable, List<String> inputParameterList){
+ cmdList.add(executable);
+ cmdList.addAll(inputParameterList);
+ }
+
+ private void initProcessBuilder(ApplicationDeploymentDescriptionType app){
+ builder = new ProcessBuilder(cmdList);
+
+ NameValuePairType[] env = app.getApplicationEnvironmentArray();
+
+ if(env != null && env.length > 0){
+ Map<String,String> builderEnv = builder.environment();
+ for (NameValuePairType entry : env) {
+ builderEnv.put(entry.getName(), entry.getValue());
+ }
+ }
+ }
+
+ public void initProperties(Map<String, String> properties) throws GFacProviderException, GFacException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputStreamToFileWriter.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputStreamToFileWriter.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputStreamToFileWriter.java
new file mode 100644
index 0000000..2c9a4d2
--- /dev/null
+++ b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputStreamToFileWriter.java
@@ -0,0 +1,68 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.local.utils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+
+public class InputStreamToFileWriter extends Thread{
+ protected final Logger log = LoggerFactory.getLogger(this.getClass());
+
+ private BufferedReader in;
+ private BufferedWriter out;
+
+ public InputStreamToFileWriter(InputStream in, String out) throws IOException {
+ this.in = new BufferedReader(new InputStreamReader(in));
+ this.out = new BufferedWriter(new FileWriter(out));
+ }
+
+ public void run() {
+ try {
+ String line = null;
+ while ((line = in.readLine()) != null) {
+ if (log.isDebugEnabled()) {
+ log.debug(line);
+ }
+ out.write(line);
+ out.newLine();
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ if (in != null) {
+ try {
+ in.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ if (out != null) {
+ try {
+ out.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputUtils.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputUtils.java
new file mode 100644
index 0000000..ba65753
--- /dev/null
+++ b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/InputUtils.java
@@ -0,0 +1,39 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.local.utils;
+
+import java.util.List;
+
+public class InputUtils {
+ private static final String SPACE = " ";
+
+ private InputUtils() {
+ }
+
+ public static String buildCommand(List<String> cmdList) {
+ StringBuffer buff = new StringBuffer();
+ for (String string : cmdList) {
+ buff.append(string);
+ buff.append(SPACE);
+ }
+ return buff.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/LocalProviderUtil.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/LocalProviderUtil.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/LocalProviderUtil.java
new file mode 100644
index 0000000..4a19b65
--- /dev/null
+++ b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/local/utils/LocalProviderUtil.java
@@ -0,0 +1,54 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.local.utils;
+
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.provider.GFacProviderException;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+
+public class LocalProviderUtil {
+ private static final Logger log = LoggerFactory.getLogger(LocalProviderUtil.class);
+
+ private void makeFileSystemDir(String dir) throws GFacProviderException {
+ File f = new File(dir);
+ if (f.isDirectory() && f.exists()) {
+ return;
+ } else if (!new File(dir).mkdir()) {
+ throw new GFacProviderException("Cannot make directory " + dir);
+ }
+ }
+
+ public void makeDirectory(JobExecutionContext jobExecutionContext) throws GFacProviderException {
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.
+ getApplicationContext().getApplicationDeploymentDescription().getType();
+ log.info("working diectroy = " + app.getStaticWorkingDirectory());
+ log.info("temp directory = " + app.getScratchWorkingDirectory());
+ makeFileSystemDir(app.getStaticWorkingDirectory());
+ makeFileSystemDir(app.getScratchWorkingDirectory());
+ makeFileSystemDir(app.getInputDataDirectory());
+ makeFileSystemDir(app.getOutputDataDirectory());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/provider/impl/LocalProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/provider/impl/LocalProvider.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/provider/impl/LocalProvider.java
deleted file mode 100644
index 6a92e53..0000000
--- a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/provider/impl/LocalProvider.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.provider.impl;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.airavata.gfac.Constants;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.notification.events.StartExecutionEvent;
-import org.apache.airavata.gfac.provider.AbstractProvider;
-import org.apache.airavata.gfac.provider.GFacProviderException;
-import org.apache.airavata.gfac.provider.utils.ProviderUtils;
-import org.apache.airavata.gfac.utils.GFacUtils;
-import org.apache.airavata.gfac.utils.InputStreamToFileWriter;
-import org.apache.airavata.gfac.utils.InputUtils;
-import org.apache.airavata.gfac.utils.OutputUtils;
-import org.apache.airavata.model.workspace.experiment.JobDetails;
-import org.apache.airavata.model.workspace.experiment.JobState;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.apache.airavata.schemas.gfac.NameValuePairType;
-import org.apache.xmlbeans.XmlException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import sun.reflect.generics.reflectiveObjects.NotImplementedException;
-
-public class LocalProvider extends AbstractProvider {
- private static final Logger log = LoggerFactory.getLogger(LocalProvider.class);
- private ProcessBuilder builder;
- private List<String> cmdList;
- private String jobId;
-
- public static class LocalProviderJobData{
- private String applicationName;
- private List<String> inputParameters;
- private String workingDir;
- private String inputDir;
- private String outputDir;
- public String getApplicationName() {
- return applicationName;
- }
- public void setApplicationName(String applicationName) {
- this.applicationName = applicationName;
- }
- public List<String> getInputParameters() {
- return inputParameters;
- }
- public void setInputParameters(List<String> inputParameters) {
- this.inputParameters = inputParameters;
- }
- public String getWorkingDir() {
- return workingDir;
- }
- public void setWorkingDir(String workingDir) {
- this.workingDir = workingDir;
- }
- public String getInputDir() {
- return inputDir;
- }
- public void setInputDir(String inputDir) {
- this.inputDir = inputDir;
- }
- public String getOutputDir() {
- return outputDir;
- }
- public void setOutputDir(String outputDir) {
- this.outputDir = outputDir;
- }
- }
- public LocalProvider(){
- cmdList = new ArrayList<String>();
- }
-
- public void initialize(JobExecutionContext jobExecutionContext) throws GFacProviderException,GFacException {
- super.initialize(jobExecutionContext);
- ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().
- getApplicationDeploymentDescription().getType();
-
- buildCommand(app.getExecutableLocation(), ProviderUtils.getInputParameters(jobExecutionContext));
- initProcessBuilder(app);
-
- // extra environment variables
- builder.environment().put(Constants.INPUT_DATA_DIR_VAR_NAME, app.getInputDataDirectory());
- builder.environment().put(Constants.OUTPUT_DATA_DIR_VAR_NAME, app.getOutputDataDirectory());
-
- // set working directory
- builder.directory(new File(app.getStaticWorkingDirectory()));
-
- // log info
- log.info("Command = " + InputUtils.buildCommand(cmdList));
- log.info("Working dir = " + builder.directory());
- for (String key : builder.environment().keySet()) {
- log.info("Env[" + key + "] = " + builder.environment().get(key));
- }
- }
-
- public void execute(JobExecutionContext jobExecutionContext) throws GFacProviderException {
- jobExecutionContext.getNotifier().publish(new StartExecutionEvent());
- ApplicationDeploymentDescriptionType app = jobExecutionContext.
- getApplicationContext().getApplicationDeploymentDescription().getType();
- JobDetails jobDetails = new JobDetails();
- try {
- jobId = jobExecutionContext.getTaskData().getTaskID();
- jobDetails.setJobID(jobId);
- jobDetails.setJobDescription(app.toString());
- jobExecutionContext.setJobDetails(jobDetails);
- jobDetails.setJobDescription(app.toString());
- GFacUtils.saveJobStatus(jobExecutionContext,jobDetails, JobState.SETUP);
- // running cmd
- Process process = builder.start();
-
- Thread standardOutWriter = new InputStreamToFileWriter(process.getInputStream(), app.getStandardOutput());
- Thread standardErrorWriter = new InputStreamToFileWriter(process.getErrorStream(), app.getStandardError());
-
- // start output threads
- standardOutWriter.setDaemon(true);
- standardErrorWriter.setDaemon(true);
- standardOutWriter.start();
- standardErrorWriter.start();
-
- int returnValue = process.waitFor();
-
- // make sure other two threads are done
- standardOutWriter.join();
- standardErrorWriter.join();
-
- /*
- * check return value. usually not very helpful to draw conclusions based on return values so don't bother.
- * just provide warning in the log messages
- */
- if (returnValue != 0) {
- log.error("Process finished with non zero return value. Process may have failed");
- } else {
- log.info("Process finished with return value of zero.");
- }
-
- StringBuffer buf = new StringBuffer();
- buf.append("Executed ").append(InputUtils.buildCommand(cmdList))
- .append(" on the localHost, working directory = ").append(app.getStaticWorkingDirectory())
- .append(" tempDirectory = ").append(app.getScratchWorkingDirectory()).append(" With the status ")
- .append(String.valueOf(returnValue));
- log.info(buf.toString());
- } catch (IOException io) {
- throw new GFacProviderException(io.getMessage(), io);
- } catch (InterruptedException e) {
- throw new GFacProviderException(e.getMessage(), e);
- }catch (GFacException e) {
- throw new GFacProviderException(e.getMessage(), e);
- }
- }
-
-// private void saveApplicationJob(JobExecutionContext jobExecutionContext)
-// throws GFacProviderException {
-// ApplicationDeploymentDescriptionType app = jobExecutionContext.
-// getApplicationContext().getApplicationDeploymentDescription().getType();
-// ApplicationJob appJob = GFacUtils.createApplicationJob(jobExecutionContext);
-// appJob.setJobId(jobId);
-// LocalProviderJobData data = new LocalProviderJobData();
-// data.setApplicationName(app.getExecutableLocation());
-// data.setInputDir(app.getInputDataDirectory());
-// data.setOutputDir(app.getOutputDataDirectory());
-// data.setWorkingDir(builder.directory().toString());
-// data.setInputParameters(ProviderUtils.getInputParameters(jobExecutionContext));
-// ByteArrayOutputStream stream = new ByteArrayOutputStream();
-// JAXB.marshal(data, stream);
-// appJob.setJobData(stream.toString());
-// appJob.setSubmittedTime(Calendar.getInstance().getTime());
-// appJob.setStatus(ApplicationJobStatus.SUBMITTED);
-// appJob.setStatusUpdateTime(appJob.getSubmittedTime());
-// GFacUtils.recordApplicationJob(jobExecutionContext, appJob);
-// }
-
- public void dispose(JobExecutionContext jobExecutionContext) throws GFacProviderException {
- ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
-
- try {
- String stdOutStr = GFacUtils.readFileToString(app.getStandardOutput());
- String stdErrStr = GFacUtils.readFileToString(app.getStandardError());
- Map<String, Object> output = jobExecutionContext.getOutMessageContext().getParameters();
- OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr);
- } catch (XmlException e) {
- throw new GFacProviderException("Cannot read output:" + e.getMessage(), e);
- } catch (IOException io) {
- throw new GFacProviderException(io.getMessage(), io);
- } catch (Exception e){
- throw new GFacProviderException("Error in retrieving results",e);
- }
- }
-
- public void cancelJob(String jobId, JobExecutionContext jobExecutionContext) throws GFacException {
- throw new NotImplementedException();
- }
-
-
- private void buildCommand(String executable, List<String> inputParameterList){
- cmdList.add(executable);
- cmdList.addAll(inputParameterList);
- }
-
- private void initProcessBuilder(ApplicationDeploymentDescriptionType app){
- builder = new ProcessBuilder(cmdList);
-
- NameValuePairType[] env = app.getApplicationEnvironmentArray();
-
- if(env != null && env.length > 0){
- Map<String,String> builderEnv = builder.environment();
- for (NameValuePairType entry : env) {
- builderEnv.put(entry.getName(), entry.getValue());
- }
- }
- }
-
- public void initProperties(Map<String, String> properties) throws GFacProviderException, GFacException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputStreamToFileWriter.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputStreamToFileWriter.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputStreamToFileWriter.java
deleted file mode 100644
index 691e11d..0000000
--- a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputStreamToFileWriter.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.utils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.*;
-
-public class InputStreamToFileWriter extends Thread{
- protected final Logger log = LoggerFactory.getLogger(this.getClass());
-
- private BufferedReader in;
- private BufferedWriter out;
-
- public InputStreamToFileWriter(InputStream in, String out) throws IOException {
- this.in = new BufferedReader(new InputStreamReader(in));
- this.out = new BufferedWriter(new FileWriter(out));
- }
-
- public void run() {
- try {
- String line = null;
- while ((line = in.readLine()) != null) {
- if (log.isDebugEnabled()) {
- log.debug(line);
- }
- out.write(line);
- out.newLine();
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- if (in != null) {
- try {
- in.close();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- if (out != null) {
- try {
- out.close();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputUtils.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputUtils.java
deleted file mode 100644
index dedfa29..0000000
--- a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/InputUtils.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.utils;
-
-import java.util.List;
-
-public class InputUtils {
- private static final String SPACE = " ";
-
- private InputUtils() {
- }
-
- public static String buildCommand(List<String> cmdList) {
- StringBuffer buff = new StringBuffer();
- for (String string : cmdList) {
- buff.append(string);
- buff.append(SPACE);
- }
- return buff.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/LocalProviderUtil.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/LocalProviderUtil.java b/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/LocalProviderUtil.java
deleted file mode 100644
index c80eeda..0000000
--- a/modules/gfac/gfac-local/src/main/java/org/apache/airavata/gfac/utils/LocalProviderUtil.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.utils;
-
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.provider.GFacProviderException;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-
-public class LocalProviderUtil {
- private static final Logger log = LoggerFactory.getLogger(LocalProviderUtil.class);
-
- private void makeFileSystemDir(String dir) throws GFacProviderException {
- File f = new File(dir);
- if (f.isDirectory() && f.exists()) {
- return;
- } else if (!new File(dir).mkdir()) {
- throw new GFacProviderException("Cannot make directory " + dir);
- }
- }
-
- public void makeDirectory(JobExecutionContext jobExecutionContext) throws GFacProviderException {
- ApplicationDeploymentDescriptionType app = jobExecutionContext.
- getApplicationContext().getApplicationDeploymentDescription().getType();
- log.info("working diectroy = " + app.getStaticWorkingDirectory());
- log.info("temp directory = " + app.getScratchWorkingDirectory());
- makeFileSystemDir(app.getStaticWorkingDirectory());
- makeFileSystemDir(app.getScratchWorkingDirectory());
- makeFileSystemDir(app.getInputDataDirectory());
- makeFileSystemDir(app.getOutputDataDirectory());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/test/java/org/apache/airavata/core/gfac/services/impl/LocalProviderTest.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/test/java/org/apache/airavata/core/gfac/services/impl/LocalProviderTest.java b/modules/gfac/gfac-local/src/test/java/org/apache/airavata/core/gfac/services/impl/LocalProviderTest.java
index bade5f0..902e424 100644
--- a/modules/gfac/gfac-local/src/test/java/org/apache/airavata/core/gfac/services/impl/LocalProviderTest.java
+++ b/modules/gfac/gfac-local/src/test/java/org/apache/airavata/core/gfac/services/impl/LocalProviderTest.java
@@ -26,9 +26,9 @@ import org.apache.airavata.gfac.GFacException;
import org.apache.airavata.gfac.context.ApplicationContext;
import org.apache.airavata.gfac.context.JobExecutionContext;
import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.gfac.handler.LocalDirectorySetupHandler;
+import org.apache.airavata.gfac.local.handler.LocalDirectorySetupHandler;
+import org.apache.airavata.gfac.local.impl.LocalProvider;
import org.apache.airavata.gfac.provider.GFacProviderException;
-import org.apache.airavata.gfac.provider.impl.LocalProvider;
import org.apache.airavata.model.workspace.experiment.TaskDetails;
import org.apache.airavata.persistance.registry.jpa.impl.LoggingRegistryImpl;
import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-local/src/test/resources/gfac-config.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-local/src/test/resources/gfac-config.xml b/modules/gfac/gfac-local/src/test/resources/gfac-config.xml
index f16460f..b9432da 100644
--- a/modules/gfac/gfac-local/src/test/resources/gfac-config.xml
+++ b/modules/gfac/gfac-local/src/test/resources/gfac-config.xml
@@ -19,9 +19,9 @@
</InHandlers>
<OutHandlers></OutHandlers>
</GlobalHandlers>
- <Provider class="org.apache.airavata.gfac.provider.impl.LocalProvider" host="org.apache.airavata.schemas.gfac.impl.HostDescriptionTypeImpl">
+ <Provider class="org.apache.airavata.gfac.local.impl.LocalProvider" host="org.apache.airavata.schemas.gfac.impl.HostDescriptionTypeImpl">
<InHandlers>
- <Handler class="org.apache.airavata.gfac.handler.LocalDirectorySetupHandler"/>
+ <Handler class="org.apache.airavata.gfac.local.handler.LocalDirectorySetupHandler"/>
</InHandlers>
</Provider>
</GFac>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java
index ec14c08..6ab86af 100644
--- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java
+++ b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/ResourceConnection.java
@@ -21,19 +21,13 @@
package org.apache.airavata.gfac.monitor.impl.pull.qstat;
import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.SecurityContext;
-import org.apache.airavata.gfac.context.security.GSISecurityContext;
+import org.apache.airavata.gfac.gsissh.security.GSISecurityContext;
import org.apache.airavata.gfac.monitor.HostMonitorData;
import org.apache.airavata.gfac.monitor.MonitorID;
import org.apache.airavata.gsi.ssh.api.SSHApiException;
-import org.apache.airavata.gsi.ssh.api.ServerInfo;
-import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
-import org.apache.airavata.gsi.ssh.api.job.JobManagerConfiguration;
import org.apache.airavata.gsi.ssh.impl.JobStatus;
import org.apache.airavata.gsi.ssh.impl.PBSCluster;
-import org.apache.airavata.gsi.ssh.util.CommonUtils;
import org.apache.airavata.model.workspace.experiment.JobState;
-import org.apache.airavata.schemas.gfac.GsisshHostType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/context/security/SSHSecurityContext.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/context/security/SSHSecurityContext.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/context/security/SSHSecurityContext.java
deleted file mode 100644
index 2d71b7d..0000000
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/context/security/SSHSecurityContext.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package org.apache.airavata.gfac.context.security;
-
-import java.io.IOException;
-
-import net.schmizz.sshj.SSHClient;
-import net.schmizz.sshj.connection.channel.direct.Session;
-import net.schmizz.sshj.userauth.keyprovider.KeyProvider;
-
-import org.apache.airavata.gfac.SecurityContext;
-import org.apache.airavata.gsi.ssh.api.Cluster;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Handle SSH security
- */
-public class SSHSecurityContext implements SecurityContext {
- private static final Logger log = LoggerFactory.getLogger(SSHSecurityContext.class);
-
- public static final String SSH_SECURITY_CONTEXT = "ssh";
-
- private String username;
- private String privateKeyLoc;
- private String keyPass;
- private SSHClient sshClient;
- private Session session;
-
- private Cluster pbsCluster;
-
- public String getUsername() {
- return username;
- }
-
- public void setUsername(String username) {
- this.username = username;
- }
-
- public String getPrivateKeyLoc() {
- return privateKeyLoc;
- }
-
- public void setPrivateKeyLoc(String privateKeyLoc) {
- this.privateKeyLoc = privateKeyLoc;
- }
-
- public String getKeyPass() {
- return keyPass;
- }
-
- public void setKeyPass(String keyPass) {
- this.keyPass = keyPass;
- }
-
- public void closeSession(Session session) {
- if (session != null) {
- try {
- session.close();
- } catch (Exception e) {
- log.warn("Cannot Close SSH Session");
- }
- }
- }
-
- public Session getSession(String hostAddress) throws IOException {
- try {
- if (sshClient == null) {
- sshClient = new SSHClient();
- }
- if (getSSHClient().isConnected())
- return getSSHClient().startSession();
-
- KeyProvider pkey = getSSHClient().loadKeys(getPrivateKeyLoc(), getKeyPass());
-
- getSSHClient().loadKnownHosts();
-
- getSSHClient().connect(hostAddress);
- getSSHClient().authPublickey(getUsername(), pkey);
- session = getSSHClient().startSession();
- return session;
-
- } catch (NullPointerException ne) {
- throw new SecurityException("Cannot load security context for SSH", ne);
- }
- }
-
- public SSHClient getSSHClient() {
- if (sshClient == null) {
- sshClient = new SSHClient();
- }
- return sshClient;
- }
-
- public void setPbsCluster(Cluster pbsCluster) {
- this.pbsCluster = pbsCluster;
- }
-
- public Cluster getPbsCluster() {
- return this.pbsCluster;
- }
-}
[2/8] changing package names of gfac implementations
Posted by la...@apache.org.
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/AdvancedSCPInputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/AdvancedSCPInputHandler.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/AdvancedSCPInputHandler.java
deleted file mode 100644
index 13c325d..0000000
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/AdvancedSCPInputHandler.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.handler;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.StringUtil;
-import org.apache.airavata.commons.gfac.type.ActualParameter;
-import org.apache.airavata.commons.gfac.type.MappingFactory;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.gfac.context.security.SSHSecurityContext;
-import org.apache.airavata.gfac.util.GFACSSHUtils;
-import org.apache.airavata.gsi.ssh.api.Cluster;
-import org.apache.airavata.gsi.ssh.api.SSHApiException;
-import org.apache.airavata.gsi.ssh.api.ServerInfo;
-import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
-import org.apache.airavata.gsi.ssh.impl.PBSCluster;
-import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPasswordAuthenticationInfo;
-import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPublicKeyFileAuthentication;
-import org.apache.airavata.gsi.ssh.util.CommonUtils;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.apache.airavata.schemas.gfac.URIArrayType;
-import org.apache.airavata.schemas.gfac.URIParameterType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.*;
-
-/**
- * This handler will copy input data from gateway machine to airavata
- * installed machine, later running handlers can copy the input files to computing resource
- * <Handler class="org.apache.airavata.gfac.handler.AdvancedSCPOutputHandler">
- <property name="privateKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa"/>
- <property name="publicKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa.pub"/>
- <property name="userName" value="airavata"/>
- <property name="hostName" value="gw98.iu.xsede.org"/>
- <property name="inputPath" value="/home/airavata/outputData"/>
- */
-public class AdvancedSCPInputHandler extends AbstractHandler{
- private static final Logger log = LoggerFactory.getLogger(AdvancedSCPInputHandler.class);
-
- private String password = null;
-
- private String publicKeyPath;
-
- private String passPhrase;
-
- private String privateKeyPath;
-
- private String userName;
-
- private String hostName;
-
- private String inputPath;
-
- public void initProperties(Map<String, String> properties) throws GFacHandlerException {
- password = properties.get("password");
- passPhrase = properties.get("passPhrase");
- privateKeyPath = properties.get("privateKeyPath");
- publicKeyPath = properties.get("publicKeyPath");
- userName = properties.get("userName");
- hostName = properties.get("hostName");
- inputPath = properties.get("inputPath");
- }
-
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- MessageContext inputNew = new MessageContext();
- try{
- if(jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT) == null){
- try {
- GFACSSHUtils.addSecurityContext(jobExecutionContext);
- } catch (ApplicationSettingsException e) {
- log.error(e.getMessage());
- throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
- }
- }
- ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext()
- .getApplicationDeploymentDescription().getType();
-
- AuthenticationInfo authenticationInfo = null;
- if (password != null) {
- authenticationInfo = new DefaultPasswordAuthenticationInfo(this.password);
- } else {
- authenticationInfo = new DefaultPublicKeyFileAuthentication(this.publicKeyPath, this.privateKeyPath,
- this.passPhrase);
- }
- // Server info
- ServerInfo serverInfo = new ServerInfo(this.userName, this.hostName);
- Cluster pbsCluster = null;
- // here doesn't matter what the job manager is because we are only doing some file handling
- // not really dealing with monitoring or job submission, so we pa
- pbsCluster = new PBSCluster(serverInfo, authenticationInfo, CommonUtils.getPBSJobManager("/opt/torque/torque-4.2.3.1/bin/"));
- String parentPath = inputPath + File.separator + jobExecutionContext.getExperimentID() + File.separator + jobExecutionContext.getTaskData().getTaskID();
- (new File(parentPath)).mkdirs();
- MessageContext input = jobExecutionContext.getInMessageContext();
- Set<String> parameters = input.getParameters().keySet();
- for (String paramName : parameters) {
- ActualParameter actualParameter = (ActualParameter) input.getParameters().get(paramName);
- String paramValue = MappingFactory.toString(actualParameter);
- //TODO: Review this with type
- if ("URI".equals(actualParameter.getType().getType().toString())) {
- ((URIParameterType) actualParameter.getType()).setValue(stageInputFiles(pbsCluster, paramValue, parentPath));
- } else if ("URIArray".equals(actualParameter.getType().getType().toString())) {
- List<String> split = Arrays.asList(StringUtil.getElementsFromString(paramValue));
- List<String> newFiles = new ArrayList<String>();
- for (String paramValueEach : split) {
- String stageInputFiles = stageInputFiles(pbsCluster, paramValueEach, parentPath);
- newFiles.add(stageInputFiles);
- }
- ((URIArrayType) actualParameter.getType()).setValueArray(newFiles.toArray(new String[newFiles.size()]));
- }
- inputNew.getParameters().put(paramName, actualParameter);
- }
- } catch (Exception e) {
- log.error(e.getMessage());
- throw new GFacHandlerException("Error while input File Staging", e, e.getLocalizedMessage());
- }
- jobExecutionContext.setInMessageContext(inputNew);
- }
-
- private String stageInputFiles(Cluster cluster, String paramValue, String parentPath) throws GFacException {
- try {
- cluster.scpFrom(paramValue, parentPath);
- return "file://" + parentPath + File.separator + (new File(paramValue)).getName();
- } catch (SSHApiException e) {
- log.error("Error tranfering remote file to local file, remote path: " + paramValue);
- throw new GFacException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/AdvancedSCPOutputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/AdvancedSCPOutputHandler.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/AdvancedSCPOutputHandler.java
deleted file mode 100644
index dba3e69..0000000
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/AdvancedSCPOutputHandler.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.handler;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.security.SSHSecurityContext;
-import org.apache.airavata.gfac.util.GFACSSHUtils;
-import org.apache.airavata.gsi.ssh.api.Cluster;
-import org.apache.airavata.gsi.ssh.api.SSHApiException;
-import org.apache.airavata.gsi.ssh.api.ServerInfo;
-import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
-import org.apache.airavata.gsi.ssh.impl.PBSCluster;
-import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPasswordAuthenticationInfo;
-import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPublicKeyFileAuthentication;
-import org.apache.airavata.gsi.ssh.util.CommonUtils;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.Map;
-
-/**
- * This handler will copy outputs from airavata installed local directory
- * to a remote location, prior to this handler SCPOutputHandler should be invoked
- * Should add following configuration to gfac-config.xml and configure the keys properly
- * <Handler class="org.apache.airavata.gfac.handler.AdvancedSCPOutputHandler">
- <property name="privateKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa"/>
- <property name="publicKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa.pub"/>
- <property name="userName" value="airavata"/>
- <property name="hostName" value="gw98.iu.xsede.org"/>
- <property name="outputPath" value="/home/airavata/outputData"/>
- <property name="passPhrase" value="/home/airavata/outputData"/>
- <property name="password" value="/home/airavata/outputData"/>
-
- */
-public class AdvancedSCPOutputHandler extends AbstractHandler {
- private static final Logger log = LoggerFactory.getLogger(AdvancedSCPOutputHandler.class);
-
- private String password = null;
-
- private String publicKeyPath;
-
- private String passPhrase;
-
- private String privateKeyPath;
-
- private String userName;
-
- private String hostName;
-
- private String outputPath;
-
- public void initProperties(Map<String, String> properties) throws GFacHandlerException {
- password = properties.get("password");
- passPhrase = properties.get("passPhrase");
- privateKeyPath = properties.get("privateKeyPath");
- publicKeyPath = properties.get("publicKeyPath");
- userName = properties.get("userName");
- hostName = properties.get("hostName");
- outputPath = properties.get("outputPath");
- }
-
- @Override
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- try {
- if (jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT) == null) {
- try {
- GFACSSHUtils.addSecurityContext(jobExecutionContext);
- } catch (ApplicationSettingsException e) {
- log.error(e.getMessage());
- throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
- }
- }
- ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext()
- .getApplicationDeploymentDescription().getType();
- String standardError = app.getStandardError();
- String standardOutput = app.getStandardOutput();
- String outputDataDirectory = app.getOutputDataDirectory();
-
- AuthenticationInfo authenticationInfo = null;
- if (password != null) {
- authenticationInfo = new DefaultPasswordAuthenticationInfo(this.password);
- } else {
- authenticationInfo = new DefaultPublicKeyFileAuthentication(this.publicKeyPath, this.privateKeyPath,
- this.passPhrase);
- }
- // Server info
- ServerInfo serverInfo = new ServerInfo(this.userName, this.hostName);
-
- Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, CommonUtils.getPBSJobManager("/opt/torque/torque-4.2.3.1/bin/"));
- outputPath = outputPath + File.separator + jobExecutionContext.getExperimentID() + "-" + jobExecutionContext.getTaskData().getTaskID()
- + File.separator;
- pbsCluster.makeDirectory(outputPath);
- pbsCluster.scpTo(outputPath, standardError);
- pbsCluster.scpTo(outputPath, standardOutput);
- for (String files : jobExecutionContext.getOutputFiles()) {
- pbsCluster.scpTo(outputPath, files);
- }
- } catch (SSHApiException e) {
- log.error("Error transfering files to remote host : " + hostName + " with the user: " + userName);
- log.error(e.getMessage());
- throw new GFacHandlerException(e);
- } catch (GFacException e) {
- throw new GFacHandlerException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHDirectorySetupHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHDirectorySetupHandler.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHDirectorySetupHandler.java
deleted file mode 100644
index 1f99a98..0000000
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHDirectorySetupHandler.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.handler;
-
-import java.util.Map;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.security.SSHSecurityContext;
-import org.apache.airavata.gfac.util.GFACSSHUtils;
-import org.apache.airavata.gfac.utils.GFacUtils;
-import org.apache.airavata.gsi.ssh.api.Cluster;
-import org.apache.airavata.gsi.ssh.api.SSHApiException;
-import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
-import org.apache.airavata.model.workspace.experiment.DataTransferDetails;
-import org.apache.airavata.model.workspace.experiment.ErrorCategory;
-import org.apache.airavata.model.workspace.experiment.TransferState;
-import org.apache.airavata.model.workspace.experiment.TransferStatus;
-import org.apache.airavata.registry.cpi.ChildDataType;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class SSHDirectorySetupHandler extends AbstractHandler{
- private static final Logger log = LoggerFactory.getLogger(SSHDirectorySetupHandler.class);
-
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- try {
- if (jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT) == null) {
- GFACSSHUtils.addSecurityContext(jobExecutionContext);
- }
- } catch (ApplicationSettingsException e) {
- log.error(e.getMessage());
- throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
- } catch (GFacException e) {
- e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
- }
-
- log.info("Setup SSH job directorties");
- super.invoke(jobExecutionContext);
- makeDirectory(jobExecutionContext);
-
- }
- private void makeDirectory(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- try{
- Cluster cluster = ((SSHSecurityContext) jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT)).getPbsCluster();
- if (cluster == null) {
- throw new GFacHandlerException("Security context is not set properly");
- } else {
- log.info("Successfully retrieved the Security Context");
- }
- ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
- String workingDirectory = app.getScratchWorkingDirectory();
- cluster.makeDirectory(workingDirectory);
- cluster.makeDirectory(app.getScratchWorkingDirectory());
- cluster.makeDirectory(app.getInputDataDirectory());
- cluster.makeDirectory(app.getOutputDataDirectory());
- DataTransferDetails detail = new DataTransferDetails();
- TransferStatus status = new TransferStatus();
- status.setTransferState(TransferState.DIRECTORY_SETUP);
- detail.setTransferStatus(status);
- detail.setTransferDescription("Working directory = " + workingDirectory);
-
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
-
- } catch (SSHApiException e) {
- throw new GFacHandlerException("Error executing the Handler: " + SSHDirectorySetupHandler.class, e);
- } catch (Exception e) {
- DataTransferDetails detail = new DataTransferDetails();
- TransferStatus status = new TransferStatus();
- status.setTransferState(TransferState.FAILED);
- detail.setTransferStatus(status);
- try {
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
- GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
- } catch (Exception e1) {
- throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
- }
- throw new GFacHandlerException("Error executing the Handler: " + SSHDirectorySetupHandler.class, e);
- }
- }
-
- public void initProperties(Map<String, String> properties) throws GFacHandlerException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHInputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHInputHandler.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHInputHandler.java
deleted file mode 100644
index 5cc0ffb..0000000
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHInputHandler.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.handler;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.StringUtil;
-import org.apache.airavata.commons.gfac.type.ActualParameter;
-import org.apache.airavata.commons.gfac.type.MappingFactory;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.gfac.context.security.SSHSecurityContext;
-import org.apache.airavata.gfac.util.GFACSSHUtils;
-import org.apache.airavata.gfac.utils.GFacUtils;
-import org.apache.airavata.gsi.ssh.api.Cluster;
-import org.apache.airavata.gsi.ssh.api.SSHApiException;
-import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
-import org.apache.airavata.model.workspace.experiment.DataTransferDetails;
-import org.apache.airavata.model.workspace.experiment.ErrorCategory;
-import org.apache.airavata.model.workspace.experiment.TransferState;
-import org.apache.airavata.model.workspace.experiment.TransferStatus;
-import org.apache.airavata.registry.cpi.ChildDataType;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.apache.airavata.schemas.gfac.URIArrayType;
-import org.apache.airavata.schemas.gfac.URIParameterType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class SSHInputHandler extends AbstractHandler {
-
- private static final Logger log = LoggerFactory.getLogger(SSHInputHandler.class);
-
-
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- DataTransferDetails detail = new DataTransferDetails();
- TransferStatus status = new TransferStatus();
- MessageContext inputNew = new MessageContext();
- try {
-
- if (jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT) == null) {
- try {
- GFACSSHUtils.addSecurityContext(jobExecutionContext);
- } catch (ApplicationSettingsException e) {
- log.error(e.getMessage());
- throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
- }
- }
- log.info("Invoking SCPInputHandler");
- super.invoke(jobExecutionContext);
-
-
- MessageContext input = jobExecutionContext.getInMessageContext();
- Set<String> parameters = input.getParameters().keySet();
- for (String paramName : parameters) {
- ActualParameter actualParameter = (ActualParameter) input.getParameters().get(paramName);
- String paramValue = MappingFactory.toString(actualParameter);
- //TODO: Review this with type
- if ("URI".equals(actualParameter.getType().getType().toString())) {
- ((URIParameterType) actualParameter.getType()).setValue(stageInputFiles(jobExecutionContext, paramValue));
- } else if ("URIArray".equals(actualParameter.getType().getType().toString())) {
- List<String> split = Arrays.asList(StringUtil.getElementsFromString(paramValue));
- List<String> newFiles = new ArrayList<String>();
- for (String paramValueEach : split) {
- String stageInputFiles = stageInputFiles(jobExecutionContext, paramValueEach);
- status.setTransferState(TransferState.UPLOAD);
- detail.setTransferStatus(status);
- detail.setTransferDescription("Input Data Staged: " + stageInputFiles);
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
- newFiles.add(stageInputFiles);
- }
- ((URIArrayType) actualParameter.getType()).setValueArray(newFiles.toArray(new String[newFiles.size()]));
- }
- inputNew.getParameters().put(paramName, actualParameter);
- }
- } catch (Exception e) {
- log.error(e.getMessage());
- status.setTransferState(TransferState.FAILED);
- detail.setTransferStatus(status);
- try {
- GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
- } catch (Exception e1) {
- throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
- }
- throw new GFacHandlerException("Error while input File Staging", e, e.getLocalizedMessage());
- }
- jobExecutionContext.setInMessageContext(inputNew);
- }
-
- private static String stageInputFiles(JobExecutionContext jobExecutionContext, String paramValue) throws IOException, GFacException {
- Cluster cluster = ((SSHSecurityContext) jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT)).getPbsCluster();
- if (cluster == null) {
- throw new GFacException("Security context is not set properly");
- } else {
- log.info("Successfully retrieved the Security Context");
- }
- ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
- int i = paramValue.lastIndexOf(File.separator);
- String substring = paramValue.substring(i + 1);
- try {
- String targetFile = app.getInputDataDirectory() + File.separator + substring;
- if(paramValue.startsWith("file")){
- paramValue = paramValue.substring(paramValue.indexOf(":") + 1, paramValue.length());
- }
- cluster.scpTo(targetFile, paramValue);
- return targetFile;
- } catch (SSHApiException e) {
- throw new GFacHandlerException("Error while input File Staging", e, e.getLocalizedMessage());
- }
- }
-
- public void initProperties(Map<String, String> properties) throws GFacHandlerException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHOutputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHOutputHandler.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHOutputHandler.java
deleted file mode 100644
index 9e1cfb2..0000000
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHOutputHandler.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.handler;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import net.schmizz.sshj.connection.ConnectionException;
-import net.schmizz.sshj.transport.TransportException;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.Constants;
-import org.apache.airavata.commons.gfac.type.ActualParameter;
-import org.apache.airavata.commons.gfac.type.ApplicationDescription;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.security.SSHSecurityContext;
-import org.apache.airavata.gfac.provider.GFacProviderException;
-import org.apache.airavata.gfac.util.GFACSSHUtils;
-import org.apache.airavata.gfac.utils.GFacUtils;
-import org.apache.airavata.gfac.utils.OutputUtils;
-import org.apache.airavata.gsi.ssh.api.Cluster;
-import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
-import org.apache.airavata.model.workspace.experiment.*;
-import org.apache.airavata.registry.cpi.ChildDataType;
-import org.apache.airavata.registry.cpi.DataType;
-import org.apache.airavata.registry.cpi.RegistryException;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.apache.airavata.schemas.gfac.GsisshHostType;
-import org.apache.airavata.schemas.gfac.URIParameterType;
-import org.apache.xmlbeans.XmlException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class SSHOutputHandler extends AbstractHandler{
- private static final Logger log = LoggerFactory.getLogger(SSHOutputHandler.class);
-
- public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- if (jobExecutionContext.getApplicationContext().getHostDescription().getType() instanceof GsisshHostType) { // this is because we don't have the right jobexecution context
- // so attempting to get it from the registry
- if (Constants.PUSH.equals(((GsisshHostType) jobExecutionContext.getApplicationContext().getHostDescription().getType()).getMonitorMode())) { // this is because we don't have the right jobexecution context
- // so attempting to get it from the registry
- log.warn("During the out handler chain jobExecution context came null, so trying to handler");
- ApplicationDescription applicationDeploymentDescription = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription();
- TaskDetails taskData = null;
- try {
- taskData = (TaskDetails) registry.get(DataType.TASK_DETAIL, jobExecutionContext.getTaskData().getTaskID());
- } catch (RegistryException e) {
- log.error("Error retrieving job details from Registry");
- throw new GFacHandlerException("Error retrieving job details from Registry", e);
- }
- JobDetails jobDetails = taskData.getJobDetailsList().get(0);
- String jobDescription = jobDetails.getJobDescription();
- if (jobDescription != null) {
- JobDescriptor jobDescriptor = null;
- try {
- jobDescriptor = JobDescriptor.fromXML(jobDescription);
- } catch (XmlException e1) {
- e1.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
- }
- applicationDeploymentDescription.getType().setScratchWorkingDirectory(
- jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getWorkingDirectory());
- applicationDeploymentDescription.getType().setInputDataDirectory(jobDescriptor.getInputDirectory());
- applicationDeploymentDescription.getType().setOutputDataDirectory(jobDescriptor.getOutputDirectory());
- applicationDeploymentDescription.getType().setStandardError(jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getStandardErrorFile());
- applicationDeploymentDescription.getType().setStandardOutput(jobDescriptor.getJobDescriptorDocument().getJobDescriptor().getStandardOutFile());
- }
- }
- }
-
- try {
- if (jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT) == null) {
-
- GFACSSHUtils.addSecurityContext(jobExecutionContext);
- }
- } catch (ApplicationSettingsException e) {
- log.error(e.getMessage());
- throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
- } catch (GFacException e1) {
- log.error(e1.getMessage());
- throw new GFacHandlerException("Error while creating SSHSecurityContext", e1, e1.getLocalizedMessage());
- }
-
- super.invoke(jobExecutionContext);
- DataTransferDetails detail = new DataTransferDetails();
- TransferStatus status = new TransferStatus();
-
- ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext()
- .getApplicationDeploymentDescription().getType();
- try {
- Cluster cluster = ((SSHSecurityContext) jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT)).getPbsCluster();
- if (cluster == null) {
- throw new GFacProviderException("Security context is not set properly");
- } else {
- log.info("Successfully retrieved the Security Context");
- }
-
- // Get the Stdouts and StdErrs
- String timeStampedServiceName = GFacUtils.createUniqueNameForService(jobExecutionContext.getServiceName());
-
- TaskDetails taskData = jobExecutionContext.getTaskData();
- String outputDataDir = null;
- File localStdOutFile;
- File localStdErrFile;
-
- if (taskData.getAdvancedOutputDataHandling() != null) {
- outputDataDir = taskData.getAdvancedOutputDataHandling().getOutputDataDir();
- }
- if (outputDataDir == null) {
- outputDataDir = File.separator + "tmp";
- }
- outputDataDir = outputDataDir + File.separator + jobExecutionContext.getExperimentID() + "-" + jobExecutionContext.getTaskData().getTaskID();
- (new File(outputDataDir)).mkdirs();
-
-
- localStdOutFile = new File(outputDataDir + File.separator + timeStampedServiceName + "stdout");
- localStdErrFile = new File(outputDataDir + File.separator + timeStampedServiceName + "stderr");
-// cluster.makeDirectory(outputDataDir);
- cluster.scpFrom(app.getStandardOutput(), localStdOutFile.getAbsolutePath());
- Thread.sleep(1000);
- cluster.scpFrom(app.getStandardError(), localStdErrFile.getAbsolutePath());
- Thread.sleep(1000);
-
- String stdOutStr = GFacUtils.readFileToString(localStdOutFile.getAbsolutePath());
- String stdErrStr = GFacUtils.readFileToString(localStdErrFile.getAbsolutePath());
- status.setTransferState(TransferState.COMPLETE);
- detail.setTransferStatus(status);
- detail.setTransferDescription("STDOUT:" + stdOutStr);
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
-
- status.setTransferState(TransferState.COMPLETE);
- detail.setTransferStatus(status);
- detail.setTransferDescription("STDERR:" + stdErrStr);
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
-
-
- Map<String, ActualParameter> stringMap = new HashMap<String, ActualParameter>();
- Map<String, Object> output = jobExecutionContext.getOutMessageContext().getParameters();
- Set<String> keys = output.keySet();
- for (String paramName : keys) {
- ActualParameter actualParameter = (ActualParameter) output.get(paramName);
- if ("URI".equals(actualParameter.getType().getType().toString())) {
-
- List<String> outputList = cluster.listDirectory(app.getOutputDataDirectory());
- if (outputList.size() == 0 || outputList.get(0).isEmpty()) {
- stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr);
- } else {
- String valueList = outputList.get(0);
- cluster.scpFrom(app.getOutputDataDirectory() + File.separator + valueList, outputDataDir);
- jobExecutionContext.addOutputFile(outputDataDir + File.separator + valueList);
- ((URIParameterType) actualParameter.getType()).setValue(valueList);
- stringMap = new HashMap<String, ActualParameter>();
- stringMap.put(paramName, actualParameter);
- }
- } else {
- stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr);
- }
- }
- if (stringMap == null || stringMap.isEmpty()) {
- throw new GFacHandlerException(
- "Empty Output returned from the Application, Double check the application"
- + "and ApplicationDescriptor output Parameter Names");
- }
- status.setTransferState(TransferState.DOWNLOAD);
- detail.setTransferStatus(status);
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
-
- app.setStandardError(localStdErrFile.getAbsolutePath());
- app.setStandardOutput(localStdOutFile.getAbsolutePath());
- app.setOutputDataDirectory(outputDataDir);
- } catch (XmlException e) {
- throw new GFacHandlerException("Cannot read output:" + e.getMessage(), e);
- } catch (ConnectionException e) {
- throw new GFacHandlerException(e.getMessage(), e);
- } catch (TransportException e) {
- throw new GFacHandlerException(e.getMessage(), e);
- } catch (IOException e) {
- throw new GFacHandlerException(e.getMessage(), e);
- } catch (Exception e) {
- try {
- status.setTransferState(TransferState.FAILED);
- detail.setTransferStatus(status);
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
- GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
- } catch (Exception e1) {
- throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
- }
- throw new GFacHandlerException("Error in retrieving results", e);
- }
-
- }
-
- public void initProperties(Map<String, String> properties) throws GFacHandlerException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/provider/impl/SSHProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/provider/impl/SSHProvider.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/provider/impl/SSHProvider.java
deleted file mode 100644
index e972f0d..0000000
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/provider/impl/SSHProvider.java
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.provider.impl;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Calendar;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.Set;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.commons.gfac.type.ActualParameter;
-import org.apache.airavata.commons.gfac.type.MappingFactory;
-import org.apache.airavata.gfac.Constants;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.gfac.context.security.SSHSecurityContext;
-import org.apache.airavata.gfac.handler.GFacHandlerException;
-import org.apache.airavata.gfac.notification.events.StartExecutionEvent;
-import org.apache.airavata.gfac.provider.AbstractProvider;
-import org.apache.airavata.gfac.provider.GFacProviderException;
-import org.apache.airavata.gfac.util.GFACSSHUtils;
-import org.apache.airavata.gfac.utils.GFacUtils;
-import org.apache.airavata.gsi.ssh.api.Cluster;
-import org.apache.airavata.gsi.ssh.api.CommandExecutor;
-import org.apache.airavata.gsi.ssh.api.SSHApiException;
-import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
-import org.apache.airavata.gsi.ssh.impl.RawCommandInfo;
-import org.apache.airavata.gsi.ssh.impl.StandardOutReader;
-import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
-import org.apache.airavata.model.workspace.experiment.ErrorCategory;
-import org.apache.airavata.model.workspace.experiment.JobDetails;
-import org.apache.airavata.model.workspace.experiment.JobState;
-import org.apache.airavata.schemas.gfac.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import sun.reflect.generics.reflectiveObjects.NotImplementedException;
-
-/**
- * Execute application using remote SSH
- */
-public class SSHProvider extends AbstractProvider {
- private static final Logger log = LoggerFactory.getLogger(SSHProvider.class);
- private Cluster cluster;
- private String jobID = null;
- private String taskID = null;
- // we keep gsisshprovider to support qsub submission incase of hpc scenario with ssh
- private boolean hpcType = false;
-
- public void initialize(JobExecutionContext jobExecutionContext) throws GFacProviderException, GFacException {
- super.initialize(jobExecutionContext);
- if(jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT) == null){
- try {
- GFACSSHUtils.addSecurityContext(jobExecutionContext);
- } catch (ApplicationSettingsException e) {
- log.error(e.getMessage());
- throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
- }
- }
- taskID = jobExecutionContext.getTaskData().getTaskID();
- if (!((SSHHostType) jobExecutionContext.getApplicationContext().getHostDescription().getType()).getHpcResource()) {
- jobID = "SSH_" + jobExecutionContext.getApplicationContext().getHostDescription().getType().getHostAddress() + "_" + Calendar.getInstance().getTimeInMillis();
- cluster = ((SSHSecurityContext) jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT)).getPbsCluster();
-
- ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
- String remoteFile = app.getStaticWorkingDirectory() + File.separatorChar + Constants.EXECUTABLE_NAME;
- details.setJobID(taskID);
- details.setJobDescription(remoteFile);
- jobExecutionContext.setJobDetails(details);
- JobDescriptor jobDescriptor = GFACSSHUtils.createJobDescriptor(jobExecutionContext, app, null);
- details.setJobDescription(jobDescriptor.toXML());
-
- GFacUtils.saveJobStatus(jobExecutionContext, details, JobState.SETUP);
- log.info(remoteFile);
- try {
- File runscript = createShellScript(jobExecutionContext);
- cluster.scpTo(remoteFile, runscript.getAbsolutePath());
- } catch (Exception e) {
- throw new GFacProviderException(e.getLocalizedMessage(), e);
- }
- }else{
- hpcType = true;
- }
- }
-
-
- public void execute(JobExecutionContext jobExecutionContext) throws GFacProviderException {
- if (!hpcType) {
- ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
- try {
- /*
- * Execute
- */
- String execuable = app.getStaticWorkingDirectory() + File.separatorChar + Constants.EXECUTABLE_NAME;
- details.setJobDescription(execuable);
-
-// GFacUtils.updateJobStatus(details, JobState.SUBMITTED);
- RawCommandInfo rawCommandInfo = new RawCommandInfo("/bin/chmod 755 " + execuable + "; " + execuable);
-
- StandardOutReader jobIDReaderCommandOutput = new StandardOutReader();
-
- CommandExecutor.executeCommand(rawCommandInfo, cluster.getSession(), jobIDReaderCommandOutput);
- String stdOutputString = getOutputifAvailable(jobIDReaderCommandOutput, "Error submitting job to resource");
-
- log.info("stdout=" + stdOutputString);
-
-// GFacUtils.updateJobStatus(details, JobState.COMPLETE);
- } catch (Exception e) {
- throw new GFacProviderException(e.getMessage(), e);
- } finally {
- if (cluster != null) {
- try {
- cluster.disconnect();
- } catch (SSHApiException e) {
- throw new GFacProviderException(e.getMessage(), e);
- }
- }
- }
- } else {
- try {
- jobExecutionContext.getNotifier().publish(new StartExecutionEvent());
- HostDescriptionType host = jobExecutionContext.getApplicationContext().
- getHostDescription().getType();
- HpcApplicationDeploymentType app = (HpcApplicationDeploymentType) jobExecutionContext.getApplicationContext().
- getApplicationDeploymentDescription().getType();
- JobDetails jobDetails = new JobDetails();
- String taskID = jobExecutionContext.getTaskData().getTaskID();
- try {
- Cluster cluster = null;
- if (jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT) != null) {
- cluster = ((SSHSecurityContext) jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT)).getPbsCluster();
- }
- if (cluster == null) {
- throw new GFacProviderException("Security context is not set properly");
- } else {
- log.info("Successfully retrieved the Security Context");
- }
- // This installed path is a mandetory field, because this could change based on the computing resource
- JobDescriptor jobDescriptor = GFACSSHUtils.createJobDescriptor(jobExecutionContext, app, cluster);
-
- log.info(jobDescriptor.toXML());
-
- jobDetails.setJobDescription(jobDescriptor.toXML());
-
- String jobID = cluster.submitBatchJob(jobDescriptor);
- jobExecutionContext.setJobDetails(jobDetails);
- if (jobID == null) {
- jobDetails.setJobID("none");
- GFacUtils.saveJobStatus(jobExecutionContext, jobDetails, JobState.FAILED);
- } else {
- jobDetails.setJobID(jobID);
- GFacUtils.saveJobStatus(jobExecutionContext, jobDetails, JobState.SUBMITTED);
- }
-
- } catch (SSHApiException e) {
- String error = "Error submitting the job to host " + host.getHostAddress() + " message: " + e.getMessage();
- log.error(error);
- jobDetails.setJobID("none");
- GFacUtils.saveJobStatus(jobExecutionContext, jobDetails, JobState.FAILED);
- GFacUtils.saveErrorDetails(jobExecutionContext, error, CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
- throw new GFacProviderException(error, e);
- } catch (Exception e) {
- String error = "Error submitting the job to host " + host.getHostAddress() + " message: " + e.getMessage();
- log.error(error);
- jobDetails.setJobID("none");
- GFacUtils.saveJobStatus(jobExecutionContext, jobDetails, JobState.FAILED);
- GFacUtils.saveErrorDetails(jobExecutionContext, error, CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
- throw new GFacProviderException(error, e);
- }
- } catch (GFacException e) {
- throw new GFacProviderException(e.getMessage(), e);
- }
- }
- }
-
- public void dispose(JobExecutionContext jobExecutionContext) throws GFacProviderException {
-
- }
-
-
- public void cancelJob(String jobId, JobExecutionContext jobExecutionContext) throws GFacException {
- throw new NotImplementedException();
- }
-
-
- private File createShellScript(JobExecutionContext context) throws IOException {
- ApplicationDeploymentDescriptionType app = context.getApplicationContext()
- .getApplicationDeploymentDescription().getType();
- String uniqueDir = app.getApplicationName().getStringValue() + System.currentTimeMillis()
- + new Random().nextLong();
-
- File shellScript = File.createTempFile(uniqueDir, "sh");
- OutputStream out = new FileOutputStream(shellScript);
-
- out.write("#!/bin/bash\n".getBytes());
- out.write(("cd " + app.getStaticWorkingDirectory() + "\n").getBytes());
- out.write(("export " + Constants.INPUT_DATA_DIR_VAR_NAME + "=" + app.getInputDataDirectory() + "\n").getBytes());
- out.write(("export " + Constants.OUTPUT_DATA_DIR_VAR_NAME + "=" + app.getOutputDataDirectory() + "\n")
- .getBytes());
- // get the env of the host and the application
- NameValuePairType[] env = app.getApplicationEnvironmentArray();
-
- Map<String, String> nv = new HashMap<String, String>();
- if (env != null) {
- for (int i = 0; i < env.length; i++) {
- String key = env[i].getName();
- String value = env[i].getValue();
- nv.put(key, value);
- }
- }
- for (Entry<String, String> entry : nv.entrySet()) {
- log.debug("Env[" + entry.getKey() + "] = " + entry.getValue());
- out.write(("export " + entry.getKey() + "=" + entry.getValue() + "\n").getBytes());
-
- }
-
- // prepare the command
- final String SPACE = " ";
- StringBuffer cmd = new StringBuffer();
- cmd.append(app.getExecutableLocation());
- cmd.append(SPACE);
-
- MessageContext input = context.getInMessageContext();
- ;
- Map<String, Object> inputs = input.getParameters();
- Set<String> keys = inputs.keySet();
- for (String paramName : keys) {
- ActualParameter actualParameter = (ActualParameter) inputs.get(paramName);
- if ("URIArray".equals(actualParameter.getType().getType().toString())) {
- String[] values = ((URIArrayType) actualParameter.getType()).getValueArray();
- for (String value : values) {
- cmd.append(value);
- cmd.append(SPACE);
- }
- } else {
- String paramValue = MappingFactory.toString(actualParameter);
- cmd.append(paramValue);
- cmd.append(SPACE);
- }
- }
- // We redirect the error and stdout to remote files, they will be read
- // in later
- cmd.append(SPACE);
- cmd.append("1>");
- cmd.append(SPACE);
- cmd.append(app.getStandardOutput());
- cmd.append(SPACE);
- cmd.append("2>");
- cmd.append(SPACE);
- cmd.append(app.getStandardError());
-
- String cmdStr = cmd.toString();
- log.info("Command = " + cmdStr);
- out.write((cmdStr + "\n").getBytes());
- String message = "\"execuationSuceeded\"";
- out.write(("echo " + message + "\n").getBytes());
- out.close();
-
- return shellScript;
- }
-
- public void initProperties(Map<String, String> properties) throws GFacProviderException, GFacException {
-
- }
- /**
- * This method will read standard output and if there's any it will be parsed
- * @param jobIDReaderCommandOutput
- * @param errorMsg
- * @return
- * @throws SSHApiException
- */
- private String getOutputifAvailable(StandardOutReader jobIDReaderCommandOutput, String errorMsg) throws SSHApiException {
- String stdOutputString = jobIDReaderCommandOutput.getStdOutputString();
- String stdErrorString = jobIDReaderCommandOutput.getStdErrorString();
-
- if(stdOutputString == null || stdOutputString.isEmpty() || (stdErrorString != null && !stdErrorString.isEmpty())){
- log.error("Standard Error output : " + stdErrorString);
- throw new SSHApiException(errorMsg + stdErrorString);
- }
- return stdOutputString;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPInputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPInputHandler.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPInputHandler.java
new file mode 100644
index 0000000..cba476e
--- /dev/null
+++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPInputHandler.java
@@ -0,0 +1,152 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.ssh.handler;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.StringUtil;
+import org.apache.airavata.commons.gfac.type.ActualParameter;
+import org.apache.airavata.commons.gfac.type.MappingFactory;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.context.MessageContext;
+import org.apache.airavata.gfac.ssh.security.SSHSecurityContext;
+import org.apache.airavata.gfac.handler.AbstractHandler;
+import org.apache.airavata.gfac.handler.GFacHandlerException;
+import org.apache.airavata.gfac.ssh.util.GFACSSHUtils;
+import org.apache.airavata.gsi.ssh.api.Cluster;
+import org.apache.airavata.gsi.ssh.api.SSHApiException;
+import org.apache.airavata.gsi.ssh.api.ServerInfo;
+import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
+import org.apache.airavata.gsi.ssh.impl.PBSCluster;
+import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPasswordAuthenticationInfo;
+import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPublicKeyFileAuthentication;
+import org.apache.airavata.gsi.ssh.util.CommonUtils;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.URIArrayType;
+import org.apache.airavata.schemas.gfac.URIParameterType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.*;
+
+/**
+ * This handler will copy input data from gateway machine to airavata
+ * installed machine, later running handlers can copy the input files to computing resource
+ * <Handler class="AdvancedSCPOutputHandler">
+ <property name="privateKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa"/>
+ <property name="publicKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa.pub"/>
+ <property name="userName" value="airavata"/>
+ <property name="hostName" value="gw98.iu.xsede.org"/>
+ <property name="inputPath" value="/home/airavata/outputData"/>
+ */
+public class AdvancedSCPInputHandler extends AbstractHandler {
+ private static final Logger log = LoggerFactory.getLogger(AdvancedSCPInputHandler.class);
+
+ private String password = null;
+
+ private String publicKeyPath;
+
+ private String passPhrase;
+
+ private String privateKeyPath;
+
+ private String userName;
+
+ private String hostName;
+
+ private String inputPath;
+
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
+ password = properties.get("password");
+ passPhrase = properties.get("passPhrase");
+ privateKeyPath = properties.get("privateKeyPath");
+ publicKeyPath = properties.get("publicKeyPath");
+ userName = properties.get("userName");
+ hostName = properties.get("hostName");
+ inputPath = properties.get("inputPath");
+ }
+
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ MessageContext inputNew = new MessageContext();
+ try{
+ if(jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT) == null){
+ try {
+ GFACSSHUtils.addSecurityContext(jobExecutionContext);
+ } catch (ApplicationSettingsException e) {
+ log.error(e.getMessage());
+ throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
+ }
+ }
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext()
+ .getApplicationDeploymentDescription().getType();
+
+ AuthenticationInfo authenticationInfo = null;
+ if (password != null) {
+ authenticationInfo = new DefaultPasswordAuthenticationInfo(this.password);
+ } else {
+ authenticationInfo = new DefaultPublicKeyFileAuthentication(this.publicKeyPath, this.privateKeyPath,
+ this.passPhrase);
+ }
+ // Server info
+ ServerInfo serverInfo = new ServerInfo(this.userName, this.hostName);
+ Cluster pbsCluster = null;
+ // here doesn't matter what the job manager is because we are only doing some file handling
+ // not really dealing with monitoring or job submission, so we pa
+ pbsCluster = new PBSCluster(serverInfo, authenticationInfo, CommonUtils.getPBSJobManager("/opt/torque/torque-4.2.3.1/bin/"));
+ String parentPath = inputPath + File.separator + jobExecutionContext.getExperimentID() + File.separator + jobExecutionContext.getTaskData().getTaskID();
+ (new File(parentPath)).mkdirs();
+ MessageContext input = jobExecutionContext.getInMessageContext();
+ Set<String> parameters = input.getParameters().keySet();
+ for (String paramName : parameters) {
+ ActualParameter actualParameter = (ActualParameter) input.getParameters().get(paramName);
+ String paramValue = MappingFactory.toString(actualParameter);
+ //TODO: Review this with type
+ if ("URI".equals(actualParameter.getType().getType().toString())) {
+ ((URIParameterType) actualParameter.getType()).setValue(stageInputFiles(pbsCluster, paramValue, parentPath));
+ } else if ("URIArray".equals(actualParameter.getType().getType().toString())) {
+ List<String> split = Arrays.asList(StringUtil.getElementsFromString(paramValue));
+ List<String> newFiles = new ArrayList<String>();
+ for (String paramValueEach : split) {
+ String stageInputFiles = stageInputFiles(pbsCluster, paramValueEach, parentPath);
+ newFiles.add(stageInputFiles);
+ }
+ ((URIArrayType) actualParameter.getType()).setValueArray(newFiles.toArray(new String[newFiles.size()]));
+ }
+ inputNew.getParameters().put(paramName, actualParameter);
+ }
+ } catch (Exception e) {
+ log.error(e.getMessage());
+ throw new GFacHandlerException("Error while input File Staging", e, e.getLocalizedMessage());
+ }
+ jobExecutionContext.setInMessageContext(inputNew);
+ }
+
+ private String stageInputFiles(Cluster cluster, String paramValue, String parentPath) throws GFacException {
+ try {
+ cluster.scpFrom(paramValue, parentPath);
+ return "file://" + parentPath + File.separator + (new File(paramValue)).getName();
+ } catch (SSHApiException e) {
+ log.error("Error tranfering remote file to local file, remote path: " + paramValue);
+ throw new GFacException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java
new file mode 100644
index 0000000..79ad49a
--- /dev/null
+++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java
@@ -0,0 +1,130 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.ssh.handler;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.ssh.security.SSHSecurityContext;
+import org.apache.airavata.gfac.handler.AbstractHandler;
+import org.apache.airavata.gfac.handler.GFacHandlerException;
+import org.apache.airavata.gfac.ssh.util.GFACSSHUtils;
+import org.apache.airavata.gsi.ssh.api.Cluster;
+import org.apache.airavata.gsi.ssh.api.SSHApiException;
+import org.apache.airavata.gsi.ssh.api.ServerInfo;
+import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
+import org.apache.airavata.gsi.ssh.impl.PBSCluster;
+import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPasswordAuthenticationInfo;
+import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPublicKeyFileAuthentication;
+import org.apache.airavata.gsi.ssh.util.CommonUtils;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.Map;
+
+/**
+ * This handler will copy outputs from airavata installed local directory
+ * to a remote location, prior to this handler SCPOutputHandler should be invoked
+ * Should add following configuration to gfac-config.xml and configure the keys properly
+ * <Handler class="AdvancedSCPOutputHandler">
+ <property name="privateKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa"/>
+ <property name="publicKeyPath" value="/Users/lahirugunathilake/.ssh/id_dsa.pub"/>
+ <property name="userName" value="airavata"/>
+ <property name="hostName" value="gw98.iu.xsede.org"/>
+ <property name="outputPath" value="/home/airavata/outputData"/>
+ <property name="passPhrase" value="/home/airavata/outputData"/>
+ <property name="password" value="/home/airavata/outputData"/>
+
+ */
+public class AdvancedSCPOutputHandler extends AbstractHandler {
+ private static final Logger log = LoggerFactory.getLogger(AdvancedSCPOutputHandler.class);
+
+ private String password = null;
+
+ private String publicKeyPath;
+
+ private String passPhrase;
+
+ private String privateKeyPath;
+
+ private String userName;
+
+ private String hostName;
+
+ private String outputPath;
+
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
+ password = properties.get("password");
+ passPhrase = properties.get("passPhrase");
+ privateKeyPath = properties.get("privateKeyPath");
+ publicKeyPath = properties.get("publicKeyPath");
+ userName = properties.get("userName");
+ hostName = properties.get("hostName");
+ outputPath = properties.get("outputPath");
+ }
+
+ @Override
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ try {
+ if (jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT) == null) {
+ try {
+ GFACSSHUtils.addSecurityContext(jobExecutionContext);
+ } catch (ApplicationSettingsException e) {
+ log.error(e.getMessage());
+ throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
+ }
+ }
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext()
+ .getApplicationDeploymentDescription().getType();
+ String standardError = app.getStandardError();
+ String standardOutput = app.getStandardOutput();
+ String outputDataDirectory = app.getOutputDataDirectory();
+
+ AuthenticationInfo authenticationInfo = null;
+ if (password != null) {
+ authenticationInfo = new DefaultPasswordAuthenticationInfo(this.password);
+ } else {
+ authenticationInfo = new DefaultPublicKeyFileAuthentication(this.publicKeyPath, this.privateKeyPath,
+ this.passPhrase);
+ }
+ // Server info
+ ServerInfo serverInfo = new ServerInfo(this.userName, this.hostName);
+
+ Cluster pbsCluster = new PBSCluster(serverInfo, authenticationInfo, CommonUtils.getPBSJobManager("/opt/torque/torque-4.2.3.1/bin/"));
+ outputPath = outputPath + File.separator + jobExecutionContext.getExperimentID() + "-" + jobExecutionContext.getTaskData().getTaskID()
+ + File.separator;
+ pbsCluster.makeDirectory(outputPath);
+ pbsCluster.scpTo(outputPath, standardError);
+ pbsCluster.scpTo(outputPath, standardOutput);
+ for (String files : jobExecutionContext.getOutputFiles()) {
+ pbsCluster.scpTo(outputPath, files);
+ }
+ } catch (SSHApiException e) {
+ log.error("Error transfering files to remote host : " + hostName + " with the user: " + userName);
+ log.error(e.getMessage());
+ throw new GFacHandlerException(e);
+ } catch (GFacException e) {
+ throw new GFacHandlerException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHDirectorySetupHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHDirectorySetupHandler.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHDirectorySetupHandler.java
new file mode 100644
index 0000000..0040de0
--- /dev/null
+++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHDirectorySetupHandler.java
@@ -0,0 +1,107 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.ssh.handler;
+
+import java.util.Map;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.ssh.security.SSHSecurityContext;
+import org.apache.airavata.gfac.handler.AbstractHandler;
+import org.apache.airavata.gfac.handler.GFacHandlerException;
+import org.apache.airavata.gfac.ssh.util.GFACSSHUtils;
+import org.apache.airavata.gfac.utils.GFacUtils;
+import org.apache.airavata.gsi.ssh.api.Cluster;
+import org.apache.airavata.gsi.ssh.api.SSHApiException;
+import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
+import org.apache.airavata.model.workspace.experiment.DataTransferDetails;
+import org.apache.airavata.model.workspace.experiment.ErrorCategory;
+import org.apache.airavata.model.workspace.experiment.TransferState;
+import org.apache.airavata.model.workspace.experiment.TransferStatus;
+import org.apache.airavata.registry.cpi.ChildDataType;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SSHDirectorySetupHandler extends AbstractHandler {
+ private static final Logger log = LoggerFactory.getLogger(SSHDirectorySetupHandler.class);
+
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ try {
+ if (jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT) == null) {
+ GFACSSHUtils.addSecurityContext(jobExecutionContext);
+ }
+ } catch (ApplicationSettingsException e) {
+ log.error(e.getMessage());
+ throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
+ } catch (GFacException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+
+ log.info("Setup SSH job directorties");
+ super.invoke(jobExecutionContext);
+ makeDirectory(jobExecutionContext);
+
+ }
+ private void makeDirectory(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ try{
+ Cluster cluster = ((SSHSecurityContext) jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT)).getPbsCluster();
+ if (cluster == null) {
+ throw new GFacHandlerException("Security context is not set properly");
+ } else {
+ log.info("Successfully retrieved the Security Context");
+ }
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
+ String workingDirectory = app.getScratchWorkingDirectory();
+ cluster.makeDirectory(workingDirectory);
+ cluster.makeDirectory(app.getScratchWorkingDirectory());
+ cluster.makeDirectory(app.getInputDataDirectory());
+ cluster.makeDirectory(app.getOutputDataDirectory());
+ DataTransferDetails detail = new DataTransferDetails();
+ TransferStatus status = new TransferStatus();
+ status.setTransferState(TransferState.DIRECTORY_SETUP);
+ detail.setTransferStatus(status);
+ detail.setTransferDescription("Working directory = " + workingDirectory);
+
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
+
+ } catch (SSHApiException e) {
+ throw new GFacHandlerException("Error executing the Handler: " + SSHDirectorySetupHandler.class, e);
+ } catch (Exception e) {
+ DataTransferDetails detail = new DataTransferDetails();
+ TransferStatus status = new TransferStatus();
+ status.setTransferState(TransferState.FAILED);
+ detail.setTransferStatus(status);
+ try {
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
+ } catch (Exception e1) {
+ throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
+ }
+ throw new GFacHandlerException("Error executing the Handler: " + SSHDirectorySetupHandler.class, e);
+ }
+ }
+
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9bb8c2be/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHInputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHInputHandler.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHInputHandler.java
new file mode 100644
index 0000000..3d445e0
--- /dev/null
+++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHInputHandler.java
@@ -0,0 +1,143 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.ssh.handler;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.StringUtil;
+import org.apache.airavata.commons.gfac.type.ActualParameter;
+import org.apache.airavata.commons.gfac.type.MappingFactory;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.context.MessageContext;
+import org.apache.airavata.gfac.ssh.security.SSHSecurityContext;
+import org.apache.airavata.gfac.handler.AbstractHandler;
+import org.apache.airavata.gfac.handler.GFacHandlerException;
+import org.apache.airavata.gfac.ssh.util.GFACSSHUtils;
+import org.apache.airavata.gfac.utils.GFacUtils;
+import org.apache.airavata.gsi.ssh.api.Cluster;
+import org.apache.airavata.gsi.ssh.api.SSHApiException;
+import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
+import org.apache.airavata.model.workspace.experiment.DataTransferDetails;
+import org.apache.airavata.model.workspace.experiment.ErrorCategory;
+import org.apache.airavata.model.workspace.experiment.TransferState;
+import org.apache.airavata.model.workspace.experiment.TransferStatus;
+import org.apache.airavata.registry.cpi.ChildDataType;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.URIArrayType;
+import org.apache.airavata.schemas.gfac.URIParameterType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SSHInputHandler extends AbstractHandler {
+
+ private static final Logger log = LoggerFactory.getLogger(SSHInputHandler.class);
+
+
+ public void invoke(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
+ DataTransferDetails detail = new DataTransferDetails();
+ TransferStatus status = new TransferStatus();
+ MessageContext inputNew = new MessageContext();
+ try {
+
+ if (jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT) == null) {
+ try {
+ GFACSSHUtils.addSecurityContext(jobExecutionContext);
+ } catch (ApplicationSettingsException e) {
+ log.error(e.getMessage());
+ throw new GFacHandlerException("Error while creating SSHSecurityContext", e, e.getLocalizedMessage());
+ }
+ }
+ log.info("Invoking SCPInputHandler");
+ super.invoke(jobExecutionContext);
+
+
+ MessageContext input = jobExecutionContext.getInMessageContext();
+ Set<String> parameters = input.getParameters().keySet();
+ for (String paramName : parameters) {
+ ActualParameter actualParameter = (ActualParameter) input.getParameters().get(paramName);
+ String paramValue = MappingFactory.toString(actualParameter);
+ //TODO: Review this with type
+ if ("URI".equals(actualParameter.getType().getType().toString())) {
+ ((URIParameterType) actualParameter.getType()).setValue(stageInputFiles(jobExecutionContext, paramValue));
+ } else if ("URIArray".equals(actualParameter.getType().getType().toString())) {
+ List<String> split = Arrays.asList(StringUtil.getElementsFromString(paramValue));
+ List<String> newFiles = new ArrayList<String>();
+ for (String paramValueEach : split) {
+ String stageInputFiles = stageInputFiles(jobExecutionContext, paramValueEach);
+ status.setTransferState(TransferState.UPLOAD);
+ detail.setTransferStatus(status);
+ detail.setTransferDescription("Input Data Staged: " + stageInputFiles);
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
+ newFiles.add(stageInputFiles);
+ }
+ ((URIArrayType) actualParameter.getType()).setValueArray(newFiles.toArray(new String[newFiles.size()]));
+ }
+ inputNew.getParameters().put(paramName, actualParameter);
+ }
+ } catch (Exception e) {
+ log.error(e.getMessage());
+ status.setTransferState(TransferState.FAILED);
+ detail.setTransferStatus(status);
+ try {
+ GFacUtils.saveErrorDetails(jobExecutionContext, e.getLocalizedMessage(), CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.FILE_SYSTEM_FAILURE);
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
+ } catch (Exception e1) {
+ throw new GFacHandlerException("Error persisting status", e1, e1.getLocalizedMessage());
+ }
+ throw new GFacHandlerException("Error while input File Staging", e, e.getLocalizedMessage());
+ }
+ jobExecutionContext.setInMessageContext(inputNew);
+ }
+
+ private static String stageInputFiles(JobExecutionContext jobExecutionContext, String paramValue) throws IOException, GFacException {
+ Cluster cluster = ((SSHSecurityContext) jobExecutionContext.getSecurityContext(SSHSecurityContext.SSH_SECURITY_CONTEXT)).getPbsCluster();
+ if (cluster == null) {
+ throw new GFacException("Security context is not set properly");
+ } else {
+ log.info("Successfully retrieved the Security Context");
+ }
+ ApplicationDeploymentDescriptionType app = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription().getType();
+ int i = paramValue.lastIndexOf(File.separator);
+ String substring = paramValue.substring(i + 1);
+ try {
+ String targetFile = app.getInputDataDirectory() + File.separator + substring;
+ if(paramValue.startsWith("file")){
+ paramValue = paramValue.substring(paramValue.indexOf(":") + 1, paramValue.length());
+ }
+ cluster.scpTo(targetFile, paramValue);
+ return targetFile;
+ } catch (SSHApiException e) {
+ throw new GFacHandlerException("Error while input File Staging", e, e.getLocalizedMessage());
+ }
+ }
+
+ public void initProperties(Map<String, String> properties) throws GFacHandlerException {
+
+ }
+}