You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ch...@apache.org on 2015/12/03 20:26:39 UTC
[3/5] airavata git commit: credential store related changes,
BES changes
http://git-wip-us.apache.org/repos/asf/airavata/blob/0d9a6126/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java
new file mode 100644
index 0000000..cf651cf
--- /dev/null
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java
@@ -0,0 +1,194 @@
+///*
+// *
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements. See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership. The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing,
+// * software distributed under the License is distributed on an
+// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// * KIND, either express or implied. See the License for the
+// * specific language governing permissions and limitations
+// * under the License.
+// *
+//*/
+//
+//package org.apache.airavata.gfac.impl.task;
+//
+//import de.fzj.unicore.bes.client.FactoryClient;
+//import de.fzj.unicore.uas.client.StorageClient;
+//import de.fzj.unicore.wsrflite.xmlbeans.WSUtilities;
+//import eu.unicore.util.httpclient.DefaultClientConfiguration;
+//import org.apache.airavata.gfac.core.GFacUtils;
+//import org.apache.airavata.gfac.core.context.ProcessContext;
+//import org.apache.airavata.gfac.core.context.TaskContext;
+//import org.apache.airavata.gfac.core.task.JobSubmissionTask;
+//import org.apache.airavata.gfac.core.task.TaskException;
+//import org.apache.airavata.gfac.impl.task.utils.bes.DataTransferrer;
+//import org.apache.airavata.gfac.impl.task.utils.bes.JSDLGenerator;
+//import org.apache.airavata.gfac.impl.task.utils.bes.StorageCreator;
+//import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
+//import org.apache.airavata.model.appcatalog.computeresource.UnicoreJobSubmission;
+//import org.apache.airavata.model.job.JobModel;
+//import org.apache.airavata.model.status.JobState;
+//import org.apache.airavata.model.status.JobStatus;
+//import org.apache.airavata.model.status.TaskStatus;
+//import org.apache.airavata.model.task.TaskTypes;
+//import org.ggf.schemas.bes.x2006.x08.besFactory.CreateActivityDocument;
+//import org.ggf.schemas.bes.x2006.x08.besFactory.CreateActivityResponseDocument;
+//import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
+//import org.slf4j.Logger;
+//import org.slf4j.LoggerFactory;
+//import org.w3.x2005.x08.addressing.EndpointReferenceType;
+//
+//import java.util.Calendar;
+//import java.util.Map;
+//
+//public class BESJobSubmissionTask implements JobSubmissionTask {
+// private static final Logger log = LoggerFactory.getLogger(BESJobSubmissionTask.class);
+// private DefaultClientConfiguration secProperties;
+//
+// private String jobId;
+// @Override
+// public JobStatus cancel(TaskContext taskcontext) throws TaskException {
+// return null;
+// }
+//
+// @Override
+// public void init(Map<String, String> propertyMap) throws TaskException {
+// }
+//
+// @Override
+// public TaskStatus execute(TaskContext taskContext) {
+// StorageClient sc = null;
+// try {
+// ProcessContext processContext = taskContext.getParentProcessContext();
+// JobSubmissionProtocol protocol = processContext.getJobSubmissionProtocol();
+// String interfaceId = processContext.getApplicationInterfaceDescription().getApplicationInterfaceId();
+// String factoryUrl = null;
+// if (protocol.equals(JobSubmissionProtocol.UNICORE)) {
+// UnicoreJobSubmission unicoreJobSubmission = GFacUtils.getUnicoreJobSubmission(interfaceId);
+// factoryUrl = unicoreJobSubmission.getUnicoreEndPointURL();
+// }
+// EndpointReferenceType eprt = EndpointReferenceType.Factory.newInstance();
+// eprt.addNewAddress().setStringValue(factoryUrl);
+// String userDN = processContext.getProcessModel().getUserDn();
+//
+// CreateActivityDocument cad = CreateActivityDocument.Factory.newInstance();
+//
+// // create storage
+// StorageCreator storageCreator = new StorageCreator(secProperties, factoryUrl, 5, null);
+// sc = storageCreator.createStorage();
+//
+// JobDefinitionType jobDefinition = JSDLGenerator.buildJSDLInstance(processContext, sc.getUrl()).getJobDefinition();
+// cad.addNewCreateActivity().addNewActivityDocument().setJobDefinition(jobDefinition);
+//
+// log.info("Submitted JSDL: " + jobDefinition.getJobDescription());
+//
+// // upload files if any
+// DataTransferrer dt = new DataTransferrer(processContext, sc);
+// dt.uploadLocalFiles();
+//
+// JobModel jobDetails = new JobModel();
+// FactoryClient factory = new FactoryClient(eprt, secProperties);
+//
+// log.info(String.format("Activity Submitting to %s ... \n",
+// factoryUrl));
+// monitorPublisher.publish(new StartExecutionEvent());
+// CreateActivityResponseDocument response = factory.createActivity(cad);
+// log.info(String.format("Activity Submitted to %s \n", factoryUrl));
+//
+// EndpointReferenceType activityEpr = response.getCreateActivityResponse().getActivityIdentifier();
+//
+// log.info("Activity : " + activityEpr.getAddress().getStringValue() + " Submitted.");
+//
+// // factory.waitWhileActivityIsDone(activityEpr, 1000);
+// jobId = WSUtilities.extractResourceID(activityEpr);
+// if (jobId == null) {
+// jobId = new Long(Calendar.getInstance().getTimeInMillis())
+// .toString();
+// }
+// log.info("JobID: " + jobId);
+// jobDetails.setJobId(jobId);
+// jobDetails.setJobDescription(activityEpr.toString());
+// jobDetails.setJobStatus(new JobStatus(JobState.SUBMITTED));
+// processContext.setJobModel(jobDetails);
+// GFacUtils.saveJobStatus(processContext, jobDetails);
+// log.info(formatStatusMessage(activityEpr.getAddress()
+// .getStringValue(), factory.getActivityStatus(activityEpr)
+// .toString()));
+//
+// waitUntilDone(eprt, activityEpr, jobDetails, secProperties);
+//
+// ActivityStatusType activityStatus = null;
+// activityStatus = getStatus(factory, activityEpr);
+// log.info(formatStatusMessage(activityEpr.getAddress().getStringValue(), activityStatus.getState().toString()));
+// ActivityClient activityClient;
+// activityClient = new ActivityClient(activityEpr, secProperties);
+// // now use the activity working directory property
+// dt.setStorageClient(activityClient.getUspaceClient());
+//
+// if ((activityStatus.getState() == ActivityStateEnumeration.FAILED)) {
+// String error = activityStatus.getFault().getFaultcode()
+// .getLocalPart()
+// + "\n"
+// + activityStatus.getFault().getFaultstring()
+// + "\n EXITCODE: " + activityStatus.getExitCode();
+// log.info(error);
+//
+// JobState applicationJobStatus = JobState.FAILED;
+// sendNotification(jobExecutionContext,applicationJobStatus);
+// GFacUtils.updateJobStatus(jobExecutionContext, jobDetails, applicationJobStatus);
+// try {Thread.sleep(5000);} catch (InterruptedException e) {}
+//
+// //What if job is failed before execution and there are not stdouts generated yet?
+// log.debug("Downloading any standard output and error files, if they were produced.");
+// dt.downloadStdOuts();
+//
+// } else if (activityStatus.getState() == ActivityStateEnumeration.CANCELLED) {
+// JobState applicationJobStatus = JobState.CANCELED;
+// sendNotification(jobExecutionContext,applicationJobStatus);
+// GFacUtils.updateJobStatus(jobExecutionContext, jobDetails, applicationJobStatus);
+// throw new GFacProviderException(
+// jobExecutionContext.getExperimentID() + "Job Canceled");
+// } else if (activityStatus.getState() == ActivityStateEnumeration.FINISHED) {
+// try {
+// Thread.sleep(5000);
+// JobState applicationJobStatus = JobState.COMPLETE;
+// sendNotification(jobExecutionContext,applicationJobStatus);
+//
+// } catch (InterruptedException e) {
+// }
+// if (activityStatus.getExitCode() == 0) {
+// dt.downloadRemoteFiles();
+// } else {
+// dt.downloadStdOuts();
+// }
+// }
+//
+// dt.publishFinalOutputs();
+// } catch (AppCatalogException e) {
+// log.error("Error while retrieving UNICORE job submission..");
+// throw new GFacProviderException("Error while retrieving UNICORE job submission..", e);
+// } catch (Exception e) {
+// log.error("Cannot create storage..");
+// throw new GFacProviderException("Cannot create storage..", e);
+// }
+// }
+//
+// @Override
+// public TaskStatus recover(TaskContext taskContext) {
+// return null;
+// }
+//
+// @Override
+// public TaskTypes getType() {
+// return null;
+// }
+//}
http://git-wip-us.apache.org/repos/asf/airavata/blob/0d9a6126/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/ActivityInfo.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/ActivityInfo.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/ActivityInfo.java
new file mode 100644
index 0000000..cac235c
--- /dev/null
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/ActivityInfo.java
@@ -0,0 +1,50 @@
+/*
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*/
+package org.apache.airavata.gfac.impl.task.utils.bes;
+
+import org.ggf.schemas.bes.x2006.x08.besFactory.ActivityStatusType;
+import org.w3.x2005.x08.addressing.EndpointReferenceType;
+
+import java.io.Serializable;
+
+public class ActivityInfo implements Serializable{
+
+ private static final long serialVersionUID = 1L;
+
+ private EndpointReferenceType activityEPR;
+
+ private ActivityStatusType activityStatusDoc;
+
+
+ public EndpointReferenceType getActivityEPR() {
+ return activityEPR;
+ }
+ public void setActivityEPR(EndpointReferenceType activityEPR) {
+ this.activityEPR = activityEPR;
+ }
+ public ActivityStatusType getActivityStatus() {
+ return activityStatusDoc;
+ }
+ public void setActivityStatusDoc(ActivityStatusType activityStatusDoc) {
+ this.activityStatusDoc = activityStatusDoc;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/0d9a6126/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/ApplicationProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/ApplicationProcessor.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/ApplicationProcessor.java
new file mode 100644
index 0000000..b17b92e
--- /dev/null
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/ApplicationProcessor.java
@@ -0,0 +1,221 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.impl.task.utils.bes;
+
+import org.apache.airavata.gfac.core.context.ProcessContext;
+import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
+import org.apache.airavata.model.appcatalog.appdeployment.ApplicationParallelismType;
+import org.apache.airavata.model.application.io.InputDataObjectType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.ApplicationType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.FileNameType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.UserNameType;
+import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.NumberOfProcessesType;
+import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.ProcessesPerHostType;
+import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.ThreadsPerProcessType;
+
+import java.util.Iterator;
+import java.util.List;
+
+
+public class ApplicationProcessor {
+
+ public static void generateJobSpecificAppElements(JobDefinitionType value, ProcessContext context){
+
+ String userName = getUserNameFromContext(context);
+// if (userName.equalsIgnoreCase("admin")){
+// userName = "CN=zdv575, O=Ultrascan Gateway, C=DE";
+// }
+
+ ApplicationDeploymentDescription appDep= context.getApplicationDeploymentDescription();
+ String appname = context.getApplicationInterfaceDescription().getApplicationName();
+ ApplicationParallelismType parallelism = appDep.getParallelism();
+ ApplicationType appType = JSDLUtils.getOrCreateApplication(value);
+ appType.setApplicationName(appname);
+
+
+// if (appDep.getSetEnvironment().size() > 0) {
+// createApplicationEnvironment(value, appDep.getSetEnvironment(), parallelism);
+// }
+//
+
+ String stdout = context.getStdoutLocation();
+ String stderr = context.getStderrLocation();
+ if(stdout != null) {
+ stdout = stdout.substring(stdout.lastIndexOf('/')+1);
+ }
+
+ if(stderr != null) {
+ stderr = stderr.substring(stderr.lastIndexOf('/')+1);
+ }
+
+ stdout = (stdout == null || stdout.equals("")) ? "stdout":stdout;
+ stderr = (stdout == null || stderr.equals("")) ? "stderr":stderr;
+
+ if (appDep.getExecutablePath() != null) {
+ FileNameType fNameType = FileNameType.Factory.newInstance();
+ fNameType.setStringValue(appDep.getExecutablePath());
+ if(isParallelJob(context)) {
+ JSDLUtils.getOrCreateSPMDApplication(value).setExecutable(fNameType);
+ if (parallelism.equals(ApplicationParallelismType.OPENMP_MPI)){
+ JSDLUtils.getSPMDApplication(value).setSPMDVariation(SPMDVariations.OpenMPI.value());
+ }else if (parallelism.equals(ApplicationParallelismType.MPI)){
+ JSDLUtils.getSPMDApplication(value).setSPMDVariation(SPMDVariations.MPI.value());
+ }
+
+ // setting number of processes
+ try {
+ String np = getInputAsString(context, BESConstants.NUMBER_OF_PROCESSES);
+ if((np != null) && (Integer.parseInt(np) > 0)){
+ NumberOfProcessesType num = NumberOfProcessesType.Factory.newInstance();
+ num.setStringValue(np);
+ JSDLUtils.getSPMDApplication(value).setNumberOfProcesses(num);
+ }
+
+ }catch(RuntimeException np) {
+ // do nothing
+ }
+
+
+ try {
+ // setting processes per host
+ String pphost = getInputAsString(context, BESConstants.PROCESSES_PER_HOST);
+ if((pphost != null) && (Integer.parseInt(pphost) > 0)){
+ ProcessesPerHostType pph = ProcessesPerHostType.Factory.newInstance();
+ pph.setStringValue(String.valueOf(pphost));
+ JSDLUtils.getSPMDApplication(value).setProcessesPerHost(pph);
+ }
+ }catch(RuntimeException np) {
+ // do nothing
+ }
+
+ int totalThreadCount = context.getProcessModel().getResourceSchedule().getNumberOfThreads();
+ // we take it as threads per processes
+ if(totalThreadCount > 0){
+ ThreadsPerProcessType tpp = ThreadsPerProcessType.Factory.newInstance();
+ tpp.setStringValue(String.valueOf(totalThreadCount));
+ JSDLUtils.getSPMDApplication(value).setThreadsPerProcess(tpp);
+ }
+
+ if(userName != null) {
+ UserNameType userNameType = UserNameType.Factory.newInstance();
+ userNameType.setStringValue(userName);
+ JSDLUtils.getSPMDApplication(value).setUserName(userNameType);
+ }
+ if (stdout != null){
+ FileNameType fName = FileNameType.Factory.newInstance();
+ fName.setStringValue(stdout);
+ JSDLUtils.getOrCreateSPMDApplication(value).setOutput(fName);
+ }
+ if (stderr != null){
+ FileNameType fName = FileNameType.Factory.newInstance();
+ fName.setStringValue(stderr);
+ JSDLUtils.getOrCreateSPMDApplication(value).setError(fName);
+ }
+
+
+ }
+ else {
+ JSDLUtils.getOrCreatePOSIXApplication(value).setExecutable(fNameType);
+ if(userName != null) {
+ UserNameType userNameType = UserNameType.Factory.newInstance();
+ userNameType.setStringValue(userName);
+ JSDLUtils.getOrCreatePOSIXApplication(value).setUserName(userNameType);
+ }
+ if (stdout != null){
+ FileNameType fName = FileNameType.Factory.newInstance();
+ fName.setStringValue(stdout);
+ JSDLUtils.getOrCreatePOSIXApplication(value).setOutput(fName);
+ }
+ if (stderr != null){
+ FileNameType fName = FileNameType.Factory.newInstance();
+ fName.setStringValue(stderr);
+ JSDLUtils.getOrCreatePOSIXApplication(value).setError(fName);
+ }
+ }
+ }
+ }
+
+ public static String getUserNameFromContext(ProcessContext jobContext) {
+ if(jobContext.getProcessModel() == null)
+ return null;
+ //TODO: Extend unicore model to specify optional unix user id (allocation account)
+ return "admin";
+ }
+
+ public static void addApplicationArgument(JobDefinitionType value, ProcessContext context, String stringPrm) {
+ if(isParallelJob(context)){
+ JSDLUtils.getOrCreateSPMDApplication(value).addNewArgument().setStringValue(stringPrm);
+ }
+ else {
+ JSDLUtils.getOrCreatePOSIXApplication(value).addNewArgument().setStringValue(stringPrm);
+ }
+ }
+
+ public static String getApplicationStdOut(JobDefinitionType value, ProcessContext context) throws RuntimeException {
+ if (isParallelJob(context)) return JSDLUtils.getOrCreateSPMDApplication(value).getOutput().getStringValue();
+ else return JSDLUtils.getOrCreatePOSIXApplication(value).getOutput().getStringValue();
+ }
+
+ public static String getApplicationStdErr(JobDefinitionType value, ProcessContext context) throws RuntimeException {
+ if (isParallelJob(context)) return JSDLUtils.getOrCreateSPMDApplication(value).getError().getStringValue();
+ else return JSDLUtils.getOrCreatePOSIXApplication(value).getError().getStringValue();
+ }
+
+ public static void createGenericApplication(JobDefinitionType value, String appName) {
+ ApplicationType appType = JSDLUtils.getOrCreateApplication(value);
+ appType.setApplicationName(appName);
+ }
+
+ public static boolean isParallelJob(ProcessContext context) {
+
+ ApplicationDeploymentDescription appDep = context.getApplicationDeploymentDescription();
+ ApplicationParallelismType parallelism = appDep.getParallelism();
+
+ boolean isParallel = false;
+
+ if(parallelism.equals(ApplicationParallelismType.MPI) ||
+ parallelism.equals(ApplicationParallelismType.OPENMP_MPI) ||
+ parallelism.equals(ApplicationParallelismType.OPENMP )) {
+ isParallel = true;
+ }
+
+ return isParallel;
+ }
+
+ private static String getInputAsString(ProcessContext context, String name) {
+ List<InputDataObjectType> inputList = context.getProcessModel().getProcessInputs();
+ String value = null;
+ for (Iterator<InputDataObjectType> iterator = inputList.iterator(); iterator.hasNext();) {
+ InputDataObjectType inputDataObjectType = iterator
+ .next();
+ if (inputDataObjectType.getName().equals(name)) {
+ value = inputDataObjectType.getValue();
+ break;
+ }
+ }
+ return value;
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/0d9a6126/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/BESConstants.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/BESConstants.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/BESConstants.java
new file mode 100644
index 0000000..655095b
--- /dev/null
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/BESConstants.java
@@ -0,0 +1,45 @@
+/*
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*/
+
+package org.apache.airavata.gfac.impl.task.utils.bes;
+
+public interface BESConstants {
+
+ public static final String PROP_SMS_EPR = "unicore.sms.epr";
+
+ public static final String PROP_BES_URL = "bes.factory.url";
+
+ public static final String PROP_ACTIVITY_INFO = "bes.activity.info";
+
+ public static final String PROP_CLIENT_CONF = "bes.client.config";
+
+ public static final String PROP_CA_CERT_PATH = "bes.ca.cert.path";
+
+ public static final String PROP_CA_KEY_PATH = "bes.ca.key.path";
+
+ public static final String PROP_CA_KEY_PASS = "bes.ca.key.pass";
+
+ public static final String NUMBER_OF_PROCESSES = "NumberOfProcesses";
+
+ public static final String PROCESSES_PER_HOST = "ProcessesPerHost";
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/0d9a6126/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/DataTransferrer.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/DataTransferrer.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/DataTransferrer.java
new file mode 100644
index 0000000..945c8a2
--- /dev/null
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/DataTransferrer.java
@@ -0,0 +1,150 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.impl.task.utils.bes;
+
+import de.fzj.unicore.uas.client.StorageClient;
+import org.apache.airavata.gfac.core.context.ProcessContext;
+import org.apache.airavata.model.application.io.DataType;
+import org.apache.airavata.model.application.io.OutputDataObjectType;
+import org.apache.airavata.model.process.ProcessModel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Data movement utility class for transferring files before and after the job execution phase.
+ *
+ * */
+public class DataTransferrer {
+
+ protected final Logger log = LoggerFactory.getLogger(this.getClass());
+
+ protected ProcessContext processContext;
+
+ protected StorageClient storageClient;
+
+ protected List<OutputDataObjectType> resultantOutputsLst;
+
+ protected String gatewayDownloadLocation, stdoutLocation, stderrLocation;
+
+ public DataTransferrer(ProcessContext jobContext, StorageClient storageClient) {
+ this.processContext = jobContext;
+ this.storageClient = storageClient;
+ resultantOutputsLst = new ArrayList<OutputDataObjectType>();
+ initStdoutsLocation();
+ }
+
+ private void initStdoutsLocation() {
+
+ gatewayDownloadLocation = getDownloadLocation();
+
+ String stdout = processContext.getStdoutLocation();
+ String stderr = processContext.getStderrLocation();
+
+ if(stdout != null) {
+ stdout = stdout.substring(stdout.lastIndexOf('/')+1);
+ }
+
+ if(stderr != null) {
+ stderr = stderr.substring(stderr.lastIndexOf('/')+1);
+ }
+
+ String stdoutFileName = (stdout == null || stdout.equals("")) ? "stdout"
+ : stdout;
+ String stderrFileName = (stdout == null || stderr.equals("")) ? "stderr"
+ : stderr;
+
+ stdoutLocation = gatewayDownloadLocation+File.separator+stdoutFileName;
+
+ stderrLocation = gatewayDownloadLocation+File.separator+stderrFileName;
+
+ List<OutputDataObjectType> processOutputs = processContext.getProcessModel().getProcessOutputs();
+ if (processOutputs != null && !processOutputs.isEmpty()){
+ for (OutputDataObjectType processOutput : processOutputs){
+ if (processOutput.getType().equals(DataType.STDOUT)){
+ processOutput.setValue(stdoutLocation);
+ }
+ if (processOutput.getType().equals(DataType.STDERR)){
+ processOutput.setValue(stderrLocation);
+ }
+
+ }
+ }
+ }
+
+ private String getDownloadLocation() {
+ ProcessModel processModel = processContext.getProcessModel();
+ String outputDataDir = "";
+
+ if (processContext.getOutputDir() != null ) {
+
+ outputDataDir = processContext.getOutputDir();
+
+
+ if ("".equals(outputDataDir)) {
+ outputDataDir = getTempPath();
+ }
+
+ else {
+
+ // in case of remote locations use the tmp location
+ if (outputDataDir.startsWith("scp:") ||
+ outputDataDir.startsWith("ftp:") ||
+ outputDataDir.startsWith("gsiftp:")) {
+ outputDataDir = getTempPath();
+ } else if ( outputDataDir.startsWith("file:") &&
+ outputDataDir.contains("@")){
+ outputDataDir = getTempPath();
+
+ } else {
+ try {
+ URI u = new URI(outputDataDir);
+ outputDataDir = u.getPath();
+ } catch (URISyntaxException e) {
+ outputDataDir = getTempPath();
+ }
+ }
+ }
+ }
+
+ File file = new File(outputDataDir);
+ if(!file.exists()){
+ file.mkdirs();
+ }
+
+
+ return outputDataDir;
+ }
+
+ private String getTempPath() {
+ String tmpOutputDir = File.separator + "tmp" + File.separator
+ + processContext.getProcessId();
+ (new File(tmpOutputDir)).mkdirs();
+ return tmpOutputDir;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/0d9a6126/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/FileDownloader.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/FileDownloader.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/FileDownloader.java
new file mode 100644
index 0000000..400bea6
--- /dev/null
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/FileDownloader.java
@@ -0,0 +1,255 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.impl.task.utils.bes;
+
+import de.fzj.unicore.uas.client.FileTransferClient;
+import de.fzj.unicore.uas.client.StorageClient;
+import de.fzj.unicore.uas.client.UFTPConstants;
+import de.fzj.unicore.uas.client.UFTPFileTransferClient;
+import de.fzj.unicore.uas.fts.FiletransferOptions.IMonitorable;
+import de.fzj.unicore.uas.fts.FiletransferOptions.SupportsPartialRead;
+import org.unigrids.services.atomic.types.GridFileType;
+import org.unigrids.services.atomic.types.ProtocolType;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Map;
+
+/**
+ * helper that exports remote files from a UNICORE Storage
+ * to the local client machine.<br/>
+ * Simple wildcards ("*" and "?") and download of
+ * directories are supported.
+ *
+ * TODO this should be refactored so the single-file download logic
+ * is separated from the wildcard/directory/provided outputStream logic
+ *
+ * @author schuller
+ */
+public class FileDownloader extends FileTransferBase{
+
+ private boolean showProgress=true;
+
+ private boolean forceFileOnly=false;
+
+ private OutputStream targetStream=null;
+
+ public FileDownloader(String from, String to, Mode mode){
+ this(from,to,mode,true);
+ }
+
+ public FileDownloader(String from, String to, Mode mode, boolean failOnError){
+ this.to=to;
+ this.from=from;
+ this.mode=mode;
+ this.failOnError=failOnError;
+ }
+
+ public void perform(StorageClient sms)throws Exception{
+ boolean isWildcard=hasWildCards(from);
+ boolean isDirectory=false;
+ GridFileType gridSource=null;
+ if(isWildcard){
+ performWildCardExport(sms);
+ }
+ else {
+ //check if source is a directory
+ gridSource=sms.listProperties(from);
+ isDirectory=gridSource.getIsDirectory();
+ if(isDirectory){
+ if(forceFileOnly){
+ throw new IOException("Source is a directory");
+ }
+ performDirectoryExport(gridSource, new File(to), sms);
+ }
+ else{
+ download(gridSource,new File(to),sms);
+ }
+ }
+ }
+
+ protected void performDirectoryExport(GridFileType directory, File targetDirectory, StorageClient sms)throws Exception{
+ if(!targetDirectory.exists()|| !targetDirectory.canWrite()){
+ throw new IOException("Target directory <"+to+"> does not exist or is not writable!");
+ }
+ if(!targetDirectory.isDirectory()){
+ throw new IOException("Target <"+to+"> is not a directory!");
+ }
+ GridFileType[]gridFiles=sms.listDirectory(directory.getPath());
+ for(GridFileType file: gridFiles){
+ if(file.getIsDirectory()){
+ if(!recurse) {
+ System.out.println("Skipping directory "+file.getPath());
+ continue;
+ }
+ else{
+ File newTargetDirectory=new File(targetDirectory,getName(file.getPath()));
+ boolean success=newTargetDirectory.mkdirs();
+ if(!success)throw new IOException("Can create directory: "+newTargetDirectory.getAbsolutePath());
+ performDirectoryExport(file, newTargetDirectory, sms);
+ continue;
+ }
+ }
+ download(file, new File(targetDirectory,getName(file.getPath())), sms);
+ }
+ }
+
+ protected void performWildCardExport(StorageClient sms)throws Exception{
+ String dir=getDir(from);
+ if(dir==null)dir="/";
+ GridFileType[] files=sms.find(dir, false, from, false, null, null);
+ File targetDir=targetStream==null?new File(to):null;
+ if(targetStream==null){
+ if(!targetDir.isDirectory())throw new IOException("Target is not a directory.");
+ }
+ for(GridFileType f: files){
+ download(f, targetDir, sms);
+ }
+ }
+
+ private String getDir(String path){
+ return new File(path).getParent();
+ }
+
+ private String getName(String path){
+ return new File(path).getName();
+ }
+
+ /**
+ * download a single regular file
+ *
+ * @param source - grid file descriptor
+ * @param localFile - local file or directory to write to
+ * @param sms
+ * @throws Exception
+ */
+ private void download(GridFileType source, File localFile, StorageClient sms)throws Exception{
+ if(source==null || source.getIsDirectory()){
+ throw new IllegalStateException("Source="+source);
+ }
+
+ OutputStream os=targetStream!=null?targetStream:null;
+ FileTransferClient ftc=null;
+ try{
+ String path=source.getPath();
+ if(targetStream==null){
+ if(localFile.isDirectory()){
+ localFile=new File(localFile,getName(source.getPath()));
+ }
+ if(mode.equals(Mode.nooverwrite) && localFile.exists()){
+ System.out.println("File exists and creation mode was set to 'nooverwrite'.");
+ return;
+ }
+ System.out.println("Downloading remote file '"+sms.getUrl()+"#/"+path+"' -> "+localFile.getAbsolutePath());
+ os=new FileOutputStream(localFile.getAbsolutePath(), mode.equals(Mode.append));
+ }
+
+ chosenProtocol=sms.findSupportedProtocol(preferredProtocols.toArray(new ProtocolType.Enum[preferredProtocols.size()]));
+ Map<String,String>extraParameters=makeExtraParameters(chosenProtocol);
+ ftc=sms.getExport(path,extraParameters,chosenProtocol);
+ configure(ftc, extraParameters);
+ System.out.println("DEB:File transfer URL : "+ftc.getUrl());
+// ProgressBar p=null;
+ if(ftc instanceof IMonitorable && showProgress){
+ long size=ftc.getSourceFileSize();
+ if(isRange()){
+ size=getRangeSize();
+ }
+// p=new ProgressBar(localFile.getName(),size,msg);
+// ((IMonitorable) ftc).setProgressListener(p);
+ }
+ long startTime=System.currentTimeMillis();
+ if(isRange()){
+ if(!(ftc instanceof SupportsPartialRead)){
+ throw new Exception("Byte range is defined but protocol does not allow " +
+ "partial read! Please choose a different protocol!");
+ }
+ System.out.println("Byte range: "+startByte+" - "+(getRangeSize()>0?endByte:""));
+ SupportsPartialRead pReader=(SupportsPartialRead)ftc;
+ pReader.readPartial(startByte, endByte-startByte+1, os);
+ }
+ else{
+ ftc.readAllData(os);
+ }
+// if(p!=null){
+// p.finish();
+// }
+ if(timing){
+ long duration=System.currentTimeMillis()-startTime;
+ double rate=(double)localFile.length()/(double)duration;
+ System.out.println("Rate: " +rate+ " kB/sec.");
+ }
+ if(targetStream==null)copyProperties(source, localFile);
+ }
+ finally{
+ try{
+ if(targetStream==null && os!=null){
+ os.close();
+ }
+ }catch(Exception ignored){}
+ if(ftc!=null){
+ try{
+ ftc.destroy();
+ }catch(Exception e1){
+// System.out.println("Could not destroy the filetransfer client",e1);
+ }
+ }
+ }
+ }
+
+ /**
+ * if possible, copy the remote executable flag to the local file
+ * @throws Exception
+ */
+ private void copyProperties(GridFileType source, File localFile)throws Exception{
+ try{
+ localFile.setExecutable(source.getPermissions().getExecutable());
+ }
+ catch(Exception ex){
+ //TODO: logging
+// ("Can't set 'executable' flag for "+localFile.getName(), ex);
+ }
+ }
+
+ private void configure(FileTransferClient ftc, Map<String,String>params){
+ if(ftc instanceof UFTPFileTransferClient){
+ UFTPFileTransferClient u=(UFTPFileTransferClient)ftc;
+ String secret=params.get(UFTPConstants.PARAM_SECRET);
+ u.setSecret(secret);
+ }
+ }
+
+ public void setShowProgress(boolean showProgress) {
+ this.showProgress = showProgress;
+ }
+
+ public void setForceFileOnly(boolean forceFileOnly) {
+ this.forceFileOnly = forceFileOnly;
+ }
+
+ public void setTargetStream(OutputStream targetStream) {
+ this.targetStream = targetStream;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/0d9a6126/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/FileTransferBase.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/FileTransferBase.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/FileTransferBase.java
new file mode 100644
index 0000000..3459cce
--- /dev/null
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/FileTransferBase.java
@@ -0,0 +1,223 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.impl.task.utils.bes;
+
+import de.fzj.unicore.uas.client.StorageClient;
+import de.fzj.unicore.uas.util.PropertyHelper;
+import org.unigrids.services.atomic.types.GridFileType;
+import org.unigrids.services.atomic.types.ProtocolType;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.util.*;
+import java.util.regex.Pattern;
+
+public class FileTransferBase {
+
+ protected Properties extraParameterSource;
+
+ protected boolean timing=false;
+
+ protected boolean recurse=false;
+
+ protected String from;
+
+ protected String to;
+
+ //index of first byte to download
+ protected Long startByte;
+
+ //index of last byte to download
+ protected Long endByte;
+
+ /**
+ * the creation mode
+ */
+ protected Mode mode;
+
+ /**
+ * whether the job processing should fail if an error occurs
+ */
+ protected boolean failOnError;
+
+ protected List<ProtocolType.Enum> preferredProtocols=new ArrayList<ProtocolType.Enum>();
+
+ public FileTransferBase(){
+ preferredProtocols.add(ProtocolType.BFT);
+ }
+
+ protected Map<String,String>makeExtraParameters(ProtocolType.Enum protocol){
+ Map<String, String> res;
+ if(extraParameterSource==null){
+ res=new HashMap<String, String>();
+ }
+ else{
+ String p=String.valueOf(protocol);
+ PropertyHelper ph=new PropertyHelper(extraParameterSource, new String[]{p,p.toLowerCase()});
+ res= ph.getFilteredMap();
+ }
+ if(res.size()>0){
+ // TODO: change it to logger
+ System.out.println("Have "+res.size()+" extra parameters for protocol "+protocol);
+ }
+ return res;
+ }
+
+
+ public String getTo() {
+ return to;
+ }
+
+ public String getFrom() {
+ return from;
+ }
+
+ public void setTo(String to) {
+ this.to = to;
+ }
+
+ public void setFrom(String from) {
+ this.from = from;
+ }
+
+ public Mode getMode() {
+ return mode;
+ }
+
+ public boolean isFailOnError() {
+ return failOnError;
+ }
+
+ public boolean isTiming() {
+ return timing;
+ }
+
+ public void setTiming(boolean timing) {
+ this.timing = timing;
+ }
+
+ public void setFailOnError(boolean failOnError) {
+ this.failOnError = failOnError;
+ }
+
+ public List<ProtocolType.Enum> getPreferredProtocols() {
+ return preferredProtocols;
+ }
+
+ public void setPreferredProtocols(List<ProtocolType.Enum> preferredProtocols) {
+ this.preferredProtocols = preferredProtocols;
+ }
+
+ public void setExtraParameterSource(Properties properties){
+ this.extraParameterSource=properties;
+ }
+
+ public void setRecurse(boolean recurse) {
+ this.recurse = recurse;
+ }
+ /**
+ * check if the given path denotes a valid remote directory
+ * @param remotePath - the path
+ * @param sms - the storage
+ * @return <code>true</code> if the remote directory exists and is a directory
+ */
+ protected boolean isValidDirectory(String remotePath, StorageClient sms){
+ boolean result=false;
+ if(! ("/".equals(remotePath) || ".".equals(remotePath)) ){
+ try{
+ GridFileType gft=sms.listProperties(remotePath);
+ result=gft.getIsDirectory();
+ }catch(Exception ex){
+ result=false;
+ }
+ }
+ else result=true;
+
+ return result;
+ }
+
+ public File[] resolveWildCards(File original){
+ final String name=original.getName();
+ if(!hasWildCards(original))return new File[]{original};
+ File parent=original.getParentFile();
+ if(parent==null)parent=new File(".");
+ FilenameFilter filter=new FilenameFilter(){
+ Pattern p=createPattern(name);
+ public boolean accept(File file, String name){
+ return p.matcher(name).matches();
+ }
+ };
+ return parent.listFiles(filter);
+ }
+
+ protected boolean hasWildCards(File file){
+ return hasWildCards(file.getName());
+ }
+
+ public boolean hasWildCards(String name){
+ return name.contains("*") || name.contains("?");
+ }
+
+ private Pattern createPattern(String nameWithWildcards){
+ String regex=nameWithWildcards.replace("?",".").replace("*", ".*");
+ return Pattern.compile(regex);
+ }
+
+ protected ProtocolType.Enum chosenProtocol=null;
+
+ public ProtocolType.Enum getChosenProtocol(){
+ return chosenProtocol;
+ }
+
+ public Long getStartByte() {
+ return startByte;
+ }
+
+ public void setStartByte(Long startByte) {
+ this.startByte = startByte;
+ }
+
+ public Long getEndByte() {
+ return endByte;
+ }
+
+ public void setEndByte(Long endByte) {
+ this.endByte = endByte;
+ }
+
+ /**
+ * checks if a byte range is defined
+ * @return <code>true</code> iff both startByte and endByte are defined
+ */
+ protected boolean isRange(){
+ return startByte!=null && endByte!=null;
+ }
+
+ /**
+ * get the number of bytes in the byte range, or "-1" if the range is open-ended
+ * @return
+ */
+ protected long getRangeSize(){
+ if(Long.MAX_VALUE==endByte)return -1;
+ return endByte-startByte;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/0d9a6126/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/FileUploader.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/FileUploader.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/FileUploader.java
new file mode 100644
index 0000000..f690091
--- /dev/null
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/FileUploader.java
@@ -0,0 +1,242 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.impl.task.utils.bes;
+
+import de.fzj.unicore.uas.client.FileTransferClient;
+import de.fzj.unicore.uas.client.StorageClient;
+import de.fzj.unicore.uas.client.UFTPConstants;
+import de.fzj.unicore.uas.client.UFTPFileTransferClient;
+import de.fzj.unicore.uas.fts.FiletransferOptions.IMonitorable;
+import org.unigrids.services.atomic.types.ProtocolType;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * upload local file(s) to a remote location
+ *
+ * @author schuller
+ */
+public class FileUploader extends FileTransferBase{
+
+ public FileUploader(String from, String to, Mode mode)throws FileNotFoundException{
+ this(from,to,mode,true);
+ }
+
+ public FileUploader(String from, String to, Mode mode, boolean failOnError)throws FileNotFoundException{
+ this.to=to;
+ this.from=from;
+ this.mode=mode;
+ this.failOnError=failOnError;
+ checkOK();
+ }
+
+ public String getFrom() {
+ return from;
+ }
+
+ public String getTo() {
+ return to;
+ }
+
+
+ public void perform(StorageClient sms)throws Exception{
+ File fileSpec=new File(from);
+ boolean hasWildCards=false;
+ boolean isDirectory=fileSpec.isDirectory();
+ File[] fileset=null;
+
+ if(!isDirectory){
+ hasWildCards=hasWildCards(fileSpec);
+ }
+
+ chosenProtocol=sms.findSupportedProtocol(preferredProtocols.toArray(new ProtocolType.Enum[preferredProtocols.size()]));
+ Map<String,String>extraParameters=makeExtraParameters(chosenProtocol);
+
+ if(!hasWildCards && !isDirectory){
+ //single regular file
+ uploadFile(fileSpec,to,sms,chosenProtocol,extraParameters);
+ return;
+ }
+
+ //handle wildcards or directory
+ if(hasWildCards){
+ fileset=resolveWildCards(fileSpec);
+ }
+ else{
+ fileset=fileSpec.listFiles();
+ }
+
+ if(!isValidDirectory(to, sms)){
+ throw new IOException("The specified remote target '"+to+"' is not a directory");
+ }
+ if(to==null)to="/";
+ String target=isDirectory?to+"/"+fileSpec.getName():to;
+ sms.createDirectory(target);
+ uploadFiles(fileset,target,sms,chosenProtocol,extraParameters);
+ }
+
+ /**
+ * upload a set of files to a remote directory (which must exist)
+ *
+ * @param files
+ * @param remoteDirectory
+ * @param sms
+ * @param protocol
+ * @param extraParameters
+ * @throws Exception
+ */
+ private void uploadFiles(File[]files, String remoteDirectory, StorageClient sms, ProtocolType.Enum protocol,
+ Map<String,String>extraParameters)throws Exception{
+ for(File localFile: files){
+ String target=remoteDirectory+"/"+localFile.getName();
+ if(localFile.isDirectory()){
+ if(!recurse){
+ System.out.println("Skipping directory "+localFile.getAbsolutePath());
+ }else{
+ File[] fileset=localFile.listFiles();
+ sms.createDirectory(target);
+ uploadFiles(fileset,target,sms,protocol,extraParameters);
+ }
+ }else{
+ uploadFile(localFile,target,sms,protocol,extraParameters);
+ }
+ }
+ }
+
+ /**
+ * uploads a single regular file
+ *
+ * @param localFile
+ * @param remotePath
+ * @param sms
+ * @param protocol
+ * @param extraParameters
+ * @throws Exception
+ */
+ private void uploadFile(File localFile, String remotePath, StorageClient sms, ProtocolType.Enum protocol,
+ Map<String,String>extraParameters) throws Exception{
+ long startTime=System.currentTimeMillis();
+ FileInputStream is=null;
+ FileTransferClient ftc=null;
+ try{
+ if(remotePath==null){
+ remotePath="/"+localFile.getName();
+ }
+ else if(remotePath.endsWith("/")){
+ remotePath+=localFile.getName();
+ }
+ System.out.println("Uploading local file '"+localFile.getAbsolutePath()+"' -> '"+sms.getUrl()+"#"+remotePath+"'");
+ is=new FileInputStream(localFile.getAbsolutePath());
+ boolean append=Mode.append.equals(mode);
+ ftc=sms.getImport(remotePath, append, extraParameters, protocol);
+ configure(ftc, extraParameters);
+ if(append)ftc.setAppend(true);
+ String url=ftc.getUrl();
+ System.out.println("File transfer URL : "+url);
+// ProgressBar p=null;
+ if(ftc instanceof IMonitorable){
+ long size=localFile.length();
+ if(isRange()){
+ size=getRangeSize();
+ }
+// p=new ProgressBar(localFile.getName(),size,msg);
+// ((IMonitorable) ftc).setProgressListener(p);
+ }
+ if(isRange()){
+ System.out.println("Byte range: "+startByte+" - "+(getRangeSize()>0?endByte:""));
+ long skipped=0;
+ while(skipped<startByte){
+ skipped+=is.skip(startByte);
+ }
+ ftc.writeAllData(is, endByte-startByte+1);
+
+ }else{
+ ftc.writeAllData(is);
+ }
+ copyProperties(localFile, sms, remotePath);
+
+// if(ftc instanceof IMonitorable){
+// p.finish();
+// }
+
+ }finally{
+ if(ftc!=null){
+ try{
+ ftc.destroy();
+ }catch(Exception e1){
+// msg.error("Could not clean-up the filetransfer at <"+ftc.getUrl()+">",e1);
+ }
+ }
+ try{ if(is!=null)is.close(); }catch(Exception ignored){}
+ }
+ if(timing){
+ long duration=System.currentTimeMillis()-startTime;
+ double rate=(double)localFile.length()/(double)duration;
+ System.out.println("Rate: "+rate+ " kB/sec.");
+ }
+ }
+
+ /**
+ * if possible, copy the local executable flag to the remote file
+ * @param sourceFile - local file
+ * @throws Exception
+ */
+ private void copyProperties(File sourceFile, StorageClient sms, String target)throws Exception{
+ boolean x=sourceFile.canExecute();
+ try{
+ if(x){
+ sms.changePermissions(target, true, true, x);
+ }
+ }catch(Exception ex){
+// System.out.println("Can't set exectuable flag on remote file.",ex);
+ }
+ }
+
+ private void checkOK()throws FileNotFoundException{
+ if(!failOnError){
+ return;
+ }
+ File orig=new File(from);
+ if(!orig.isAbsolute()){
+ orig=new File(System.getProperty("user.dir"),from);
+ }
+ File[] files=resolveWildCards(orig);
+ if(files==null){
+ throw new FileNotFoundException("Local import '"+from+"' does not exist.");
+ }
+ for(File f: files){
+ if(!f.exists())throw new FileNotFoundException("Local import '"+from+"' does not exist.");
+ }
+ }
+
+ private void configure(FileTransferClient ftc, Map<String,String>params){
+ if(ftc instanceof UFTPFileTransferClient){
+ UFTPFileTransferClient u=(UFTPFileTransferClient)ftc;
+ String secret=params.get(UFTPConstants.PARAM_SECRET);
+ u.setSecret(secret);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/0d9a6126/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/JSDLGenerator.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/JSDLGenerator.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/JSDLGenerator.java
new file mode 100644
index 0000000..3403240
--- /dev/null
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/JSDLGenerator.java
@@ -0,0 +1,115 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.gfac.impl.task.utils.bes;
+
+
+import org.apache.airavata.gfac.core.context.ProcessContext;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionDocument;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ *
+ * Utility class generates a JSDL instance from JobExecutionContext instance
+ *
+ * @author shahbaz memon
+ *
+ * */
+
+public class JSDLGenerator implements BESConstants {
+
+ protected final Logger log = LoggerFactory.getLogger(this.getClass());
+
+ public synchronized static JobDefinitionDocument buildJSDLInstance(ProcessContext context) throws Exception {
+
+ JobDefinitionDocument jobDefDoc = JobDefinitionDocument.Factory
+ .newInstance();
+ JobDefinitionType value = jobDefDoc.addNewJobDefinition();
+
+
+ // build Identification
+ createJobIdentification(value, context);
+
+ ResourceProcessor.generateResourceElements(value, context);
+
+ ApplicationProcessor.generateJobSpecificAppElements(value, context);
+
+
+ return jobDefDoc;
+ }
+
+ public synchronized static JobDefinitionDocument buildJSDLInstance(ProcessContext context, String smsUrl) throws Exception {
+
+ JobDefinitionDocument jobDefDoc = JobDefinitionDocument.Factory
+ .newInstance();
+ JobDefinitionType value = jobDefDoc.addNewJobDefinition();
+
+
+ // build Identification
+ createJobIdentification(value, context);
+
+ ResourceProcessor.generateResourceElements(value, context);
+
+ ApplicationProcessor.generateJobSpecificAppElements(value, context);
+
+ UASDataStagingProcessor.generateDataStagingElements(value, context, smsUrl);
+
+ return jobDefDoc;
+ }
+
+ public synchronized static JobDefinitionDocument buildJSDLInstance(
+ ProcessContext context, String smsUrl, Object jobDirectoryMode)
+ throws Exception {
+
+ JobDefinitionDocument jobDefDoc = JobDefinitionDocument.Factory
+ .newInstance();
+ JobDefinitionType value = jobDefDoc.addNewJobDefinition();
+
+ // build Identification
+ createJobIdentification(value, context);
+
+ ResourceProcessor.generateResourceElements(value, context);
+
+ ApplicationProcessor.generateJobSpecificAppElements(value, context);
+
+ UASDataStagingProcessor.generateDataStagingElements(value, context,
+ smsUrl);
+
+ return jobDefDoc;
+ }
+
+ private static void createJobIdentification(JobDefinitionType value, ProcessContext context) {
+
+ if (context != null) {
+ if (context.getComputeResourcePreference() != null && context.getComputeResourcePreference().getAllocationProjectNumber() != null)
+ JSDLUtils.addProjectName(value, context.getComputeResourcePreference().getAllocationProjectNumber());
+
+ if (context.getApplicationInterfaceDescription() != null && context.getApplicationInterfaceDescription().getApplicationDescription() != null)
+ JSDLUtils.getOrCreateJobIdentification(value).setDescription(context.getApplicationInterfaceDescription().getApplicationDescription());
+
+ if (context.getApplicationInterfaceDescription() != null && context.getApplicationInterfaceDescription().getApplicationName() != null)
+ JSDLUtils.getOrCreateJobIdentification(value).setJobName(context.getApplicationInterfaceDescription().getApplicationName());
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/0d9a6126/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/JSDLUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/JSDLUtils.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/JSDLUtils.java
new file mode 100644
index 0000000..204edfb
--- /dev/null
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/JSDLUtils.java
@@ -0,0 +1,517 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.impl.task.utils.bes;
+
+
+import org.apache.commons.httpclient.URIException;
+import org.apache.xmlbeans.XmlCursor;
+import org.apache.xmlbeans.XmlObject;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.*;
+import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.EnvironmentType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.POSIXApplicationDocument;
+import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.POSIXApplicationType;
+import org.ggf.schemas.jsdl.x2006.x07.jsdlHpcpa.HPCProfileApplicationDocument;
+import org.ggf.schemas.jsdl.x2006.x07.jsdlHpcpa.HPCProfileApplicationType;
+import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.SPMDApplicationDocument;
+import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.SPMDApplicationType;
+
+import javax.xml.namespace.QName;
+
+
+/**
+ *
+ * @author shahbaz memon, bastian demuth
+ *
+ */
+public class JSDLUtils
+{
+
+ public static final int FLAG_OVERWRITE = 1;
+ public static final int FLAG_APPEND = 2;
+ public static final int FLAG_DELETE_ON_TERMINATE = 32;
+
+ public static final QName POSIX_APPLICATION=POSIXApplicationDocument.type.getDocumentElementName();
+
+ public static final QName HPC_PROFILE_APPLICATION=HPCProfileApplicationDocument.type.getDocumentElementName();
+
+ public static final QName SPMD_APPLICATION=SPMDApplicationDocument.type.getDocumentElementName();
+
+ public static final String PROCESSESPERHOST = "ProcessesPerHost";
+ public static final String NUMBEROFPROCESSES = "NumberOfProcesses";
+ public static final String THREADSPERHOST = "ThreadsPerHost";
+
+
+
+ public static EnvironmentType addEnvVariable(JobDefinitionType def,String name, String value) {
+ POSIXApplicationType posixApp = getOrCreatePOSIXApplication(def);
+ EnvironmentType newEnv = posixApp.addNewEnvironment();
+ newEnv.setName(name);
+ newEnv.setStringValue(value);
+ return newEnv;
+ }
+
+ public static void setApplicationName(JobDefinitionType value, String applicationName) {
+ getOrCreateApplication(value).setApplicationName(applicationName);
+ }
+
+ public static void setApplicationVersion(JobDefinitionType value, String applicationVersion) {
+ getOrCreateApplication(value).setApplicationVersion(applicationVersion);
+ }
+
+ public static void addProjectName(JobDefinitionType value, String projectName) {
+ getOrCreateJobIdentification(value).addNewJobProject().setStringValue(projectName);
+ }
+
+ public static void addMultipleProjectNames(JobDefinitionType value, String[] projectNames) {
+ for (String name : projectNames) {
+ getOrCreateJobIdentification(value).addNewJobProject().setStringValue(name);
+ }
+ }
+
+ public static void addCandidateHost(JobDefinitionType value, String host) {
+ getOrCreateCandidateHosts(value).addHostName(host);
+
+ }
+ public static void addDataStagingTargetElement(JobDefinitionType value, String fileSystem, String file, String uri) {
+ addDataStagingTargetElement(value,fileSystem, file, uri, 1);
+ }
+
+ public static void addDataStagingTargetElement(JobDefinitionType value, String fileSystem, String file, String uri, int flags) {
+ JobDescriptionType jobDescr = getOrCreateJobDescription(value);
+ DataStagingType newDS = jobDescr.addNewDataStaging();
+ CreationFlagEnumeration.Enum creationFlag = CreationFlagEnumeration.DONT_OVERWRITE;
+ if((flags & FLAG_OVERWRITE) != 0) creationFlag = CreationFlagEnumeration.OVERWRITE;
+ if((flags & FLAG_APPEND) != 0) creationFlag = CreationFlagEnumeration.APPEND;
+ boolean deleteOnTerminate = (flags & FLAG_DELETE_ON_TERMINATE) != 0;
+ newDS.setCreationFlag(creationFlag);
+ newDS.setDeleteOnTermination(deleteOnTerminate);
+ SourceTargetType target = newDS.addNewTarget();
+
+ try {
+ if (uri != null) {
+ URIUtils.encodeAll(uri);
+ target.setURI(uri);
+ }
+ } catch (URIException e) {
+ }
+ newDS.setFileName(file);
+ if (fileSystem != null && !fileSystem.equals("Work")) { //$NON-NLS-1$
+ newDS.setFilesystemName(fileSystem);
+ }
+ }
+
+ public static void addDataStagingSourceElement(JobDefinitionType value, String uri, String fileSystem, String file) {
+ addDataStagingSourceElement(value, uri, fileSystem, file, 1);
+ }
+
+ public static void addDataStagingSourceElement(JobDefinitionType value, String uri, String fileSystem, String file, int flags) {
+ JobDescriptionType jobDescr = getOrCreateJobDescription(value);
+
+ try {
+ uri = (uri == null) ? null : URIUtils.encodeAll(uri);
+ } catch (URIException e) {
+ }
+ DataStagingType newDS = jobDescr.addNewDataStaging();
+ CreationFlagEnumeration.Enum creationFlag = CreationFlagEnumeration.DONT_OVERWRITE;
+ if((flags & FLAG_OVERWRITE) != 0) creationFlag = CreationFlagEnumeration.OVERWRITE;
+ if((flags & FLAG_APPEND) != 0) creationFlag = CreationFlagEnumeration.APPEND;
+ boolean deleteOnTerminate = (flags & FLAG_DELETE_ON_TERMINATE) != 0;
+ newDS.setCreationFlag(creationFlag);
+ newDS.setDeleteOnTermination(deleteOnTerminate);
+ SourceTargetType source = newDS.addNewSource();
+ source.setURI(uri);
+ newDS.setFileName(file);
+ if (fileSystem != null && !fileSystem.equals("Work")) { //$NON-NLS-1$
+ newDS.setFilesystemName(fileSystem);
+ }
+ }
+
+
+ public static ApplicationType getOrCreateApplication(JobDefinitionType value) {
+ JobDescriptionType jobDescr = getOrCreateJobDescription(value);
+ if (!jobDescr.isSetApplication()) {
+ jobDescr.addNewApplication();
+ }
+ return jobDescr.getApplication();
+ }
+
+ public static CandidateHostsType getOrCreateCandidateHosts(JobDefinitionType value) {
+ ResourcesType resources = getOrCreateResources(value);
+ if (!resources.isSetCandidateHosts()) {
+ resources.addNewCandidateHosts();
+ }
+ return resources.getCandidateHosts();
+ }
+
+ public static CPUArchitectureType getOrCreateCPUArchitecture(JobDefinitionType value) {
+
+ ResourcesType jobResources = getOrCreateResources(value);
+ if (!jobResources.isSetCPUArchitecture()) {
+ jobResources.addNewCPUArchitecture();
+ }
+ return jobResources.getCPUArchitecture();
+ }
+
+ public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateIndividualCPUCount(JobDefinitionType value) {
+ ResourcesType jobResources = getOrCreateResources(value);
+ if (!jobResources.isSetIndividualCPUCount()) {
+ jobResources.addNewIndividualCPUCount();
+ }
+ return jobResources.getIndividualCPUCount();
+ }
+
+
+ public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateIndividualCPUSpeed(JobDefinitionType value) {
+
+ ResourcesType jobResources = getOrCreateResources(value);
+ if (!jobResources.isSetIndividualCPUSpeed()) {
+ jobResources.addNewIndividualCPUSpeed();
+ }
+ return jobResources.getIndividualCPUSpeed();
+ }
+
+ public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateIndividualCPUTime(JobDefinitionType value) {
+
+ ResourcesType jobResources = getOrCreateResources(value);
+ if ( !jobResources.isSetIndividualCPUTime() ) {
+ jobResources.addNewIndividualCPUTime();
+ }
+ return jobResources.getIndividualCPUTime();
+ }
+
+ public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateIndividualDiskSpace(JobDefinitionType value) {
+
+ ResourcesType jobResources = getOrCreateResources(value);
+ if (!jobResources.isSetIndividualDiskSpace()) {
+ jobResources.addNewIndividualDiskSpace();
+ }
+ return jobResources.getIndividualDiskSpace();
+ }
+
+ public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateIndividualPhysicalMemory(JobDefinitionType value) {
+
+ ResourcesType jobResources = getOrCreateResources(value);
+ if (!jobResources.isSetIndividualPhysicalMemory()) {
+ jobResources.addNewIndividualPhysicalMemory();
+ }
+ return jobResources.getIndividualPhysicalMemory();
+ }
+
+ public static JobDescriptionType getOrCreateJobDescription(JobDefinitionType value) {
+ if (value.getJobDescription() == null) {
+ return value.addNewJobDescription();
+ }
+ return value.getJobDescription();
+ }
+
+ public static JobIdentificationType getOrCreateJobIdentification(JobDefinitionType value) {
+ JobDescriptionType descr = getOrCreateJobDescription(value);
+ if (descr.getJobIdentification() == null) {
+ return descr.addNewJobIdentification();
+ }
+ return descr.getJobIdentification();
+ }
+
+ public static OperatingSystemType getOrCreateOperatingSystem(JobDefinitionType value)
+ {
+ ResourcesType jobResources = getOrCreateResources(value);
+ if(!jobResources.isSetOperatingSystem()) {
+ jobResources.addNewOperatingSystem();
+ }
+ return jobResources.getOperatingSystem();
+ }
+
+ public static ResourcesType getOrCreateResources(JobDefinitionType value) {
+ JobDescriptionType jobDescr = getOrCreateJobDescription(value);
+ if (!jobDescr.isSetResources()) {
+ jobDescr.addNewResources();
+ }
+ return jobDescr.getResources();
+ }
+
+
+ public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateTotalCPUCount(JobDefinitionType value) {
+
+ ResourcesType jobResources = getOrCreateResources(value);
+ if ( !jobResources.isSetTotalCPUCount() ) {
+ jobResources.addNewTotalCPUCount();
+ }
+ return jobResources.getTotalCPUCount();
+ }
+
+
+ public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateTotalResourceCount(JobDefinitionType value) {
+
+ ResourcesType jobResources = getOrCreateResources(value);
+ if ( !jobResources.isSetTotalResourceCount())
+ {
+ jobResources.addNewTotalResourceCount();
+ }
+ return jobResources.getTotalResourceCount();
+ }
+
+ public static POSIXApplicationType getOrCreatePOSIXApplication(JobDefinitionType value) {
+
+ ApplicationType application = getOrCreateApplication(value);
+
+ if(getHPCProfileApplication(value) != null){
+ //TODO handle: not creating POSIX element if HPCProfile already exists
+ return getPOSIXApplication(value);
+ }
+
+ if (getPOSIXApplication(value) == null) {
+ XmlCursor acursor = application.newCursor();
+ acursor.toEndToken();
+ acursor.insertElement(POSIX_APPLICATION);
+ acursor.dispose();
+ }
+ return getPOSIXApplication(value);
+ }
+
+
+ public static SPMDApplicationType getOrCreateSPMDApplication(JobDefinitionType value) {
+
+ ApplicationType application = getOrCreateApplication(value);
+
+ if (getSPMDApplication(value) == null) {
+ XmlCursor acursor = application.newCursor();
+ acursor.toEndToken();
+ acursor.insertElement(SPMD_APPLICATION);
+ acursor.dispose();
+ }
+ return getSPMDApplication(value);
+ }
+
+ public static SPMDApplicationType getSPMDApplication(JobDefinitionType value) {
+ if (value != null &&
+ value.getJobDescription() != null &&
+ value.getJobDescription().isSetApplication() ) {
+ XmlCursor acursor = value.getJobDescription().getApplication().newCursor();
+ if (acursor.toFirstChild()) {
+ do {
+ if(acursor.getName().equals(SPMD_APPLICATION)) {
+ XmlObject result = acursor.getObject();
+ acursor.dispose();
+ return (SPMDApplicationType) result;
+ }
+ } while (acursor.toNextSibling());
+ acursor.dispose();
+ return null;
+ } else {
+ acursor.dispose();
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+
+
+
+ public static POSIXApplicationType getPOSIXApplication(JobDefinitionType value) {
+ if (value != null &&
+ value.getJobDescription() != null &&
+ value.getJobDescription().isSetApplication() ) {
+ XmlCursor acursor = value.getJobDescription().getApplication().newCursor();
+ if (acursor.toFirstChild()) {
+ do {
+ if(acursor.getName().equals(POSIX_APPLICATION)) {
+ XmlObject result = acursor.getObject();
+ acursor.dispose();
+ return (POSIXApplicationType) result;
+ }
+ } while (acursor.toNextSibling());
+ acursor.dispose();
+ return null;
+ } else {
+ acursor.dispose();
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+
+
+
+ public static HPCProfileApplicationType getOrCreateHPCProfileApplication(JobDefinitionType value) {
+
+ ApplicationType application = getOrCreateApplication(value);
+
+ if(getPOSIXApplication(value) != null){
+ //TODO handle: creating HPC element if POSIX already exists
+ return getHPCProfileApplication(value);
+ }
+
+ if (getHPCProfileApplication(value) == null) {
+ XmlCursor acursor = application.newCursor();
+ acursor.toEndToken();
+ acursor.insertElement(HPC_PROFILE_APPLICATION);
+ acursor.dispose();
+ }
+ return getHPCProfileApplication(value);
+ }
+
+
+ public static HPCProfileApplicationType getHPCProfileApplication(JobDefinitionType value) {
+ if (value != null &&
+ value.getJobDescription() != null &&
+ value.getJobDescription().isSetApplication() ) {
+ XmlCursor acursor = value.getJobDescription().getApplication().newCursor();
+ if (acursor.toFirstChild()) {
+ do {
+ if(acursor.getName().equals(HPC_PROFILE_APPLICATION)) {
+ XmlObject result = acursor.getObject();
+ acursor.dispose();
+ return (HPCProfileApplicationType) result;
+ }
+ } while (acursor.toNextSibling());
+ acursor.dispose();
+ return null;
+ } else {
+ acursor.dispose();
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+
+
+
+
+ public static RangeValueType getTotalCPUCountRequirements(JobDefinitionType value) {
+ if(value != null && value.getJobDescription() != null && value.getJobDescription().isSetResources() &&
+ value.getJobDescription().getResources().isSetTotalCPUCount()){
+ return toU6RangeValue(value.getJobDescription().getResources().getTotalCPUCount());
+ }
+ else
+ return null;
+ }
+
+ public static RangeValueType getTotalResourceCountRequirements(JobDefinitionType value) {
+ if(value != null && value.getJobDescription() != null && value.getJobDescription().isSetResources() &&
+ value.getJobDescription().getResources().isSetTotalResourceCount()){
+ return toU6RangeValue(value.getJobDescription().getResources().getTotalResourceCount());
+ }
+ else
+ return null;
+ }
+
+
+ public static RangeValueType toU6RangeValue(org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType jsdlType) {
+ RangeValueType result = new RangeValueType();
+ if(jsdlType.getExactArray().length > 0){
+ result.setExact(jsdlType.getExactArray(0).getDoubleValue());
+ }
+ if(jsdlType.isSetLowerBoundedRange()){
+ result.setLowerBound(jsdlType.getLowerBoundedRange().getDoubleValue());
+ }
+ if(jsdlType.isSetUpperBoundedRange()){
+ result.setUpperBound(jsdlType.getUpperBoundedRange().getDoubleValue());
+ }
+ return result;
+ }
+
+
+
+ public static void setCPUArchitectureRequirements(JobDefinitionType value, ProcessorRequirement cpuArchitecture) {
+ if(cpuArchitecture == null || cpuArchitecture.getValue() == null) return;
+ CPUArchitectureType cpuArch = getOrCreateCPUArchitecture(value);
+ cpuArch.setCPUArchitectureName(ProcessorArchitectureEnumeration.Enum.forString(cpuArchitecture.getValue()));
+ }
+
+ public static void setIndividualCPUCountRequirements(JobDefinitionType value, RangeValueType cpuCount) {
+ org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType individualCPUCount = getOrCreateIndividualCPUCount(value);
+ setRangeValue(cpuCount, individualCPUCount);
+ }
+
+ public static void setIndividualCPUSpeedRequirements(JobDefinitionType value, RangeValueType cpuSpeed) {
+ org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType individualCPUSpeed = getOrCreateIndividualCPUSpeed(value);
+ setRangeValue(cpuSpeed, individualCPUSpeed);
+ }
+
+ public static void setIndividualCPUTimeRequirements(JobDefinitionType value, RangeValueType cpuTime) {
+ org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType cpuIndividualTime = getOrCreateIndividualCPUTime(value);
+ setRangeValue(cpuTime, cpuIndividualTime);
+ }
+
+ public static void setIndividualDiskSpaceRequirements(JobDefinitionType value, RangeValueType diskSpace) {
+ org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType individualDiskSpace = getOrCreateIndividualDiskSpace(value);
+ setRangeValue(diskSpace, individualDiskSpace);
+ }
+
+ public static void setIndividualPhysicalMemoryRequirements(JobDefinitionType value, RangeValueType physicalMemory) {
+ org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType individualPhysicalMemory = getOrCreateIndividualPhysicalMemory(value);
+ setRangeValue(physicalMemory, individualPhysicalMemory);
+ }
+
+
+ public static void setName(JobDefinitionType value, String name) {
+ getOrCreateJobIdentification(value).setJobName(name);
+ }
+
+
+ public static void setRangeValue(RangeValueType u6Type, org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType jsdlType) {
+ Double exact = u6Type.getExact();
+ Double epsilon = u6Type.getEpsilon();
+ Double lower = u6Type.getLowerBound();
+ Double upper = u6Type.getUpperBound();
+
+
+ if(lower.isNaN() && upper.isNaN())
+ {
+ ExactType exactType = jsdlType.getExactArray().length > 0 ? jsdlType.getExactArray(0) : jsdlType.addNewExact();
+ exactType.setDoubleValue(exact);
+ if(!epsilon.isNaN() && epsilon != 0)
+ {
+ exactType.setEpsilon(epsilon);
+ }
+ }
+ else
+ {
+ if(!lower.isNaN())
+ {
+ BoundaryType lowerBound = jsdlType.isSetLowerBoundedRange() ? jsdlType.getLowerBoundedRange() : jsdlType.addNewLowerBoundedRange();
+ lowerBound.setDoubleValue(lower);
+ lowerBound.setExclusiveBound(!u6Type.isIncludeLowerBound());
+ }
+
+ if(!upper.isNaN())
+ {
+ BoundaryType upperBound = jsdlType.isSetUpperBoundedRange() ? jsdlType.getUpperBoundedRange() : jsdlType.addNewUpperBoundedRange();
+ upperBound.setDoubleValue(upper);
+ upperBound.setExclusiveBound(!u6Type.isIncludeUpperBound());
+ }
+ }
+ }
+
+ public static void setTotalCPUCountRequirements(JobDefinitionType value, RangeValueType cpuCount) {
+ org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType cpuTotalCount = getOrCreateTotalCPUCount(value);
+ setRangeValue(cpuCount, cpuTotalCount);
+ }
+
+ public static void setTotalResourceCountRequirements(JobDefinitionType value, RangeValueType resourceCount) {
+ org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType totalCount = getOrCreateTotalResourceCount(value);
+ setRangeValue(resourceCount, totalCount);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/0d9a6126/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/Mode.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/Mode.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/Mode.java
new file mode 100644
index 0000000..f1a7eb7
--- /dev/null
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/utils/bes/Mode.java
@@ -0,0 +1,45 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.impl.task.utils.bes;
+
+/**
+ * file creation modes
+ */
+public enum Mode {
+
+ /**
+ * overwrite any existing file
+ */
+ overwrite,
+
+ /**
+ * append to an existing file
+ */
+ append,
+
+ /**
+ * do NOT overwrite and fail if the file exists
+ */
+ nooverwrite
+
+
+}
\ No newline at end of file