You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2014/05/05 16:49:58 UTC
[1/4] moving gfac-bes classes to new package.
Repository: airavata
Updated Branches:
refs/heads/master 9bb8c2bef -> 49b6987f3
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/FileUploader.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/FileUploader.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/FileUploader.java
deleted file mode 100644
index 4ab1737..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/FileUploader.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.utils;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.Map;
-
-import org.unigrids.services.atomic.types.ProtocolType;
-
-import de.fzj.unicore.uas.client.FileTransferClient;
-import de.fzj.unicore.uas.client.StorageClient;
-import de.fzj.unicore.uas.client.UFTPConstants;
-import de.fzj.unicore.uas.client.UFTPFileTransferClient;
-import de.fzj.unicore.uas.fts.FiletransferOptions.IMonitorable;
-
-/**
- * upload local file(s) to a remote location
- *
- * @author schuller
- */
-public class FileUploader extends FileTransferBase{
-
- public FileUploader(String from, String to, Mode mode)throws FileNotFoundException{
- this(from,to,mode,true);
- }
-
- public FileUploader(String from, String to, Mode mode, boolean failOnError)throws FileNotFoundException{
- this.to=to;
- this.from=from;
- this.mode=mode;
- this.failOnError=failOnError;
- checkOK();
- }
-
- public String getFrom() {
- return from;
- }
-
- public String getTo() {
- return to;
- }
-
-
- public void perform(StorageClient sms)throws Exception{
- File fileSpec=new File(from);
- boolean hasWildCards=false;
- boolean isDirectory=fileSpec.isDirectory();
- File[] fileset=null;
-
- if(!isDirectory){
- hasWildCards=hasWildCards(fileSpec);
- }
-
- chosenProtocol=sms.findSupportedProtocol(preferredProtocols.toArray(new ProtocolType.Enum[preferredProtocols.size()]));
- Map<String,String>extraParameters=makeExtraParameters(chosenProtocol);
-
- if(!hasWildCards && !isDirectory){
- //single regular file
- uploadFile(fileSpec,to,sms,chosenProtocol,extraParameters);
- return;
- }
-
- //handle wildcards or directory
- if(hasWildCards){
- fileset=resolveWildCards(fileSpec);
- }
- else{
- fileset=fileSpec.listFiles();
- }
-
- if(!isValidDirectory(to, sms)){
- throw new IOException("The specified remote target '"+to+"' is not a directory");
- }
- if(to==null)to="/";
- String target=isDirectory?to+"/"+fileSpec.getName():to;
- sms.createDirectory(target);
- uploadFiles(fileset,target,sms,chosenProtocol,extraParameters);
- }
-
- /**
- * upload a set of files to a remote directory (which must exist)
- *
- * @param files
- * @param remoteDirectory
- * @param sms
- * @param protocol
- * @param extraParameters
- * @param msg
- * @throws Exception
- */
- private void uploadFiles(File[]files, String remoteDirectory, StorageClient sms, ProtocolType.Enum protocol,
- Map<String,String>extraParameters)throws Exception{
- for(File localFile: files){
- String target=remoteDirectory+"/"+localFile.getName();
- if(localFile.isDirectory()){
- if(!recurse){
- System.out.println("Skipping directory "+localFile.getAbsolutePath());
- }else{
- File[] fileset=localFile.listFiles();
- sms.createDirectory(target);
- uploadFiles(fileset,target,sms,protocol,extraParameters);
- }
- }else{
- uploadFile(localFile,target,sms,protocol,extraParameters);
- }
- }
- }
-
- /**
- * uploads a single regular file
- *
- * @param localFile
- * @param remotePath
- * @param sms
- * @param protocol
- * @param extraParameters
- * @param msg
- * @throws Exception
- */
- private void uploadFile(File localFile, String remotePath, StorageClient sms, ProtocolType.Enum protocol,
- Map<String,String>extraParameters) throws Exception{
- long startTime=System.currentTimeMillis();
- FileInputStream is=null;
- FileTransferClient ftc=null;
- try{
- if(remotePath==null){
- remotePath="/"+localFile.getName();
- }
- else if(remotePath.endsWith("/")){
- remotePath+=localFile.getName();
- }
- System.out.println("Uploading local file '"+localFile.getAbsolutePath()+"' -> '"+sms.getUrl()+"#"+remotePath+"'");
- is=new FileInputStream(localFile.getAbsolutePath());
- boolean append=Mode.append.equals(mode);
- ftc=sms.getImport(remotePath, append, extraParameters, protocol);
- configure(ftc, extraParameters);
- if(append)ftc.setAppend(true);
- String url=ftc.getUrl();
- System.out.println("File transfer URL : "+url);
-// ProgressBar p=null;
- if(ftc instanceof IMonitorable){
- long size=localFile.length();
- if(isRange()){
- size=getRangeSize();
- }
-// p=new ProgressBar(localFile.getName(),size,msg);
-// ((IMonitorable) ftc).setProgressListener(p);
- }
- if(isRange()){
- System.out.println("Byte range: "+startByte+" - "+(getRangeSize()>0?endByte:""));
- long skipped=0;
- while(skipped<startByte){
- skipped+=is.skip(startByte);
- }
- ftc.writeAllData(is, endByte-startByte+1);
-
- }else{
- ftc.writeAllData(is);
- }
- copyProperties(localFile, sms, remotePath);
-
-// if(ftc instanceof IMonitorable){
-// p.finish();
-// }
-
- }finally{
- if(ftc!=null){
- try{
- ftc.destroy();
- }catch(Exception e1){
-// msg.error("Could not clean-up the filetransfer at <"+ftc.getUrl()+">",e1);
- }
- }
- try{ if(is!=null)is.close(); }catch(Exception ignored){}
- }
- if(timing){
- long duration=System.currentTimeMillis()-startTime;
- double rate=(double)localFile.length()/(double)duration;
- System.out.println("Rate: "+rate+ " kB/sec.");
- }
- }
-
- /**
- * if possible, copy the local executable flag to the remote file
- * @param sourceFile - local file
- * @throws Exception
- */
- private void copyProperties(File sourceFile, StorageClient sms, String target)throws Exception{
- boolean x=sourceFile.canExecute();
- try{
- if(x){
- sms.changePermissions(target, true, true, x);
- }
- }catch(Exception ex){
-// System.out.println("Can't set exectuable flag on remote file.",ex);
- }
- }
-
- private void checkOK()throws FileNotFoundException{
- if(!failOnError){
- return;
- }
- File orig=new File(from);
- if(!orig.isAbsolute()){
- orig=new File(System.getProperty("user.dir"),from);
- }
- File[] files=resolveWildCards(orig);
- if(files==null){
- throw new FileNotFoundException("Local import '"+from+"' does not exist.");
- }
- for(File f: files){
- if(!f.exists())throw new FileNotFoundException("Local import '"+from+"' does not exist.");
- }
- }
-
- private void configure(FileTransferClient ftc, Map<String,String>params){
- if(ftc instanceof UFTPFileTransferClient){
- UFTPFileTransferClient u=(UFTPFileTransferClient)ftc;
- String secret=params.get(UFTPConstants.PARAM_SECRET);
- u.setSecret(secret);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/JSDLGenerator.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/JSDLGenerator.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/JSDLGenerator.java
deleted file mode 100644
index 8754b17..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/JSDLGenerator.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package org.apache.airavata.gfac.utils;
-
-
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.utils.ResourceProcessor;
-import org.apache.airavata.gfac.utils.UASDataStagingProcessor;
-import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionDocument;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- * Utility class generates a JSDL instance from JobExecutionContext instance
- * @author shahbaz memon
- *
- * */
-
-public class JSDLGenerator {
-
- protected final Logger log = LoggerFactory.getLogger(this.getClass());
-
-
- public synchronized static JobDefinitionDocument buildJSDLInstance(JobExecutionContext context) throws Exception {
-
- JobDefinitionDocument jobDefDoc = JobDefinitionDocument.Factory
- .newInstance();
- JobDefinitionType value = jobDefDoc.addNewJobDefinition();
-
- HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
- .getApplicationContext().getApplicationDeploymentDescription()
- .getType();
-
- // build Identification
- createJobIdentification(value, appDepType);
-
- ResourceProcessor.generateResourceElements(value, context);
-
- ApplicationProcessor.generateJobSpecificAppElements(value, context);
-
- DataStagingProcessor.generateDataStagingElements(value, context);
-
-
- return jobDefDoc;
- }
-
-
- public synchronized static JobDefinitionDocument buildJSDLInstance(JobExecutionContext context, String smsUrl) throws Exception {
-
- JobDefinitionDocument jobDefDoc = JobDefinitionDocument.Factory
- .newInstance();
- JobDefinitionType value = jobDefDoc.addNewJobDefinition();
-
- HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
- .getApplicationContext().getApplicationDeploymentDescription()
- .getType();
-
- // build Identification
- createJobIdentification(value, appDepType);
-
- ResourceProcessor.generateResourceElements(value, context);
-
- ApplicationProcessor.generateJobSpecificAppElements(value, context);
-
- UASDataStagingProcessor.generateDataStagingElements(value, context, smsUrl);
-
- return jobDefDoc;
- }
-
- private static void createJobIdentification(JobDefinitionType value, HpcApplicationDeploymentType appDepType){
- if( appDepType.getProjectAccount() != null ){
-
- if (appDepType.getProjectAccount().getProjectAccountNumber() != null)
- JSDLUtils.addProjectName(value, appDepType.getProjectAccount()
- .getProjectAccountNumber());
-
- if (appDepType.getProjectAccount().getProjectAccountDescription() != null)
- JSDLUtils.getOrCreateJobIdentification(value).setDescription(
- appDepType.getProjectAccount()
- .getProjectAccountDescription());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/JSDLUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/JSDLUtils.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/JSDLUtils.java
deleted file mode 100644
index 438fc34..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/JSDLUtils.java
+++ /dev/null
@@ -1,540 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.utils;
-
-
-import javax.xml.namespace.QName;
-
-import org.apache.commons.httpclient.URIException;
-import org.apache.xmlbeans.XmlCursor;
-import org.apache.xmlbeans.XmlObject;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.ApplicationType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.BoundaryType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.CPUArchitectureType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.CandidateHostsType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.CreationFlagEnumeration;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.DataStagingType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.ExactType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDescriptionType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobIdentificationType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.OperatingSystemType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.OperatingSystemTypeEnumeration;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.OperatingSystemTypeType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.ProcessorArchitectureEnumeration;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.ResourcesType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.SourceTargetType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.EnvironmentType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.POSIXApplicationDocument;
-import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.POSIXApplicationType;
-import org.ggf.schemas.jsdl.x2006.x07.jsdlHpcpa.HPCProfileApplicationDocument;
-import org.ggf.schemas.jsdl.x2006.x07.jsdlHpcpa.HPCProfileApplicationType;
-import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.SPMDApplicationDocument;
-import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.SPMDApplicationType;
-
-
-/**
- *
- * @author shahbaz memon, bastian demuth
- *
- */
-public class JSDLUtils
-{
-
- public static final int FLAG_OVERWRITE = 1;
- public static final int FLAG_APPEND = 2;
- public static final int FLAG_DELETE_ON_TERMINATE = 32;
-
- public static final QName POSIX_APPLICATION=POSIXApplicationDocument.type.getDocumentElementName();
-
- public static final QName HPC_PROFILE_APPLICATION=HPCProfileApplicationDocument.type.getDocumentElementName();
-
- public static final QName SPMD_APPLICATION=SPMDApplicationDocument.type.getDocumentElementName();
-
- public static final String PROCESSESPERHOST = "ProcessesPerHost";
- public static final String NUMBEROFPROCESSES = "NumberOfProcesses";
- public static final String THREADSPERHOST = "ThreadsPerHost";
-
-
-
- public static EnvironmentType addEnvVariable(JobDefinitionType def,String name, String value) {
- POSIXApplicationType posixApp = getOrCreatePOSIXApplication(def);
- EnvironmentType newEnv = posixApp.addNewEnvironment();
- newEnv.setName(name);
- newEnv.setStringValue(value);
- return newEnv;
- }
-
- public static void setApplicationName(JobDefinitionType value, String applicationName) {
- getOrCreateApplication(value).setApplicationName(applicationName);
- }
-
- public static void setApplicationVersion(JobDefinitionType value, String applicationVersion) {
- getOrCreateApplication(value).setApplicationVersion(applicationVersion);
- }
-
- public static void addProjectName(JobDefinitionType value, String projectName) {
- getOrCreateJobIdentification(value).addNewJobProject().setStringValue(projectName);
- }
-
- public static void addMultipleProjectNames(JobDefinitionType value, String[] projectNames) {
- for (String name : projectNames) {
- getOrCreateJobIdentification(value).addNewJobProject().setStringValue(name);
- }
- }
-
- public static void addCandidateHost(JobDefinitionType value, String host) {
- getOrCreateCandidateHosts(value).addHostName(host);
-
- }
- public static void addDataStagingTargetElement(JobDefinitionType value, String fileSystem, String file, String uri) {
- addDataStagingTargetElement(value,fileSystem, file, uri, 0);
- }
-
- public static void addDataStagingTargetElement(JobDefinitionType value, String fileSystem, String file, String uri, int flags) {
- JobDescriptionType jobDescr = getOrCreateJobDescription(value);
- DataStagingType newDS = jobDescr.addNewDataStaging();
- CreationFlagEnumeration.Enum creationFlag = CreationFlagEnumeration.DONT_OVERWRITE;
- if((flags & FLAG_OVERWRITE) != 0) creationFlag = CreationFlagEnumeration.OVERWRITE;
- if((flags & FLAG_APPEND) != 0) creationFlag = CreationFlagEnumeration.APPEND;
- boolean deleteOnTerminate = (flags & FLAG_DELETE_ON_TERMINATE) != 0;
- newDS.setCreationFlag(creationFlag);
- newDS.setDeleteOnTermination(deleteOnTerminate);
- SourceTargetType target = newDS.addNewTarget();
-
- try {
- uri = (uri == null) ? null : URIUtils.encodeAll(uri);
- } catch (URIException e) {
- }
- target.setURI(uri);
- newDS.setFileName(file);
- if (fileSystem != null && !fileSystem.equals("Work")) { //$NON-NLS-1$
- newDS.setFilesystemName(fileSystem);
- }
- }
-
- public static void addDataStagingSourceElement(JobDefinitionType value, String uri, String fileSystem, String file) {
- addDataStagingSourceElement(value, uri, fileSystem, file, 0);
- }
-
- public static void addDataStagingSourceElement(JobDefinitionType value, String uri, String fileSystem, String file, int flags) {
- JobDescriptionType jobDescr = getOrCreateJobDescription(value);
-
- try {
- uri = (uri == null) ? null : URIUtils.encodeAll(uri);
- } catch (URIException e) {
- }
- DataStagingType newDS = jobDescr.addNewDataStaging();
- CreationFlagEnumeration.Enum creationFlag = CreationFlagEnumeration.DONT_OVERWRITE;
- if((flags & FLAG_OVERWRITE) != 0) creationFlag = CreationFlagEnumeration.OVERWRITE;
- if((flags & FLAG_APPEND) != 0) creationFlag = CreationFlagEnumeration.APPEND;
- boolean deleteOnTerminate = (flags & FLAG_DELETE_ON_TERMINATE) != 0;
- newDS.setCreationFlag(creationFlag);
- newDS.setDeleteOnTermination(deleteOnTerminate);
- SourceTargetType source = newDS.addNewSource();
- source.setURI(uri);
- newDS.setFileName(file);
- if (fileSystem != null && !fileSystem.equals("Work")) { //$NON-NLS-1$
- newDS.setFilesystemName(fileSystem);
- }
- }
-
-
- public static ApplicationType getOrCreateApplication(JobDefinitionType value) {
- JobDescriptionType jobDescr = getOrCreateJobDescription(value);
- if (!jobDescr.isSetApplication()) {
- jobDescr.addNewApplication();
- }
- return jobDescr.getApplication();
- }
-
- public static CandidateHostsType getOrCreateCandidateHosts(JobDefinitionType value) {
- ResourcesType resources = getOrCreateResources(value);
- if (!resources.isSetCandidateHosts()) {
- resources.addNewCandidateHosts();
- }
- return resources.getCandidateHosts();
- }
-
- public static CPUArchitectureType getOrCreateCPUArchitecture(JobDefinitionType value) {
-
- ResourcesType jobResources = getOrCreateResources(value);
- if (!jobResources.isSetCPUArchitecture()) {
- jobResources.addNewCPUArchitecture();
- }
- return jobResources.getCPUArchitecture();
- }
-
- public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateIndividualCPUCount(JobDefinitionType value) {
- ResourcesType jobResources = getOrCreateResources(value);
- if (!jobResources.isSetIndividualCPUCount()) {
- jobResources.addNewIndividualCPUCount();
- }
- return jobResources.getIndividualCPUCount();
- }
-
-
- public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateIndividualCPUSpeed(JobDefinitionType value) {
-
- ResourcesType jobResources = getOrCreateResources(value);
- if (!jobResources.isSetIndividualCPUSpeed()) {
- jobResources.addNewIndividualCPUSpeed();
- }
- return jobResources.getIndividualCPUSpeed();
- }
-
- public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateIndividualCPUTime(JobDefinitionType value) {
-
- ResourcesType jobResources = getOrCreateResources(value);
- if ( !jobResources.isSetIndividualCPUTime() ) {
- jobResources.addNewIndividualCPUTime();
- }
- return jobResources.getIndividualCPUTime();
- }
-
- public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateIndividualDiskSpace(JobDefinitionType value) {
-
- ResourcesType jobResources = getOrCreateResources(value);
- if (!jobResources.isSetIndividualDiskSpace()) {
- jobResources.addNewIndividualDiskSpace();
- }
- return jobResources.getIndividualDiskSpace();
- }
-
- public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateIndividualPhysicalMemory(JobDefinitionType value) {
-
- ResourcesType jobResources = getOrCreateResources(value);
- if (!jobResources.isSetIndividualPhysicalMemory()) {
- jobResources.addNewIndividualPhysicalMemory();
- }
- return jobResources.getIndividualPhysicalMemory();
- }
-
- public static JobDescriptionType getOrCreateJobDescription(JobDefinitionType value) {
- if (value.getJobDescription() == null) {
- return value.addNewJobDescription();
- }
- return value.getJobDescription();
- }
-
- public static JobIdentificationType getOrCreateJobIdentification(JobDefinitionType value) {
- JobDescriptionType descr = getOrCreateJobDescription(value);
- if (descr.getJobIdentification() == null) {
- return descr.addNewJobIdentification();
- }
- return descr.getJobIdentification();
- }
-
- public static OperatingSystemType getOrCreateOperatingSystem(JobDefinitionType value)
- {
- ResourcesType jobResources = getOrCreateResources(value);
- if(!jobResources.isSetOperatingSystem()) {
- jobResources.addNewOperatingSystem();
- }
- return jobResources.getOperatingSystem();
- }
-
- public static ResourcesType getOrCreateResources(JobDefinitionType value) {
- JobDescriptionType jobDescr = getOrCreateJobDescription(value);
- if (!jobDescr.isSetResources()) {
- jobDescr.addNewResources();
- }
- return jobDescr.getResources();
- }
-
-
- public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateTotalCPUCount(JobDefinitionType value) {
-
- ResourcesType jobResources = getOrCreateResources(value);
- if ( !jobResources.isSetTotalCPUCount() ) {
- jobResources.addNewTotalCPUCount();
- }
- return jobResources.getTotalCPUCount();
- }
-
-
- public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateTotalResourceCount(JobDefinitionType value) {
-
- ResourcesType jobResources = getOrCreateResources(value);
- if ( !jobResources.isSetTotalResourceCount())
- {
- jobResources.addNewTotalResourceCount();
- }
- return jobResources.getTotalResourceCount();
- }
-
- public static POSIXApplicationType getOrCreatePOSIXApplication(JobDefinitionType value) {
-
- ApplicationType application = getOrCreateApplication(value);
-
- if(getHPCProfileApplication(value) != null){
- //TODO handle: not creating POSIX element if HPCProfile already exists
- return getPOSIXApplication(value);
- }
-
- if (getPOSIXApplication(value) == null) {
- XmlCursor acursor = application.newCursor();
- acursor.toEndToken();
- acursor.insertElement(POSIX_APPLICATION);
- acursor.dispose();
- }
- return getPOSIXApplication(value);
- }
-
-
- public static SPMDApplicationType getOrCreateSPMDApplication(JobDefinitionType value) {
-
- ApplicationType application = getOrCreateApplication(value);
-
- if (getSPMDApplication(value) == null) {
- XmlCursor acursor = application.newCursor();
- acursor.toEndToken();
- acursor.insertElement(SPMD_APPLICATION);
- acursor.dispose();
- }
- return getSPMDApplication(value);
- }
-
- public static SPMDApplicationType getSPMDApplication(JobDefinitionType value) {
- if (value != null &&
- value.getJobDescription() != null &&
- value.getJobDescription().isSetApplication() ) {
- XmlCursor acursor = value.getJobDescription().getApplication().newCursor();
- if (acursor.toFirstChild()) {
- do {
- if(acursor.getName().equals(SPMD_APPLICATION)) {
- XmlObject result = acursor.getObject();
- acursor.dispose();
- return (SPMDApplicationType) result;
- }
- } while (acursor.toNextSibling());
- acursor.dispose();
- return null;
- } else {
- acursor.dispose();
- return null;
- }
- } else {
- return null;
- }
- }
-
-
-
- public static POSIXApplicationType getPOSIXApplication(JobDefinitionType value) {
- if (value != null &&
- value.getJobDescription() != null &&
- value.getJobDescription().isSetApplication() ) {
- XmlCursor acursor = value.getJobDescription().getApplication().newCursor();
- if (acursor.toFirstChild()) {
- do {
- if(acursor.getName().equals(POSIX_APPLICATION)) {
- XmlObject result = acursor.getObject();
- acursor.dispose();
- return (POSIXApplicationType) result;
- }
- } while (acursor.toNextSibling());
- acursor.dispose();
- return null;
- } else {
- acursor.dispose();
- return null;
- }
- } else {
- return null;
- }
- }
-
-
-
- public static HPCProfileApplicationType getOrCreateHPCProfileApplication(JobDefinitionType value) {
-
- ApplicationType application = getOrCreateApplication(value);
-
- if(getPOSIXApplication(value) != null){
- //TODO handle: creating HPC element if POSIX already exists
- return getHPCProfileApplication(value);
- }
-
- if (getHPCProfileApplication(value) == null) {
- XmlCursor acursor = application.newCursor();
- acursor.toEndToken();
- acursor.insertElement(HPC_PROFILE_APPLICATION);
- acursor.dispose();
- }
- return getHPCProfileApplication(value);
- }
-
-
- public static HPCProfileApplicationType getHPCProfileApplication(JobDefinitionType value) {
- if (value != null &&
- value.getJobDescription() != null &&
- value.getJobDescription().isSetApplication() ) {
- XmlCursor acursor = value.getJobDescription().getApplication().newCursor();
- if (acursor.toFirstChild()) {
- do {
- if(acursor.getName().equals(HPC_PROFILE_APPLICATION)) {
- XmlObject result = acursor.getObject();
- acursor.dispose();
- return (HPCProfileApplicationType) result;
- }
- } while (acursor.toNextSibling());
- acursor.dispose();
- return null;
- } else {
- acursor.dispose();
- return null;
- }
- } else {
- return null;
- }
- }
-
-
-
-
- public static RangeValueType getTotalCPUCountRequirements(JobDefinitionType value) {
- if(value != null && value.getJobDescription() != null && value.getJobDescription().isSetResources() &&
- value.getJobDescription().getResources().isSetTotalCPUCount()){
- return toU6RangeValue(value.getJobDescription().getResources().getTotalCPUCount());
- }
- else
- return null;
- }
-
- public static RangeValueType getTotalResourceCountRequirements(JobDefinitionType value) {
- if(value != null && value.getJobDescription() != null && value.getJobDescription().isSetResources() &&
- value.getJobDescription().getResources().isSetTotalResourceCount()){
- return toU6RangeValue(value.getJobDescription().getResources().getTotalResourceCount());
- }
- else
- return null;
- }
-
-
- public static RangeValueType toU6RangeValue(org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType jsdlType) {
- RangeValueType result = new RangeValueType();
- if(jsdlType.getExactArray().length > 0){
- result.setExact(jsdlType.getExactArray(0).getDoubleValue());
- }
- if(jsdlType.isSetLowerBoundedRange()){
- result.setLowerBound(jsdlType.getLowerBoundedRange().getDoubleValue());
- }
- if(jsdlType.isSetUpperBoundedRange()){
- result.setUpperBound(jsdlType.getUpperBoundedRange().getDoubleValue());
- }
- return result;
- }
-
-
-
- public static void setCPUArchitectureRequirements(JobDefinitionType value, ProcessorRequirement cpuArchitecture) {
- if(cpuArchitecture == null || cpuArchitecture.getValue() == null) return;
- CPUArchitectureType cpuArch = getOrCreateCPUArchitecture(value);
- cpuArch.setCPUArchitectureName(ProcessorArchitectureEnumeration.Enum.forString(cpuArchitecture.getValue()));
- }
-
- public static void setIndividualCPUCountRequirements(JobDefinitionType value, RangeValueType cpuCount) {
- org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType individualCPUCount = getOrCreateIndividualCPUCount(value);
- setRangeValue(cpuCount, individualCPUCount);
- }
-
- public static void setIndividualCPUSpeedRequirements(JobDefinitionType value, RangeValueType cpuSpeed) {
- org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType individualCPUSpeed = getOrCreateIndividualCPUSpeed(value);
- setRangeValue(cpuSpeed, individualCPUSpeed);
- }
-
- public static void setIndividualCPUTimeRequirements(JobDefinitionType value, RangeValueType cpuTime) {
- org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType cpuIndividualTime = getOrCreateIndividualCPUTime(value);
- setRangeValue(cpuTime, cpuIndividualTime);
- }
-
- public static void setIndividualDiskSpaceRequirements(JobDefinitionType value, RangeValueType diskSpace) {
- org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType individualDiskSpace = getOrCreateIndividualDiskSpace(value);
- setRangeValue(diskSpace, individualDiskSpace);
- }
-
- public static void setIndividualPhysicalMemoryRequirements(JobDefinitionType value, RangeValueType physicalMemory) {
- org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType individualPhysicalMemory = getOrCreateIndividualPhysicalMemory(value);
- setRangeValue(physicalMemory, individualPhysicalMemory);
- }
-
-
- public static void setName(JobDefinitionType value, String name) {
- getOrCreateJobIdentification(value).setJobName(name);
- }
-
- public static void setOperatingSystemRequirements(JobDefinitionType value, OSRequirement osType) {
- if(osType == null || osType.getOSType() == null) return;
- OperatingSystemType os_Type = getOrCreateOperatingSystem(value);
- OperatingSystemTypeType ostt = os_Type.addNewOperatingSystemType();
- ostt.setOperatingSystemName(OperatingSystemTypeEnumeration.Enum.forString(osType.getOSType().getValue()));
- if(osType.getOSVersion() != null)
- {
- os_Type.setOperatingSystemVersion(osType.getOSVersion());
- }
- }
-
- public static void setRangeValue(RangeValueType u6Type, org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType jsdlType) {
- Double exact = u6Type.getExact();
- Double epsilon = u6Type.getEpsilon();
- Double lower = u6Type.getLowerBound();
- Double upper = u6Type.getUpperBound();
-
-
- if(lower.isNaN() && upper.isNaN())
- {
- ExactType exactType = jsdlType.getExactArray().length > 0 ? jsdlType.getExactArray(0) : jsdlType.addNewExact();
- exactType.setDoubleValue(exact);
- if(!epsilon.isNaN() && epsilon != 0)
- {
- exactType.setEpsilon(epsilon);
- }
- }
- else
- {
- if(!lower.isNaN())
- {
- BoundaryType lowerBound = jsdlType.isSetLowerBoundedRange() ? jsdlType.getLowerBoundedRange() : jsdlType.addNewLowerBoundedRange();
- lowerBound.setDoubleValue(lower);
- lowerBound.setExclusiveBound(!u6Type.isIncludeLowerBound());
- }
-
- if(!upper.isNaN())
- {
- BoundaryType upperBound = jsdlType.isSetUpperBoundedRange() ? jsdlType.getUpperBoundedRange() : jsdlType.addNewUpperBoundedRange();
- upperBound.setDoubleValue(upper);
- upperBound.setExclusiveBound(!u6Type.isIncludeUpperBound());
- }
- }
- }
-
- public static void setTotalCPUCountRequirements(JobDefinitionType value, RangeValueType cpuCount) {
- org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType cpuTotalCount = getOrCreateTotalCPUCount(value);
- setRangeValue(cpuCount, cpuTotalCount);
- }
-
- public static void setTotalResourceCountRequirements(JobDefinitionType value, RangeValueType resourceCount) {
- org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType totalCount = getOrCreateTotalResourceCount(value);
- setRangeValue(resourceCount, totalCount);
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/Mode.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/Mode.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/Mode.java
deleted file mode 100644
index f47b75d..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/Mode.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.utils;
-
-/**
- * file creation modes
- */
-public enum Mode {
-
- /**
- * overwrite any existing file
- */
- overwrite,
-
- /**
- * append to an existing file
- */
- append,
-
- /**
- * do NOT overwrite and fail if the file exists
- */
- nooverwrite
-
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/OSRequirement.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/OSRequirement.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/OSRequirement.java
deleted file mode 100644
index aa59de9..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/OSRequirement.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.utils;
-
-import org.apache.airavata.gfac.provider.utils.ResourceRequirement;
-
-public class OSRequirement implements ResourceRequirement {
- private OSType osType;
- private String version;
- protected boolean enabled;
-
-
- public OSRequirement() {
- }
-
- /**
- *
- * @param type -
- * the type of the O/S
- * @param version -
- * the version of the O/S
- */
- public OSRequirement(OSType osType, String osVersion) {
- setOSType(osType);
- setOSVersion(osVersion);
- }
-
- /**
- * Set the type of the O/S
- *
- * @param type -
- * the type of the O/S
- */
- public void setOSType(OSType osType) {
- this.osType = osType;
- }
-
- /**
- * Get the type of the O/S
- *
- * @return the type of the O/S
- */
- public OSType getOSType() {
- return osType;
- }
-
- /**
- * Set the version of the O/S
- *
- * @param version -
- * the version of the O/S
- */
- public void setOSVersion(String version) {
- this.version = version;
- }
-
- /**
- * Get the version of the O/S
- *
- * @return the version of the O/S
- */
- public String getOSVersion() {
- return version;
- }
-
- /**
- *
- * equals this instance of class with another instance
- */
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj==null || getClass() != obj.getClass()) return false;
- final OSRequirement other = (OSRequirement) obj;
- boolean typeEqual = osType == null ? other.osType == null : osType.equals(other.osType);
- boolean versionEqual = version == null ? other.version == null : version.equals(other.version);
- return typeEqual && versionEqual && isEnabled() == other.isEnabled();
- }
-
-
-
- public boolean isEnabled() {
- return enabled;
- }
-
- public void setEnabled(boolean enabled) {
- this.enabled = enabled;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/OSType.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/OSType.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/OSType.java
deleted file mode 100644
index b720f95..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/OSType.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.utils;
-
-public enum OSType {
-
- unknown("Unknown"), //$NON-NLS-1$
- linux("LINUX"), //$NON-NLS-1$
- mac_os("MACOS"), //$NON-NLS-1$
- win95("WIN95"), //$NON-NLS-1$
- win98("WIN98"), //$NON-NLS-1$
- windows_R_Me("Windows_R_Me"), //$NON-NLS-1$
- winNT("WINNT"), //$NON-NLS-1$
- windows_2000("Windows_2000"), //$NON-NLS-1$
- windows_XP("Windows_XP"), //$NON-NLS-1$
- msdos("MSDOS"), //$NON-NLS-1$
- solaris("Solaris"), //$NON-NLS-1$
- sunOS("SunOS"), //$NON-NLS-1$
- freeBSD("FreeBSD"), //$NON-NLS-1$
- netBSD("NetBSD"), //$NON-NLS-1$
- openBSD("OpenBSD"), //$NON-NLS-1$
- bsdunix("BSDUNIX"), //$NON-NLS-1$
- aix("AIX"), //$NON-NLS-1$
- z_OS("z_OS"), //$NON-NLS-1$
- os_2("OS_2"), //$NON-NLS-1$
- os9("OS9"), //$NON-NLS-1$
- netWare("NetWare"), //$NON-NLS-1$
- tru64_unix("Tru64_UNIX"), //$NON-NLS-1$
- irix("IRIX"), //$NON-NLS-1$
- osf("OSF"), //$NON-NLS-1$
-
- mvs("MVS"), //$NON-NLS-1$
- os400("OS400"), //$NON-NLS-1$
- javaVM("JavaVM"), //$NON-NLS-1$
- win3x("WIN3x"), //$NON-NLS-1$
- winCE("WINCE"), //$NON-NLS-1$
- NCR3000("NCR3000"), //$NON-NLS-1$
- dc_os("DC_OS"), //$NON-NLS-1$
- reliant_unix("Reliant_UNIX"), //$NON-NLS-1$
- sco_unixWare("SCO_UnixWare"), //$NON-NLS-1$
- sco_openServer("SCO_OpenServer"), //$NON-NLS-1$
- sequent("Sequent"), //$NON-NLS-1$
- u6000("U6000"), //$NON-NLS-1$
- aseries("ASERIES"), //$NON-NLS-1$
- tandemNSK("TandemNSK"), //$NON-NLS-1$
- tandemNT("TandemNT"), //$NON-NLS-1$
- bs2000("BS2000"), //$NON-NLS-1$
- lynx("Lynx"), //$NON-NLS-1$
- xenix("XENIX"), //$NON-NLS-1$
- vm("VM"), //$NON-NLS-1$
- interactive_unix("Interactive_UNIX"), //$NON-NLS-1$
- gnu_hurd("GNU_Hurd"), //$NON-NLS-1$
- mach_kernel("MACH_Kernel"), //$NON-NLS-1$
- inferno("Inferno"), //$NON-NLS-1$
- qnx("QNX"), //$NON-NLS-1$
- epoc("EPOC"), //$NON-NLS-1$
- ixWorks("IxWorks"), //$NON-NLS-1$
- vxWorks("VxWorks"), //$NON-NLS-1$
- mint("MiNT"), //$NON-NLS-1$
- beOS("BeOS"), //$NON-NLS-1$
- hp_mpe("HP_MPE"), //$NON-NLS-1$
- nextStep("NextStep"), //$NON-NLS-1$
- palmPilot("PalmPilot"), //$NON-NLS-1$
- rhapsody("Rhapsody"), //$NON-NLS-1$
- dedicated("Dedicated"), //$NON-NLS-1$
- os_390("OS_390"), //$NON-NLS-1$
- vse("VSE"), //$NON-NLS-1$
- tpf("TPF"), //$NON-NLS-1$
- caldera_open_unix("Caldera_Open_UNIX"), //$NON-NLS-1$
- attunix("ATTUNIX"), //$NON-NLS-1$
- dgux("DGUX"), //$NON-NLS-1$
- decnt("DECNT"), //$NON-NLS-1$
- openVMS("OpenVMS"), //$NON-NLS-1$
- hpux("HPUX"), //$NON-NLS-1$
- other("other"); //$NON-NLS-1$
-
-
- private OSType(String value) {
- this.value = value;
- }
-
- private final String value;
-
- public String getValue() {
- return value;
- }
-
- public static OSType fromString(String value)
- {
- for(OSType type : values())
- {
- if(type.value.equals(value))
- {
- return type;
- }
- }
- return null;
- }
-
- public String toString()
- {
- return value;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/ProcessorRequirement.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/ProcessorRequirement.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/ProcessorRequirement.java
deleted file mode 100644
index 82bf043..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/ProcessorRequirement.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-
-package org.apache.airavata.gfac.utils;
-
-public enum ProcessorRequirement{
- sparc("sparc"), //$NON-NLS-1$
- powerpc("powerpc"), //$NON-NLS-1$
- x86("x86"), //$NON-NLS-1$
- x86_32("x86_32"), //$NON-NLS-1$
- x86_64("x86_64"), //$NON-NLS-1$
- parisc("parisc"), //$NON-NLS-1$
- mips("mips"), //$NON-NLS-1$
- ia64("ia64"), //$NON-NLS-1$
- arm("arm"), //$NON-NLS-1$
- other("other"); //$NON-NLS-1$
-
- ProcessorRequirement(String value) {
- this.value = value;
- }
-
- private final String value;
-
- public String getValue() {
- return value;
- }
-
- public static ProcessorRequirement fromString(String value)
- {
- for (ProcessorRequirement type : values()) {
- if (type.value.equals(value)) {
- return type;
- }
- }
- return other;
- }
-
- public String toString()
- {
- return value;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/RangeValueType.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/RangeValueType.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/RangeValueType.java
deleted file mode 100644
index 114ea51..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/RangeValueType.java
+++ /dev/null
@@ -1,274 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.utils;
-
-import org.apache.airavata.gfac.provider.utils.ResourceRequirement;
-
-public class RangeValueType implements ResourceRequirement {
-
-
- private double exact = Double.NaN;
- private double lowerBound = Double.NEGATIVE_INFINITY;
- private double upperBound = Double.POSITIVE_INFINITY;
-
- private double epsilon = Double.NaN;
- private boolean includeLowerBound = true;
- private boolean includeUpperBound = true;
-
- private boolean enabled = false;
-
-
- public RangeValueType(double exact, double epsilon, double lowerBound, boolean includeLowerBound, double upperBound, boolean includeUpperBound, boolean enabled) {
- this.exact = exact;
- this.epsilon = epsilon;
- this.lowerBound = lowerBound;
- this.includeLowerBound = includeLowerBound;
- this.upperBound = upperBound;
- this.includeUpperBound = includeUpperBound;
- this.enabled = enabled;
- }
-
-
-
- /**
- * Create the range requirements
- *
- * @param exact -
- * the exact value
- * @param lowerBound -
- * the lower bound
- * @param upperBound -
- * the upper bound
- * @param includelowerBound -
- * true, if lowerBound should be included in range
- * @param includeUpperBound -
- * true, if upperBound should be included in range
- *
- */
- public RangeValueType(double exact, double epsilon, double lowerBound, boolean includeLowerBound, double upperBound, boolean includeUpperBound) {
- this(exact,epsilon,lowerBound,includeLowerBound,upperBound,includeUpperBound,false);
-
- }
-
-
- /**
- * Create the range requirements
- *
- * @param exact -
- * the exact value
- * @param lowerBound -
- * the lower bound
- * @param upperBound -
- * the upper bound
- */
- public RangeValueType(double exact, double epsilon, double lowerBound, double upperBound) {
- this(exact,epsilon,lowerBound,true,upperBound,true);
- }
-
-
- public RangeValueType(double exact, double lowerBound, double upperBound) {
- this(exact,Double.NaN,lowerBound,true,upperBound,true);
- }
-
- /**
- * Create the exact requirements
- *
- * @param exact -
- * the exact value
- * @param epsilon -
- * the epsilon arround exact
- *
- */
- public RangeValueType(double exact, double epsilon) {
- this(exact,epsilon,Double.NaN,Double.NaN);
- }
-
-
- /**
- * Create the exact requirements
- *
- * @param exact -
- * the exact value
- */
- public RangeValueType(double exact) {
- this(exact,Double.NaN);
- }
-
- public RangeValueType() {
- }
-
- /**
- * Get exact requirements
- *
- * @return the exact requirements
- */
- public double getExact() {
- return exact;
- }
-
- /**
- * Set exact requirements
- *
- * @param exact -
- * the exact requirements
- */
- public void setExact(double exact) {
- this.exact = exact;
- }
-
- /**
- * Get epsilon
- *
- * @return the epsilon
- */
- public double getEpsilon() {
- return epsilon;
- }
-
- /**
- * Set epsilon
- *
- * @param epsilon -
- * epsilon belonging to to exact requirements
- */
- public void setEpsilon(double epsilon) {
- this.epsilon = epsilon;
- }
-
- /**
- * Get lower bound
- *
- * @return the lower bound
- */
- public double getLowerBound() {
- return lowerBound;
- }
-
- /**
- * Set lower bound
- *
- * @param lowerBound -
- * the lower bound
- */
- public void setLowerBound(double lowerBound) {
- this.lowerBound = lowerBound;
- }
-
- /**
- * Get upper bound
- *
- * @return the upper bound
- */
- public double getUpperBound() {
- return upperBound;
- }
-
- /**
- * Set upper bound
- *
- * @param upperBound -
- * the upper bound
- */
- public void setUpperBound(double upperBound) {
- this.upperBound = upperBound;
- }
-
- /**
- * Test if requirements are met
- *
- * @param value -
- * the tested value
- * @return <code>true</code> if value is in the range and not less than
- * the exact value
- */
- public boolean lowerThanDouble(double value) {
- return (value >= exact && value >= lowerBound && value <= upperBound) ? true : false;
- }
-
- public String toString() {
- if (lowerBound == Double.NEGATIVE_INFINITY && upperBound == Double.POSITIVE_INFINITY) {
- return Double.toString(exact);
- }
- else {
- return "(e=" + Double.toString(exact) + ",l=" + Double.toString(lowerBound) + ",u=" //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
- + Double.toString(upperBound) + ")"; //$NON-NLS-1$
- }
- }
-
-
- public boolean isIncludeLowerBound() {
- return includeLowerBound;
- }
-
-
- public void setIncludeLowerBound(boolean includeLowerBound) {
- this.includeLowerBound = includeLowerBound;
- }
-
-
- public boolean isIncludeUpperBound() {
- return includeUpperBound;
- }
-
-
- public void setIncludeUpperBound(boolean includeUpperBound) {
- this.includeUpperBound = includeUpperBound;
- }
-
- public RangeValueType clone(){
- return new RangeValueType(this.exact, this.epsilon, this.lowerBound, this.includeLowerBound, this.upperBound, this.includeUpperBound,this.enabled);
- }
-
-
-
- public boolean isEnabled() {
- return enabled;
- }
-
-
-
- public void setEnabled(boolean enabled) {
- this.enabled = enabled;
- }
-
-
- public boolean equals(Object o)
- {
- if(! (o instanceof RangeValueType)) return false;
- RangeValueType other = (RangeValueType) o;
- return doublesEqual(getExact(),other.getExact())
- && doublesEqual(getEpsilon(), other.getEpsilon())
- && doublesEqual(getLowerBound(), other.getLowerBound())
- && doublesEqual(getUpperBound(), other.getUpperBound())
- && isIncludeLowerBound() == other.isIncludeLowerBound()
- && isIncludeUpperBound() == other.isIncludeUpperBound()
- && isEnabled() == other.isEnabled();
- }
-
-
- private boolean doublesEqual(double a, double b)
- {
- Double A = new Double(a);
- Double B = new Double(b);
- return A.equals(B);
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/ResourceProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/ResourceProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/ResourceProcessor.java
deleted file mode 100644
index cc98f96..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/ResourceProcessor.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.utils;
-
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.provider.GFacProviderException;
-import org.apache.airavata.model.workspace.experiment.ComputationalResourceScheduling;
-import org.apache.airavata.model.workspace.experiment.TaskDetails;
-import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
-import org.apache.airavata.schemas.gfac.QueueType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
-import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.NumberOfProcessesType;
-
-public class ResourceProcessor {
-
-
- public static void generateResourceElements(JobDefinitionType value, JobExecutionContext context) throws Exception{
-
- HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
- .getApplicationContext().getApplicationDeploymentDescription()
- .getType();
-
- createMemory(value, appDepType);
- TaskDetails taskData = context.getTaskData();
- if(taskData != null && taskData.isSetTaskScheduling()){
- ComputationalResourceScheduling computionResource= taskData.getTaskScheduling();
- try {
- int cpuCount = computionResource.getTotalCPUCount();
- if(cpuCount>0){
-// appDepType.setCpuCount(cpuCount);
- NumberOfProcessesType num = NumberOfProcessesType.Factory.newInstance();
- String processers = Integer.toString(cpuCount);
- num.setStringValue(processers);
- JSDLUtils.getOrCreateSPMDApplication(value).setNumberOfProcesses(num);
- }
- } catch (NullPointerException e) {
- new GFacProviderException("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used",e);
- }
- try {
- int nodeCount = computionResource.getNodeCount();
- if(nodeCount>0){
- appDepType.setNodeCount(nodeCount);
- }
- } catch (NullPointerException e) {
- new GFacProviderException("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used",e);
- }
- try {
- String queueName = computionResource.getQueueName();
- if (queueName != null) {
- if(appDepType.getQueue() == null){
- QueueType queueType = appDepType.addNewQueue();
- queueType.setQueueName(queueName);
- }else{
- appDepType.getQueue().setQueueName(queueName);
- }
- }
- } catch (NullPointerException e) {
- new GFacProviderException("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used",e);
- }
- try {
- int maxwallTime = computionResource.getWallTimeLimit();
- if(maxwallTime>0){
- appDepType.setMaxWallTime(maxwallTime);
- }
- } catch (NullPointerException e) {
- new GFacProviderException("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used",e);
- }
- }
-
- if (appDepType.getCpuCount() > 0) {
- RangeValueType rangeType = new RangeValueType();
- rangeType.setLowerBound(Double.NaN);
- rangeType.setUpperBound(Double.NaN);
- rangeType.setExact(appDepType.getCpuCount());
- JSDLUtils.setTotalCPUCountRequirements(value, rangeType);
- }
-
- if (appDepType.getProcessorsPerNode() > 0) {
- RangeValueType rangeType = new RangeValueType();
- rangeType.setLowerBound(Double.NaN);
- rangeType.setUpperBound(Double.NaN);
- rangeType.setExact(appDepType.getProcessorsPerNode());
- JSDLUtils.setIndividualCPUCountRequirements(value, rangeType);
- }
-
- if (appDepType.getNodeCount() > 0) {
- RangeValueType rangeType = new RangeValueType();
- rangeType.setLowerBound(Double.NaN);
- rangeType.setUpperBound(Double.NaN);
- rangeType.setExact(appDepType.getNodeCount());
- JSDLUtils.setTotalResourceCountRequirements(value, rangeType);
- }
-
- if(appDepType.getMaxWallTime() > 0) {
- RangeValueType cpuTime = new RangeValueType();
- cpuTime.setLowerBound(Double.NaN);
- cpuTime.setUpperBound(Double.NaN);
- long wallTime = appDepType.getMaxWallTime() * 60;
- cpuTime.setExact(wallTime);
- JSDLUtils.setIndividualCPUTimeRequirements(value, cpuTime);
- }
- }
-
-
- private static void createMemory(JobDefinitionType value, HpcApplicationDeploymentType appDepType){
- if (appDepType.getMinMemory() > 0 && appDepType.getMaxMemory() > 0) {
- RangeValueType rangeType = new RangeValueType();
- rangeType.setLowerBound(appDepType.getMinMemory());
- rangeType.setUpperBound(appDepType.getMaxMemory());
- JSDLUtils.setIndividualPhysicalMemoryRequirements(value, rangeType);
- }
-
- else if (appDepType.getMinMemory() > 0 && appDepType.getMaxMemory() <= 0) {
- // TODO set Wall time
- RangeValueType rangeType = new RangeValueType();
- rangeType.setLowerBound(appDepType.getMinMemory());
- JSDLUtils.setIndividualPhysicalMemoryRequirements(value, rangeType);
- }
-
- else if (appDepType.getMinMemory() <= 0 && appDepType.getMaxMemory() > 0) {
- // TODO set Wall time
- RangeValueType rangeType = new RangeValueType();
- rangeType.setUpperBound(appDepType.getMinMemory());
- JSDLUtils.setIndividualPhysicalMemoryRequirements(value, rangeType);
- }
-
- }
-
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/SPMDProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/SPMDProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/SPMDProcessor.java
deleted file mode 100644
index 170d60d..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/SPMDProcessor.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.utils;
-
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
-
-public class SPMDProcessor {
-
- public static void generateSPMDElements(JobDefinitionType value, JobExecutionContext context) {
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/SPMDVariations.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/SPMDVariations.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/SPMDVariations.java
deleted file mode 100644
index 4b73fba..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/SPMDVariations.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.utils;
-
-public enum SPMDVariations {
-
- MPI ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MPI"),
- GridMPI ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/GridMPI"),
- IntelMPI ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/IntelMPI"),
- LAMMPI ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/LAM-MPI"),
- MPICH1 ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MPICH1"),
- MPICH2 ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MPICH2"),
- MPICHGM ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MPICH-GM"),
- MPICHMX ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MPICH-MX"),
- MVAPICH ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MVAPICH"),
- MVAPICH2 ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MVAPICH2"),
- OpenMPI ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/OpenMPI"),
- POE ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/POE"),
- PVM ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/PVM");
-
- private final String variation;
-
- private SPMDVariations(String variation) {
- this.variation = variation;
- }
-
- public String value(){
- return variation;
- }
-
-}
-
-
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/StorageCreator.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/StorageCreator.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/StorageCreator.java
deleted file mode 100644
index 40729c6..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/StorageCreator.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.utils;
-
-import java.util.Calendar;
-
-import javax.security.auth.x500.X500Principal;
-
-import org.oasisOpen.docs.wsrf.sg2.EntryType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.unigrids.services.atomic.types.PropertyType;
-import org.unigrids.x2006.x04.services.smf.CreateSMSDocument;
-import org.unigrids.x2006.x04.services.smf.StorageBackendParametersDocument.StorageBackendParameters;
-import org.unigrids.x2006.x04.services.smf.StorageDescriptionType;
-import org.w3.x2005.x08.addressing.EndpointReferenceType;
-
-import de.fzj.unicore.uas.StorageFactory;
-import de.fzj.unicore.uas.client.StorageClient;
-import de.fzj.unicore.uas.client.StorageFactoryClient;
-import de.fzj.unicore.wsrflite.xmlbeans.WSUtilities;
-import de.fzj.unicore.wsrflite.xmlbeans.client.RegistryClient;
-import de.fzj.unicore.wsrflite.xmlbeans.sg.Registry;
-
-
-import eu.unicore.util.httpclient.DefaultClientConfiguration;
-
-public class StorageCreator {
- protected final Logger log = LoggerFactory.getLogger(this.getClass());
-
- /**
- * the initial lifetime (in days) for newly created SMSs
- */
- private int initialLifeTime;
-
- /**
- * factory URL to use
- */
- private String factoryUrl;
-
- /**
- * site where to create the storage
- */
- private String siteName;
-
- /**
- * storage type to create
- */
- private String storageType;
-
- private DefaultClientConfiguration secProps;
-
- private String userName;
-
- public StorageCreator(DefaultClientConfiguration secProps, String besUrl, int initialLifetime, String storageType, String userName) {
- this.secProps = secProps;
- this.factoryUrl = getStorageFactoryUrl(besUrl);
- this.storageType = storageType;
- this.initialLifeTime = initialLifetime;
- this.userName = userName;
- }
-
-
- public StorageCreator(DefaultClientConfiguration secProps, String besUrl, int initialLifetime, String userName) {
- this.secProps = secProps;
- this.factoryUrl = getStorageFactoryUrl(besUrl);
- this.initialLifeTime = initialLifetime;
- this.userName = userName;
- }
-
-
- // The target site must have storage factory deployed with bes factory
- public StorageClient createStorage() throws Exception{
-
- if(factoryUrl == null) {
- throw new Exception("Cannot create Storage Factory Url");
- }
-
- EndpointReferenceType sfEpr= WSUtilities.makeServiceEPR(factoryUrl, StorageFactory.SMF_PORT);
-
- String dn = findServerName(factoryUrl, sfEpr);
-
- WSUtilities.addServerIdentity(sfEpr, dn);
-
- secProps.getETDSettings().setReceiver(new X500Principal(dn));
- secProps.getETDSettings().setIssuerCertificateChain(secProps.getCredential().getCertificateChain());
-
- // TODO: remove it afterwards
- if(userName != null) {
- secProps.getETDSettings().getRequestedUserAttributes2().put("xlogin", new String[]{userName});
- }
-
- StorageFactoryClient sfc = new StorageFactoryClient(sfEpr, secProps);
-
- if (log.isDebugEnabled()){
- log.debug("Using storage factory at <"+sfc.getUrl()+">");
- }
-
- StorageClient sc = null;
- try{
- sc=sfc.createSMS(getCreateSMSDocument());
-
- String addr=sc.getEPR().getAddress().getStringValue();
- log.info(addr);
-
- }catch(Exception ex){
- log.error("Could not create storage",ex);
- throw new Exception(ex);
- }
-
- return sc;
- }
-
- protected String findServerName(String besUrl, EndpointReferenceType smsEpr)throws Exception{
-
- int besIndex = besUrl.indexOf("StorageFactory?res");
- String ss = besUrl.substring(0, besIndex);
- ss = ss + "Registry";
-
- EndpointReferenceType eprt = WSUtilities.makeServiceEPR(ss, "default_registry", Registry.REGISTRY_PORT);
-
- RegistryClient registry = new RegistryClient(eprt, secProps);
-
- //first, check if server name is already in the EPR...
- String dn=WSUtilities.extractServerIDFromEPR(smsEpr);
- if(dn!=null){
- return dn;
- }
- //otherwise find a matching service in the registry
- String url=smsEpr.getAddress().getStringValue();
- if(url.contains("/services/"))url=url.substring(0,url.indexOf("/services"));
- if(log.isDebugEnabled()) log.debug("Checking for services at "+url);
- for(EntryType entry:registry.listEntries()){
- if(entry.getMemberServiceEPR().getAddress().getStringValue().startsWith(url)){
- dn=WSUtilities.extractServerIDFromEPR(entry.getMemberServiceEPR());
- if(dn!=null){
- return dn;
- }
- }
- }
- return null;
- }
-
-
- public static String getStorageFactoryUrl(String besUrl){
- int besIndex = besUrl.indexOf("BESFactory?res");
- String ss = besUrl.substring(0, besIndex);
- ss = ss + "StorageFactory?res=default_storage_factory";
- return ss;
- }
-
- /**
- * prepare request
- * */
- protected CreateSMSDocument getCreateSMSDocument(String ...keyValueParams){
- CreateSMSDocument in=CreateSMSDocument.Factory.newInstance();
- in.addNewCreateSMS();
- if(initialLifeTime>0){
- in.getCreateSMS().addNewTerminationTime().setCalendarValue(getTermTime());
- }
- if(storageType!=null){
- if(log.isDebugEnabled()) {
- log.debug("Will create storage of type : "+storageType);
- }
- StorageDescriptionType desc=in.getCreateSMS().addNewStorageDescription();
- desc.setStorageBackendType(storageType);
- if(keyValueParams.length>1){
- //other parameters from the cmdline as key=value
- StorageBackendParameters params=desc.addNewStorageBackendParameters();
- for(int i=1;i<keyValueParams.length;i++){
- String arg=keyValueParams[i];
- String[]sp=arg.split("=",2);
- PropertyType prop=params.addNewProperty();
- prop.setName(sp[0]);
- prop.setValue(sp[1]);
- if(log.isDebugEnabled()) {
- log.debug("Have parameter : "+arg);
- }
- }
- }
- }
- return in;
- }
-
- protected Calendar getTermTime(){
- Calendar c = Calendar.getInstance();
- c.add(Calendar.DATE, initialLifeTime);
- return c;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/UASDataStagingProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/UASDataStagingProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/UASDataStagingProcessor.java
deleted file mode 100644
index 461ee0b..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/UASDataStagingProcessor.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.utils;
-
-import java.io.File;
-import java.util.Map;
-
-import org.apache.airavata.commons.gfac.type.ActualParameter;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
-import org.apache.airavata.schemas.gfac.StringArrayType;
-import org.apache.airavata.schemas.gfac.StringParameterType;
-import org.apache.airavata.schemas.gfac.URIArrayType;
-import org.apache.airavata.schemas.gfac.URIParameterType;
-import org.apache.airavata.schemas.gfac.UnicoreHostType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
-
-public class UASDataStagingProcessor {
-
- public static void generateDataStagingElements(JobDefinitionType value, JobExecutionContext context, String smsUrl) throws Exception{
-
- HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
- .getApplicationContext().getApplicationDeploymentDescription()
- .getType();
-
- smsUrl = "BFT:"+smsUrl;
-
- if (context.getInMessageContext().getParameters().size() > 0) {
- buildDataStagingFromInputContext(context, value, smsUrl, appDepType);
- }
- MessageContext outMessage = new MessageContext();
- ActualParameter a1 = new ActualParameter();
- a1.getType().changeType(StringParameterType.type);
- ((StringParameterType)a1.getType()).setValue("output/analysis-results.tar");
- outMessage.addParameter("o1", a1);
- context.setOutMessageContext(outMessage);
-
- if (context.getOutMessageContext().getParameters().size() > 0) {
- buildFromOutputContext(context, value, smsUrl, appDepType);
- }
- createStdOutURIs(value, appDepType, smsUrl, isUnicoreEndpoint(context));
- }
-
- private static void createInURISMSElement(JobDefinitionType value,
- String smsUrl, String inputDir, ActualParameter inParam)
- throws Exception {
-
- String uri = ((URIParameterType) inParam.getType()).getValue();
- //TODO: To add this input file name setting part of Airavata API
- String fileName = "input/" + new File(uri).getName();
- if (uri.startsWith("file")) {
- String fileUri = smsUrl+"#/"+fileName;
-
- JSDLUtils.addDataStagingSourceElement(value, fileUri, null, fileName);
- } else if (uri.startsWith("gsiftp") || uri.startsWith("http")
- || uri.startsWith("rns")) {
- // no need to stage-in those files to the input
- // directory because unicore site will fetch them for the user
- JSDLUtils.addDataStagingSourceElement(value, uri, null, fileName);
- }
-
- }
-
- private static void createStdOutURIs(JobDefinitionType value,
- HpcApplicationDeploymentType appDepType, String smsUrl,
- boolean isUnicore) throws Exception {
-
-
- String stdout = ApplicationProcessor.getApplicationStdOut(value, appDepType);
-
- String stderr = ApplicationProcessor.getApplicationStdErr(value, appDepType);
-
- String stdoutFileName = (stdout == null || stdout.equals("")) ? "stdout"
- : stdout;
- String stdoutURI = smsUrl+"#/output/"+stdoutFileName;
- JSDLUtils.addDataStagingTargetElement(value, null, stdoutFileName,
- stdoutURI);
-
- String stderrFileName = (stdout == null || stderr.equals("")) ? "stderr"
- : stderr;
- String stderrURI = smsUrl+"#/output/"+stderrFileName;
- JSDLUtils.addDataStagingTargetElement(value, null, stderrFileName,
- stderrURI);
-
- if(isUnicore) {
- String scriptExitCodeFName = "UNICORE_SCRIPT_EXIT_CODE";
- String scriptExitCode = smsUrl+"#/output/"+scriptExitCodeFName;
- JSDLUtils.addDataStagingTargetElement(value, null,
- scriptExitCodeFName, scriptExitCode.toString());
- }
-
- }
-
-
- private static void createOutStringElements(JobDefinitionType value,
- HpcApplicationDeploymentType appDeptype, String smsUrl, String prmValue) throws Exception {
-
- if(prmValue == null || "".equals(prmValue)) return;
-
- String finalSMSPath = smsUrl + "#/output/"+prmValue;
-
- JSDLUtils.addDataStagingTargetElement(value, null, prmValue, finalSMSPath);
- }
-
-
- private static void createOutURIElement(JobDefinitionType value,
- String prmValue) throws Exception {
- String fileName = new File(prmValue.toString()).getName();
- JSDLUtils.addDataStagingTargetElement(value, null, fileName, prmValue);
- }
-
-
- private static JobDefinitionType buildFromOutputContext(JobExecutionContext context,
- JobDefinitionType value, String smsUrl,
- HpcApplicationDeploymentType appDepType) throws Exception {
-
- Map<String, Object> outputParams = context.getOutMessageContext()
- .getParameters();
-
- for (String paramKey : outputParams.keySet()) {
-
- ActualParameter outParam = (ActualParameter) outputParams
- .get(paramKey);
-
- // if single urls then convert each url into jsdl source
- // elements,
- // that are formed by concat of gridftpurl+inputdir+filename
-
- String paramDataType = outParam.getType().getType().toString();
-
- if ("URI".equals(paramDataType)) {
- String uriPrm = ((URIParameterType) outParam.getType())
- .getValue();
- createOutURIElement(value, uriPrm);
- }
-
- // string params are converted into the job arguments
-
- else if (("URIArray").equals(paramDataType)) {
- String[] uriArray = ((URIArrayType) outParam.getType())
- .getValueArray();
- for (String u : uriArray) {
-
- createOutURIElement(value, u);
- }
-
- }
- else if ("String".equals(paramDataType)) {
- String stringPrm = ((StringParameterType) outParam
- .getType()).getValue();
- createOutStringElements(value, appDepType, smsUrl, stringPrm);
- }
-
- else if ("StringArray".equals(paramDataType)) {
- String[] valueArray = ((StringArrayType) outParam.getType())
- .getValueArray();
- for (String v : valueArray) {
- createOutStringElements(value, appDepType, smsUrl, v);
- }
- }
- }
-
- return value;
- }
-
-
- private static void buildDataStagingFromInputContext(JobExecutionContext context, JobDefinitionType value, String smsUrl, HpcApplicationDeploymentType appDepType)
- throws Exception {
-
- // TODO set data directory
- Map<String, Object> inputParams = context.getInMessageContext()
- .getParameters();
-
- for (String paramKey : inputParams.keySet()) {
-
- ActualParameter inParam = (ActualParameter) inputParams
- .get(paramKey);
-
- // if single urls then convert each url into jsdl source
- // elements,
- // that are formed by concat of gridftpurl+inputdir+filename
-
- String paramDataType = inParam.getType().getType().toString();
-
- if ("URI".equals(paramDataType)) {
- createInURISMSElement(value, smsUrl,
- appDepType.getInputDataDirectory(), inParam);
- }
-
- // string params are converted into the job arguments
-
- else if ("String".equals(paramDataType)) {
- String stringPrm = ((StringParameterType) inParam.getType())
- .getValue();
- ApplicationProcessor.addApplicationArgument(value, appDepType, stringPrm);
- }
- }
-
- }
-
- public static boolean isUnicoreEndpoint(JobExecutionContext context) {
- return ( (context.getApplicationContext().getHostDescription().getType() instanceof UnicoreHostType)?true:false );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/URIUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/URIUtils.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/URIUtils.java
deleted file mode 100644
index 34d0da1..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/URIUtils.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.utils;
-
-import org.apache.commons.httpclient.URI;
-import org.apache.commons.httpclient.URIException;
-import org.apache.commons.httpclient.util.URIUtil;
-
-import java.net.URISyntaxException;
-
-public class URIUtils {
-
- public static String encodeAll(String uri) throws URIException
- {
- String result = encodeAuthority(uri);
- result = encodePath(uri);
- result = encodeQuery(result );
- result = encodeFragment(result );
- return result;
- }
-
- public static String encodeAuthority(String uri) throws URIException
- {
- int start = uri.indexOf("//");
- if(start == -1) return uri;
- start++;
- int end = uri.indexOf("/",start+1);
- if(end == -1) end = uri.indexOf("?",start+1);
- if(end == -1) end = uri.indexOf("#",start+1);
- if(end == -1) end = uri.length();
- String before = uri.substring(0, start+1);
- String authority= uri.substring(start+1,end);
- String after = uri.substring(end);
- authority = URIUtil.encode(authority, URI.allowed_authority);
-
- return before+authority+after;
- }
-
- public static String encodePath(String uri) throws URIException
- {
- int doubleSlashIndex = uri.indexOf("//");
- boolean hasAuthority = doubleSlashIndex >= 0;
- int start = -1;
- if(hasAuthority)
- {
- start = uri.indexOf("/",doubleSlashIndex+2);
- }
- else
- {
- start = uri.indexOf(":");
- }
- if(start == -1) return uri;
-
- int end = uri.indexOf("?",start+1);
- if(end == -1) end = uri.indexOf("#",start+1);
- if(end == -1) end = uri.length();
- String before = uri.substring(0, start+1);
- String path= uri.substring(start+1,end);
- String after = uri.substring(end);
- path = URIUtil.encode(path, URI.allowed_abs_path);
- return before+path+after;
- }
-
-
- public static String encodeQuery(String uri) throws URIException
- {
- int queryStart = uri.indexOf("?");
- if(queryStart == -1) return uri;
- int queryEnd = uri.indexOf("#");
- if(queryEnd == -1) queryEnd = uri.length();
-
- String beforeQuery = uri.substring(0, queryStart+1);
- String query = uri.substring(queryStart+1,queryEnd);
- String afterQuery = uri.substring(queryEnd);
- query = URIUtil.encode(query, URI.allowed_query);
- return beforeQuery+query+afterQuery;
- }
-
-
- public static String encodeFragment(String uri) throws URIException
- {
- int fragmentStart = uri.indexOf("#");
- if(fragmentStart == -1) return uri;
-
- String beforeFragment = uri.substring(0, fragmentStart+1);
- String fragment = uri.substring(fragmentStart+1);
- fragment = URIUtil.encode(fragment, URI.allowed_fragment);
- return beforeFragment+fragment;
- }
- public static java.net.URI createGsiftpURI(String host, String localPath) throws URISyntaxException {
- StringBuffer buf = new StringBuffer();
- if (!host.startsWith("gsiftp://"))
- buf.append("gsiftp://");
- buf.append(host);
- if (!host.endsWith("/"))
- buf.append("/");
- buf.append(localPath);
- return new java.net.URI(buf.toString());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTestWithMyProxyAuth.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTestWithMyProxyAuth.java b/modules/gfac/gfac-bes/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTestWithMyProxyAuth.java
index d048f67..cf9b82a 100644
--- a/modules/gfac/gfac-bes/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTestWithMyProxyAuth.java
+++ b/modules/gfac/gfac-bes/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTestWithMyProxyAuth.java
@@ -25,38 +25,6 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-import java.io.File;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.UUID;
-
-import org.apache.airavata.commons.gfac.type.ActualParameter;
-import org.apache.airavata.commons.gfac.type.ApplicationDescription;
-import org.apache.airavata.commons.gfac.type.HostDescription;
-import org.apache.airavata.commons.gfac.type.ServiceDescription;
-import org.apache.airavata.gfac.GFacConfiguration;
-import org.apache.airavata.gfac.context.ApplicationContext;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.MessageContext;
-import org.apache.airavata.gfac.utils.JSDLGenerator;
-import org.apache.airavata.gfac.utils.JSDLUtils;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
-import org.apache.airavata.schemas.gfac.InputParameterType;
-import org.apache.airavata.schemas.gfac.JobTypeType;
-import org.apache.airavata.schemas.gfac.OutputParameterType;
-import org.apache.airavata.schemas.gfac.ProjectAccountType;
-import org.apache.airavata.schemas.gfac.QueueType;
-import org.apache.airavata.schemas.gfac.StringParameterType;
-import org.apache.airavata.schemas.gfac.URIParameterType;
-import org.apache.airavata.schemas.gfac.UnicoreHostType;
-import org.apache.log4j.PropertyConfigurator;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionDocument;
-import org.junit.Before;
-import org.junit.Test;
-
//public class JSDLGeneratorTestWithMyProxyAuth {
//
// public static final String[] hostArray = new String[] { "https://zam1161v01.zam.kfa-juelich.de:8002/INTEROP1/services/BESFactory?res=default_bes_factory" };
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/orchestrator/airavata-orchestrator-service/src/test/resources/gfac-config.xml
----------------------------------------------------------------------
diff --git a/modules/orchestrator/airavata-orchestrator-service/src/test/resources/gfac-config.xml b/modules/orchestrator/airavata-orchestrator-service/src/test/resources/gfac-config.xml
index 61dca4f..901f72e 100644
--- a/modules/orchestrator/airavata-orchestrator-service/src/test/resources/gfac-config.xml
+++ b/modules/orchestrator/airavata-orchestrator-service/src/test/resources/gfac-config.xml
@@ -36,7 +36,7 @@
<Handler class="org.apache.airavata.gfac.handler.GridFTPOutputHandler"/>
</OutHandlers>
</Provider>
- <Provider class="org.apache.airavata.gfac.provider.impl.BESProvider" host="org.apache.airavata.schemas.gfac.impl.UnicoreHostTypeImpl">
+ <Provider class="BESProvider" host="org.apache.airavata.schemas.gfac.impl.UnicoreHostTypeImpl">
<InHandlers>
<Handler class="org.apache.airavata.gfac.handler.GramDirectorySetupHandler"/>
<Handler class="org.apache.airavata.gfac.handler.GridFTPInputHandler"/>
[3/4] moving gfac-bes classes to new package.
Posted by la...@apache.org.
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/FileUploader.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/FileUploader.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/FileUploader.java
new file mode 100644
index 0000000..6c62053
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/FileUploader.java
@@ -0,0 +1,245 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.bes.utils;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Map;
+
+import org.unigrids.services.atomic.types.ProtocolType;
+
+import de.fzj.unicore.uas.client.FileTransferClient;
+import de.fzj.unicore.uas.client.StorageClient;
+import de.fzj.unicore.uas.client.UFTPConstants;
+import de.fzj.unicore.uas.client.UFTPFileTransferClient;
+import de.fzj.unicore.uas.fts.FiletransferOptions.IMonitorable;
+
+/**
+ * upload local file(s) to a remote location
+ *
+ * @author schuller
+ */
+public class FileUploader extends FileTransferBase{
+
+ public FileUploader(String from, String to, Mode mode)throws FileNotFoundException{
+ this(from,to,mode,true);
+ }
+
+ public FileUploader(String from, String to, Mode mode, boolean failOnError)throws FileNotFoundException{
+ this.to=to;
+ this.from=from;
+ this.mode=mode;
+ this.failOnError=failOnError;
+ checkOK();
+ }
+
+ public String getFrom() {
+ return from;
+ }
+
+ public String getTo() {
+ return to;
+ }
+
+
+ public void perform(StorageClient sms)throws Exception{
+ File fileSpec=new File(from);
+ boolean hasWildCards=false;
+ boolean isDirectory=fileSpec.isDirectory();
+ File[] fileset=null;
+
+ if(!isDirectory){
+ hasWildCards=hasWildCards(fileSpec);
+ }
+
+ chosenProtocol=sms.findSupportedProtocol(preferredProtocols.toArray(new ProtocolType.Enum[preferredProtocols.size()]));
+ Map<String,String>extraParameters=makeExtraParameters(chosenProtocol);
+
+ if(!hasWildCards && !isDirectory){
+ //single regular file
+ uploadFile(fileSpec,to,sms,chosenProtocol,extraParameters);
+ return;
+ }
+
+ //handle wildcards or directory
+ if(hasWildCards){
+ fileset=resolveWildCards(fileSpec);
+ }
+ else{
+ fileset=fileSpec.listFiles();
+ }
+
+ if(!isValidDirectory(to, sms)){
+ throw new IOException("The specified remote target '"+to+"' is not a directory");
+ }
+ if(to==null)to="/";
+ String target=isDirectory?to+"/"+fileSpec.getName():to;
+ sms.createDirectory(target);
+ uploadFiles(fileset,target,sms,chosenProtocol,extraParameters);
+ }
+
+ /**
+ * upload a set of files to a remote directory (which must exist)
+ *
+ * @param files
+ * @param remoteDirectory
+ * @param sms
+ * @param protocol
+ * @param extraParameters
+ * @param msg
+ * @throws Exception
+ */
+ private void uploadFiles(File[]files, String remoteDirectory, StorageClient sms, ProtocolType.Enum protocol,
+ Map<String,String>extraParameters)throws Exception{
+ for(File localFile: files){
+ String target=remoteDirectory+"/"+localFile.getName();
+ if(localFile.isDirectory()){
+ if(!recurse){
+ System.out.println("Skipping directory "+localFile.getAbsolutePath());
+ }else{
+ File[] fileset=localFile.listFiles();
+ sms.createDirectory(target);
+ uploadFiles(fileset,target,sms,protocol,extraParameters);
+ }
+ }else{
+ uploadFile(localFile,target,sms,protocol,extraParameters);
+ }
+ }
+ }
+
+ /**
+ * uploads a single regular file
+ *
+ * @param localFile
+ * @param remotePath
+ * @param sms
+ * @param protocol
+ * @param extraParameters
+ * @param msg
+ * @throws Exception
+ */
+ private void uploadFile(File localFile, String remotePath, StorageClient sms, ProtocolType.Enum protocol,
+ Map<String,String>extraParameters) throws Exception{
+ long startTime=System.currentTimeMillis();
+ FileInputStream is=null;
+ FileTransferClient ftc=null;
+ try{
+ if(remotePath==null){
+ remotePath="/"+localFile.getName();
+ }
+ else if(remotePath.endsWith("/")){
+ remotePath+=localFile.getName();
+ }
+ System.out.println("Uploading local file '"+localFile.getAbsolutePath()+"' -> '"+sms.getUrl()+"#"+remotePath+"'");
+ is=new FileInputStream(localFile.getAbsolutePath());
+ boolean append=Mode.append.equals(mode);
+ ftc=sms.getImport(remotePath, append, extraParameters, protocol);
+ configure(ftc, extraParameters);
+ if(append)ftc.setAppend(true);
+ String url=ftc.getUrl();
+ System.out.println("File transfer URL : "+url);
+// ProgressBar p=null;
+ if(ftc instanceof IMonitorable){
+ long size=localFile.length();
+ if(isRange()){
+ size=getRangeSize();
+ }
+// p=new ProgressBar(localFile.getName(),size,msg);
+// ((IMonitorable) ftc).setProgressListener(p);
+ }
+ if(isRange()){
+ System.out.println("Byte range: "+startByte+" - "+(getRangeSize()>0?endByte:""));
+ long skipped=0;
+ while(skipped<startByte){
+ skipped+=is.skip(startByte);
+ }
+ ftc.writeAllData(is, endByte-startByte+1);
+
+ }else{
+ ftc.writeAllData(is);
+ }
+ copyProperties(localFile, sms, remotePath);
+
+// if(ftc instanceof IMonitorable){
+// p.finish();
+// }
+
+ }finally{
+ if(ftc!=null){
+ try{
+ ftc.destroy();
+ }catch(Exception e1){
+// msg.error("Could not clean-up the filetransfer at <"+ftc.getUrl()+">",e1);
+ }
+ }
+ try{ if(is!=null)is.close(); }catch(Exception ignored){}
+ }
+ if(timing){
+ long duration=System.currentTimeMillis()-startTime;
+ double rate=(double)localFile.length()/(double)duration;
+ System.out.println("Rate: "+rate+ " kB/sec.");
+ }
+ }
+
+ /**
+ * if possible, copy the local executable flag to the remote file
+ * @param sourceFile - local file
+ * @throws Exception
+ */
+ private void copyProperties(File sourceFile, StorageClient sms, String target)throws Exception{
+ boolean x=sourceFile.canExecute();
+ try{
+ if(x){
+ sms.changePermissions(target, true, true, x);
+ }
+ }catch(Exception ex){
+// System.out.println("Can't set exectuable flag on remote file.",ex);
+ }
+ }
+
+ private void checkOK()throws FileNotFoundException{
+ if(!failOnError){
+ return;
+ }
+ File orig=new File(from);
+ if(!orig.isAbsolute()){
+ orig=new File(System.getProperty("user.dir"),from);
+ }
+ File[] files=resolveWildCards(orig);
+ if(files==null){
+ throw new FileNotFoundException("Local import '"+from+"' does not exist.");
+ }
+ for(File f: files){
+ if(!f.exists())throw new FileNotFoundException("Local import '"+from+"' does not exist.");
+ }
+ }
+
+ private void configure(FileTransferClient ftc, Map<String,String>params){
+ if(ftc instanceof UFTPFileTransferClient){
+ UFTPFileTransferClient u=(UFTPFileTransferClient)ftc;
+ String secret=params.get(UFTPConstants.PARAM_SECRET);
+ u.setSecret(secret);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/JSDLGenerator.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/JSDLGenerator.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/JSDLGenerator.java
new file mode 100644
index 0000000..5c0b832
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/JSDLGenerator.java
@@ -0,0 +1,103 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.gfac.bes.utils;
+
+
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionDocument;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * Utility class generates a JSDL instance from JobExecutionContext instance
+ * @author shahbaz memon
+ *
+ * */
+
+public class JSDLGenerator {
+
+ protected final Logger log = LoggerFactory.getLogger(this.getClass());
+
+
+ public synchronized static JobDefinitionDocument buildJSDLInstance(JobExecutionContext context) throws Exception {
+
+ JobDefinitionDocument jobDefDoc = JobDefinitionDocument.Factory
+ .newInstance();
+ JobDefinitionType value = jobDefDoc.addNewJobDefinition();
+
+ HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
+ .getApplicationContext().getApplicationDeploymentDescription()
+ .getType();
+
+ // build Identification
+ createJobIdentification(value, appDepType);
+
+ ResourceProcessor.generateResourceElements(value, context);
+
+ ApplicationProcessor.generateJobSpecificAppElements(value, context);
+
+ DataStagingProcessor.generateDataStagingElements(value, context);
+
+
+ return jobDefDoc;
+ }
+
+
+ public synchronized static JobDefinitionDocument buildJSDLInstance(JobExecutionContext context, String smsUrl) throws Exception {
+
+ JobDefinitionDocument jobDefDoc = JobDefinitionDocument.Factory
+ .newInstance();
+ JobDefinitionType value = jobDefDoc.addNewJobDefinition();
+
+ HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
+ .getApplicationContext().getApplicationDeploymentDescription()
+ .getType();
+
+ // build Identification
+ createJobIdentification(value, appDepType);
+
+ ResourceProcessor.generateResourceElements(value, context);
+
+ ApplicationProcessor.generateJobSpecificAppElements(value, context);
+
+ UASDataStagingProcessor.generateDataStagingElements(value, context, smsUrl);
+
+ return jobDefDoc;
+ }
+
+ private static void createJobIdentification(JobDefinitionType value, HpcApplicationDeploymentType appDepType){
+ if( appDepType.getProjectAccount() != null ){
+
+ if (appDepType.getProjectAccount().getProjectAccountNumber() != null)
+ JSDLUtils.addProjectName(value, appDepType.getProjectAccount()
+ .getProjectAccountNumber());
+
+ if (appDepType.getProjectAccount().getProjectAccountDescription() != null)
+ JSDLUtils.getOrCreateJobIdentification(value).setDescription(
+ appDepType.getProjectAccount()
+ .getProjectAccountDescription());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/JSDLUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/JSDLUtils.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/JSDLUtils.java
new file mode 100644
index 0000000..a5cd2a2
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/JSDLUtils.java
@@ -0,0 +1,540 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.bes.utils;
+
+
+import javax.xml.namespace.QName;
+
+import org.apache.commons.httpclient.URIException;
+import org.apache.xmlbeans.XmlCursor;
+import org.apache.xmlbeans.XmlObject;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.ApplicationType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.BoundaryType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.CPUArchitectureType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.CandidateHostsType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.CreationFlagEnumeration;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.DataStagingType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.ExactType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDescriptionType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobIdentificationType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.OperatingSystemType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.OperatingSystemTypeEnumeration;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.OperatingSystemTypeType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.ProcessorArchitectureEnumeration;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.ResourcesType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.SourceTargetType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.EnvironmentType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.POSIXApplicationDocument;
+import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.POSIXApplicationType;
+import org.ggf.schemas.jsdl.x2006.x07.jsdlHpcpa.HPCProfileApplicationDocument;
+import org.ggf.schemas.jsdl.x2006.x07.jsdlHpcpa.HPCProfileApplicationType;
+import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.SPMDApplicationDocument;
+import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.SPMDApplicationType;
+
+
+/**
+ *
+ * @author shahbaz memon, bastian demuth
+ *
+ */
+public class JSDLUtils
+{
+
+ public static final int FLAG_OVERWRITE = 1;
+ public static final int FLAG_APPEND = 2;
+ public static final int FLAG_DELETE_ON_TERMINATE = 32;
+
+ public static final QName POSIX_APPLICATION=POSIXApplicationDocument.type.getDocumentElementName();
+
+ public static final QName HPC_PROFILE_APPLICATION=HPCProfileApplicationDocument.type.getDocumentElementName();
+
+ public static final QName SPMD_APPLICATION=SPMDApplicationDocument.type.getDocumentElementName();
+
+ public static final String PROCESSESPERHOST = "ProcessesPerHost";
+ public static final String NUMBEROFPROCESSES = "NumberOfProcesses";
+ public static final String THREADSPERHOST = "ThreadsPerHost";
+
+
+
+ public static EnvironmentType addEnvVariable(JobDefinitionType def,String name, String value) {
+ POSIXApplicationType posixApp = getOrCreatePOSIXApplication(def);
+ EnvironmentType newEnv = posixApp.addNewEnvironment();
+ newEnv.setName(name);
+ newEnv.setStringValue(value);
+ return newEnv;
+ }
+
+ public static void setApplicationName(JobDefinitionType value, String applicationName) {
+ getOrCreateApplication(value).setApplicationName(applicationName);
+ }
+
+ public static void setApplicationVersion(JobDefinitionType value, String applicationVersion) {
+ getOrCreateApplication(value).setApplicationVersion(applicationVersion);
+ }
+
+ public static void addProjectName(JobDefinitionType value, String projectName) {
+ getOrCreateJobIdentification(value).addNewJobProject().setStringValue(projectName);
+ }
+
+ public static void addMultipleProjectNames(JobDefinitionType value, String[] projectNames) {
+ for (String name : projectNames) {
+ getOrCreateJobIdentification(value).addNewJobProject().setStringValue(name);
+ }
+ }
+
+ public static void addCandidateHost(JobDefinitionType value, String host) {
+ getOrCreateCandidateHosts(value).addHostName(host);
+
+ }
+ public static void addDataStagingTargetElement(JobDefinitionType value, String fileSystem, String file, String uri) {
+ addDataStagingTargetElement(value,fileSystem, file, uri, 0);
+ }
+
+ public static void addDataStagingTargetElement(JobDefinitionType value, String fileSystem, String file, String uri, int flags) {
+ JobDescriptionType jobDescr = getOrCreateJobDescription(value);
+ DataStagingType newDS = jobDescr.addNewDataStaging();
+ CreationFlagEnumeration.Enum creationFlag = CreationFlagEnumeration.DONT_OVERWRITE;
+ if((flags & FLAG_OVERWRITE) != 0) creationFlag = CreationFlagEnumeration.OVERWRITE;
+ if((flags & FLAG_APPEND) != 0) creationFlag = CreationFlagEnumeration.APPEND;
+ boolean deleteOnTerminate = (flags & FLAG_DELETE_ON_TERMINATE) != 0;
+ newDS.setCreationFlag(creationFlag);
+ newDS.setDeleteOnTermination(deleteOnTerminate);
+ SourceTargetType target = newDS.addNewTarget();
+
+ try {
+ uri = (uri == null) ? null : URIUtils.encodeAll(uri);
+ } catch (URIException e) {
+ }
+ target.setURI(uri);
+ newDS.setFileName(file);
+ if (fileSystem != null && !fileSystem.equals("Work")) { //$NON-NLS-1$
+ newDS.setFilesystemName(fileSystem);
+ }
+ }
+
+ public static void addDataStagingSourceElement(JobDefinitionType value, String uri, String fileSystem, String file) {
+ addDataStagingSourceElement(value, uri, fileSystem, file, 0);
+ }
+
+ public static void addDataStagingSourceElement(JobDefinitionType value, String uri, String fileSystem, String file, int flags) {
+ JobDescriptionType jobDescr = getOrCreateJobDescription(value);
+
+ try {
+ uri = (uri == null) ? null : URIUtils.encodeAll(uri);
+ } catch (URIException e) {
+ }
+ DataStagingType newDS = jobDescr.addNewDataStaging();
+ CreationFlagEnumeration.Enum creationFlag = CreationFlagEnumeration.DONT_OVERWRITE;
+ if((flags & FLAG_OVERWRITE) != 0) creationFlag = CreationFlagEnumeration.OVERWRITE;
+ if((flags & FLAG_APPEND) != 0) creationFlag = CreationFlagEnumeration.APPEND;
+ boolean deleteOnTerminate = (flags & FLAG_DELETE_ON_TERMINATE) != 0;
+ newDS.setCreationFlag(creationFlag);
+ newDS.setDeleteOnTermination(deleteOnTerminate);
+ SourceTargetType source = newDS.addNewSource();
+ source.setURI(uri);
+ newDS.setFileName(file);
+ if (fileSystem != null && !fileSystem.equals("Work")) { //$NON-NLS-1$
+ newDS.setFilesystemName(fileSystem);
+ }
+ }
+
+
+ public static ApplicationType getOrCreateApplication(JobDefinitionType value) {
+ JobDescriptionType jobDescr = getOrCreateJobDescription(value);
+ if (!jobDescr.isSetApplication()) {
+ jobDescr.addNewApplication();
+ }
+ return jobDescr.getApplication();
+ }
+
+ public static CandidateHostsType getOrCreateCandidateHosts(JobDefinitionType value) {
+ ResourcesType resources = getOrCreateResources(value);
+ if (!resources.isSetCandidateHosts()) {
+ resources.addNewCandidateHosts();
+ }
+ return resources.getCandidateHosts();
+ }
+
+ public static CPUArchitectureType getOrCreateCPUArchitecture(JobDefinitionType value) {
+
+ ResourcesType jobResources = getOrCreateResources(value);
+ if (!jobResources.isSetCPUArchitecture()) {
+ jobResources.addNewCPUArchitecture();
+ }
+ return jobResources.getCPUArchitecture();
+ }
+
+ public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateIndividualCPUCount(JobDefinitionType value) {
+ ResourcesType jobResources = getOrCreateResources(value);
+ if (!jobResources.isSetIndividualCPUCount()) {
+ jobResources.addNewIndividualCPUCount();
+ }
+ return jobResources.getIndividualCPUCount();
+ }
+
+
+ public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateIndividualCPUSpeed(JobDefinitionType value) {
+
+ ResourcesType jobResources = getOrCreateResources(value);
+ if (!jobResources.isSetIndividualCPUSpeed()) {
+ jobResources.addNewIndividualCPUSpeed();
+ }
+ return jobResources.getIndividualCPUSpeed();
+ }
+
+ public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateIndividualCPUTime(JobDefinitionType value) {
+
+ ResourcesType jobResources = getOrCreateResources(value);
+ if ( !jobResources.isSetIndividualCPUTime() ) {
+ jobResources.addNewIndividualCPUTime();
+ }
+ return jobResources.getIndividualCPUTime();
+ }
+
+ public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateIndividualDiskSpace(JobDefinitionType value) {
+
+ ResourcesType jobResources = getOrCreateResources(value);
+ if (!jobResources.isSetIndividualDiskSpace()) {
+ jobResources.addNewIndividualDiskSpace();
+ }
+ return jobResources.getIndividualDiskSpace();
+ }
+
+ public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateIndividualPhysicalMemory(JobDefinitionType value) {
+
+ ResourcesType jobResources = getOrCreateResources(value);
+ if (!jobResources.isSetIndividualPhysicalMemory()) {
+ jobResources.addNewIndividualPhysicalMemory();
+ }
+ return jobResources.getIndividualPhysicalMemory();
+ }
+
+ public static JobDescriptionType getOrCreateJobDescription(JobDefinitionType value) {
+ if (value.getJobDescription() == null) {
+ return value.addNewJobDescription();
+ }
+ return value.getJobDescription();
+ }
+
+ public static JobIdentificationType getOrCreateJobIdentification(JobDefinitionType value) {
+ JobDescriptionType descr = getOrCreateJobDescription(value);
+ if (descr.getJobIdentification() == null) {
+ return descr.addNewJobIdentification();
+ }
+ return descr.getJobIdentification();
+ }
+
+ public static OperatingSystemType getOrCreateOperatingSystem(JobDefinitionType value)
+ {
+ ResourcesType jobResources = getOrCreateResources(value);
+ if(!jobResources.isSetOperatingSystem()) {
+ jobResources.addNewOperatingSystem();
+ }
+ return jobResources.getOperatingSystem();
+ }
+
+ public static ResourcesType getOrCreateResources(JobDefinitionType value) {
+ JobDescriptionType jobDescr = getOrCreateJobDescription(value);
+ if (!jobDescr.isSetResources()) {
+ jobDescr.addNewResources();
+ }
+ return jobDescr.getResources();
+ }
+
+
+ public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateTotalCPUCount(JobDefinitionType value) {
+
+ ResourcesType jobResources = getOrCreateResources(value);
+ if ( !jobResources.isSetTotalCPUCount() ) {
+ jobResources.addNewTotalCPUCount();
+ }
+ return jobResources.getTotalCPUCount();
+ }
+
+
+ public static org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType getOrCreateTotalResourceCount(JobDefinitionType value) {
+
+ ResourcesType jobResources = getOrCreateResources(value);
+ if ( !jobResources.isSetTotalResourceCount())
+ {
+ jobResources.addNewTotalResourceCount();
+ }
+ return jobResources.getTotalResourceCount();
+ }
+
+ public static POSIXApplicationType getOrCreatePOSIXApplication(JobDefinitionType value) {
+
+ ApplicationType application = getOrCreateApplication(value);
+
+ if(getHPCProfileApplication(value) != null){
+ //TODO handle: not creating POSIX element if HPCProfile already exists
+ return getPOSIXApplication(value);
+ }
+
+ if (getPOSIXApplication(value) == null) {
+ XmlCursor acursor = application.newCursor();
+ acursor.toEndToken();
+ acursor.insertElement(POSIX_APPLICATION);
+ acursor.dispose();
+ }
+ return getPOSIXApplication(value);
+ }
+
+
+ public static SPMDApplicationType getOrCreateSPMDApplication(JobDefinitionType value) {
+
+ ApplicationType application = getOrCreateApplication(value);
+
+ if (getSPMDApplication(value) == null) {
+ XmlCursor acursor = application.newCursor();
+ acursor.toEndToken();
+ acursor.insertElement(SPMD_APPLICATION);
+ acursor.dispose();
+ }
+ return getSPMDApplication(value);
+ }
+
+ public static SPMDApplicationType getSPMDApplication(JobDefinitionType value) {
+ if (value != null &&
+ value.getJobDescription() != null &&
+ value.getJobDescription().isSetApplication() ) {
+ XmlCursor acursor = value.getJobDescription().getApplication().newCursor();
+ if (acursor.toFirstChild()) {
+ do {
+ if(acursor.getName().equals(SPMD_APPLICATION)) {
+ XmlObject result = acursor.getObject();
+ acursor.dispose();
+ return (SPMDApplicationType) result;
+ }
+ } while (acursor.toNextSibling());
+ acursor.dispose();
+ return null;
+ } else {
+ acursor.dispose();
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+
+
+
+ public static POSIXApplicationType getPOSIXApplication(JobDefinitionType value) {
+ if (value != null &&
+ value.getJobDescription() != null &&
+ value.getJobDescription().isSetApplication() ) {
+ XmlCursor acursor = value.getJobDescription().getApplication().newCursor();
+ if (acursor.toFirstChild()) {
+ do {
+ if(acursor.getName().equals(POSIX_APPLICATION)) {
+ XmlObject result = acursor.getObject();
+ acursor.dispose();
+ return (POSIXApplicationType) result;
+ }
+ } while (acursor.toNextSibling());
+ acursor.dispose();
+ return null;
+ } else {
+ acursor.dispose();
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+
+
+
+ public static HPCProfileApplicationType getOrCreateHPCProfileApplication(JobDefinitionType value) {
+
+ ApplicationType application = getOrCreateApplication(value);
+
+ if(getPOSIXApplication(value) != null){
+ //TODO handle: creating HPC element if POSIX already exists
+ return getHPCProfileApplication(value);
+ }
+
+ if (getHPCProfileApplication(value) == null) {
+ XmlCursor acursor = application.newCursor();
+ acursor.toEndToken();
+ acursor.insertElement(HPC_PROFILE_APPLICATION);
+ acursor.dispose();
+ }
+ return getHPCProfileApplication(value);
+ }
+
+
+ public static HPCProfileApplicationType getHPCProfileApplication(JobDefinitionType value) {
+ if (value != null &&
+ value.getJobDescription() != null &&
+ value.getJobDescription().isSetApplication() ) {
+ XmlCursor acursor = value.getJobDescription().getApplication().newCursor();
+ if (acursor.toFirstChild()) {
+ do {
+ if(acursor.getName().equals(HPC_PROFILE_APPLICATION)) {
+ XmlObject result = acursor.getObject();
+ acursor.dispose();
+ return (HPCProfileApplicationType) result;
+ }
+ } while (acursor.toNextSibling());
+ acursor.dispose();
+ return null;
+ } else {
+ acursor.dispose();
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+
+
+
+
+ public static RangeValueType getTotalCPUCountRequirements(JobDefinitionType value) {
+ if(value != null && value.getJobDescription() != null && value.getJobDescription().isSetResources() &&
+ value.getJobDescription().getResources().isSetTotalCPUCount()){
+ return toU6RangeValue(value.getJobDescription().getResources().getTotalCPUCount());
+ }
+ else
+ return null;
+ }
+
+ public static RangeValueType getTotalResourceCountRequirements(JobDefinitionType value) {
+ if(value != null && value.getJobDescription() != null && value.getJobDescription().isSetResources() &&
+ value.getJobDescription().getResources().isSetTotalResourceCount()){
+ return toU6RangeValue(value.getJobDescription().getResources().getTotalResourceCount());
+ }
+ else
+ return null;
+ }
+
+
+ public static RangeValueType toU6RangeValue(org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType jsdlType) {
+ RangeValueType result = new RangeValueType();
+ if(jsdlType.getExactArray().length > 0){
+ result.setExact(jsdlType.getExactArray(0).getDoubleValue());
+ }
+ if(jsdlType.isSetLowerBoundedRange()){
+ result.setLowerBound(jsdlType.getLowerBoundedRange().getDoubleValue());
+ }
+ if(jsdlType.isSetUpperBoundedRange()){
+ result.setUpperBound(jsdlType.getUpperBoundedRange().getDoubleValue());
+ }
+ return result;
+ }
+
+
+
+ public static void setCPUArchitectureRequirements(JobDefinitionType value, ProcessorRequirement cpuArchitecture) {
+ if(cpuArchitecture == null || cpuArchitecture.getValue() == null) return;
+ CPUArchitectureType cpuArch = getOrCreateCPUArchitecture(value);
+ cpuArch.setCPUArchitectureName(ProcessorArchitectureEnumeration.Enum.forString(cpuArchitecture.getValue()));
+ }
+
+ public static void setIndividualCPUCountRequirements(JobDefinitionType value, RangeValueType cpuCount) {
+ org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType individualCPUCount = getOrCreateIndividualCPUCount(value);
+ setRangeValue(cpuCount, individualCPUCount);
+ }
+
+ public static void setIndividualCPUSpeedRequirements(JobDefinitionType value, RangeValueType cpuSpeed) {
+ org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType individualCPUSpeed = getOrCreateIndividualCPUSpeed(value);
+ setRangeValue(cpuSpeed, individualCPUSpeed);
+ }
+
+ public static void setIndividualCPUTimeRequirements(JobDefinitionType value, RangeValueType cpuTime) {
+ org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType cpuIndividualTime = getOrCreateIndividualCPUTime(value);
+ setRangeValue(cpuTime, cpuIndividualTime);
+ }
+
+ public static void setIndividualDiskSpaceRequirements(JobDefinitionType value, RangeValueType diskSpace) {
+ org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType individualDiskSpace = getOrCreateIndividualDiskSpace(value);
+ setRangeValue(diskSpace, individualDiskSpace);
+ }
+
+ public static void setIndividualPhysicalMemoryRequirements(JobDefinitionType value, RangeValueType physicalMemory) {
+ org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType individualPhysicalMemory = getOrCreateIndividualPhysicalMemory(value);
+ setRangeValue(physicalMemory, individualPhysicalMemory);
+ }
+
+
+ public static void setName(JobDefinitionType value, String name) {
+ getOrCreateJobIdentification(value).setJobName(name);
+ }
+
+ public static void setOperatingSystemRequirements(JobDefinitionType value, OSRequirement osType) {
+ if(osType == null || osType.getOSType() == null) return;
+ OperatingSystemType os_Type = getOrCreateOperatingSystem(value);
+ OperatingSystemTypeType ostt = os_Type.addNewOperatingSystemType();
+ ostt.setOperatingSystemName(OperatingSystemTypeEnumeration.Enum.forString(osType.getOSType().getValue()));
+ if(osType.getOSVersion() != null)
+ {
+ os_Type.setOperatingSystemVersion(osType.getOSVersion());
+ }
+ }
+
+ public static void setRangeValue(RangeValueType u6Type, org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType jsdlType) {
+ Double exact = u6Type.getExact();
+ Double epsilon = u6Type.getEpsilon();
+ Double lower = u6Type.getLowerBound();
+ Double upper = u6Type.getUpperBound();
+
+
+ if(lower.isNaN() && upper.isNaN())
+ {
+ ExactType exactType = jsdlType.getExactArray().length > 0 ? jsdlType.getExactArray(0) : jsdlType.addNewExact();
+ exactType.setDoubleValue(exact);
+ if(!epsilon.isNaN() && epsilon != 0)
+ {
+ exactType.setEpsilon(epsilon);
+ }
+ }
+ else
+ {
+ if(!lower.isNaN())
+ {
+ BoundaryType lowerBound = jsdlType.isSetLowerBoundedRange() ? jsdlType.getLowerBoundedRange() : jsdlType.addNewLowerBoundedRange();
+ lowerBound.setDoubleValue(lower);
+ lowerBound.setExclusiveBound(!u6Type.isIncludeLowerBound());
+ }
+
+ if(!upper.isNaN())
+ {
+ BoundaryType upperBound = jsdlType.isSetUpperBoundedRange() ? jsdlType.getUpperBoundedRange() : jsdlType.addNewUpperBoundedRange();
+ upperBound.setDoubleValue(upper);
+ upperBound.setExclusiveBound(!u6Type.isIncludeUpperBound());
+ }
+ }
+ }
+
+ public static void setTotalCPUCountRequirements(JobDefinitionType value, RangeValueType cpuCount) {
+ org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType cpuTotalCount = getOrCreateTotalCPUCount(value);
+ setRangeValue(cpuCount, cpuTotalCount);
+ }
+
+ public static void setTotalResourceCountRequirements(JobDefinitionType value, RangeValueType resourceCount) {
+ org.ggf.schemas.jsdl.x2005.x11.jsdl.RangeValueType totalCount = getOrCreateTotalResourceCount(value);
+ setRangeValue(resourceCount, totalCount);
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/Mode.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/Mode.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/Mode.java
new file mode 100644
index 0000000..2223ecf
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/Mode.java
@@ -0,0 +1,45 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.bes.utils;
+
+/**
+ * file creation modes
+ */
+public enum Mode {
+
+ /**
+ * overwrite any existing file
+ */
+ overwrite,
+
+ /**
+ * append to an existing file
+ */
+ append,
+
+ /**
+ * do NOT overwrite and fail if the file exists
+ */
+ nooverwrite
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/OSRequirement.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/OSRequirement.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/OSRequirement.java
new file mode 100644
index 0000000..3938598
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/OSRequirement.java
@@ -0,0 +1,108 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.bes.utils;
+
+import org.apache.airavata.gfac.provider.utils.ResourceRequirement;
+
+public class OSRequirement implements ResourceRequirement {
+ private OSType osType;
+ private String version;
+ protected boolean enabled;
+
+
+ public OSRequirement() {
+ }
+
+ /**
+ *
+ * @param type -
+ * the type of the O/S
+ * @param version -
+ * the version of the O/S
+ */
+ public OSRequirement(OSType osType, String osVersion) {
+ setOSType(osType);
+ setOSVersion(osVersion);
+ }
+
+ /**
+ * Set the type of the O/S
+ *
+ * @param type -
+ * the type of the O/S
+ */
+ public void setOSType(OSType osType) {
+ this.osType = osType;
+ }
+
+ /**
+ * Get the type of the O/S
+ *
+ * @return the type of the O/S
+ */
+ public OSType getOSType() {
+ return osType;
+ }
+
+ /**
+ * Set the version of the O/S
+ *
+ * @param version -
+ * the version of the O/S
+ */
+ public void setOSVersion(String version) {
+ this.version = version;
+ }
+
+ /**
+ * Get the version of the O/S
+ *
+ * @return the version of the O/S
+ */
+ public String getOSVersion() {
+ return version;
+ }
+
+ /**
+ *
+ * equals this instance of class with another instance
+ */
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj==null || getClass() != obj.getClass()) return false;
+ final OSRequirement other = (OSRequirement) obj;
+ boolean typeEqual = osType == null ? other.osType == null : osType.equals(other.osType);
+ boolean versionEqual = version == null ? other.version == null : version.equals(other.version);
+ return typeEqual && versionEqual && isEnabled() == other.isEnabled();
+ }
+
+
+
+ public boolean isEnabled() {
+ return enabled;
+ }
+
+ public void setEnabled(boolean enabled) {
+ this.enabled = enabled;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/OSType.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/OSType.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/OSType.java
new file mode 100644
index 0000000..b10c43a
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/OSType.java
@@ -0,0 +1,124 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.bes.utils;
+
+public enum OSType {
+
+ unknown("Unknown"), //$NON-NLS-1$
+ linux("LINUX"), //$NON-NLS-1$
+ mac_os("MACOS"), //$NON-NLS-1$
+ win95("WIN95"), //$NON-NLS-1$
+ win98("WIN98"), //$NON-NLS-1$
+ windows_R_Me("Windows_R_Me"), //$NON-NLS-1$
+ winNT("WINNT"), //$NON-NLS-1$
+ windows_2000("Windows_2000"), //$NON-NLS-1$
+ windows_XP("Windows_XP"), //$NON-NLS-1$
+ msdos("MSDOS"), //$NON-NLS-1$
+ solaris("Solaris"), //$NON-NLS-1$
+ sunOS("SunOS"), //$NON-NLS-1$
+ freeBSD("FreeBSD"), //$NON-NLS-1$
+ netBSD("NetBSD"), //$NON-NLS-1$
+ openBSD("OpenBSD"), //$NON-NLS-1$
+ bsdunix("BSDUNIX"), //$NON-NLS-1$
+ aix("AIX"), //$NON-NLS-1$
+ z_OS("z_OS"), //$NON-NLS-1$
+ os_2("OS_2"), //$NON-NLS-1$
+ os9("OS9"), //$NON-NLS-1$
+ netWare("NetWare"), //$NON-NLS-1$
+ tru64_unix("Tru64_UNIX"), //$NON-NLS-1$
+ irix("IRIX"), //$NON-NLS-1$
+ osf("OSF"), //$NON-NLS-1$
+
+ mvs("MVS"), //$NON-NLS-1$
+ os400("OS400"), //$NON-NLS-1$
+ javaVM("JavaVM"), //$NON-NLS-1$
+ win3x("WIN3x"), //$NON-NLS-1$
+ winCE("WINCE"), //$NON-NLS-1$
+ NCR3000("NCR3000"), //$NON-NLS-1$
+ dc_os("DC_OS"), //$NON-NLS-1$
+ reliant_unix("Reliant_UNIX"), //$NON-NLS-1$
+ sco_unixWare("SCO_UnixWare"), //$NON-NLS-1$
+ sco_openServer("SCO_OpenServer"), //$NON-NLS-1$
+ sequent("Sequent"), //$NON-NLS-1$
+ u6000("U6000"), //$NON-NLS-1$
+ aseries("ASERIES"), //$NON-NLS-1$
+ tandemNSK("TandemNSK"), //$NON-NLS-1$
+ tandemNT("TandemNT"), //$NON-NLS-1$
+ bs2000("BS2000"), //$NON-NLS-1$
+ lynx("Lynx"), //$NON-NLS-1$
+ xenix("XENIX"), //$NON-NLS-1$
+ vm("VM"), //$NON-NLS-1$
+ interactive_unix("Interactive_UNIX"), //$NON-NLS-1$
+ gnu_hurd("GNU_Hurd"), //$NON-NLS-1$
+ mach_kernel("MACH_Kernel"), //$NON-NLS-1$
+ inferno("Inferno"), //$NON-NLS-1$
+ qnx("QNX"), //$NON-NLS-1$
+ epoc("EPOC"), //$NON-NLS-1$
+ ixWorks("IxWorks"), //$NON-NLS-1$
+ vxWorks("VxWorks"), //$NON-NLS-1$
+ mint("MiNT"), //$NON-NLS-1$
+ beOS("BeOS"), //$NON-NLS-1$
+ hp_mpe("HP_MPE"), //$NON-NLS-1$
+ nextStep("NextStep"), //$NON-NLS-1$
+ palmPilot("PalmPilot"), //$NON-NLS-1$
+ rhapsody("Rhapsody"), //$NON-NLS-1$
+ dedicated("Dedicated"), //$NON-NLS-1$
+ os_390("OS_390"), //$NON-NLS-1$
+ vse("VSE"), //$NON-NLS-1$
+ tpf("TPF"), //$NON-NLS-1$
+ caldera_open_unix("Caldera_Open_UNIX"), //$NON-NLS-1$
+ attunix("ATTUNIX"), //$NON-NLS-1$
+ dgux("DGUX"), //$NON-NLS-1$
+ decnt("DECNT"), //$NON-NLS-1$
+ openVMS("OpenVMS"), //$NON-NLS-1$
+ hpux("HPUX"), //$NON-NLS-1$
+ other("other"); //$NON-NLS-1$
+
+
+ private OSType(String value) {
+ this.value = value;
+ }
+
+ private final String value;
+
+ public String getValue() {
+ return value;
+ }
+
+ public static OSType fromString(String value)
+ {
+ for(OSType type : values())
+ {
+ if(type.value.equals(value))
+ {
+ return type;
+ }
+ }
+ return null;
+ }
+
+ public String toString()
+ {
+ return value;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ProcessorRequirement.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ProcessorRequirement.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ProcessorRequirement.java
new file mode 100644
index 0000000..0b48dd7
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ProcessorRequirement.java
@@ -0,0 +1,61 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+
+package org.apache.airavata.gfac.bes.utils;
+
+public enum ProcessorRequirement{
+ sparc("sparc"), //$NON-NLS-1$
+ powerpc("powerpc"), //$NON-NLS-1$
+ x86("x86"), //$NON-NLS-1$
+ x86_32("x86_32"), //$NON-NLS-1$
+ x86_64("x86_64"), //$NON-NLS-1$
+ parisc("parisc"), //$NON-NLS-1$
+ mips("mips"), //$NON-NLS-1$
+ ia64("ia64"), //$NON-NLS-1$
+ arm("arm"), //$NON-NLS-1$
+ other("other"); //$NON-NLS-1$
+
+ ProcessorRequirement(String value) {
+ this.value = value;
+ }
+
+ private final String value;
+
+ public String getValue() {
+ return value;
+ }
+
+ public static ProcessorRequirement fromString(String value)
+ {
+ for (ProcessorRequirement type : values()) {
+ if (type.value.equals(value)) {
+ return type;
+ }
+ }
+ return other;
+ }
+
+ public String toString()
+ {
+ return value;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/RangeValueType.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/RangeValueType.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/RangeValueType.java
new file mode 100644
index 0000000..6aec82a
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/RangeValueType.java
@@ -0,0 +1,274 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.bes.utils;
+
+import org.apache.airavata.gfac.provider.utils.ResourceRequirement;
+
+public class RangeValueType implements ResourceRequirement {
+
+
+ private double exact = Double.NaN;
+ private double lowerBound = Double.NEGATIVE_INFINITY;
+ private double upperBound = Double.POSITIVE_INFINITY;
+
+ private double epsilon = Double.NaN;
+ private boolean includeLowerBound = true;
+ private boolean includeUpperBound = true;
+
+ private boolean enabled = false;
+
+
+ public RangeValueType(double exact, double epsilon, double lowerBound, boolean includeLowerBound, double upperBound, boolean includeUpperBound, boolean enabled) {
+ this.exact = exact;
+ this.epsilon = epsilon;
+ this.lowerBound = lowerBound;
+ this.includeLowerBound = includeLowerBound;
+ this.upperBound = upperBound;
+ this.includeUpperBound = includeUpperBound;
+ this.enabled = enabled;
+ }
+
+
+
+ /**
+ * Create the range requirements
+ *
+ * @param exact -
+ * the exact value
+ * @param lowerBound -
+ * the lower bound
+ * @param upperBound -
+ * the upper bound
+ * @param includelowerBound -
+ * true, if lowerBound should be included in range
+ * @param includeUpperBound -
+ * true, if upperBound should be included in range
+ *
+ */
+ public RangeValueType(double exact, double epsilon, double lowerBound, boolean includeLowerBound, double upperBound, boolean includeUpperBound) {
+ this(exact,epsilon,lowerBound,includeLowerBound,upperBound,includeUpperBound,false);
+
+ }
+
+
+ /**
+ * Create the range requirements
+ *
+ * @param exact -
+ * the exact value
+ * @param lowerBound -
+ * the lower bound
+ * @param upperBound -
+ * the upper bound
+ */
+ public RangeValueType(double exact, double epsilon, double lowerBound, double upperBound) {
+ this(exact,epsilon,lowerBound,true,upperBound,true);
+ }
+
+
+ public RangeValueType(double exact, double lowerBound, double upperBound) {
+ this(exact,Double.NaN,lowerBound,true,upperBound,true);
+ }
+
+ /**
+ * Create the exact requirements
+ *
+ * @param exact -
+ * the exact value
+ * @param epsilon -
+ * the epsilon arround exact
+ *
+ */
+ public RangeValueType(double exact, double epsilon) {
+ this(exact,epsilon,Double.NaN,Double.NaN);
+ }
+
+
+ /**
+ * Create the exact requirements
+ *
+ * @param exact -
+ * the exact value
+ */
+ public RangeValueType(double exact) {
+ this(exact,Double.NaN);
+ }
+
+ public RangeValueType() {
+ }
+
+ /**
+ * Get exact requirements
+ *
+ * @return the exact requirements
+ */
+ public double getExact() {
+ return exact;
+ }
+
+ /**
+ * Set exact requirements
+ *
+ * @param exact -
+ * the exact requirements
+ */
+ public void setExact(double exact) {
+ this.exact = exact;
+ }
+
+ /**
+ * Get epsilon
+ *
+ * @return the epsilon
+ */
+ public double getEpsilon() {
+ return epsilon;
+ }
+
+ /**
+ * Set epsilon
+ *
+ * @param epsilon -
+ * epsilon belonging to to exact requirements
+ */
+ public void setEpsilon(double epsilon) {
+ this.epsilon = epsilon;
+ }
+
+ /**
+ * Get lower bound
+ *
+ * @return the lower bound
+ */
+ public double getLowerBound() {
+ return lowerBound;
+ }
+
+ /**
+ * Set lower bound
+ *
+ * @param lowerBound -
+ * the lower bound
+ */
+ public void setLowerBound(double lowerBound) {
+ this.lowerBound = lowerBound;
+ }
+
+ /**
+ * Get upper bound
+ *
+ * @return the upper bound
+ */
+ public double getUpperBound() {
+ return upperBound;
+ }
+
+ /**
+ * Set upper bound
+ *
+ * @param upperBound -
+ * the upper bound
+ */
+ public void setUpperBound(double upperBound) {
+ this.upperBound = upperBound;
+ }
+
+ /**
+ * Test if requirements are met
+ *
+ * @param value -
+ * the tested value
+ * @return <code>true</code> if value is in the range and not less than
+ * the exact value
+ */
+ public boolean lowerThanDouble(double value) {
+ return (value >= exact && value >= lowerBound && value <= upperBound) ? true : false;
+ }
+
+ public String toString() {
+ if (lowerBound == Double.NEGATIVE_INFINITY && upperBound == Double.POSITIVE_INFINITY) {
+ return Double.toString(exact);
+ }
+ else {
+ return "(e=" + Double.toString(exact) + ",l=" + Double.toString(lowerBound) + ",u=" //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+ + Double.toString(upperBound) + ")"; //$NON-NLS-1$
+ }
+ }
+
+
+ public boolean isIncludeLowerBound() {
+ return includeLowerBound;
+ }
+
+
+ public void setIncludeLowerBound(boolean includeLowerBound) {
+ this.includeLowerBound = includeLowerBound;
+ }
+
+
+ public boolean isIncludeUpperBound() {
+ return includeUpperBound;
+ }
+
+
+ public void setIncludeUpperBound(boolean includeUpperBound) {
+ this.includeUpperBound = includeUpperBound;
+ }
+
+ public RangeValueType clone(){
+ return new RangeValueType(this.exact, this.epsilon, this.lowerBound, this.includeLowerBound, this.upperBound, this.includeUpperBound,this.enabled);
+ }
+
+
+
+ public boolean isEnabled() {
+ return enabled;
+ }
+
+
+
+ public void setEnabled(boolean enabled) {
+ this.enabled = enabled;
+ }
+
+
+ public boolean equals(Object o)
+ {
+ if(! (o instanceof RangeValueType)) return false;
+ RangeValueType other = (RangeValueType) o;
+ return doublesEqual(getExact(),other.getExact())
+ && doublesEqual(getEpsilon(), other.getEpsilon())
+ && doublesEqual(getLowerBound(), other.getLowerBound())
+ && doublesEqual(getUpperBound(), other.getUpperBound())
+ && isIncludeLowerBound() == other.isIncludeLowerBound()
+ && isIncludeUpperBound() == other.isIncludeUpperBound()
+ && isEnabled() == other.isEnabled();
+ }
+
+
+ private boolean doublesEqual(double a, double b)
+ {
+ Double A = new Double(a);
+ Double B = new Double(b);
+ return A.equals(B);
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java
new file mode 100644
index 0000000..fc8d6ec
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java
@@ -0,0 +1,152 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.bes.utils;
+
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.provider.GFacProviderException;
+import org.apache.airavata.model.workspace.experiment.ComputationalResourceScheduling;
+import org.apache.airavata.model.workspace.experiment.TaskDetails;
+import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
+import org.apache.airavata.schemas.gfac.QueueType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
+import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.NumberOfProcessesType;
+
+public class ResourceProcessor {
+
+
+ public static void generateResourceElements(JobDefinitionType value, JobExecutionContext context) throws Exception{
+
+ HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
+ .getApplicationContext().getApplicationDeploymentDescription()
+ .getType();
+
+ createMemory(value, appDepType);
+ TaskDetails taskData = context.getTaskData();
+ if(taskData != null && taskData.isSetTaskScheduling()){
+ ComputationalResourceScheduling computionResource= taskData.getTaskScheduling();
+ try {
+ int cpuCount = computionResource.getTotalCPUCount();
+ if(cpuCount>0){
+// appDepType.setCpuCount(cpuCount);
+ NumberOfProcessesType num = NumberOfProcessesType.Factory.newInstance();
+ String processers = Integer.toString(cpuCount);
+ num.setStringValue(processers);
+ JSDLUtils.getOrCreateSPMDApplication(value).setNumberOfProcesses(num);
+ }
+ } catch (NullPointerException e) {
+ new GFacProviderException("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used",e);
+ }
+ try {
+ int nodeCount = computionResource.getNodeCount();
+ if(nodeCount>0){
+ appDepType.setNodeCount(nodeCount);
+ }
+ } catch (NullPointerException e) {
+ new GFacProviderException("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used",e);
+ }
+ try {
+ String queueName = computionResource.getQueueName();
+ if (queueName != null) {
+ if(appDepType.getQueue() == null){
+ QueueType queueType = appDepType.addNewQueue();
+ queueType.setQueueName(queueName);
+ }else{
+ appDepType.getQueue().setQueueName(queueName);
+ }
+ }
+ } catch (NullPointerException e) {
+ new GFacProviderException("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used",e);
+ }
+ try {
+ int maxwallTime = computionResource.getWallTimeLimit();
+ if(maxwallTime>0){
+ appDepType.setMaxWallTime(maxwallTime);
+ }
+ } catch (NullPointerException e) {
+ new GFacProviderException("No Value sent in WorkflowContextHeader for Node Count, value in the Deployment Descriptor will be used",e);
+ }
+ }
+
+ if (appDepType.getCpuCount() > 0) {
+ RangeValueType rangeType = new RangeValueType();
+ rangeType.setLowerBound(Double.NaN);
+ rangeType.setUpperBound(Double.NaN);
+ rangeType.setExact(appDepType.getCpuCount());
+ JSDLUtils.setTotalCPUCountRequirements(value, rangeType);
+ }
+
+ if (appDepType.getProcessorsPerNode() > 0) {
+ RangeValueType rangeType = new RangeValueType();
+ rangeType.setLowerBound(Double.NaN);
+ rangeType.setUpperBound(Double.NaN);
+ rangeType.setExact(appDepType.getProcessorsPerNode());
+ JSDLUtils.setIndividualCPUCountRequirements(value, rangeType);
+ }
+
+ if (appDepType.getNodeCount() > 0) {
+ RangeValueType rangeType = new RangeValueType();
+ rangeType.setLowerBound(Double.NaN);
+ rangeType.setUpperBound(Double.NaN);
+ rangeType.setExact(appDepType.getNodeCount());
+ JSDLUtils.setTotalResourceCountRequirements(value, rangeType);
+ }
+
+ if(appDepType.getMaxWallTime() > 0) {
+ RangeValueType cpuTime = new RangeValueType();
+ cpuTime.setLowerBound(Double.NaN);
+ cpuTime.setUpperBound(Double.NaN);
+ long wallTime = appDepType.getMaxWallTime() * 60;
+ cpuTime.setExact(wallTime);
+ JSDLUtils.setIndividualCPUTimeRequirements(value, cpuTime);
+ }
+ }
+
+
+ private static void createMemory(JobDefinitionType value, HpcApplicationDeploymentType appDepType){
+ if (appDepType.getMinMemory() > 0 && appDepType.getMaxMemory() > 0) {
+ RangeValueType rangeType = new RangeValueType();
+ rangeType.setLowerBound(appDepType.getMinMemory());
+ rangeType.setUpperBound(appDepType.getMaxMemory());
+ JSDLUtils.setIndividualPhysicalMemoryRequirements(value, rangeType);
+ }
+
+ else if (appDepType.getMinMemory() > 0 && appDepType.getMaxMemory() <= 0) {
+ // TODO set Wall time
+ RangeValueType rangeType = new RangeValueType();
+ rangeType.setLowerBound(appDepType.getMinMemory());
+ JSDLUtils.setIndividualPhysicalMemoryRequirements(value, rangeType);
+ }
+
+ else if (appDepType.getMinMemory() <= 0 && appDepType.getMaxMemory() > 0) {
+ // TODO set Wall time
+ RangeValueType rangeType = new RangeValueType();
+ rangeType.setUpperBound(appDepType.getMinMemory());
+ JSDLUtils.setIndividualPhysicalMemoryRequirements(value, rangeType);
+ }
+
+ }
+
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/SPMDProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/SPMDProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/SPMDProcessor.java
new file mode 100644
index 0000000..8807026
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/SPMDProcessor.java
@@ -0,0 +1,33 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.bes.utils;
+
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
+
+public class SPMDProcessor {
+
+ public static void generateSPMDElements(JobDefinitionType value, JobExecutionContext context) {
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/SPMDVariations.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/SPMDVariations.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/SPMDVariations.java
new file mode 100644
index 0000000..755799c
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/SPMDVariations.java
@@ -0,0 +1,52 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.bes.utils;
+
+public enum SPMDVariations {
+
+ MPI ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MPI"),
+ GridMPI ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/GridMPI"),
+ IntelMPI ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/IntelMPI"),
+ LAMMPI ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/LAM-MPI"),
+ MPICH1 ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MPICH1"),
+ MPICH2 ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MPICH2"),
+ MPICHGM ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MPICH-GM"),
+ MPICHMX ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MPICH-MX"),
+ MVAPICH ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MVAPICH"),
+ MVAPICH2 ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/MVAPICH2"),
+ OpenMPI ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/OpenMPI"),
+ POE ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/POE"),
+ PVM ("http://www.ogf.org/jsdl/2007/02/jsdl-spmd/PVM");
+
+ private final String variation;
+
+ private SPMDVariations(String variation) {
+ this.variation = variation;
+ }
+
+ public String value(){
+ return variation;
+ }
+
+}
+
+
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/StorageCreator.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/StorageCreator.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/StorageCreator.java
new file mode 100644
index 0000000..4282b62
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/StorageCreator.java
@@ -0,0 +1,211 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.bes.utils;
+
+import java.util.Calendar;
+
+import javax.security.auth.x500.X500Principal;
+
+import org.oasisOpen.docs.wsrf.sg2.EntryType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.unigrids.services.atomic.types.PropertyType;
+import org.unigrids.x2006.x04.services.smf.CreateSMSDocument;
+import org.unigrids.x2006.x04.services.smf.StorageBackendParametersDocument.StorageBackendParameters;
+import org.unigrids.x2006.x04.services.smf.StorageDescriptionType;
+import org.w3.x2005.x08.addressing.EndpointReferenceType;
+
+import de.fzj.unicore.uas.StorageFactory;
+import de.fzj.unicore.uas.client.StorageClient;
+import de.fzj.unicore.uas.client.StorageFactoryClient;
+import de.fzj.unicore.wsrflite.xmlbeans.WSUtilities;
+import de.fzj.unicore.wsrflite.xmlbeans.client.RegistryClient;
+import de.fzj.unicore.wsrflite.xmlbeans.sg.Registry;
+
+
+import eu.unicore.util.httpclient.DefaultClientConfiguration;
+
+public class StorageCreator {
+ protected final Logger log = LoggerFactory.getLogger(this.getClass());
+
+ /**
+ * the initial lifetime (in days) for newly created SMSs
+ */
+ private int initialLifeTime;
+
+ /**
+ * factory URL to use
+ */
+ private String factoryUrl;
+
+ /**
+ * site where to create the storage
+ */
+ private String siteName;
+
+ /**
+ * storage type to create
+ */
+ private String storageType;
+
+ private DefaultClientConfiguration secProps;
+
+ private String userName;
+
+ public StorageCreator(DefaultClientConfiguration secProps, String besUrl, int initialLifetime, String storageType, String userName) {
+ this.secProps = secProps;
+ this.factoryUrl = getStorageFactoryUrl(besUrl);
+ this.storageType = storageType;
+ this.initialLifeTime = initialLifetime;
+ this.userName = userName;
+ }
+
+
+ public StorageCreator(DefaultClientConfiguration secProps, String besUrl, int initialLifetime, String userName) {
+ this.secProps = secProps;
+ this.factoryUrl = getStorageFactoryUrl(besUrl);
+ this.initialLifeTime = initialLifetime;
+ this.userName = userName;
+ }
+
+
+ // The target site must have storage factory deployed with bes factory
+ public StorageClient createStorage() throws Exception{
+
+ if(factoryUrl == null) {
+ throw new Exception("Cannot create Storage Factory Url");
+ }
+
+ EndpointReferenceType sfEpr= WSUtilities.makeServiceEPR(factoryUrl, StorageFactory.SMF_PORT);
+
+ String dn = findServerName(factoryUrl, sfEpr);
+
+ WSUtilities.addServerIdentity(sfEpr, dn);
+
+ secProps.getETDSettings().setReceiver(new X500Principal(dn));
+ secProps.getETDSettings().setIssuerCertificateChain(secProps.getCredential().getCertificateChain());
+
+ // TODO: remove it afterwards
+ if(userName != null) {
+ secProps.getETDSettings().getRequestedUserAttributes2().put("xlogin", new String[]{userName});
+ }
+
+ StorageFactoryClient sfc = new StorageFactoryClient(sfEpr, secProps);
+
+ if (log.isDebugEnabled()){
+ log.debug("Using storage factory at <"+sfc.getUrl()+">");
+ }
+
+ StorageClient sc = null;
+ try{
+ sc=sfc.createSMS(getCreateSMSDocument());
+
+ String addr=sc.getEPR().getAddress().getStringValue();
+ log.info(addr);
+
+ }catch(Exception ex){
+ log.error("Could not create storage",ex);
+ throw new Exception(ex);
+ }
+
+ return sc;
+ }
+
+ protected String findServerName(String besUrl, EndpointReferenceType smsEpr)throws Exception{
+
+ int besIndex = besUrl.indexOf("StorageFactory?res");
+ String ss = besUrl.substring(0, besIndex);
+ ss = ss + "Registry";
+
+ EndpointReferenceType eprt = WSUtilities.makeServiceEPR(ss, "default_registry", Registry.REGISTRY_PORT);
+
+ RegistryClient registry = new RegistryClient(eprt, secProps);
+
+ //first, check if server name is already in the EPR...
+ String dn=WSUtilities.extractServerIDFromEPR(smsEpr);
+ if(dn!=null){
+ return dn;
+ }
+ //otherwise find a matching service in the registry
+ String url=smsEpr.getAddress().getStringValue();
+ if(url.contains("/services/"))url=url.substring(0,url.indexOf("/services"));
+ if(log.isDebugEnabled()) log.debug("Checking for services at "+url);
+ for(EntryType entry:registry.listEntries()){
+ if(entry.getMemberServiceEPR().getAddress().getStringValue().startsWith(url)){
+ dn=WSUtilities.extractServerIDFromEPR(entry.getMemberServiceEPR());
+ if(dn!=null){
+ return dn;
+ }
+ }
+ }
+ return null;
+ }
+
+
+ public static String getStorageFactoryUrl(String besUrl){
+ int besIndex = besUrl.indexOf("BESFactory?res");
+ String ss = besUrl.substring(0, besIndex);
+ ss = ss + "StorageFactory?res=default_storage_factory";
+ return ss;
+ }
+
+ /**
+ * prepare request
+ * */
+ protected CreateSMSDocument getCreateSMSDocument(String ...keyValueParams){
+ CreateSMSDocument in=CreateSMSDocument.Factory.newInstance();
+ in.addNewCreateSMS();
+ if(initialLifeTime>0){
+ in.getCreateSMS().addNewTerminationTime().setCalendarValue(getTermTime());
+ }
+ if(storageType!=null){
+ if(log.isDebugEnabled()) {
+ log.debug("Will create storage of type : "+storageType);
+ }
+ StorageDescriptionType desc=in.getCreateSMS().addNewStorageDescription();
+ desc.setStorageBackendType(storageType);
+ if(keyValueParams.length>1){
+ //other parameters from the cmdline as key=value
+ StorageBackendParameters params=desc.addNewStorageBackendParameters();
+ for(int i=1;i<keyValueParams.length;i++){
+ String arg=keyValueParams[i];
+ String[]sp=arg.split("=",2);
+ PropertyType prop=params.addNewProperty();
+ prop.setName(sp[0]);
+ prop.setValue(sp[1]);
+ if(log.isDebugEnabled()) {
+ log.debug("Have parameter : "+arg);
+ }
+ }
+ }
+ }
+ return in;
+ }
+
+ protected Calendar getTermTime(){
+ Calendar c = Calendar.getInstance();
+ c.add(Calendar.DATE, initialLifeTime);
+ return c;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java
new file mode 100644
index 0000000..0284dea
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java
@@ -0,0 +1,225 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.bes.utils;
+
+import java.io.File;
+import java.util.Map;
+
+import org.apache.airavata.commons.gfac.type.ActualParameter;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.context.MessageContext;
+import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
+import org.apache.airavata.schemas.gfac.StringArrayType;
+import org.apache.airavata.schemas.gfac.StringParameterType;
+import org.apache.airavata.schemas.gfac.URIArrayType;
+import org.apache.airavata.schemas.gfac.URIParameterType;
+import org.apache.airavata.schemas.gfac.UnicoreHostType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
+
+public class UASDataStagingProcessor {
+
+ public static void generateDataStagingElements(JobDefinitionType value, JobExecutionContext context, String smsUrl) throws Exception{
+
+ HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
+ .getApplicationContext().getApplicationDeploymentDescription()
+ .getType();
+
+ smsUrl = "BFT:"+smsUrl;
+
+ if (context.getInMessageContext().getParameters().size() > 0) {
+ buildDataStagingFromInputContext(context, value, smsUrl, appDepType);
+ }
+ MessageContext outMessage = new MessageContext();
+ ActualParameter a1 = new ActualParameter();
+ a1.getType().changeType(StringParameterType.type);
+ ((StringParameterType)a1.getType()).setValue("output/analysis-results.tar");
+ outMessage.addParameter("o1", a1);
+ context.setOutMessageContext(outMessage);
+
+ if (context.getOutMessageContext().getParameters().size() > 0) {
+ buildFromOutputContext(context, value, smsUrl, appDepType);
+ }
+ createStdOutURIs(value, appDepType, smsUrl, isUnicoreEndpoint(context));
+ }
+
+ private static void createInURISMSElement(JobDefinitionType value,
+ String smsUrl, String inputDir, ActualParameter inParam)
+ throws Exception {
+
+ String uri = ((URIParameterType) inParam.getType()).getValue();
+ //TODO: To add this input file name setting part of Airavata API
+ String fileName = "input/" + new File(uri).getName();
+ if (uri.startsWith("file")) {
+ String fileUri = smsUrl+"#/"+fileName;
+
+ JSDLUtils.addDataStagingSourceElement(value, fileUri, null, fileName);
+ } else if (uri.startsWith("gsiftp") || uri.startsWith("http")
+ || uri.startsWith("rns")) {
+ // no need to stage-in those files to the input
+ // directory because unicore site will fetch them for the user
+ JSDLUtils.addDataStagingSourceElement(value, uri, null, fileName);
+ }
+
+ }
+
+ private static void createStdOutURIs(JobDefinitionType value,
+ HpcApplicationDeploymentType appDepType, String smsUrl,
+ boolean isUnicore) throws Exception {
+
+
+ String stdout = ApplicationProcessor.getApplicationStdOut(value, appDepType);
+
+ String stderr = ApplicationProcessor.getApplicationStdErr(value, appDepType);
+
+ String stdoutFileName = (stdout == null || stdout.equals("")) ? "stdout"
+ : stdout;
+ String stdoutURI = smsUrl+"#/output/"+stdoutFileName;
+ JSDLUtils.addDataStagingTargetElement(value, null, stdoutFileName,
+ stdoutURI);
+
+ String stderrFileName = (stdout == null || stderr.equals("")) ? "stderr"
+ : stderr;
+ String stderrURI = smsUrl+"#/output/"+stderrFileName;
+ JSDLUtils.addDataStagingTargetElement(value, null, stderrFileName,
+ stderrURI);
+
+ if(isUnicore) {
+ String scriptExitCodeFName = "UNICORE_SCRIPT_EXIT_CODE";
+ String scriptExitCode = smsUrl+"#/output/"+scriptExitCodeFName;
+ JSDLUtils.addDataStagingTargetElement(value, null,
+ scriptExitCodeFName, scriptExitCode.toString());
+ }
+
+ }
+
+
+ private static void createOutStringElements(JobDefinitionType value,
+ HpcApplicationDeploymentType appDeptype, String smsUrl, String prmValue) throws Exception {
+
+ if(prmValue == null || "".equals(prmValue)) return;
+
+ String finalSMSPath = smsUrl + "#/output/"+prmValue;
+
+ JSDLUtils.addDataStagingTargetElement(value, null, prmValue, finalSMSPath);
+ }
+
+
+ private static void createOutURIElement(JobDefinitionType value,
+ String prmValue) throws Exception {
+ String fileName = new File(prmValue.toString()).getName();
+ JSDLUtils.addDataStagingTargetElement(value, null, fileName, prmValue);
+ }
+
+
+ private static JobDefinitionType buildFromOutputContext(JobExecutionContext context,
+ JobDefinitionType value, String smsUrl,
+ HpcApplicationDeploymentType appDepType) throws Exception {
+
+ Map<String, Object> outputParams = context.getOutMessageContext()
+ .getParameters();
+
+ for (String paramKey : outputParams.keySet()) {
+
+ ActualParameter outParam = (ActualParameter) outputParams
+ .get(paramKey);
+
+ // if single urls then convert each url into jsdl source
+ // elements,
+ // that are formed by concat of gridftpurl+inputdir+filename
+
+ String paramDataType = outParam.getType().getType().toString();
+
+ if ("URI".equals(paramDataType)) {
+ String uriPrm = ((URIParameterType) outParam.getType())
+ .getValue();
+ createOutURIElement(value, uriPrm);
+ }
+
+ // string params are converted into the job arguments
+
+ else if (("URIArray").equals(paramDataType)) {
+ String[] uriArray = ((URIArrayType) outParam.getType())
+ .getValueArray();
+ for (String u : uriArray) {
+
+ createOutURIElement(value, u);
+ }
+
+ }
+ else if ("String".equals(paramDataType)) {
+ String stringPrm = ((StringParameterType) outParam
+ .getType()).getValue();
+ createOutStringElements(value, appDepType, smsUrl, stringPrm);
+ }
+
+ else if ("StringArray".equals(paramDataType)) {
+ String[] valueArray = ((StringArrayType) outParam.getType())
+ .getValueArray();
+ for (String v : valueArray) {
+ createOutStringElements(value, appDepType, smsUrl, v);
+ }
+ }
+ }
+
+ return value;
+ }
+
+
+ private static void buildDataStagingFromInputContext(JobExecutionContext context, JobDefinitionType value, String smsUrl, HpcApplicationDeploymentType appDepType)
+ throws Exception {
+
+ // TODO set data directory
+ Map<String, Object> inputParams = context.getInMessageContext()
+ .getParameters();
+
+ for (String paramKey : inputParams.keySet()) {
+
+ ActualParameter inParam = (ActualParameter) inputParams
+ .get(paramKey);
+
+ // if single urls then convert each url into jsdl source
+ // elements,
+ // that are formed by concat of gridftpurl+inputdir+filename
+
+ String paramDataType = inParam.getType().getType().toString();
+
+ if ("URI".equals(paramDataType)) {
+ createInURISMSElement(value, smsUrl,
+ appDepType.getInputDataDirectory(), inParam);
+ }
+
+ // string params are converted into the job arguments
+
+ else if ("String".equals(paramDataType)) {
+ String stringPrm = ((StringParameterType) inParam.getType())
+ .getValue();
+ ApplicationProcessor.addApplicationArgument(value, appDepType, stringPrm);
+ }
+ }
+
+ }
+
+ public static boolean isUnicoreEndpoint(JobExecutionContext context) {
+ return ( (context.getApplicationContext().getHostDescription().getType() instanceof UnicoreHostType)?true:false );
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/URIUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/URIUtils.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/URIUtils.java
new file mode 100644
index 0000000..fbc91da
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/URIUtils.java
@@ -0,0 +1,119 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.bes.utils;
+
+import org.apache.commons.httpclient.URI;
+import org.apache.commons.httpclient.URIException;
+import org.apache.commons.httpclient.util.URIUtil;
+
+import java.net.URISyntaxException;
+
+public class URIUtils {
+
+ public static String encodeAll(String uri) throws URIException
+ {
+ String result = encodeAuthority(uri);
+ result = encodePath(uri);
+ result = encodeQuery(result );
+ result = encodeFragment(result );
+ return result;
+ }
+
+ public static String encodeAuthority(String uri) throws URIException
+ {
+ int start = uri.indexOf("//");
+ if(start == -1) return uri;
+ start++;
+ int end = uri.indexOf("/",start+1);
+ if(end == -1) end = uri.indexOf("?",start+1);
+ if(end == -1) end = uri.indexOf("#",start+1);
+ if(end == -1) end = uri.length();
+ String before = uri.substring(0, start+1);
+ String authority= uri.substring(start+1,end);
+ String after = uri.substring(end);
+ authority = URIUtil.encode(authority, URI.allowed_authority);
+
+ return before+authority+after;
+ }
+
+ public static String encodePath(String uri) throws URIException
+ {
+ int doubleSlashIndex = uri.indexOf("//");
+ boolean hasAuthority = doubleSlashIndex >= 0;
+ int start = -1;
+ if(hasAuthority)
+ {
+ start = uri.indexOf("/",doubleSlashIndex+2);
+ }
+ else
+ {
+ start = uri.indexOf(":");
+ }
+ if(start == -1) return uri;
+
+ int end = uri.indexOf("?",start+1);
+ if(end == -1) end = uri.indexOf("#",start+1);
+ if(end == -1) end = uri.length();
+ String before = uri.substring(0, start+1);
+ String path= uri.substring(start+1,end);
+ String after = uri.substring(end);
+ path = URIUtil.encode(path, URI.allowed_abs_path);
+ return before+path+after;
+ }
+
+
+ public static String encodeQuery(String uri) throws URIException
+ {
+ int queryStart = uri.indexOf("?");
+ if(queryStart == -1) return uri;
+ int queryEnd = uri.indexOf("#");
+ if(queryEnd == -1) queryEnd = uri.length();
+
+ String beforeQuery = uri.substring(0, queryStart+1);
+ String query = uri.substring(queryStart+1,queryEnd);
+ String afterQuery = uri.substring(queryEnd);
+ query = URIUtil.encode(query, URI.allowed_query);
+ return beforeQuery+query+afterQuery;
+ }
+
+
+ public static String encodeFragment(String uri) throws URIException
+ {
+ int fragmentStart = uri.indexOf("#");
+ if(fragmentStart == -1) return uri;
+
+ String beforeFragment = uri.substring(0, fragmentStart+1);
+ String fragment = uri.substring(fragmentStart+1);
+ fragment = URIUtil.encode(fragment, URI.allowed_fragment);
+ return beforeFragment+fragment;
+ }
+ public static java.net.URI createGsiftpURI(String host, String localPath) throws URISyntaxException {
+ StringBuffer buf = new StringBuffer();
+ if (!host.startsWith("gsiftp://"))
+ buf.append("gsiftp://");
+ buf.append(host);
+ if (!host.endsWith("/"))
+ buf.append("/");
+ buf.append(localPath);
+ return new java.net.URI(buf.toString());
+ }
+}
\ No newline at end of file
[2/4] moving gfac-bes classes to new package.
Posted by la...@apache.org.
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/context/security/GSISecurityContext.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/context/security/GSISecurityContext.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/context/security/GSISecurityContext.java
deleted file mode 100644
index 3eb020f..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/context/security/GSISecurityContext.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.context.security;
-
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.credential.store.credential.Credential;
-import org.apache.airavata.credential.store.credential.impl.certificate.CertificateCredential;
-import org.apache.airavata.credential.store.store.CredentialReader;
-import org.apache.airavata.gfac.AbstractSecurityContext;
-import org.apache.airavata.gfac.Constants;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.RequestData;
-import org.apache.airavata.gsi.ssh.api.Cluster;
-import org.globus.gsi.X509Credential;
-import org.globus.gsi.gssapi.GlobusGSSCredentialImpl;
-import org.globus.gsi.provider.GlobusProvider;
-import org.globus.myproxy.GetParams;
-import org.globus.myproxy.MyProxy;
-import org.globus.myproxy.MyProxyException;
-import org.gridforum.jgss.ExtendedGSSCredential;
-import org.ietf.jgss.GSSCredential;
-import org.ietf.jgss.GSSException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.security.Security;
-import java.security.cert.X509Certificate;
-
-/**
- * Handles GRID related security.
- */
-public class GSISecurityContext extends AbstractSecurityContext {
-
- protected static final Logger log = LoggerFactory.getLogger(GSISecurityContext.class);
- /*
- * context name
- */
- public static final String GSI_SECURITY_CONTEXT = "gsi";
-
- public static int CREDENTIAL_RENEWING_THRESH_HOLD = 10 * 90;
-
- private GSSCredential gssCredentials = null;
-
- private Cluster pbsCluster = null;
-
- // Set trusted cert path and add provider
- static {
- Security.addProvider(new GlobusProvider());
- try {
- setUpTrustedCertificatePath();
- } catch (ApplicationSettingsException e) {
- log.error(e.getLocalizedMessage(), e);
- }
- }
-
- public static void setUpTrustedCertificatePath(String trustedCertificatePath) {
-
- File file = new File(trustedCertificatePath);
-
- if (!file.exists() || !file.canRead()) {
- File f = new File(".");
- log.info("Current directory " + f.getAbsolutePath());
- throw new RuntimeException("Cannot read trusted certificate path " + trustedCertificatePath);
- } else {
- System.setProperty(Constants.TRUSTED_CERTIFICATE_SYSTEM_PROPERTY, file.getAbsolutePath());
- }
- }
-
- private static void setUpTrustedCertificatePath() throws ApplicationSettingsException {
-
- String trustedCertificatePath = ServerSettings.getSetting(Constants.TRUSTED_CERT_LOCATION);
-
- setUpTrustedCertificatePath(trustedCertificatePath);
- }
-
- /**
- * Gets the trusted certificate path. Trusted certificate path is stored in "X509_CERT_DIR"
- * system property.
- * @return The trusted certificate path as a string.
- */
- public static String getTrustedCertificatePath() {
- return System.getProperty(Constants.TRUSTED_CERTIFICATE_SYSTEM_PROPERTY);
- }
-
-
- public GSISecurityContext(CredentialReader credentialReader, RequestData requestData) {
- super(credentialReader, requestData);
- }
-
-
- public GSISecurityContext(Cluster pbsCluster) {
- this.setPbsCluster(pbsCluster);
- }
-
- /**
- * Gets GSSCredentials. The process is as follows;
- * If credentials were queried for the first time create credentials.
- * 1. Try creating credentials using certificates stored in the credential store
- * 2. If 1 fails use user name and password to create credentials
- * If credentials are already created check the remaining life time of the credential. If
- * remaining life time is less than CREDENTIAL_RENEWING_THRESH_HOLD, then renew credentials.
- * @return GSSCredentials to be used.
- * @throws org.apache.airavata.gfac.GFacException If an error occurred while creating credentials.
- * @throws org.apache.airavata.common.exception.ApplicationSettingsException
- */
- public GSSCredential getGssCredentials() throws GFacException, ApplicationSettingsException {
-
- if (gssCredentials == null) {
-
- try {
- gssCredentials = getCredentialsFromStore();
- } catch (Exception e) {
- log.error("An exception occurred while retrieving credentials from the credential store. " +
- "Will continue with my proxy user name and password.", e);
- }
-
- // If store does not have credentials try to get from user name and password
- if (gssCredentials == null) {
- gssCredentials = getDefaultCredentials();
- }
-
- // if still null, throw an exception
- if (gssCredentials == null) {
- throw new GFacException("Unable to retrieve my proxy credentials to continue operation.");
- }
- } else {
- try {
- if (gssCredentials.getRemainingLifetime() < CREDENTIAL_RENEWING_THRESH_HOLD) {
- return renewCredentials();
- }
- } catch (GSSException e) {
- throw new GFacException("Unable to retrieve remaining life time from credentials.", e);
- }
- }
-
- return gssCredentials;
- }
-
- /**
- * Renews credentials. First try to renew credentials as a trusted renewer. If that failed
- * use user name and password to renew credentials.
- * @return Renewed credentials.
- * @throws org.apache.airavata.gfac.GFacException If an error occurred while renewing credentials.
- * @throws org.apache.airavata.common.exception.ApplicationSettingsException
- */
- public GSSCredential renewCredentials() throws GFacException, ApplicationSettingsException {
-
- // First try to renew credentials as a trusted renewer
- try {
- gssCredentials = renewCredentialsAsATrustedHost();
- } catch (Exception e) {
- log.warn("Renewing credentials as a trusted renewer failed", e);
- gssCredentials = getProxyCredentials();
- }
-
- return gssCredentials;
- }
-
- /**
- * Reads the credentials from credential store.
- * @return If token is found in the credential store, will return a valid credential. Else returns null.
- * @throws Exception If an error occurred while retrieving credentials.
- */
- public GSSCredential getCredentialsFromStore() throws Exception {
-
- if (getCredentialReader() == null) {
- return null;
- }
-
- Credential credential = getCredentialReader().getCredential(getRequestData().getGatewayId(),
- getRequestData().getTokenId());
-
- if (credential != null) {
- if (credential instanceof CertificateCredential) {
-
- log.info("Successfully found credentials for token id - " + getRequestData().getTokenId() +
- " gateway id - " + getRequestData().getGatewayId());
-
- CertificateCredential certificateCredential = (CertificateCredential) credential;
-
- X509Certificate[] certificates = certificateCredential.getCertificates();
- X509Credential newCredential = new X509Credential(certificateCredential.getPrivateKey(), certificates);
-
- GlobusGSSCredentialImpl cred = new GlobusGSSCredentialImpl(newCredential, GSSCredential.INITIATE_AND_ACCEPT);
- System.out.print(cred.export(ExtendedGSSCredential.IMPEXP_OPAQUE));
- return cred;
- //return new GlobusGSSCredentialImpl(newCredential,
- // GSSCredential.INITIATE_AND_ACCEPT);
- } else {
- log.info("Credential type is not CertificateCredential. Cannot create mapping globus credentials. " +
- "Credential type - " + credential.getClass().getName());
- }
- } else {
- log.info("Could not find credentials for token - " + getRequestData().getTokenId() + " and "
- + "gateway id - " + getRequestData().getGatewayId());
- }
-
- return null;
- }
-
- /**
- * Gets the default proxy certificate.
- * @return Default my proxy credentials.
- * @throws org.apache.airavata.gfac.GFacException If an error occurred while retrieving credentials.
- * @throws org.apache.airavata.common.exception.ApplicationSettingsException
- */
- public GSSCredential getDefaultCredentials() throws GFacException, ApplicationSettingsException{
- MyProxy myproxy = new MyProxy(getRequestData().getMyProxyServerUrl(), getRequestData().getMyProxyPort());
- try {
- return myproxy.get(getRequestData().getMyProxyUserName(), getRequestData().getMyProxyPassword(),
- getRequestData().getMyProxyLifeTime());
- } catch (MyProxyException e) {
- throw new GFacException("An error occurred while retrieving default security credentials.", e);
- }
- }
-
- /**
- * Gets a new proxy certificate given current credentials.
- * @return The short lived GSSCredentials
- * @throws org.apache.airavata.gfac.GFacException If an error is occurred while retrieving credentials.
- * @throws org.apache.airavata.common.exception.ApplicationSettingsException
- */
- public GSSCredential getProxyCredentials() throws GFacException, ApplicationSettingsException {
-
- MyProxy myproxy = new MyProxy(getRequestData().getMyProxyServerUrl(), getRequestData().getMyProxyPort());
- try {
- return myproxy.get(gssCredentials, getRequestData().getMyProxyUserName(), getRequestData().getMyProxyPassword(),
- getRequestData().getMyProxyLifeTime());
- } catch (MyProxyException e) {
- throw new GFacException("An error occurred while renewing security credentials using user/password.", e);
- }
- }
-
- /**
- * Renew GSSCredentials.
- * Before executing we need to add current host as a trusted renewer. Note to renew credentials
- * we dont need user name and password.
- * To do that execute following command
- * > myproxy-logon -t <LIFETIME></LIFETIME> -s <MY PROXY SERVER> -l <USER NAME>
- * E.g :- > myproxy-logon -t 264 -s myproxy.teragrid.org -l us3
- * Enter MyProxy pass phrase:
- * A credential has been received for user us3 in /tmp/x509up_u501.
- * > myproxy-init -A --cert /tmp/x509up_u501 --key /tmp/x509up_u501 -l ogce -s myproxy.teragrid.org
- * @return Renewed credentials.
- * @throws org.apache.airavata.gfac.GFacException If an error occurred while renewing credentials.
- * @throws org.apache.airavata.common.exception.ApplicationSettingsException
- */
- public GSSCredential renewCredentialsAsATrustedHost() throws GFacException, ApplicationSettingsException {
- MyProxy myproxy = new MyProxy(getRequestData().getMyProxyServerUrl(), getRequestData().getMyProxyPort());
- GetParams getParams = new GetParams();
- getParams.setAuthzCreds(gssCredentials);
- getParams.setUserName(getRequestData().getMyProxyUserName());
- getParams.setLifetime(getRequestData().getMyProxyLifeTime());
- try {
- return myproxy.get(gssCredentials, getParams);
- } catch (MyProxyException e) {
- throw new GFacException("An error occurred while renewing security credentials.", e);
- }
- }
-
- public Cluster getPbsCluster() {
- return pbsCluster;
- }
-
- public void setPbsCluster(Cluster pbsCluster) {
- this.pbsCluster = pbsCluster;
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/provider/impl/BESProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/provider/impl/BESProvider.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/provider/impl/BESProvider.java
deleted file mode 100644
index 4a531a6..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/provider/impl/BESProvider.java
+++ /dev/null
@@ -1,568 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.provider.impl;
-
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.math.BigInteger;
-import java.security.InvalidKeyException;
-import java.security.KeyPair;
-import java.security.KeyPairGenerator;
-import java.security.PrivateKey;
-import java.security.cert.X509Certificate;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Random;
-import java.util.Set;
-
-import javax.security.auth.x500.X500Principal;
-
-import org.apache.airavata.gfac.Constants;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.context.security.GSISecurityContext;
-import org.apache.airavata.gfac.notification.events.StatusChangeEvent;
-import org.apache.airavata.gfac.notification.events.UnicoreJobIDEvent;
-import org.apache.airavata.gfac.provider.AbstractProvider;
-import org.apache.airavata.gfac.provider.GFacProviderException;
-import org.apache.airavata.gfac.utils.DataTransferrer;
-import org.apache.airavata.gfac.utils.JSDLGenerator;
-import org.apache.airavata.gfac.utils.StorageCreator;
-import org.apache.airavata.gfac.utils.GFacUtils;
-import org.apache.airavata.model.workspace.experiment.JobState;
-import org.apache.airavata.registry.api.workflow.ApplicationJob;
-import org.apache.airavata.registry.api.workflow.ApplicationJob.ApplicationJobStatus;
-import org.apache.airavata.schemas.gfac.UnicoreHostType;
-import org.apache.xmlbeans.XmlCursor;
-import org.bouncycastle.asn1.ASN1InputStream;
-import org.bouncycastle.asn1.x500.X500Name;
-import org.bouncycastle.asn1.x500.style.BCStyle;
-import org.bouncycastle.asn1.x509.AlgorithmIdentifier;
-import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
-import org.ggf.schemas.bes.x2006.x08.besFactory.ActivityStateEnumeration;
-import org.ggf.schemas.bes.x2006.x08.besFactory.ActivityStateEnumeration.Enum;
-import org.ggf.schemas.bes.x2006.x08.besFactory.ActivityStatusType;
-import org.ggf.schemas.bes.x2006.x08.besFactory.CreateActivityDocument;
-import org.ggf.schemas.bes.x2006.x08.besFactory.CreateActivityResponseDocument;
-import org.ggf.schemas.bes.x2006.x08.besFactory.GetActivityStatusesDocument;
-import org.ggf.schemas.bes.x2006.x08.besFactory.GetActivityStatusesResponseDocument;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionDocument;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3.x2005.x08.addressing.EndpointReferenceType;
-
-import de.fzj.unicore.bes.client.FactoryClient;
-import de.fzj.unicore.bes.faults.UnknownActivityIdentifierFault;
-import de.fzj.unicore.uas.client.StorageClient;
-import de.fzj.unicore.wsrflite.xmlbeans.WSUtilities;
-import eu.emi.security.authn.x509.helpers.CertificateHelpers;
-import eu.emi.security.authn.x509.helpers.proxy.X509v3CertificateBuilder;
-import eu.emi.security.authn.x509.impl.CertificateUtils;
-import eu.emi.security.authn.x509.impl.CertificateUtils.Encoding;
-import eu.emi.security.authn.x509.impl.DirectoryCertChainValidator;
-import eu.emi.security.authn.x509.impl.KeyAndCertCredential;
-import eu.emi.security.authn.x509.impl.X500NameUtils;
-import eu.unicore.util.httpclient.DefaultClientConfiguration;
-
-
-
-public class BESProvider extends AbstractProvider {
- protected final Logger log = LoggerFactory.getLogger(this.getClass());
-
- private DefaultClientConfiguration secProperties;
-
- private String jobId;
-
-
-
- public void initialize(JobExecutionContext jobExecutionContext)
- throws GFacProviderException, GFacException {
- log.info("Initializing UNICORE Provider");
- super.initialize(jobExecutionContext);
- initSecurityProperties(jobExecutionContext);
- log.debug("initialized security properties");
- }
-
-
- public void execute(JobExecutionContext jobExecutionContext)
- throws GFacProviderException {
- UnicoreHostType host = (UnicoreHostType) jobExecutionContext.getApplicationContext().getHostDescription()
- .getType();
-
- String factoryUrl = host.getUnicoreBESEndPointArray()[0];
-
- EndpointReferenceType eprt = EndpointReferenceType.Factory.newInstance();
- eprt.addNewAddress().setStringValue(factoryUrl);
-
- String userDN = getUserName(jobExecutionContext);
-
- if (userDN == null || userDN.equalsIgnoreCase("admin")) {
- userDN = "CN=zdv575, O=Ultrascan Gateway, C=DE";
- }
-
- String xlogin = getCNFromUserDN(userDN);
- // create storage
- StorageCreator storageCreator = new StorageCreator(secProperties, factoryUrl, 5, xlogin);
-
- StorageClient sc = null;
- try {
- try {
- sc = storageCreator.createStorage();
- } catch (Exception e2) {
- log.error("Cannot create storage..");
- throw new GFacProviderException("Cannot create storage..", e2);
- }
-
- CreateActivityDocument cad = CreateActivityDocument.Factory.newInstance();
- JobDefinitionDocument jobDefDoc = JobDefinitionDocument.Factory.newInstance();
-
- JobDefinitionType jobDefinition = jobDefDoc.addNewJobDefinition();
- try {
- jobDefinition = JSDLGenerator.buildJSDLInstance(jobExecutionContext, sc.getUrl()).getJobDefinition();
- cad.addNewCreateActivity().addNewActivityDocument().setJobDefinition(jobDefinition);
-
- log.info("JSDL" + jobDefDoc.toString());
- } catch (Exception e1) {
- throw new GFacProviderException("Cannot generate JSDL instance from the JobExecutionContext.", e1);
- }
-
- // upload files if any
- DataTransferrer dt = new DataTransferrer(jobExecutionContext, sc);
- dt.uploadLocalFiles();
-
- FactoryClient factory = null;
- try {
- factory = new FactoryClient(eprt, secProperties);
- } catch (Exception e) {
- throw new GFacProviderException(e.getLocalizedMessage(), e);
- }
-
- CreateActivityResponseDocument response = null;
- try {
- log.info(String.format("Activity Submitting to %s ... \n", factoryUrl));
- response = factory.createActivity(cad);
- log.info(String.format("Activity Submitted to %s \n", factoryUrl));
- } catch (Exception e) {
- throw new GFacProviderException("Cannot create activity.", e);
- }
- EndpointReferenceType activityEpr = response.getCreateActivityResponse().getActivityIdentifier();
-
- log.info("Activity : " + activityEpr.getAddress().getStringValue() + " Submitted.");
-
- // factory.waitWhileActivityIsDone(activityEpr, 1000);
- jobId = WSUtilities.extractResourceID(activityEpr);
- if (jobId == null) {
- jobId = new Long(Calendar.getInstance().getTimeInMillis()).toString();
- }
- log.info("JobID: " + jobId);
- jobExecutionContext.getNotifier().publish(new UnicoreJobIDEvent(jobId));
- saveApplicationJob(jobExecutionContext, jobDefinition, activityEpr.toString());
-
- factory.getActivityStatus(activityEpr);
- log.info(formatStatusMessage(activityEpr.getAddress().getStringValue(),
- factory.getActivityStatus(activityEpr).toString()));
-
- // TODO publish the status messages to the message bus
- while ((factory.getActivityStatus(activityEpr) != ActivityStateEnumeration.FINISHED)
- && (factory.getActivityStatus(activityEpr) != ActivityStateEnumeration.FAILED)
- && (factory.getActivityStatus(activityEpr) != ActivityStateEnumeration.CANCELLED)) {
-
- ActivityStatusType activityStatus = null;
- try {
- activityStatus = getStatus(factory, activityEpr);
- JobState jobStatus = getApplicationJobStatus(activityStatus);
- String jobStatusMessage = "Status of job " + jobId + "is " + jobStatus;
- jobExecutionContext.getNotifier().publish(new StatusChangeEvent(jobStatusMessage));
- details.setJobID(jobId);
- GFacUtils.updateJobStatus(jobExecutionContext, details, jobStatus);
- } catch (UnknownActivityIdentifierFault e) {
- throw new GFacProviderException(e.getMessage(), e.getCause());
- }catch (GFacException e) {
- throw new GFacProviderException(e.getMessage(), e.getCause());
- }
-
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- }
- continue;
- }
-
- ActivityStatusType activityStatus = null;
- try {
- activityStatus = getStatus(factory, activityEpr);
- } catch (UnknownActivityIdentifierFault e) {
- throw new GFacProviderException(e.getMessage(), e.getCause());
- }
-
- log.info(formatStatusMessage(activityEpr.getAddress().getStringValue(), activityStatus.getState()
- .toString()));
-
- if ((activityStatus.getState() == ActivityStateEnumeration.FAILED)) {
- String error = activityStatus.getFault().getFaultcode().getLocalPart() + "\n"
- + activityStatus.getFault().getFaultstring() + "\n EXITCODE: " + activityStatus.getExitCode();
- log.info(error);
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- }
- dt.downloadStdOuts();
- } else if (activityStatus.getState() == ActivityStateEnumeration.CANCELLED) {
- String experimentID = (String) jobExecutionContext.getProperty(Constants.PROP_TOPIC);
- JobState jobStatus = JobState.CANCELED;
- String jobStatusMessage = "Status of job " + jobId + "is " + jobStatus;
- jobExecutionContext.getNotifier().publish(new StatusChangeEvent(jobStatusMessage));
- details.setJobID(jobId);
- try {
- GFacUtils.saveJobStatus(jobExecutionContext,details, jobStatus);
- } catch (GFacException e) {
- throw new GFacProviderException(e.getLocalizedMessage(),e);
- }
- throw new GFacProviderException(experimentID + "Job Canceled");
- }
-
- else if (activityStatus.getState() == ActivityStateEnumeration.FINISHED) {
- try {
- Thread.sleep(5000);
- } catch (InterruptedException e) {
- }
- if (activityStatus.getExitCode() == 0) {
- dt.downloadRemoteFiles();
- } else {
- dt.downloadStdOuts();
- }
- }
-
- } catch (UnknownActivityIdentifierFault e1) {
- throw new GFacProviderException(e1.getLocalizedMessage(), e1);
- } finally {
- // destroy sms instance
- try {
- if (sc != null) {
- sc.destroy();
- }
- } catch (Exception e) {
- log.warn("Cannot destroy temporary SMS instance:" + sc.getUrl(), e);
- }
- }
- }
-
- private JobState getApplicationJobStatus(ActivityStatusType activityStatus){
- if (activityStatus == null) {
- return JobState.UNKNOWN;
- }
- Enum state = activityStatus.getState();
- String status = null;
- XmlCursor acursor = activityStatus.newCursor();
- try {
- if (acursor.toFirstChild()) {
- if (acursor.getName().getNamespaceURI().equals("http://schemas.ogf.org/hpcp/2007/01/fs")) {
- status = acursor.getName().getLocalPart();
- }
- }
- if (status != null) {
- if (status.equalsIgnoreCase("Queued") || status.equalsIgnoreCase("Starting")
- || status.equalsIgnoreCase("Ready")) {
- return JobState.QUEUED;
- } else if (status.equalsIgnoreCase("Staging-In")) {
- return JobState.SUBMITTED;
- } else if (status.equalsIgnoreCase("Staging-Out") || status.equalsIgnoreCase("FINISHED")) {
- return JobState.COMPLETE;
- } else if (status.equalsIgnoreCase("Executing")) {
- return JobState.ACTIVE;
- } else if (status.equalsIgnoreCase("FAILED")) {
- return JobState.FAILED;
- } else if (status.equalsIgnoreCase("CANCELLED")) {
- return JobState.CANCELED;
- }
- } else {
- if (ActivityStateEnumeration.CANCELLED.equals(state)) {
- return JobState.CANCELED;
- } else if (ActivityStateEnumeration.FAILED.equals(state)) {
- return JobState.FAILED;
- } else if (ActivityStateEnumeration.FINISHED.equals(state)) {
- return JobState.COMPLETE;
- } else if (ActivityStateEnumeration.RUNNING.equals(state)) {
- return JobState.ACTIVE;
- }
- }
- } finally {
- if (acursor != null)
- acursor.dispose();
- }
- return JobState.UNKNOWN;
- }
-
- private void saveApplicationJob(JobExecutionContext jobExecutionContext, JobDefinitionType jobDefinition,
- String metadata) {
- ApplicationJob appJob = GFacUtils.createApplicationJob(jobExecutionContext);
- appJob.setJobId(jobId);
- appJob.setJobData(jobDefinition.toString());
- appJob.setSubmittedTime(Calendar.getInstance().getTime());
- appJob.setStatus(ApplicationJobStatus.SUBMITTED);
- appJob.setStatusUpdateTime(appJob.getSubmittedTime());
- appJob.setMetadata(metadata);
- GFacUtils.recordApplicationJob(jobExecutionContext, appJob);
- }
-
- public void dispose(JobExecutionContext jobExecutionContext) throws GFacProviderException {
- secProperties = null;
- }
-
- /**
- * EndpointReference need to be saved to make cancel work.
- *
- * @param activityEpr
- * @param jobExecutionContext
- * @throws GFacProviderException
- */
- public void cancelJob(String activityEpr, JobExecutionContext jobExecutionContext) throws GFacProviderException {
- try {
- initSecurityProperties(jobExecutionContext);
- EndpointReferenceType eprt = EndpointReferenceType.Factory.parse(activityEpr);
- UnicoreHostType host = (UnicoreHostType) jobExecutionContext.getApplicationContext().getHostDescription()
- .getType();
-
- String factoryUrl = host.getUnicoreBESEndPointArray()[0];
- EndpointReferenceType epr = EndpointReferenceType.Factory.newInstance();
- epr.addNewAddress().setStringValue(factoryUrl);
-
- FactoryClient factory = new FactoryClient(epr, secProperties);
- factory.terminateActivity(eprt);
- } catch (Exception e) {
- throw new GFacProviderException(e.getLocalizedMessage(),e);
- }
-
- }
-
- protected void downloadOffline(String smsEpr, JobExecutionContext jobExecutionContext) throws GFacProviderException {
- try {
- initSecurityProperties(jobExecutionContext);
- EndpointReferenceType eprt = EndpointReferenceType.Factory.parse(smsEpr);
- StorageClient sms = new StorageClient(eprt, secProperties);
- DataTransferrer dt = new DataTransferrer(jobExecutionContext, sms);
- // there must be output files there
- // this is also possible if client is re-connected, the jobs are
- // still
- // running and no output is produced
- dt.downloadRemoteFiles();
-
- // may be use the below method before downloading for checking
- // the number of entries
- // sms.listDirectory(".");
-
- } catch (Exception e) {
- throw new GFacProviderException(e.getLocalizedMessage(), e);
- }
- }
-
- protected void initSecurityProperties(JobExecutionContext jobExecutionContext) throws GFacProviderException,
- GFacException {
-
- if (secProperties != null)
- return;
-
- GSISecurityContext gssContext = (GSISecurityContext) jobExecutionContext
- .getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT);
-
- try {
- String certLocation = gssContext.getTrustedCertificatePath();
- List<String> trustedCert = new ArrayList<String>();
- trustedCert.add(certLocation + "/*.0");
- trustedCert.add(certLocation + "/*.pem");
-
- DirectoryCertChainValidator dcValidator = new DirectoryCertChainValidator(trustedCert, Encoding.PEM, -1,
- 60000, null);
-
- String userID = getUserName(jobExecutionContext);
-
- if ( userID == null || "".equals(userID) || userID.equalsIgnoreCase("admin") ) {
- userID = "CN=zdv575, O=Ultrascan Gateway, C=DE";
- }
-
- String userDN = userID.replaceAll("^\"|\"$", "");
-
- // TODO: should be changed to default airavata server locations
- KeyAndCertCredential cred = generateShortLivedCertificate(userDN, certLocation
- + "/cacert.pem", certLocation
- + "/cakey.pem", "ultrascan3");
- secProperties = new DefaultClientConfiguration(dcValidator, cred);
-
- // secProperties.doSSLAuthn();
- secProperties.getETDSettings().setExtendTrustDelegation(true);
-
- secProperties.setDoSignMessage(true);
-
- String[] outHandlers = secProperties.getOutHandlerClassNames();
-
- Set<String> outHandlerLst = null;
-
- // timeout in milliseconds
- Properties p = secProperties.getExtraSettings();
- p.setProperty("http.connection.timeout", "300000");
- p.setProperty("http.socket.timeout", "300000");
-
- if (outHandlers == null) {
- outHandlerLst = new HashSet<String>();
- } else {
- outHandlerLst = new HashSet<String>(Arrays.asList(outHandlers));
- }
-
- outHandlerLst.add("de.fzj.unicore.uas.security.ProxyCertOutHandler");
-
- secProperties.setOutHandlerClassNames(outHandlerLst.toArray(new String[outHandlerLst.size()]));
-
- } catch (Exception e) {
- throw new GFacProviderException(e.getMessage(), e);
- }
- }
-
- //FIXME: Get user details
- private String getUserName(JobExecutionContext context) {
-// if (context.getConfigurationData()!= null) {
-// return context.getConfigurationData().getBasicMetadata().getUserName();
-// } else {
- return "";
-// }
- }
-
- protected ActivityStatusType getStatus(FactoryClient fc, EndpointReferenceType activityEpr)
- throws UnknownActivityIdentifierFault {
-
- GetActivityStatusesDocument stats = GetActivityStatusesDocument.Factory.newInstance();
-
- stats.addNewGetActivityStatuses().setActivityIdentifierArray(new EndpointReferenceType[] { activityEpr });
-
- GetActivityStatusesResponseDocument resDoc = fc.getActivityStatuses(stats);
-
- ActivityStatusType activityStatus = resDoc.getGetActivityStatusesResponse().getResponseArray()[0]
- .getActivityStatus();
- return activityStatus;
- }
-
- protected String formatStatusMessage(String activityUrl, String status) {
- return String.format("Activity %s is %s.\n", activityUrl, status);
- }
-
- protected String subStatusAsString(ActivityStatusType statusType) {
-
- StringBuffer sb = new StringBuffer();
-
- sb.append(statusType.getState().toString());
-
- XmlCursor acursor = statusType.newCursor();
- if (acursor.toFirstChild()) {
- do {
- if (acursor.getName().getNamespaceURI().equals("http://schemas.ogf.org/hpcp/2007/01/fs")) {
- sb.append(":");
- sb.append(acursor.getName().getLocalPart());
- }
- } while (acursor.toNextSibling());
- acursor.dispose();
- return sb.toString();
- } else {
- acursor.dispose();
- return sb.toString();
- }
-
- }
-
- public void initProperties(Map<String, String> properties) throws GFacProviderException, GFacException {
-
- }
-
- protected KeyAndCertCredential generateShortLivedCertificate(String userDN, String caCertPath, String caKeyPath,
- String caPwd) throws Exception {
- final long CredentialGoodFromOffset = 1000L * 60L * 15L; // 15 minutes
- // ago
-
- final long startTime = System.currentTimeMillis() - CredentialGoodFromOffset;
- final long endTime = startTime + 30 * 3600 * 1000;
-
- String keyLengthProp = "1024";
- int keyLength = Integer.parseInt(keyLengthProp);
- String signatureAlgorithm = "SHA1withRSA";
-
- KeyAndCertCredential caCred = getCACredential(caCertPath, caKeyPath, caPwd);
-
- KeyPairGenerator kpg = KeyPairGenerator.getInstance(caCred.getKey().getAlgorithm());
- kpg.initialize(keyLength);
- KeyPair pair = kpg.generateKeyPair();
-
- X500Principal subjectDN = new X500Principal(userDN);
- Random rand = new Random();
-
- SubjectPublicKeyInfo publicKeyInfo;
- try {
- publicKeyInfo = SubjectPublicKeyInfo.getInstance(new ASN1InputStream(pair.getPublic().getEncoded())
- .readObject());
- } catch (IOException e) {
- throw new InvalidKeyException("Can not parse the public key"
- + "being included in the short lived certificate", e);
- }
-
- X500Name issuerX500Name = CertificateHelpers.toX500Name(caCred.getCertificate().getSubjectX500Principal());
-
- X500Name subjectX500Name = CertificateHelpers.toX500Name(subjectDN);
-
- X509v3CertificateBuilder certBuilder = new X509v3CertificateBuilder(issuerX500Name, new BigInteger(20, rand),
- new Date(startTime), new Date(endTime), subjectX500Name, publicKeyInfo);
-
- AlgorithmIdentifier sigAlgId = X509v3CertificateBuilder.extractAlgorithmId(caCred.getCertificate());
-
- X509Certificate certificate = certBuilder.build(caCred.getKey(), sigAlgId, signatureAlgorithm, null, null);
-
- certificate.checkValidity(new Date());
- certificate.verify(caCred.getCertificate().getPublicKey());
- KeyAndCertCredential result = new KeyAndCertCredential(pair.getPrivate(), new X509Certificate[] { certificate,
- caCred.getCertificate() });
-
- return result;
- }
-
- private KeyAndCertCredential getCACredential(String caCertPath, String caKeyPath, String password) throws Exception {
- InputStream isKey = new FileInputStream(caKeyPath);
- PrivateKey pk = CertificateUtils.loadPrivateKey(isKey, Encoding.PEM, password.toCharArray());
-
- InputStream isCert = new FileInputStream(caCertPath);
- X509Certificate caCert = CertificateUtils.loadCertificate(isCert, Encoding.PEM);
-
- if (isKey != null)
- isKey.close();
- if (isCert != null)
- isCert.close();
-
- return new KeyAndCertCredential(pk, new X509Certificate[] { caCert });
- }
-
- private String getCNFromUserDN(String userDN) {
- return X500NameUtils.getAttributeValues(userDN, BCStyle.CN)[0];
-
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/ApplicationProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/ApplicationProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/ApplicationProcessor.java
deleted file mode 100644
index 76e9818..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/ApplicationProcessor.java
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.utils;
-
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.schemas.gfac.ExtendedKeyValueType;
-import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
-import org.apache.airavata.schemas.gfac.JobTypeType;
-import org.apache.airavata.schemas.gfac.NameValuePairType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.ApplicationType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.EnvironmentType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.FileNameType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.UserNameType;
-import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.NumberOfProcessesType;
-import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.ProcessesPerHostType;
-import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.ThreadsPerProcessType;
-
-import java.io.File;
-
-
-public class ApplicationProcessor {
-
- public static void generateJobSpecificAppElements(JobDefinitionType value, JobExecutionContext context){
-
- String userName = getUserNameFromContext(context);
- if (userName.equalsIgnoreCase("admin")){
- userName = "CN=zdv575, O=Ultrascan Gateway, C=DE";
- }
-
- HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
- .getApplicationContext().getApplicationDeploymentDescription()
- .getType();
-
- createGenericApplication(value, appDepType);
-
- if (appDepType.getApplicationEnvironmentArray().length > 0) {
- createApplicationEnvironment(value,
- appDepType.getApplicationEnvironmentArray(), appDepType);
- }
-
-
- if (appDepType.getExecutableLocation() != null) {
- FileNameType fNameType = FileNameType.Factory.newInstance();
- fNameType.setStringValue(appDepType.getExecutableLocation());
- if(isParallelJob(appDepType)) {
- JSDLUtils.getOrCreateSPMDApplication(value).setExecutable(fNameType);
- JSDLUtils.getSPMDApplication(value).setSPMDVariation(getSPMDVariation(appDepType));
-
- if(getValueFromMap(appDepType, JSDLUtils.NUMBEROFPROCESSES)!=null){
- NumberOfProcessesType num = NumberOfProcessesType.Factory.newInstance();
- num.setStringValue(getValueFromMap(appDepType, JSDLUtils.NUMBEROFPROCESSES));
- JSDLUtils.getSPMDApplication(value).setNumberOfProcesses(num);
- }
-
- if(getValueFromMap(appDepType, JSDLUtils.PROCESSESPERHOST)!=null){
- ProcessesPerHostType pph = ProcessesPerHostType.Factory.newInstance();
- pph.setStringValue(getValueFromMap(appDepType, JSDLUtils.PROCESSESPERHOST));
- JSDLUtils.getSPMDApplication(value).setProcessesPerHost(pph);
- }
-
- if(getValueFromMap(appDepType, JSDLUtils.THREADSPERHOST)!=null){
- ThreadsPerProcessType tpp = ThreadsPerProcessType.Factory.newInstance();
- tpp.setStringValue(getValueFromMap(appDepType, JSDLUtils.THREADSPERHOST));
- JSDLUtils.getSPMDApplication(value).setThreadsPerProcess(tpp);
-
- }
-
- if(userName != null) {
- UserNameType userNameType = UserNameType.Factory.newInstance();
- userNameType.setStringValue(userName);
- JSDLUtils.getSPMDApplication(value).setUserName(userNameType);
- }
- }
- else {
- JSDLUtils.getOrCreatePOSIXApplication(value).setExecutable(fNameType);
- if(userName != null) {
- UserNameType userNameType = UserNameType.Factory.newInstance();
- userNameType.setStringValue(userName);
- JSDLUtils.getOrCreatePOSIXApplication(value).setUserName(userNameType);
- }
- }
- }
-
-
- String stdout = (appDepType.getStandardOutput() != null) ? new File(appDepType.getStandardOutput()).getName(): "stdout";
- ApplicationProcessor.setApplicationStdOut(value, appDepType, stdout);
-
-
- String stderr = (appDepType.getStandardError() != null) ? new File(appDepType.getStandardError()).getName() : "stderr";
- ApplicationProcessor.setApplicationStdErr(value, appDepType, stderr);
-
- }
-
- public static String getUserNameFromContext(JobExecutionContext jobContext) {
- if(jobContext.getTaskData() == null)
- return null;
- //FIXME: Discuss to get user and change this
- return "admin";
- }
- public static boolean isParallelJob(HpcApplicationDeploymentType appDepType) {
-
- boolean isParallel = false;
-
- if (appDepType.getJobType() != null) {
- // TODO set data output directory
- int status = appDepType.getJobType().intValue();
-
- switch (status) {
- // TODO: this check should be done outside this class
- case JobTypeType.INT_MPI:
- case JobTypeType.INT_OPEN_MP:
- isParallel = true;
- break;
-
- case JobTypeType.INT_SERIAL:
- case JobTypeType.INT_SINGLE:
- isParallel = false;
- break;
-
- default:
- isParallel = false;
- break;
- }
- }
- return isParallel;
- }
-
-
- public static void createApplicationEnvironment(JobDefinitionType value, NameValuePairType[] nameValuePairs, HpcApplicationDeploymentType appDepType) {
-
- if(isParallelJob(appDepType)) {
- for (NameValuePairType nv : nameValuePairs) {
- EnvironmentType envType = JSDLUtils.getOrCreateSPMDApplication(value).addNewEnvironment();
- envType.setName(nv.getName());
- envType.setStringValue(nv.getValue());
- }
- }
- else {
- for (NameValuePairType nv : nameValuePairs) {
- EnvironmentType envType = JSDLUtils.getOrCreatePOSIXApplication(value).addNewEnvironment();
- envType.setName(nv.getName());
- envType.setStringValue(nv.getValue());
- }
- }
-
- }
-
-
- public static String getSPMDVariation (HpcApplicationDeploymentType appDepType) {
-
- String variation = null;
-
- if (appDepType.getJobType() != null) {
- // TODO set data output directory
- int status = appDepType.getJobType().intValue();
-
- switch (status) {
- // TODO: this check should be done outside this class
- case JobTypeType.INT_MPI:
- variation = SPMDVariations.MPI.value();
- break;
-
- case JobTypeType.INT_OPEN_MP:
- variation = SPMDVariations.OpenMPI.value();
- break;
-
- }
- }
- return variation;
- }
-
-
- public static void addApplicationArgument(JobDefinitionType value, HpcApplicationDeploymentType appDepType, String stringPrm) {
- if(isParallelJob(appDepType))
- JSDLUtils.getOrCreateSPMDApplication(value)
- .addNewArgument().setStringValue(stringPrm);
- else
- JSDLUtils.getOrCreatePOSIXApplication(value)
- .addNewArgument().setStringValue(stringPrm);
-
- }
-
- public static void setApplicationStdErr(JobDefinitionType value, HpcApplicationDeploymentType appDepType, String stderr) {
- FileNameType fName = FileNameType.Factory.newInstance();
- fName.setStringValue(stderr);
- if (isParallelJob(appDepType))
- JSDLUtils.getOrCreateSPMDApplication(value).setError(fName);
- else
- JSDLUtils.getOrCreatePOSIXApplication(value).setError(fName);
- }
-
- public static void setApplicationStdOut(JobDefinitionType value, HpcApplicationDeploymentType appDepType, String stderr) {
- FileNameType fName = FileNameType.Factory.newInstance();
- fName.setStringValue(stderr);
- if (isParallelJob(appDepType))
- JSDLUtils.getOrCreateSPMDApplication(value).setOutput(fName);
- else
- JSDLUtils.getOrCreatePOSIXApplication(value).setOutput(fName);
- }
-
- public static String getApplicationStdOut(JobDefinitionType value, HpcApplicationDeploymentType appDepType) throws RuntimeException {
- if (isParallelJob(appDepType)) return JSDLUtils.getOrCreateSPMDApplication(value).getOutput().getStringValue();
- else return JSDLUtils.getOrCreatePOSIXApplication(value).getOutput().getStringValue();
- }
-
- public static String getApplicationStdErr(JobDefinitionType value, HpcApplicationDeploymentType appDepType) throws RuntimeException {
- if (isParallelJob(appDepType)) return JSDLUtils.getOrCreateSPMDApplication(value).getError().getStringValue();
- else return JSDLUtils.getOrCreatePOSIXApplication(value).getError().getStringValue();
- }
-
- public static void createGenericApplication(JobDefinitionType value, HpcApplicationDeploymentType appDepType) {
- if (appDepType.getApplicationName() != null) {
- ApplicationType appType = JSDLUtils.getOrCreateApplication(value);
- String appName = appDepType.getApplicationName()
- .getStringValue();
- appType.setApplicationName(appName);
- JSDLUtils.getOrCreateJobIdentification(value).setJobName(appName);
- }
- }
-
-
- public static String getValueFromMap(HpcApplicationDeploymentType appDepType, String name) {
- ExtendedKeyValueType[] extended = appDepType.getKeyValuePairsArray();
- for(ExtendedKeyValueType e: extended) {
- if(e.getName().equalsIgnoreCase(name)) {
- return e.getStringValue();
- }
- }
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/DataStagingProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/DataStagingProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/DataStagingProcessor.java
deleted file mode 100644
index adf6836..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/DataStagingProcessor.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.utils;
-
-import java.io.File;
-import java.net.URI;
-import java.util.Map;
-
-import org.apache.airavata.commons.gfac.type.ActualParameter;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
-import org.apache.airavata.schemas.gfac.StringArrayType;
-import org.apache.airavata.schemas.gfac.StringParameterType;
-import org.apache.airavata.schemas.gfac.URIArrayType;
-import org.apache.airavata.schemas.gfac.URIParameterType;
-import org.apache.airavata.schemas.gfac.UnicoreHostType;
-import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
-
-public class DataStagingProcessor {
-
- public static void generateDataStagingElements(JobDefinitionType value, JobExecutionContext context) throws Exception{
-
- HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
- .getApplicationContext().getApplicationDeploymentDescription()
- .getType();
-
-
- String gridftpEndpoint = ((UnicoreHostType) context.getApplicationContext().getHostDescription().getType())
- .getGridFTPEndPointArray()[0];
-
-
- if (context.getInMessageContext().getParameters().size() > 0) {
- buildDataStagingFromInputContext(context, value, gridftpEndpoint, appDepType);
- }
-
- if (context.getOutMessageContext().getParameters().size() > 0) {
- buildFromOutputContext(context, value, gridftpEndpoint, appDepType);
- }
-
- createStdOutURIs(value, appDepType, gridftpEndpoint, isUnicoreEndpoint(context));
-
- }
-
- private static void createInURIElement(JobDefinitionType value,
- String endpoint, String inputDir, ActualParameter inParam)
- throws Exception {
-
- String uri = ((URIParameterType) inParam.getType()).getValue();
- String fileName = new File(uri).getName();
- if (uri.startsWith("file")) {
- URI gridFTPInputDirURI = URIUtils.createGsiftpURI(endpoint,
- inputDir);
- String filePath = gridFTPInputDirURI.toString() + File.separator
- + fileName;
- JSDLUtils
- .addDataStagingSourceElement(value, filePath, null, fileName);
- } else if (uri.startsWith("gsiftp") || uri.startsWith("http")
- || uri.startsWith("rns")) {
- // no need to stage-in those files to the input
- // directory
- JSDLUtils.addDataStagingSourceElement(value, uri, null, fileName);
- }
-
- }
-
- private static void createStdOutURIs(JobDefinitionType value,
- HpcApplicationDeploymentType appDepType, String endpoint,
- boolean isUnicore) throws Exception {
-
- URI remoteOutputDir = URIUtils.createGsiftpURI(endpoint,
- appDepType.getOutputDataDirectory());
-
- String stdout = ApplicationProcessor.getApplicationStdOut(value, appDepType);
-
- String stderr = ApplicationProcessor.getApplicationStdErr(value, appDepType);
-
- String stdoutFileName = (stdout == null || stdout.equals("")) ? "stdout"
- : stdout;
- String stdoutURI = GFacUtils.createGsiftpURIAsString(
- remoteOutputDir.toString(), stdoutFileName);
- JSDLUtils.addDataStagingTargetElement(value, null, stdoutFileName,
- stdoutURI);
-
- String stderrFileName = (stdout == null || stderr.equals("")) ? "stderr"
- : stderr;
- String stderrURI = GFacUtils.createGsiftpURIAsString(
- remoteOutputDir.toString(), stderrFileName);
- JSDLUtils.addDataStagingTargetElement(value, null, stderrFileName,
- stderrURI);
-
- if(isUnicore) {
- String scriptExitCodeFName = "UNICORE_SCRIPT_EXIT_CODE";
- String scriptExitCode = GFacUtils.createGsiftpURIAsString(
- remoteOutputDir.toString(), scriptExitCodeFName);
- JSDLUtils.addDataStagingTargetElement(value, null,
- scriptExitCodeFName, scriptExitCode.toString());
- }
-
- }
-
-
- private static void createOutStringElements(JobDefinitionType value,
- HpcApplicationDeploymentType appDeptype, String endpoint, String prmValue) throws Exception {
-
- if(prmValue == null || "".equals(prmValue)) return;
-
-
- String outputUri = GFacUtils.createGsiftpURIAsString(endpoint, appDeptype.getOutputDataDirectory());
-
- URI finalOutputUri = URIUtils.createGsiftpURI(outputUri, prmValue);
- JSDLUtils.addDataStagingTargetElement(value, null, prmValue, finalOutputUri.toString());
- }
-
-
- private static void createOutURIElement(JobDefinitionType value,
- String prmValue) throws Exception {
- String fileName = new File(prmValue.toString()).getName();
- JSDLUtils.addDataStagingTargetElement(value, null, fileName, prmValue);
- }
-
-
- private static JobDefinitionType buildFromOutputContext(JobExecutionContext context,
- JobDefinitionType value, String gridftpEndpoint,
- HpcApplicationDeploymentType appDepType) throws Exception {
-
- Map<String, Object> outputParams = context.getOutMessageContext()
- .getParameters();
-
- for (String paramKey : outputParams.keySet()) {
-
- ActualParameter outParam = (ActualParameter) outputParams
- .get(paramKey);
-
- // if single urls then convert each url into jsdl source
- // elements,
- // that are formed by concat of gridftpurl+inputdir+filename
-
- String paramDataType = outParam.getType().getType().toString();
-
- if ("URI".equals(paramDataType)) {
- String uriPrm = ((URIParameterType) outParam.getType())
- .getValue();
- createOutURIElement(value, uriPrm);
- }
-
- // string params are converted into the job arguments
-
- else if (("URIArray").equals(paramDataType)) {
- String[] uriArray = ((URIArrayType) outParam.getType())
- .getValueArray();
- for (String u : uriArray) {
-
- createOutURIElement(value, u);
- }
-
- }
- else if ("String".equals(paramDataType)) {
- String stringPrm = ((StringParameterType) outParam
- .getType()).getValue();
- createOutStringElements(value, appDepType, gridftpEndpoint, stringPrm);
- }
-
- else if ("StringArray".equals(paramDataType)) {
- String[] valueArray = ((StringArrayType) outParam.getType())
- .getValueArray();
- for (String v : valueArray) {
- createOutStringElements(value, appDepType, gridftpEndpoint, v);
- }
- }
- }
-
- return value;
- }
-
-
- private static void buildDataStagingFromInputContext(JobExecutionContext context, JobDefinitionType value, String gridftpEndpoint, HpcApplicationDeploymentType appDepType)
- throws Exception {
-
- // TODO set data directory
- Map<String, Object> inputParams = context.getInMessageContext()
- .getParameters();
-
- for (String paramKey : inputParams.keySet()) {
-
- ActualParameter inParam = (ActualParameter) inputParams
- .get(paramKey);
-
- // if single urls then convert each url into jsdl source
- // elements,
- // that are formed by concat of gridftpurl+inputdir+filename
-
- String paramDataType = inParam.getType().getType().toString();
-
- if ("URI".equals(paramDataType)) {
- createInURIElement(value, gridftpEndpoint,
- appDepType.getInputDataDirectory(), inParam);
- }
-
- // string params are converted into the job arguments
-
- else if ("String".equals(paramDataType)) {
- String stringPrm = ((StringParameterType) inParam.getType())
- .getValue();
- ApplicationProcessor.addApplicationArgument(value, appDepType, stringPrm);
- }
- }
-
- }
-
-
- public static boolean isUnicoreEndpoint(JobExecutionContext context) {
- return ( (context.getApplicationContext().getHostDescription().getType() instanceof UnicoreHostType)?true:false );
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/DataTransferrer.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/DataTransferrer.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/DataTransferrer.java
deleted file mode 100644
index 6f844c7..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/DataTransferrer.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.utils;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.airavata.commons.gfac.type.ActualParameter;
-import org.apache.airavata.commons.gfac.type.ApplicationDescription;
-import org.apache.airavata.gfac.Constants;
-import org.apache.airavata.gfac.context.JobExecutionContext;
-import org.apache.airavata.gfac.provider.GFacProviderException;
-import org.apache.airavata.model.workspace.experiment.TaskDetails;
-import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
-import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
-import org.apache.airavata.schemas.gfac.StringArrayType;
-import org.apache.airavata.schemas.gfac.StringParameterType;
-import org.apache.airavata.schemas.gfac.URIParameterType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import de.fzj.unicore.uas.client.StorageClient;
-
-
-public class DataTransferrer {
- protected final Logger log = LoggerFactory.getLogger(this.getClass());
-
- private JobExecutionContext jobContext;
-
- private StorageClient storageClient;
-
- public DataTransferrer(JobExecutionContext jobContext, StorageClient storageClient) {
- this.jobContext = jobContext;
- this.storageClient = storageClient;
- }
-
-
- public void uploadLocalFiles() throws GFacProviderException {
- Map<String, Object> inputParams = jobContext.getInMessageContext()
- .getParameters();
- for (String paramKey : inputParams.keySet()) {
- ActualParameter inParam = (ActualParameter) inputParams
- .get(paramKey);
- String paramDataType = inParam.getType().getType().toString();
- if("URI".equals(paramDataType)) {
- String uri = ((URIParameterType) inParam.getType()).getValue();
- String fileName = new File(uri).getName();
- if (uri.startsWith("file")) {
- try {
- String uriWithoutProtocol = uri.substring(
- uri.lastIndexOf("://") + 1, uri.length());
- FileUploader fileUploader = new FileUploader(
- uriWithoutProtocol, "input/" + fileName,
- Mode.overwrite);
- fileUploader.perform(storageClient);
- } catch (FileNotFoundException e3) {
- throw new GFacProviderException(
- "Error while staging-in, local file "+fileName+" not found", e3);
- } catch (Exception e) {
- throw new GFacProviderException("Cannot upload files", e);
-
- }
-
- }
- }
- }
-
- }
-
- /**
- * This method will download all the remote files specified according to the output
- * context of a job.
- * */
- public void downloadRemoteFiles() throws GFacProviderException {
-
- String downloadLocation = getDownloadLocation();
-
- File file = new File(downloadLocation);
- if(!file.exists()){
- file.mkdirs();
- }
-
- Map<String, ActualParameter> stringMap = new HashMap<String, ActualParameter>();
-
- Map<String, Object> outputParams = jobContext.getOutMessageContext()
- .getParameters();
-
- for (String paramKey : outputParams.keySet()) {
-
- ActualParameter outParam = (ActualParameter) outputParams
- .get(paramKey);
-
- // if single urls then convert each url into jsdl source
- // elements,
- // that are formed by concat of gridftpurl+inputdir+filename
-
- String paramDataType = outParam.getType().getType().toString();
-
- if ("String".equals(paramDataType)) {
- String stringPrm = ((StringParameterType) outParam
- .getType()).getValue();
- String localFileName = null;
- //TODO: why analysis.tar? it wont scale to gateways..
- if(stringPrm == null || stringPrm.isEmpty()){
- localFileName = "analysis-results.tar";
- }else{
- localFileName = stringPrm.substring(stringPrm.lastIndexOf("/")+1);
- }
- String outputLocation = downloadLocation+File.separator+localFileName;
- FileDownloader fileDownloader = new FileDownloader("output/"+stringPrm,outputLocation, Mode.overwrite);
- try {
- fileDownloader.perform(storageClient);
- ((StringParameterType) outParam.getType()).setValue(outputLocation);
- stringMap.put(paramKey, outParam);
- } catch (Exception e) {
- throw new GFacProviderException(e.getLocalizedMessage(),e);
- }
- }
-
- else if ("StringArray".equals(paramDataType)) {
- String[] valueArray = ((StringArrayType) outParam.getType())
- .getValueArray();
- for (String v : valueArray) {
- String localFileName = v.substring(v.lastIndexOf("/")+1);;
- String outputLocation = downloadLocation+File.separator+localFileName;
- FileDownloader fileDownloader = new FileDownloader("output/"+v,outputLocation, Mode.overwrite);
- try {
- fileDownloader.perform(storageClient);
- ((StringParameterType) outParam.getType()).setValue(outputLocation);
- stringMap.put(paramKey, outParam);
- } catch (Exception e) {
- throw new GFacProviderException(e.getLocalizedMessage(),e);
- }
- }
- }
- }
- if (stringMap == null || stringMap.isEmpty()) {
- throw new GFacProviderException("Empty Output returned from the Application, Double check the application" +
- "and ApplicationDescriptor output Parameter Names");
- }
-
- downloadStdOuts();
- }
-
-
- public void downloadStdOuts() throws GFacProviderException{
- String downloadLocation = getDownloadLocation();
- File file = new File(downloadLocation);
- if(!file.exists()){
- file.mkdirs();
- }
-
- HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) jobContext
- .getApplicationContext().getApplicationDeploymentDescription()
- .getType();
-
- String stdout = appDepType.getStandardOutput();
- String stderr = appDepType.getStandardError();
- if(stdout != null) {
- stdout = stdout.substring(stdout.lastIndexOf('/')+1);
- }
-
- if(stderr != null) {
- stderr = stderr.substring(stderr.lastIndexOf('/')+1);
- }
-
- String stdoutFileName = (stdout == null || stdout.equals("")) ? "stdout"
- : stdout;
- String stderrFileName = (stdout == null || stderr.equals("")) ? "stderr"
- : stderr;
-
- ApplicationDescription application = jobContext.getApplicationContext().getApplicationDeploymentDescription();
- ApplicationDeploymentDescriptionType appDesc = application.getType();
-
- String stdoutLocation = downloadLocation+File.separator+stdoutFileName;
- FileDownloader f1 = new FileDownloader("output/"+stdoutFileName,stdoutLocation, Mode.overwrite);
- try {
- f1.perform(storageClient);
- String stdoutput = readFile(stdoutLocation);
- appDesc.setStandardOutput(stdoutput);
- } catch (Exception e) {
- throw new GFacProviderException(e.getLocalizedMessage(),e);
- }
- String stderrLocation = downloadLocation+File.separator+stderrFileName;
- FileDownloader f2 = new FileDownloader("output/"+stderrFileName,stderrLocation, Mode.overwrite);
- try {
- f2.perform(storageClient);
- String stderror = readFile(stderrLocation);
- appDesc.setStandardError(stderror);
- } catch (Exception e) {
- throw new GFacProviderException(e.getLocalizedMessage(),e);
- }
- }
-
- private String readFile(String localFile) throws IOException {
- BufferedReader instream = new BufferedReader(new FileReader(localFile));
- StringBuffer buff = new StringBuffer();
- String temp = null;
- while ((temp = instream.readLine()) != null) {
- buff.append(temp);
- buff.append(Constants.NEWLINE);
- }
-
- log.info("finish read file:" + localFile);
-
- return buff.toString();
- }
-
- private String getDownloadLocation() {
- TaskDetails taskData = jobContext.getTaskData();
- if (taskData != null && taskData.getAdvancedOutputDataHandling() != null) {
- String outputDataDirectory = taskData.getAdvancedOutputDataHandling().getOutputDataDir();
- return outputDataDirectory;
- }
- return null;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/FileDownloader.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/FileDownloader.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/FileDownloader.java
deleted file mode 100644
index 1eabb12..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/FileDownloader.java
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.utils;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Map;
-
-import org.unigrids.services.atomic.types.GridFileType;
-import org.unigrids.services.atomic.types.ProtocolType;
-
-import de.fzj.unicore.uas.client.FileTransferClient;
-import de.fzj.unicore.uas.client.StorageClient;
-import de.fzj.unicore.uas.client.UFTPConstants;
-import de.fzj.unicore.uas.client.UFTPFileTransferClient;
-import de.fzj.unicore.uas.fts.FiletransferOptions.IMonitorable;
-import de.fzj.unicore.uas.fts.FiletransferOptions.SupportsPartialRead;
-
-/**
- * helper that exports remote files from a UNICORE Storage
- * to the local client machine.<br/>
- * Simple wildcards ("*" and "?") and download of
- * directories are supported.
- *
- * TODO this should be refactored so the single-file download logic
- * is separated from the wildcard/directory/provided outputStream logic
- *
- * @author schuller
- */
-public class FileDownloader extends FileTransferBase{
-
- private boolean showProgress=true;
-
- private boolean forceFileOnly=false;
-
- private OutputStream targetStream=null;
-
- public FileDownloader(String from, String to, Mode mode){
- this(from,to,mode,true);
- }
-
- public FileDownloader(String from, String to, Mode mode, boolean failOnError){
- this.to=to;
- this.from=from;
- this.mode=mode;
- this.failOnError=failOnError;
- }
-
- public void perform(StorageClient sms)throws Exception{
- boolean isWildcard=hasWildCards(from);
- boolean isDirectory=false;
- GridFileType gridSource=null;
- if(isWildcard){
- performWildCardExport(sms);
- }
- else {
- //check if source is a directory
- gridSource=sms.listProperties(from);
- isDirectory=gridSource.getIsDirectory();
- if(isDirectory){
- if(forceFileOnly){
- throw new IOException("Source is a directory");
- }
- performDirectoryExport(gridSource, new File(to), sms);
- }
- else{
- download(gridSource,new File(to),sms);
- }
- }
- }
-
- protected void performDirectoryExport(GridFileType directory, File targetDirectory, StorageClient sms)throws Exception{
- if(!targetDirectory.exists()|| !targetDirectory.canWrite()){
- throw new IOException("Target directory <"+to+"> does not exist or is not writable!");
- }
- if(!targetDirectory.isDirectory()){
- throw new IOException("Target <"+to+"> is not a directory!");
- }
- GridFileType[]gridFiles=sms.listDirectory(directory.getPath());
- for(GridFileType file: gridFiles){
- if(file.getIsDirectory()){
- if(!recurse) {
- System.out.println("Skipping directory "+file.getPath());
- continue;
- }
- else{
- File newTargetDirectory=new File(targetDirectory,getName(file.getPath()));
- boolean success=newTargetDirectory.mkdirs();
- if(!success)throw new IOException("Can create directory: "+newTargetDirectory.getAbsolutePath());
- performDirectoryExport(file, newTargetDirectory, sms);
- continue;
- }
- }
- download(file, new File(targetDirectory,getName(file.getPath())), sms);
- }
- }
-
- protected void performWildCardExport(StorageClient sms)throws Exception{
- String dir=getDir(from);
- if(dir==null)dir="/";
- GridFileType[] files=sms.find(dir, false, from, false, null, null);
- File targetDir=targetStream==null?new File(to):null;
- if(targetStream==null){
- if(!targetDir.isDirectory())throw new IOException("Target is not a directory.");
- }
- for(GridFileType f: files){
- download(f, targetDir, sms);
- }
- }
-
- private String getDir(String path){
- return new File(path).getParent();
- }
-
- private String getName(String path){
- return new File(path).getName();
- }
-
- /**
- * download a single regular file
- *
- * @param source - grid file descriptor
- * @param localFile - local file or directory to write to
- * @param sms
- * @throws Exception
- */
- private void download(GridFileType source, File localFile, StorageClient sms)throws Exception{
- if(source==null || source.getIsDirectory()){
- throw new IllegalStateException("Source="+source);
- }
-
- OutputStream os=targetStream!=null?targetStream:null;
- FileTransferClient ftc=null;
- try{
- String path=source.getPath();
- if(targetStream==null){
- if(localFile.isDirectory()){
- localFile=new File(localFile,getName(source.getPath()));
- }
- if(mode.equals(Mode.nooverwrite) && localFile.exists()){
- System.out.println("File exists and creation mode was set to 'nooverwrite'.");
- return;
- }
- System.out.println("Downloading remote file '"+sms.getUrl()+"#/"+path+"' -> "+localFile.getAbsolutePath());
- os=new FileOutputStream(localFile.getAbsolutePath(), mode.equals(Mode.append));
- }
-
- chosenProtocol=sms.findSupportedProtocol(preferredProtocols.toArray(new ProtocolType.Enum[preferredProtocols.size()]));
- Map<String,String>extraParameters=makeExtraParameters(chosenProtocol);
- ftc=sms.getExport(path,extraParameters,chosenProtocol);
- configure(ftc, extraParameters);
- System.out.println("DEB:File transfer URL : "+ftc.getUrl());
-// ProgressBar p=null;
- if(ftc instanceof IMonitorable && showProgress){
- long size=ftc.getSourceFileSize();
- if(isRange()){
- size=getRangeSize();
- }
-// p=new ProgressBar(localFile.getName(),size,msg);
-// ((IMonitorable) ftc).setProgressListener(p);
- }
- long startTime=System.currentTimeMillis();
- if(isRange()){
- if(!(ftc instanceof SupportsPartialRead)){
- throw new Exception("Byte range is defined but protocol does not allow " +
- "partial read! Please choose a different protocol!");
- }
- System.out.println("Byte range: "+startByte+" - "+(getRangeSize()>0?endByte:""));
- SupportsPartialRead pReader=(SupportsPartialRead)ftc;
- pReader.readPartial(startByte, endByte-startByte+1, os);
- }
- else{
- ftc.readAllData(os);
- }
-// if(p!=null){
-// p.finish();
-// }
- if(timing){
- long duration=System.currentTimeMillis()-startTime;
- double rate=(double)localFile.length()/(double)duration;
- System.out.println("Rate: " +rate+ " kB/sec.");
- }
- if(targetStream==null)copyProperties(source, localFile);
- }
- finally{
- try{
- if(targetStream==null && os!=null){
- os.close();
- }
- }catch(Exception ignored){}
- if(ftc!=null){
- try{
- ftc.destroy();
- }catch(Exception e1){
-// System.out.println("Could not destroy the filetransfer client",e1);
- }
- }
- }
- }
-
- /**
- * if possible, copy the remote executable flag to the local file
- * @throws Exception
- */
- private void copyProperties(GridFileType source, File localFile)throws Exception{
- try{
- localFile.setExecutable(source.getPermissions().getExecutable());
- }
- catch(Exception ex){
- //TODO: logging
-// ("Can't set 'executable' flag for "+localFile.getName(), ex);
- }
- }
-
- private void configure(FileTransferClient ftc, Map<String,String>params){
- if(ftc instanceof UFTPFileTransferClient){
- UFTPFileTransferClient u=(UFTPFileTransferClient)ftc;
- String secret=params.get(UFTPConstants.PARAM_SECRET);
- u.setSecret(secret);
- }
- }
-
- public void setShowProgress(boolean showProgress) {
- this.showProgress = showProgress;
- }
-
- public void setForceFileOnly(boolean forceFileOnly) {
- this.forceFileOnly = forceFileOnly;
- }
-
- public void setTargetStream(OutputStream targetStream) {
- this.targetStream = targetStream;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/FileTransferBase.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/FileTransferBase.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/FileTransferBase.java
deleted file mode 100644
index 114ee57..0000000
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/utils/FileTransferBase.java
+++ /dev/null
@@ -1,227 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.airavata.gfac.utils;
-
-import java.io.File;
-import java.io.FilenameFilter;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.regex.Pattern;
-
-import org.unigrids.services.atomic.types.GridFileType;
-import org.unigrids.services.atomic.types.ProtocolType;
-
-import de.fzj.unicore.uas.client.StorageClient;
-import de.fzj.unicore.uas.util.PropertyHelper;
-public class FileTransferBase {
-
- protected Properties extraParameterSource;
-
- protected boolean timing=false;
-
- protected boolean recurse=false;
-
- protected String from;
-
- protected String to;
-
- //index of first byte to download
- protected Long startByte;
-
- //index of last byte to download
- protected Long endByte;
-
- /**
- * the creation mode
- */
- protected Mode mode;
-
- /**
- * whether the job processing should fail if an error occurs
- */
- protected boolean failOnError;
-
- protected List<ProtocolType.Enum> preferredProtocols=new ArrayList<ProtocolType.Enum>();
-
- public FileTransferBase(){
- preferredProtocols.add(ProtocolType.BFT);
- }
-
- protected Map<String,String>makeExtraParameters(ProtocolType.Enum protocol){
- Map<String, String> res;
- if(extraParameterSource==null){
- res=new HashMap<String, String>();
- }
- else{
- String p=String.valueOf(protocol);
- PropertyHelper ph=new PropertyHelper(extraParameterSource, new String[]{p,p.toLowerCase()});
- res= ph.getFilteredMap();
- }
- if(res.size()>0){
- // TODO: change it to logger
- System.out.println("Have "+res.size()+" extra parameters for protocol "+protocol);
- }
- return res;
- }
-
-
- public String getTo() {
- return to;
- }
-
- public String getFrom() {
- return from;
- }
-
- public void setTo(String to) {
- this.to = to;
- }
-
- public void setFrom(String from) {
- this.from = from;
- }
-
- public Mode getMode() {
- return mode;
- }
-
- public boolean isFailOnError() {
- return failOnError;
- }
-
- public boolean isTiming() {
- return timing;
- }
-
- public void setTiming(boolean timing) {
- this.timing = timing;
- }
-
- public void setFailOnError(boolean failOnError) {
- this.failOnError = failOnError;
- }
-
- public List<ProtocolType.Enum> getPreferredProtocols() {
- return preferredProtocols;
- }
-
- public void setPreferredProtocols(List<ProtocolType.Enum> preferredProtocols) {
- this.preferredProtocols = preferredProtocols;
- }
-
- public void setExtraParameterSource(Properties properties){
- this.extraParameterSource=properties;
- }
-
- public void setRecurse(boolean recurse) {
- this.recurse = recurse;
- }
- /**
- * check if the given path denotes a valid remote directory
- * @param remotePath - the path
- * @param sms - the storage
- * @return <code>true</code> if the remote directory exists and is a directory
- */
- protected boolean isValidDirectory(String remotePath, StorageClient sms){
- boolean result=false;
- if(! ("/".equals(remotePath) || ".".equals(remotePath)) ){
- try{
- GridFileType gft=sms.listProperties(remotePath);
- result=gft.getIsDirectory();
- }catch(Exception ex){
- result=false;
- }
- }
- else result=true;
-
- return result;
- }
-
- public File[] resolveWildCards(File original){
- final String name=original.getName();
- if(!hasWildCards(original))return new File[]{original};
- File parent=original.getParentFile();
- if(parent==null)parent=new File(".");
- FilenameFilter filter=new FilenameFilter(){
- Pattern p=createPattern(name);
- public boolean accept(File file, String name){
- return p.matcher(name).matches();
- }
- };
- return parent.listFiles(filter);
- }
-
- protected boolean hasWildCards(File file){
- return hasWildCards(file.getName());
- }
-
- public boolean hasWildCards(String name){
- return name.contains("*") || name.contains("?");
- }
-
- private Pattern createPattern(String nameWithWildcards){
- String regex=nameWithWildcards.replace("?",".").replace("*", ".*");
- return Pattern.compile(regex);
- }
-
- protected ProtocolType.Enum chosenProtocol=null;
-
- public ProtocolType.Enum getChosenProtocol(){
- return chosenProtocol;
- }
-
- public Long getStartByte() {
- return startByte;
- }
-
- public void setStartByte(Long startByte) {
- this.startByte = startByte;
- }
-
- public Long getEndByte() {
- return endByte;
- }
-
- public void setEndByte(Long endByte) {
- this.endByte = endByte;
- }
-
- /**
- * checks if a byte range is defined
- * @return <code>true</code> iff both startByte and endByte are defined
- */
- protected boolean isRange(){
- return startByte!=null && endByte!=null;
- }
-
- /**
- * get the number of bytes in the byte range, or "-1" if the range is open-ended
- * @return
- */
- protected long getRangeSize(){
- if(Long.MAX_VALUE==endByte)return -1;
- return endByte-startByte;
- }
-}
[4/4] git commit: moving gfac-bes classes to new package.
Posted by la...@apache.org.
moving gfac-bes classes to new package.
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/49b6987f
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/49b6987f
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/49b6987f
Branch: refs/heads/master
Commit: 49b6987f36dcde718b07fb22cc8055a7178f5035
Parents: 9bb8c2b
Author: lahiru <la...@apache.org>
Authored: Mon May 5 10:49:13 2014 -0400
Committer: lahiru <la...@apache.org>
Committed: Mon May 5 10:49:13 2014 -0400
----------------------------------------------------------------------
.../airavata/gfac/bes/impl/BESProvider.java | 568 +++++++++++++++++++
.../gfac/bes/security/GSISecurityContext.java | 288 ++++++++++
.../gfac/bes/utils/ApplicationProcessor.java | 252 ++++++++
.../gfac/bes/utils/DataStagingProcessor.java | 236 ++++++++
.../gfac/bes/utils/DataTransferrer.java | 241 ++++++++
.../airavata/gfac/bes/utils/FileDownloader.java | 256 +++++++++
.../gfac/bes/utils/FileTransferBase.java | 227 ++++++++
.../airavata/gfac/bes/utils/FileUploader.java | 245 ++++++++
.../airavata/gfac/bes/utils/JSDLGenerator.java | 103 ++++
.../airavata/gfac/bes/utils/JSDLUtils.java | 540 ++++++++++++++++++
.../apache/airavata/gfac/bes/utils/Mode.java | 45 ++
.../airavata/gfac/bes/utils/OSRequirement.java | 108 ++++
.../apache/airavata/gfac/bes/utils/OSType.java | 124 ++++
.../gfac/bes/utils/ProcessorRequirement.java | 61 ++
.../airavata/gfac/bes/utils/RangeValueType.java | 274 +++++++++
.../gfac/bes/utils/ResourceProcessor.java | 152 +++++
.../airavata/gfac/bes/utils/SPMDProcessor.java | 33 ++
.../airavata/gfac/bes/utils/SPMDVariations.java | 52 ++
.../airavata/gfac/bes/utils/StorageCreator.java | 211 +++++++
.../gfac/bes/utils/UASDataStagingProcessor.java | 225 ++++++++
.../airavata/gfac/bes/utils/URIUtils.java | 119 ++++
.../context/security/GSISecurityContext.java | 288 ----------
.../gfac/provider/impl/BESProvider.java | 568 -------------------
.../gfac/utils/ApplicationProcessor.java | 252 --------
.../gfac/utils/DataStagingProcessor.java | 235 --------
.../airavata/gfac/utils/DataTransferrer.java | 241 --------
.../airavata/gfac/utils/FileDownloader.java | 256 ---------
.../airavata/gfac/utils/FileTransferBase.java | 227 --------
.../airavata/gfac/utils/FileUploader.java | 245 --------
.../airavata/gfac/utils/JSDLGenerator.java | 105 ----
.../apache/airavata/gfac/utils/JSDLUtils.java | 540 ------------------
.../org/apache/airavata/gfac/utils/Mode.java | 45 --
.../airavata/gfac/utils/OSRequirement.java | 108 ----
.../org/apache/airavata/gfac/utils/OSType.java | 124 ----
.../gfac/utils/ProcessorRequirement.java | 61 --
.../airavata/gfac/utils/RangeValueType.java | 274 ---------
.../airavata/gfac/utils/ResourceProcessor.java | 152 -----
.../airavata/gfac/utils/SPMDProcessor.java | 33 --
.../airavata/gfac/utils/SPMDVariations.java | 52 --
.../airavata/gfac/utils/StorageCreator.java | 211 -------
.../gfac/utils/UASDataStagingProcessor.java | 225 --------
.../apache/airavata/gfac/utils/URIUtils.java | 119 ----
.../impl/JSDLGeneratorTestWithMyProxyAuth.java | 32 --
.../src/test/resources/gfac-config.xml | 2 +-
44 files changed, 4361 insertions(+), 4394 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/impl/BESProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/impl/BESProvider.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/impl/BESProvider.java
new file mode 100644
index 0000000..c41632f
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/impl/BESProvider.java
@@ -0,0 +1,568 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.bes.impl;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.math.BigInteger;
+import java.security.InvalidKeyException;
+import java.security.KeyPair;
+import java.security.KeyPairGenerator;
+import java.security.PrivateKey;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Random;
+import java.util.Set;
+
+import javax.security.auth.x500.X500Principal;
+
+import org.apache.airavata.gfac.Constants;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.bes.security.GSISecurityContext;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.notification.events.StatusChangeEvent;
+import org.apache.airavata.gfac.notification.events.UnicoreJobIDEvent;
+import org.apache.airavata.gfac.provider.AbstractProvider;
+import org.apache.airavata.gfac.provider.GFacProviderException;
+import org.apache.airavata.gfac.bes.utils.DataTransferrer;
+import org.apache.airavata.gfac.bes.utils.JSDLGenerator;
+import org.apache.airavata.gfac.bes.utils.StorageCreator;
+import org.apache.airavata.gfac.utils.GFacUtils;
+import org.apache.airavata.model.workspace.experiment.JobState;
+import org.apache.airavata.registry.api.workflow.ApplicationJob;
+import org.apache.airavata.registry.api.workflow.ApplicationJob.ApplicationJobStatus;
+import org.apache.airavata.schemas.gfac.UnicoreHostType;
+import org.apache.xmlbeans.XmlCursor;
+import org.bouncycastle.asn1.ASN1InputStream;
+import org.bouncycastle.asn1.x500.X500Name;
+import org.bouncycastle.asn1.x500.style.BCStyle;
+import org.bouncycastle.asn1.x509.AlgorithmIdentifier;
+import org.bouncycastle.asn1.x509.SubjectPublicKeyInfo;
+import org.ggf.schemas.bes.x2006.x08.besFactory.ActivityStateEnumeration;
+import org.ggf.schemas.bes.x2006.x08.besFactory.ActivityStateEnumeration.Enum;
+import org.ggf.schemas.bes.x2006.x08.besFactory.ActivityStatusType;
+import org.ggf.schemas.bes.x2006.x08.besFactory.CreateActivityDocument;
+import org.ggf.schemas.bes.x2006.x08.besFactory.CreateActivityResponseDocument;
+import org.ggf.schemas.bes.x2006.x08.besFactory.GetActivityStatusesDocument;
+import org.ggf.schemas.bes.x2006.x08.besFactory.GetActivityStatusesResponseDocument;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionDocument;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3.x2005.x08.addressing.EndpointReferenceType;
+
+import de.fzj.unicore.bes.client.FactoryClient;
+import de.fzj.unicore.bes.faults.UnknownActivityIdentifierFault;
+import de.fzj.unicore.uas.client.StorageClient;
+import de.fzj.unicore.wsrflite.xmlbeans.WSUtilities;
+import eu.emi.security.authn.x509.helpers.CertificateHelpers;
+import eu.emi.security.authn.x509.helpers.proxy.X509v3CertificateBuilder;
+import eu.emi.security.authn.x509.impl.CertificateUtils;
+import eu.emi.security.authn.x509.impl.CertificateUtils.Encoding;
+import eu.emi.security.authn.x509.impl.DirectoryCertChainValidator;
+import eu.emi.security.authn.x509.impl.KeyAndCertCredential;
+import eu.emi.security.authn.x509.impl.X500NameUtils;
+import eu.unicore.util.httpclient.DefaultClientConfiguration;
+
+
+
+public class BESProvider extends AbstractProvider {
+ protected final Logger log = LoggerFactory.getLogger(this.getClass());
+
+ private DefaultClientConfiguration secProperties;
+
+ private String jobId;
+
+
+
+ public void initialize(JobExecutionContext jobExecutionContext)
+ throws GFacProviderException, GFacException {
+ log.info("Initializing UNICORE Provider");
+ super.initialize(jobExecutionContext);
+ initSecurityProperties(jobExecutionContext);
+ log.debug("initialized security properties");
+ }
+
+
+ public void execute(JobExecutionContext jobExecutionContext)
+ throws GFacProviderException {
+ UnicoreHostType host = (UnicoreHostType) jobExecutionContext.getApplicationContext().getHostDescription()
+ .getType();
+
+ String factoryUrl = host.getUnicoreBESEndPointArray()[0];
+
+ EndpointReferenceType eprt = EndpointReferenceType.Factory.newInstance();
+ eprt.addNewAddress().setStringValue(factoryUrl);
+
+ String userDN = getUserName(jobExecutionContext);
+
+ if (userDN == null || userDN.equalsIgnoreCase("admin")) {
+ userDN = "CN=zdv575, O=Ultrascan Gateway, C=DE";
+ }
+
+ String xlogin = getCNFromUserDN(userDN);
+ // create storage
+ StorageCreator storageCreator = new StorageCreator(secProperties, factoryUrl, 5, xlogin);
+
+ StorageClient sc = null;
+ try {
+ try {
+ sc = storageCreator.createStorage();
+ } catch (Exception e2) {
+ log.error("Cannot create storage..");
+ throw new GFacProviderException("Cannot create storage..", e2);
+ }
+
+ CreateActivityDocument cad = CreateActivityDocument.Factory.newInstance();
+ JobDefinitionDocument jobDefDoc = JobDefinitionDocument.Factory.newInstance();
+
+ JobDefinitionType jobDefinition = jobDefDoc.addNewJobDefinition();
+ try {
+ jobDefinition = JSDLGenerator.buildJSDLInstance(jobExecutionContext, sc.getUrl()).getJobDefinition();
+ cad.addNewCreateActivity().addNewActivityDocument().setJobDefinition(jobDefinition);
+
+ log.info("JSDL" + jobDefDoc.toString());
+ } catch (Exception e1) {
+ throw new GFacProviderException("Cannot generate JSDL instance from the JobExecutionContext.", e1);
+ }
+
+ // upload files if any
+ DataTransferrer dt = new DataTransferrer(jobExecutionContext, sc);
+ dt.uploadLocalFiles();
+
+ FactoryClient factory = null;
+ try {
+ factory = new FactoryClient(eprt, secProperties);
+ } catch (Exception e) {
+ throw new GFacProviderException(e.getLocalizedMessage(), e);
+ }
+
+ CreateActivityResponseDocument response = null;
+ try {
+ log.info(String.format("Activity Submitting to %s ... \n", factoryUrl));
+ response = factory.createActivity(cad);
+ log.info(String.format("Activity Submitted to %s \n", factoryUrl));
+ } catch (Exception e) {
+ throw new GFacProviderException("Cannot create activity.", e);
+ }
+ EndpointReferenceType activityEpr = response.getCreateActivityResponse().getActivityIdentifier();
+
+ log.info("Activity : " + activityEpr.getAddress().getStringValue() + " Submitted.");
+
+ // factory.waitWhileActivityIsDone(activityEpr, 1000);
+ jobId = WSUtilities.extractResourceID(activityEpr);
+ if (jobId == null) {
+ jobId = new Long(Calendar.getInstance().getTimeInMillis()).toString();
+ }
+ log.info("JobID: " + jobId);
+ jobExecutionContext.getNotifier().publish(new UnicoreJobIDEvent(jobId));
+ saveApplicationJob(jobExecutionContext, jobDefinition, activityEpr.toString());
+
+ factory.getActivityStatus(activityEpr);
+ log.info(formatStatusMessage(activityEpr.getAddress().getStringValue(),
+ factory.getActivityStatus(activityEpr).toString()));
+
+ // TODO publish the status messages to the message bus
+ while ((factory.getActivityStatus(activityEpr) != ActivityStateEnumeration.FINISHED)
+ && (factory.getActivityStatus(activityEpr) != ActivityStateEnumeration.FAILED)
+ && (factory.getActivityStatus(activityEpr) != ActivityStateEnumeration.CANCELLED)) {
+
+ ActivityStatusType activityStatus = null;
+ try {
+ activityStatus = getStatus(factory, activityEpr);
+ JobState jobStatus = getApplicationJobStatus(activityStatus);
+ String jobStatusMessage = "Status of job " + jobId + "is " + jobStatus;
+ jobExecutionContext.getNotifier().publish(new StatusChangeEvent(jobStatusMessage));
+ details.setJobID(jobId);
+ GFacUtils.updateJobStatus(jobExecutionContext, details, jobStatus);
+ } catch (UnknownActivityIdentifierFault e) {
+ throw new GFacProviderException(e.getMessage(), e.getCause());
+ }catch (GFacException e) {
+ throw new GFacProviderException(e.getMessage(), e.getCause());
+ }
+
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ }
+ continue;
+ }
+
+ ActivityStatusType activityStatus = null;
+ try {
+ activityStatus = getStatus(factory, activityEpr);
+ } catch (UnknownActivityIdentifierFault e) {
+ throw new GFacProviderException(e.getMessage(), e.getCause());
+ }
+
+ log.info(formatStatusMessage(activityEpr.getAddress().getStringValue(), activityStatus.getState()
+ .toString()));
+
+ if ((activityStatus.getState() == ActivityStateEnumeration.FAILED)) {
+ String error = activityStatus.getFault().getFaultcode().getLocalPart() + "\n"
+ + activityStatus.getFault().getFaultstring() + "\n EXITCODE: " + activityStatus.getExitCode();
+ log.info(error);
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ }
+ dt.downloadStdOuts();
+ } else if (activityStatus.getState() == ActivityStateEnumeration.CANCELLED) {
+ String experimentID = (String) jobExecutionContext.getProperty(Constants.PROP_TOPIC);
+ JobState jobStatus = JobState.CANCELED;
+ String jobStatusMessage = "Status of job " + jobId + "is " + jobStatus;
+ jobExecutionContext.getNotifier().publish(new StatusChangeEvent(jobStatusMessage));
+ details.setJobID(jobId);
+ try {
+ GFacUtils.saveJobStatus(jobExecutionContext,details, jobStatus);
+ } catch (GFacException e) {
+ throw new GFacProviderException(e.getLocalizedMessage(),e);
+ }
+ throw new GFacProviderException(experimentID + "Job Canceled");
+ }
+
+ else if (activityStatus.getState() == ActivityStateEnumeration.FINISHED) {
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ }
+ if (activityStatus.getExitCode() == 0) {
+ dt.downloadRemoteFiles();
+ } else {
+ dt.downloadStdOuts();
+ }
+ }
+
+ } catch (UnknownActivityIdentifierFault e1) {
+ throw new GFacProviderException(e1.getLocalizedMessage(), e1);
+ } finally {
+ // destroy sms instance
+ try {
+ if (sc != null) {
+ sc.destroy();
+ }
+ } catch (Exception e) {
+ log.warn("Cannot destroy temporary SMS instance:" + sc.getUrl(), e);
+ }
+ }
+ }
+
+ private JobState getApplicationJobStatus(ActivityStatusType activityStatus){
+ if (activityStatus == null) {
+ return JobState.UNKNOWN;
+ }
+ Enum state = activityStatus.getState();
+ String status = null;
+ XmlCursor acursor = activityStatus.newCursor();
+ try {
+ if (acursor.toFirstChild()) {
+ if (acursor.getName().getNamespaceURI().equals("http://schemas.ogf.org/hpcp/2007/01/fs")) {
+ status = acursor.getName().getLocalPart();
+ }
+ }
+ if (status != null) {
+ if (status.equalsIgnoreCase("Queued") || status.equalsIgnoreCase("Starting")
+ || status.equalsIgnoreCase("Ready")) {
+ return JobState.QUEUED;
+ } else if (status.equalsIgnoreCase("Staging-In")) {
+ return JobState.SUBMITTED;
+ } else if (status.equalsIgnoreCase("Staging-Out") || status.equalsIgnoreCase("FINISHED")) {
+ return JobState.COMPLETE;
+ } else if (status.equalsIgnoreCase("Executing")) {
+ return JobState.ACTIVE;
+ } else if (status.equalsIgnoreCase("FAILED")) {
+ return JobState.FAILED;
+ } else if (status.equalsIgnoreCase("CANCELLED")) {
+ return JobState.CANCELED;
+ }
+ } else {
+ if (ActivityStateEnumeration.CANCELLED.equals(state)) {
+ return JobState.CANCELED;
+ } else if (ActivityStateEnumeration.FAILED.equals(state)) {
+ return JobState.FAILED;
+ } else if (ActivityStateEnumeration.FINISHED.equals(state)) {
+ return JobState.COMPLETE;
+ } else if (ActivityStateEnumeration.RUNNING.equals(state)) {
+ return JobState.ACTIVE;
+ }
+ }
+ } finally {
+ if (acursor != null)
+ acursor.dispose();
+ }
+ return JobState.UNKNOWN;
+ }
+
+ private void saveApplicationJob(JobExecutionContext jobExecutionContext, JobDefinitionType jobDefinition,
+ String metadata) {
+ ApplicationJob appJob = GFacUtils.createApplicationJob(jobExecutionContext);
+ appJob.setJobId(jobId);
+ appJob.setJobData(jobDefinition.toString());
+ appJob.setSubmittedTime(Calendar.getInstance().getTime());
+ appJob.setStatus(ApplicationJobStatus.SUBMITTED);
+ appJob.setStatusUpdateTime(appJob.getSubmittedTime());
+ appJob.setMetadata(metadata);
+ GFacUtils.recordApplicationJob(jobExecutionContext, appJob);
+ }
+
+ public void dispose(JobExecutionContext jobExecutionContext) throws GFacProviderException {
+ secProperties = null;
+ }
+
+ /**
+ * EndpointReference need to be saved to make cancel work.
+ *
+ * @param activityEpr
+ * @param jobExecutionContext
+ * @throws GFacProviderException
+ */
+ public void cancelJob(String activityEpr, JobExecutionContext jobExecutionContext) throws GFacProviderException {
+ try {
+ initSecurityProperties(jobExecutionContext);
+ EndpointReferenceType eprt = EndpointReferenceType.Factory.parse(activityEpr);
+ UnicoreHostType host = (UnicoreHostType) jobExecutionContext.getApplicationContext().getHostDescription()
+ .getType();
+
+ String factoryUrl = host.getUnicoreBESEndPointArray()[0];
+ EndpointReferenceType epr = EndpointReferenceType.Factory.newInstance();
+ epr.addNewAddress().setStringValue(factoryUrl);
+
+ FactoryClient factory = new FactoryClient(epr, secProperties);
+ factory.terminateActivity(eprt);
+ } catch (Exception e) {
+ throw new GFacProviderException(e.getLocalizedMessage(),e);
+ }
+
+ }
+
+ protected void downloadOffline(String smsEpr, JobExecutionContext jobExecutionContext) throws GFacProviderException {
+ try {
+ initSecurityProperties(jobExecutionContext);
+ EndpointReferenceType eprt = EndpointReferenceType.Factory.parse(smsEpr);
+ StorageClient sms = new StorageClient(eprt, secProperties);
+ DataTransferrer dt = new DataTransferrer(jobExecutionContext, sms);
+ // there must be output files there
+ // this is also possible if client is re-connected, the jobs are
+ // still
+ // running and no output is produced
+ dt.downloadRemoteFiles();
+
+ // may be use the below method before downloading for checking
+ // the number of entries
+ // sms.listDirectory(".");
+
+ } catch (Exception e) {
+ throw new GFacProviderException(e.getLocalizedMessage(), e);
+ }
+ }
+
+ protected void initSecurityProperties(JobExecutionContext jobExecutionContext) throws GFacProviderException,
+ GFacException {
+
+ if (secProperties != null)
+ return;
+
+ GSISecurityContext gssContext = (GSISecurityContext) jobExecutionContext
+ .getSecurityContext(GSISecurityContext.GSI_SECURITY_CONTEXT);
+
+ try {
+ String certLocation = gssContext.getTrustedCertificatePath();
+ List<String> trustedCert = new ArrayList<String>();
+ trustedCert.add(certLocation + "/*.0");
+ trustedCert.add(certLocation + "/*.pem");
+
+ DirectoryCertChainValidator dcValidator = new DirectoryCertChainValidator(trustedCert, Encoding.PEM, -1,
+ 60000, null);
+
+ String userID = getUserName(jobExecutionContext);
+
+ if ( userID == null || "".equals(userID) || userID.equalsIgnoreCase("admin") ) {
+ userID = "CN=zdv575, O=Ultrascan Gateway, C=DE";
+ }
+
+ String userDN = userID.replaceAll("^\"|\"$", "");
+
+ // TODO: should be changed to default airavata server locations
+ KeyAndCertCredential cred = generateShortLivedCertificate(userDN, certLocation
+ + "/cacert.pem", certLocation
+ + "/cakey.pem", "ultrascan3");
+ secProperties = new DefaultClientConfiguration(dcValidator, cred);
+
+ // secProperties.doSSLAuthn();
+ secProperties.getETDSettings().setExtendTrustDelegation(true);
+
+ secProperties.setDoSignMessage(true);
+
+ String[] outHandlers = secProperties.getOutHandlerClassNames();
+
+ Set<String> outHandlerLst = null;
+
+ // timeout in milliseconds
+ Properties p = secProperties.getExtraSettings();
+ p.setProperty("http.connection.timeout", "300000");
+ p.setProperty("http.socket.timeout", "300000");
+
+ if (outHandlers == null) {
+ outHandlerLst = new HashSet<String>();
+ } else {
+ outHandlerLst = new HashSet<String>(Arrays.asList(outHandlers));
+ }
+
+ outHandlerLst.add("de.fzj.unicore.uas.security.ProxyCertOutHandler");
+
+ secProperties.setOutHandlerClassNames(outHandlerLst.toArray(new String[outHandlerLst.size()]));
+
+ } catch (Exception e) {
+ throw new GFacProviderException(e.getMessage(), e);
+ }
+ }
+
+ //FIXME: Get user details
+ private String getUserName(JobExecutionContext context) {
+// if (context.getConfigurationData()!= null) {
+// return context.getConfigurationData().getBasicMetadata().getUserName();
+// } else {
+ return "";
+// }
+ }
+
+ protected ActivityStatusType getStatus(FactoryClient fc, EndpointReferenceType activityEpr)
+ throws UnknownActivityIdentifierFault {
+
+ GetActivityStatusesDocument stats = GetActivityStatusesDocument.Factory.newInstance();
+
+ stats.addNewGetActivityStatuses().setActivityIdentifierArray(new EndpointReferenceType[] { activityEpr });
+
+ GetActivityStatusesResponseDocument resDoc = fc.getActivityStatuses(stats);
+
+ ActivityStatusType activityStatus = resDoc.getGetActivityStatusesResponse().getResponseArray()[0]
+ .getActivityStatus();
+ return activityStatus;
+ }
+
+ protected String formatStatusMessage(String activityUrl, String status) {
+ return String.format("Activity %s is %s.\n", activityUrl, status);
+ }
+
+ protected String subStatusAsString(ActivityStatusType statusType) {
+
+ StringBuffer sb = new StringBuffer();
+
+ sb.append(statusType.getState().toString());
+
+ XmlCursor acursor = statusType.newCursor();
+ if (acursor.toFirstChild()) {
+ do {
+ if (acursor.getName().getNamespaceURI().equals("http://schemas.ogf.org/hpcp/2007/01/fs")) {
+ sb.append(":");
+ sb.append(acursor.getName().getLocalPart());
+ }
+ } while (acursor.toNextSibling());
+ acursor.dispose();
+ return sb.toString();
+ } else {
+ acursor.dispose();
+ return sb.toString();
+ }
+
+ }
+
+ public void initProperties(Map<String, String> properties) throws GFacProviderException, GFacException {
+
+ }
+
+ protected KeyAndCertCredential generateShortLivedCertificate(String userDN, String caCertPath, String caKeyPath,
+ String caPwd) throws Exception {
+ final long CredentialGoodFromOffset = 1000L * 60L * 15L; // 15 minutes
+ // ago
+
+ final long startTime = System.currentTimeMillis() - CredentialGoodFromOffset;
+ final long endTime = startTime + 30 * 3600 * 1000;
+
+ String keyLengthProp = "1024";
+ int keyLength = Integer.parseInt(keyLengthProp);
+ String signatureAlgorithm = "SHA1withRSA";
+
+ KeyAndCertCredential caCred = getCACredential(caCertPath, caKeyPath, caPwd);
+
+ KeyPairGenerator kpg = KeyPairGenerator.getInstance(caCred.getKey().getAlgorithm());
+ kpg.initialize(keyLength);
+ KeyPair pair = kpg.generateKeyPair();
+
+ X500Principal subjectDN = new X500Principal(userDN);
+ Random rand = new Random();
+
+ SubjectPublicKeyInfo publicKeyInfo;
+ try {
+ publicKeyInfo = SubjectPublicKeyInfo.getInstance(new ASN1InputStream(pair.getPublic().getEncoded())
+ .readObject());
+ } catch (IOException e) {
+ throw new InvalidKeyException("Can not parse the public key"
+ + "being included in the short lived certificate", e);
+ }
+
+ X500Name issuerX500Name = CertificateHelpers.toX500Name(caCred.getCertificate().getSubjectX500Principal());
+
+ X500Name subjectX500Name = CertificateHelpers.toX500Name(subjectDN);
+
+ X509v3CertificateBuilder certBuilder = new X509v3CertificateBuilder(issuerX500Name, new BigInteger(20, rand),
+ new Date(startTime), new Date(endTime), subjectX500Name, publicKeyInfo);
+
+ AlgorithmIdentifier sigAlgId = X509v3CertificateBuilder.extractAlgorithmId(caCred.getCertificate());
+
+ X509Certificate certificate = certBuilder.build(caCred.getKey(), sigAlgId, signatureAlgorithm, null, null);
+
+ certificate.checkValidity(new Date());
+ certificate.verify(caCred.getCertificate().getPublicKey());
+ KeyAndCertCredential result = new KeyAndCertCredential(pair.getPrivate(), new X509Certificate[] { certificate,
+ caCred.getCertificate() });
+
+ return result;
+ }
+
+ private KeyAndCertCredential getCACredential(String caCertPath, String caKeyPath, String password) throws Exception {
+ InputStream isKey = new FileInputStream(caKeyPath);
+ PrivateKey pk = CertificateUtils.loadPrivateKey(isKey, Encoding.PEM, password.toCharArray());
+
+ InputStream isCert = new FileInputStream(caCertPath);
+ X509Certificate caCert = CertificateUtils.loadCertificate(isCert, Encoding.PEM);
+
+ if (isKey != null)
+ isKey.close();
+ if (isCert != null)
+ isCert.close();
+
+ return new KeyAndCertCredential(pk, new X509Certificate[] { caCert });
+ }
+
+ private String getCNFromUserDN(String userDN) {
+ return X500NameUtils.getAttributeValues(userDN, BCStyle.CN)[0];
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/security/GSISecurityContext.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/security/GSISecurityContext.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/security/GSISecurityContext.java
new file mode 100644
index 0000000..22d2e13
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/security/GSISecurityContext.java
@@ -0,0 +1,288 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.bes.security;
+
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.credential.store.credential.Credential;
+import org.apache.airavata.credential.store.credential.impl.certificate.CertificateCredential;
+import org.apache.airavata.credential.store.store.CredentialReader;
+import org.apache.airavata.gfac.AbstractSecurityContext;
+import org.apache.airavata.gfac.Constants;
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.RequestData;
+import org.apache.airavata.gsi.ssh.api.Cluster;
+import org.globus.gsi.X509Credential;
+import org.globus.gsi.gssapi.GlobusGSSCredentialImpl;
+import org.globus.gsi.provider.GlobusProvider;
+import org.globus.myproxy.GetParams;
+import org.globus.myproxy.MyProxy;
+import org.globus.myproxy.MyProxyException;
+import org.gridforum.jgss.ExtendedGSSCredential;
+import org.ietf.jgss.GSSCredential;
+import org.ietf.jgss.GSSException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.security.Security;
+import java.security.cert.X509Certificate;
+
+/**
+ * Handles GRID related security.
+ */
+public class GSISecurityContext extends AbstractSecurityContext {
+
+ protected static final Logger log = LoggerFactory.getLogger(GSISecurityContext.class);
+ /*
+ * context name
+ */
+ public static final String GSI_SECURITY_CONTEXT = "gsi";
+
+ public static int CREDENTIAL_RENEWING_THRESH_HOLD = 10 * 90;
+
+ private GSSCredential gssCredentials = null;
+
+ private Cluster pbsCluster = null;
+
+ // Set trusted cert path and add provider
+ static {
+ Security.addProvider(new GlobusProvider());
+ try {
+ setUpTrustedCertificatePath();
+ } catch (ApplicationSettingsException e) {
+ log.error(e.getLocalizedMessage(), e);
+ }
+ }
+
+ public static void setUpTrustedCertificatePath(String trustedCertificatePath) {
+
+ File file = new File(trustedCertificatePath);
+
+ if (!file.exists() || !file.canRead()) {
+ File f = new File(".");
+ log.info("Current directory " + f.getAbsolutePath());
+ throw new RuntimeException("Cannot read trusted certificate path " + trustedCertificatePath);
+ } else {
+ System.setProperty(Constants.TRUSTED_CERTIFICATE_SYSTEM_PROPERTY, file.getAbsolutePath());
+ }
+ }
+
+ private static void setUpTrustedCertificatePath() throws ApplicationSettingsException {
+
+ String trustedCertificatePath = ServerSettings.getSetting(Constants.TRUSTED_CERT_LOCATION);
+
+ setUpTrustedCertificatePath(trustedCertificatePath);
+ }
+
+ /**
+ * Gets the trusted certificate path. Trusted certificate path is stored in "X509_CERT_DIR"
+ * system property.
+ * @return The trusted certificate path as a string.
+ */
+ public static String getTrustedCertificatePath() {
+ return System.getProperty(Constants.TRUSTED_CERTIFICATE_SYSTEM_PROPERTY);
+ }
+
+
+ public GSISecurityContext(CredentialReader credentialReader, RequestData requestData) {
+ super(credentialReader, requestData);
+ }
+
+
+ public GSISecurityContext(Cluster pbsCluster) {
+ this.setPbsCluster(pbsCluster);
+ }
+
+ /**
+ * Gets GSSCredentials. The process is as follows;
+ * If credentials were queried for the first time create credentials.
+ * 1. Try creating credentials using certificates stored in the credential store
+ * 2. If 1 fails use user name and password to create credentials
+ * If credentials are already created check the remaining life time of the credential. If
+ * remaining life time is less than CREDENTIAL_RENEWING_THRESH_HOLD, then renew credentials.
+ * @return GSSCredentials to be used.
+ * @throws org.apache.airavata.gfac.GFacException If an error occurred while creating credentials.
+ * @throws org.apache.airavata.common.exception.ApplicationSettingsException
+ */
+ public GSSCredential getGssCredentials() throws GFacException, ApplicationSettingsException {
+
+ if (gssCredentials == null) {
+
+ try {
+ gssCredentials = getCredentialsFromStore();
+ } catch (Exception e) {
+ log.error("An exception occurred while retrieving credentials from the credential store. " +
+ "Will continue with my proxy user name and password.", e);
+ }
+
+ // If store does not have credentials try to get from user name and password
+ if (gssCredentials == null) {
+ gssCredentials = getDefaultCredentials();
+ }
+
+ // if still null, throw an exception
+ if (gssCredentials == null) {
+ throw new GFacException("Unable to retrieve my proxy credentials to continue operation.");
+ }
+ } else {
+ try {
+ if (gssCredentials.getRemainingLifetime() < CREDENTIAL_RENEWING_THRESH_HOLD) {
+ return renewCredentials();
+ }
+ } catch (GSSException e) {
+ throw new GFacException("Unable to retrieve remaining life time from credentials.", e);
+ }
+ }
+
+ return gssCredentials;
+ }
+
+ /**
+ * Renews credentials. First try to renew credentials as a trusted renewer. If that failed
+ * use user name and password to renew credentials.
+ * @return Renewed credentials.
+ * @throws org.apache.airavata.gfac.GFacException If an error occurred while renewing credentials.
+ * @throws org.apache.airavata.common.exception.ApplicationSettingsException
+ */
+ public GSSCredential renewCredentials() throws GFacException, ApplicationSettingsException {
+
+ // First try to renew credentials as a trusted renewer
+ try {
+ gssCredentials = renewCredentialsAsATrustedHost();
+ } catch (Exception e) {
+ log.warn("Renewing credentials as a trusted renewer failed", e);
+ gssCredentials = getProxyCredentials();
+ }
+
+ return gssCredentials;
+ }
+
+ /**
+ * Reads the credentials from credential store.
+ * @return If token is found in the credential store, will return a valid credential. Else returns null.
+ * @throws Exception If an error occurred while retrieving credentials.
+ */
+ public GSSCredential getCredentialsFromStore() throws Exception {
+
+ if (getCredentialReader() == null) {
+ return null;
+ }
+
+ Credential credential = getCredentialReader().getCredential(getRequestData().getGatewayId(),
+ getRequestData().getTokenId());
+
+ if (credential != null) {
+ if (credential instanceof CertificateCredential) {
+
+ log.info("Successfully found credentials for token id - " + getRequestData().getTokenId() +
+ " gateway id - " + getRequestData().getGatewayId());
+
+ CertificateCredential certificateCredential = (CertificateCredential) credential;
+
+ X509Certificate[] certificates = certificateCredential.getCertificates();
+ X509Credential newCredential = new X509Credential(certificateCredential.getPrivateKey(), certificates);
+
+ GlobusGSSCredentialImpl cred = new GlobusGSSCredentialImpl(newCredential, GSSCredential.INITIATE_AND_ACCEPT);
+ System.out.print(cred.export(ExtendedGSSCredential.IMPEXP_OPAQUE));
+ return cred;
+ //return new GlobusGSSCredentialImpl(newCredential,
+ // GSSCredential.INITIATE_AND_ACCEPT);
+ } else {
+ log.info("Credential type is not CertificateCredential. Cannot create mapping globus credentials. " +
+ "Credential type - " + credential.getClass().getName());
+ }
+ } else {
+ log.info("Could not find credentials for token - " + getRequestData().getTokenId() + " and "
+ + "gateway id - " + getRequestData().getGatewayId());
+ }
+
+ return null;
+ }
+
+ /**
+ * Gets the default proxy certificate.
+ * @return Default my proxy credentials.
+ * @throws org.apache.airavata.gfac.GFacException If an error occurred while retrieving credentials.
+ * @throws org.apache.airavata.common.exception.ApplicationSettingsException
+ */
+ public GSSCredential getDefaultCredentials() throws GFacException, ApplicationSettingsException{
+ MyProxy myproxy = new MyProxy(getRequestData().getMyProxyServerUrl(), getRequestData().getMyProxyPort());
+ try {
+ return myproxy.get(getRequestData().getMyProxyUserName(), getRequestData().getMyProxyPassword(),
+ getRequestData().getMyProxyLifeTime());
+ } catch (MyProxyException e) {
+ throw new GFacException("An error occurred while retrieving default security credentials.", e);
+ }
+ }
+
+ /**
+ * Gets a new proxy certificate given current credentials.
+ * @return The short lived GSSCredentials
+ * @throws org.apache.airavata.gfac.GFacException If an error is occurred while retrieving credentials.
+ * @throws org.apache.airavata.common.exception.ApplicationSettingsException
+ */
+ public GSSCredential getProxyCredentials() throws GFacException, ApplicationSettingsException {
+
+ MyProxy myproxy = new MyProxy(getRequestData().getMyProxyServerUrl(), getRequestData().getMyProxyPort());
+ try {
+ return myproxy.get(gssCredentials, getRequestData().getMyProxyUserName(), getRequestData().getMyProxyPassword(),
+ getRequestData().getMyProxyLifeTime());
+ } catch (MyProxyException e) {
+ throw new GFacException("An error occurred while renewing security credentials using user/password.", e);
+ }
+ }
+
+ /**
+ * Renew GSSCredentials.
+ * Before executing we need to add current host as a trusted renewer. Note to renew credentials
+ * we dont need user name and password.
+ * To do that execute following command
+ * > myproxy-logon -t <LIFETIME></LIFETIME> -s <MY PROXY SERVER> -l <USER NAME>
+ * E.g :- > myproxy-logon -t 264 -s myproxy.teragrid.org -l us3
+ * Enter MyProxy pass phrase:
+ * A credential has been received for user us3 in /tmp/x509up_u501.
+ * > myproxy-init -A --cert /tmp/x509up_u501 --key /tmp/x509up_u501 -l ogce -s myproxy.teragrid.org
+ * @return Renewed credentials.
+ * @throws org.apache.airavata.gfac.GFacException If an error occurred while renewing credentials.
+ * @throws org.apache.airavata.common.exception.ApplicationSettingsException
+ */
+ public GSSCredential renewCredentialsAsATrustedHost() throws GFacException, ApplicationSettingsException {
+ MyProxy myproxy = new MyProxy(getRequestData().getMyProxyServerUrl(), getRequestData().getMyProxyPort());
+ GetParams getParams = new GetParams();
+ getParams.setAuthzCreds(gssCredentials);
+ getParams.setUserName(getRequestData().getMyProxyUserName());
+ getParams.setLifetime(getRequestData().getMyProxyLifeTime());
+ try {
+ return myproxy.get(gssCredentials, getParams);
+ } catch (MyProxyException e) {
+ throw new GFacException("An error occurred while renewing security credentials.", e);
+ }
+ }
+
+ public Cluster getPbsCluster() {
+ return pbsCluster;
+ }
+
+ public void setPbsCluster(Cluster pbsCluster) {
+ this.pbsCluster = pbsCluster;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java
new file mode 100644
index 0000000..77f8a3a
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java
@@ -0,0 +1,252 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.bes.utils;
+
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.schemas.gfac.ExtendedKeyValueType;
+import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
+import org.apache.airavata.schemas.gfac.JobTypeType;
+import org.apache.airavata.schemas.gfac.NameValuePairType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.ApplicationType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.EnvironmentType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.FileNameType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.UserNameType;
+import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.NumberOfProcessesType;
+import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.ProcessesPerHostType;
+import org.ogf.schemas.jsdl.x2007.x02.jsdlSpmd.ThreadsPerProcessType;
+
+import java.io.File;
+
+
+public class ApplicationProcessor {
+
+ public static void generateJobSpecificAppElements(JobDefinitionType value, JobExecutionContext context){
+
+ String userName = getUserNameFromContext(context);
+ if (userName.equalsIgnoreCase("admin")){
+ userName = "CN=zdv575, O=Ultrascan Gateway, C=DE";
+ }
+
+ HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
+ .getApplicationContext().getApplicationDeploymentDescription()
+ .getType();
+
+ createGenericApplication(value, appDepType);
+
+ if (appDepType.getApplicationEnvironmentArray().length > 0) {
+ createApplicationEnvironment(value,
+ appDepType.getApplicationEnvironmentArray(), appDepType);
+ }
+
+
+ if (appDepType.getExecutableLocation() != null) {
+ FileNameType fNameType = FileNameType.Factory.newInstance();
+ fNameType.setStringValue(appDepType.getExecutableLocation());
+ if(isParallelJob(appDepType)) {
+ JSDLUtils.getOrCreateSPMDApplication(value).setExecutable(fNameType);
+ JSDLUtils.getSPMDApplication(value).setSPMDVariation(getSPMDVariation(appDepType));
+
+ if(getValueFromMap(appDepType, JSDLUtils.NUMBEROFPROCESSES)!=null){
+ NumberOfProcessesType num = NumberOfProcessesType.Factory.newInstance();
+ num.setStringValue(getValueFromMap(appDepType, JSDLUtils.NUMBEROFPROCESSES));
+ JSDLUtils.getSPMDApplication(value).setNumberOfProcesses(num);
+ }
+
+ if(getValueFromMap(appDepType, JSDLUtils.PROCESSESPERHOST)!=null){
+ ProcessesPerHostType pph = ProcessesPerHostType.Factory.newInstance();
+ pph.setStringValue(getValueFromMap(appDepType, JSDLUtils.PROCESSESPERHOST));
+ JSDLUtils.getSPMDApplication(value).setProcessesPerHost(pph);
+ }
+
+ if(getValueFromMap(appDepType, JSDLUtils.THREADSPERHOST)!=null){
+ ThreadsPerProcessType tpp = ThreadsPerProcessType.Factory.newInstance();
+ tpp.setStringValue(getValueFromMap(appDepType, JSDLUtils.THREADSPERHOST));
+ JSDLUtils.getSPMDApplication(value).setThreadsPerProcess(tpp);
+
+ }
+
+ if(userName != null) {
+ UserNameType userNameType = UserNameType.Factory.newInstance();
+ userNameType.setStringValue(userName);
+ JSDLUtils.getSPMDApplication(value).setUserName(userNameType);
+ }
+ }
+ else {
+ JSDLUtils.getOrCreatePOSIXApplication(value).setExecutable(fNameType);
+ if(userName != null) {
+ UserNameType userNameType = UserNameType.Factory.newInstance();
+ userNameType.setStringValue(userName);
+ JSDLUtils.getOrCreatePOSIXApplication(value).setUserName(userNameType);
+ }
+ }
+ }
+
+
+ String stdout = (appDepType.getStandardOutput() != null) ? new File(appDepType.getStandardOutput()).getName(): "stdout";
+ ApplicationProcessor.setApplicationStdOut(value, appDepType, stdout);
+
+
+ String stderr = (appDepType.getStandardError() != null) ? new File(appDepType.getStandardError()).getName() : "stderr";
+ ApplicationProcessor.setApplicationStdErr(value, appDepType, stderr);
+
+ }
+
+ public static String getUserNameFromContext(JobExecutionContext jobContext) {
+ if(jobContext.getTaskData() == null)
+ return null;
+ //FIXME: Discuss to get user and change this
+ return "admin";
+ }
+ public static boolean isParallelJob(HpcApplicationDeploymentType appDepType) {
+
+ boolean isParallel = false;
+
+ if (appDepType.getJobType() != null) {
+ // TODO set data output directory
+ int status = appDepType.getJobType().intValue();
+
+ switch (status) {
+ // TODO: this check should be done outside this class
+ case JobTypeType.INT_MPI:
+ case JobTypeType.INT_OPEN_MP:
+ isParallel = true;
+ break;
+
+ case JobTypeType.INT_SERIAL:
+ case JobTypeType.INT_SINGLE:
+ isParallel = false;
+ break;
+
+ default:
+ isParallel = false;
+ break;
+ }
+ }
+ return isParallel;
+ }
+
+
+ public static void createApplicationEnvironment(JobDefinitionType value, NameValuePairType[] nameValuePairs, HpcApplicationDeploymentType appDepType) {
+
+ if(isParallelJob(appDepType)) {
+ for (NameValuePairType nv : nameValuePairs) {
+ EnvironmentType envType = JSDLUtils.getOrCreateSPMDApplication(value).addNewEnvironment();
+ envType.setName(nv.getName());
+ envType.setStringValue(nv.getValue());
+ }
+ }
+ else {
+ for (NameValuePairType nv : nameValuePairs) {
+ EnvironmentType envType = JSDLUtils.getOrCreatePOSIXApplication(value).addNewEnvironment();
+ envType.setName(nv.getName());
+ envType.setStringValue(nv.getValue());
+ }
+ }
+
+ }
+
+
+ public static String getSPMDVariation (HpcApplicationDeploymentType appDepType) {
+
+ String variation = null;
+
+ if (appDepType.getJobType() != null) {
+ // TODO set data output directory
+ int status = appDepType.getJobType().intValue();
+
+ switch (status) {
+ // TODO: this check should be done outside this class
+ case JobTypeType.INT_MPI:
+ variation = SPMDVariations.MPI.value();
+ break;
+
+ case JobTypeType.INT_OPEN_MP:
+ variation = SPMDVariations.OpenMPI.value();
+ break;
+
+ }
+ }
+ return variation;
+ }
+
+
+ public static void addApplicationArgument(JobDefinitionType value, HpcApplicationDeploymentType appDepType, String stringPrm) {
+ if(isParallelJob(appDepType))
+ JSDLUtils.getOrCreateSPMDApplication(value)
+ .addNewArgument().setStringValue(stringPrm);
+ else
+ JSDLUtils.getOrCreatePOSIXApplication(value)
+ .addNewArgument().setStringValue(stringPrm);
+
+ }
+
+ public static void setApplicationStdErr(JobDefinitionType value, HpcApplicationDeploymentType appDepType, String stderr) {
+ FileNameType fName = FileNameType.Factory.newInstance();
+ fName.setStringValue(stderr);
+ if (isParallelJob(appDepType))
+ JSDLUtils.getOrCreateSPMDApplication(value).setError(fName);
+ else
+ JSDLUtils.getOrCreatePOSIXApplication(value).setError(fName);
+ }
+
+ public static void setApplicationStdOut(JobDefinitionType value, HpcApplicationDeploymentType appDepType, String stderr) {
+ FileNameType fName = FileNameType.Factory.newInstance();
+ fName.setStringValue(stderr);
+ if (isParallelJob(appDepType))
+ JSDLUtils.getOrCreateSPMDApplication(value).setOutput(fName);
+ else
+ JSDLUtils.getOrCreatePOSIXApplication(value).setOutput(fName);
+ }
+
+ public static String getApplicationStdOut(JobDefinitionType value, HpcApplicationDeploymentType appDepType) throws RuntimeException {
+ if (isParallelJob(appDepType)) return JSDLUtils.getOrCreateSPMDApplication(value).getOutput().getStringValue();
+ else return JSDLUtils.getOrCreatePOSIXApplication(value).getOutput().getStringValue();
+ }
+
+ public static String getApplicationStdErr(JobDefinitionType value, HpcApplicationDeploymentType appDepType) throws RuntimeException {
+ if (isParallelJob(appDepType)) return JSDLUtils.getOrCreateSPMDApplication(value).getError().getStringValue();
+ else return JSDLUtils.getOrCreatePOSIXApplication(value).getError().getStringValue();
+ }
+
+ public static void createGenericApplication(JobDefinitionType value, HpcApplicationDeploymentType appDepType) {
+ if (appDepType.getApplicationName() != null) {
+ ApplicationType appType = JSDLUtils.getOrCreateApplication(value);
+ String appName = appDepType.getApplicationName()
+ .getStringValue();
+ appType.setApplicationName(appName);
+ JSDLUtils.getOrCreateJobIdentification(value).setJobName(appName);
+ }
+ }
+
+
+ public static String getValueFromMap(HpcApplicationDeploymentType appDepType, String name) {
+ ExtendedKeyValueType[] extended = appDepType.getKeyValuePairsArray();
+ for(ExtendedKeyValueType e: extended) {
+ if(e.getName().equalsIgnoreCase(name)) {
+ return e.getStringValue();
+ }
+ }
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataStagingProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataStagingProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataStagingProcessor.java
new file mode 100644
index 0000000..1ff5504
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataStagingProcessor.java
@@ -0,0 +1,236 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.bes.utils;
+
+import java.io.File;
+import java.net.URI;
+import java.util.Map;
+
+import org.apache.airavata.commons.gfac.type.ActualParameter;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.utils.GFacUtils;
+import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
+import org.apache.airavata.schemas.gfac.StringArrayType;
+import org.apache.airavata.schemas.gfac.StringParameterType;
+import org.apache.airavata.schemas.gfac.URIArrayType;
+import org.apache.airavata.schemas.gfac.URIParameterType;
+import org.apache.airavata.schemas.gfac.UnicoreHostType;
+import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
+
+public class DataStagingProcessor {
+
+ public static void generateDataStagingElements(JobDefinitionType value, JobExecutionContext context) throws Exception{
+
+ HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) context
+ .getApplicationContext().getApplicationDeploymentDescription()
+ .getType();
+
+
+ String gridftpEndpoint = ((UnicoreHostType) context.getApplicationContext().getHostDescription().getType())
+ .getGridFTPEndPointArray()[0];
+
+
+ if (context.getInMessageContext().getParameters().size() > 0) {
+ buildDataStagingFromInputContext(context, value, gridftpEndpoint, appDepType);
+ }
+
+ if (context.getOutMessageContext().getParameters().size() > 0) {
+ buildFromOutputContext(context, value, gridftpEndpoint, appDepType);
+ }
+
+ createStdOutURIs(value, appDepType, gridftpEndpoint, isUnicoreEndpoint(context));
+
+ }
+
+ private static void createInURIElement(JobDefinitionType value,
+ String endpoint, String inputDir, ActualParameter inParam)
+ throws Exception {
+
+ String uri = ((URIParameterType) inParam.getType()).getValue();
+ String fileName = new File(uri).getName();
+ if (uri.startsWith("file")) {
+ URI gridFTPInputDirURI = URIUtils.createGsiftpURI(endpoint,
+ inputDir);
+ String filePath = gridFTPInputDirURI.toString() + File.separator
+ + fileName;
+ JSDLUtils
+ .addDataStagingSourceElement(value, filePath, null, fileName);
+ } else if (uri.startsWith("gsiftp") || uri.startsWith("http")
+ || uri.startsWith("rns")) {
+ // no need to stage-in those files to the input
+ // directory
+ JSDLUtils.addDataStagingSourceElement(value, uri, null, fileName);
+ }
+
+ }
+
+ private static void createStdOutURIs(JobDefinitionType value,
+ HpcApplicationDeploymentType appDepType, String endpoint,
+ boolean isUnicore) throws Exception {
+
+ URI remoteOutputDir = URIUtils.createGsiftpURI(endpoint,
+ appDepType.getOutputDataDirectory());
+
+ String stdout = ApplicationProcessor.getApplicationStdOut(value, appDepType);
+
+ String stderr = ApplicationProcessor.getApplicationStdErr(value, appDepType);
+
+ String stdoutFileName = (stdout == null || stdout.equals("")) ? "stdout"
+ : stdout;
+ String stdoutURI = GFacUtils.createGsiftpURIAsString(
+ remoteOutputDir.toString(), stdoutFileName);
+ JSDLUtils.addDataStagingTargetElement(value, null, stdoutFileName,
+ stdoutURI);
+
+ String stderrFileName = (stdout == null || stderr.equals("")) ? "stderr"
+ : stderr;
+ String stderrURI = GFacUtils.createGsiftpURIAsString(
+ remoteOutputDir.toString(), stderrFileName);
+ JSDLUtils.addDataStagingTargetElement(value, null, stderrFileName,
+ stderrURI);
+
+ if(isUnicore) {
+ String scriptExitCodeFName = "UNICORE_SCRIPT_EXIT_CODE";
+ String scriptExitCode = GFacUtils.createGsiftpURIAsString(
+ remoteOutputDir.toString(), scriptExitCodeFName);
+ JSDLUtils.addDataStagingTargetElement(value, null,
+ scriptExitCodeFName, scriptExitCode.toString());
+ }
+
+ }
+
+
+ private static void createOutStringElements(JobDefinitionType value,
+ HpcApplicationDeploymentType appDeptype, String endpoint, String prmValue) throws Exception {
+
+ if(prmValue == null || "".equals(prmValue)) return;
+
+
+ String outputUri = GFacUtils.createGsiftpURIAsString(endpoint, appDeptype.getOutputDataDirectory());
+
+ URI finalOutputUri = URIUtils.createGsiftpURI(outputUri, prmValue);
+ JSDLUtils.addDataStagingTargetElement(value, null, prmValue, finalOutputUri.toString());
+ }
+
+
+ private static void createOutURIElement(JobDefinitionType value,
+ String prmValue) throws Exception {
+ String fileName = new File(prmValue.toString()).getName();
+ JSDLUtils.addDataStagingTargetElement(value, null, fileName, prmValue);
+ }
+
+
+ private static JobDefinitionType buildFromOutputContext(JobExecutionContext context,
+ JobDefinitionType value, String gridftpEndpoint,
+ HpcApplicationDeploymentType appDepType) throws Exception {
+
+ Map<String, Object> outputParams = context.getOutMessageContext()
+ .getParameters();
+
+ for (String paramKey : outputParams.keySet()) {
+
+ ActualParameter outParam = (ActualParameter) outputParams
+ .get(paramKey);
+
+ // if single urls then convert each url into jsdl source
+ // elements,
+ // that are formed by concat of gridftpurl+inputdir+filename
+
+ String paramDataType = outParam.getType().getType().toString();
+
+ if ("URI".equals(paramDataType)) {
+ String uriPrm = ((URIParameterType) outParam.getType())
+ .getValue();
+ createOutURIElement(value, uriPrm);
+ }
+
+ // string params are converted into the job arguments
+
+ else if (("URIArray").equals(paramDataType)) {
+ String[] uriArray = ((URIArrayType) outParam.getType())
+ .getValueArray();
+ for (String u : uriArray) {
+
+ createOutURIElement(value, u);
+ }
+
+ }
+ else if ("String".equals(paramDataType)) {
+ String stringPrm = ((StringParameterType) outParam
+ .getType()).getValue();
+ createOutStringElements(value, appDepType, gridftpEndpoint, stringPrm);
+ }
+
+ else if ("StringArray".equals(paramDataType)) {
+ String[] valueArray = ((StringArrayType) outParam.getType())
+ .getValueArray();
+ for (String v : valueArray) {
+ createOutStringElements(value, appDepType, gridftpEndpoint, v);
+ }
+ }
+ }
+
+ return value;
+ }
+
+
+ private static void buildDataStagingFromInputContext(JobExecutionContext context, JobDefinitionType value, String gridftpEndpoint, HpcApplicationDeploymentType appDepType)
+ throws Exception {
+
+ // TODO set data directory
+ Map<String, Object> inputParams = context.getInMessageContext()
+ .getParameters();
+
+ for (String paramKey : inputParams.keySet()) {
+
+ ActualParameter inParam = (ActualParameter) inputParams
+ .get(paramKey);
+
+ // if single urls then convert each url into jsdl source
+ // elements,
+ // that are formed by concat of gridftpurl+inputdir+filename
+
+ String paramDataType = inParam.getType().getType().toString();
+
+ if ("URI".equals(paramDataType)) {
+ createInURIElement(value, gridftpEndpoint,
+ appDepType.getInputDataDirectory(), inParam);
+ }
+
+ // string params are converted into the job arguments
+
+ else if ("String".equals(paramDataType)) {
+ String stringPrm = ((StringParameterType) inParam.getType())
+ .getValue();
+ ApplicationProcessor.addApplicationArgument(value, appDepType, stringPrm);
+ }
+ }
+
+ }
+
+
+ public static boolean isUnicoreEndpoint(JobExecutionContext context) {
+ return ( (context.getApplicationContext().getHostDescription().getType() instanceof UnicoreHostType)?true:false );
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java
new file mode 100644
index 0000000..a1a884b
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java
@@ -0,0 +1,241 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.bes.utils;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.airavata.commons.gfac.type.ActualParameter;
+import org.apache.airavata.commons.gfac.type.ApplicationDescription;
+import org.apache.airavata.gfac.Constants;
+import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.gfac.provider.GFacProviderException;
+import org.apache.airavata.model.workspace.experiment.TaskDetails;
+import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.HpcApplicationDeploymentType;
+import org.apache.airavata.schemas.gfac.StringArrayType;
+import org.apache.airavata.schemas.gfac.StringParameterType;
+import org.apache.airavata.schemas.gfac.URIParameterType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import de.fzj.unicore.uas.client.StorageClient;
+
+
+public class DataTransferrer {
+ protected final Logger log = LoggerFactory.getLogger(this.getClass());
+
+ private JobExecutionContext jobContext;
+
+ private StorageClient storageClient;
+
+ public DataTransferrer(JobExecutionContext jobContext, StorageClient storageClient) {
+ this.jobContext = jobContext;
+ this.storageClient = storageClient;
+ }
+
+
+ public void uploadLocalFiles() throws GFacProviderException {
+ Map<String, Object> inputParams = jobContext.getInMessageContext()
+ .getParameters();
+ for (String paramKey : inputParams.keySet()) {
+ ActualParameter inParam = (ActualParameter) inputParams
+ .get(paramKey);
+ String paramDataType = inParam.getType().getType().toString();
+ if("URI".equals(paramDataType)) {
+ String uri = ((URIParameterType) inParam.getType()).getValue();
+ String fileName = new File(uri).getName();
+ if (uri.startsWith("file")) {
+ try {
+ String uriWithoutProtocol = uri.substring(
+ uri.lastIndexOf("://") + 1, uri.length());
+ FileUploader fileUploader = new FileUploader(
+ uriWithoutProtocol, "input/" + fileName,
+ Mode.overwrite);
+ fileUploader.perform(storageClient);
+ } catch (FileNotFoundException e3) {
+ throw new GFacProviderException(
+ "Error while staging-in, local file "+fileName+" not found", e3);
+ } catch (Exception e) {
+ throw new GFacProviderException("Cannot upload files", e);
+
+ }
+
+ }
+ }
+ }
+
+ }
+
+ /**
+ * This method will download all the remote files specified according to the output
+ * context of a job.
+ * */
+ public void downloadRemoteFiles() throws GFacProviderException {
+
+ String downloadLocation = getDownloadLocation();
+
+ File file = new File(downloadLocation);
+ if(!file.exists()){
+ file.mkdirs();
+ }
+
+ Map<String, ActualParameter> stringMap = new HashMap<String, ActualParameter>();
+
+ Map<String, Object> outputParams = jobContext.getOutMessageContext()
+ .getParameters();
+
+ for (String paramKey : outputParams.keySet()) {
+
+ ActualParameter outParam = (ActualParameter) outputParams
+ .get(paramKey);
+
+ // if single urls then convert each url into jsdl source
+ // elements,
+ // that are formed by concat of gridftpurl+inputdir+filename
+
+ String paramDataType = outParam.getType().getType().toString();
+
+ if ("String".equals(paramDataType)) {
+ String stringPrm = ((StringParameterType) outParam
+ .getType()).getValue();
+ String localFileName = null;
+ //TODO: why analysis.tar? it wont scale to gateways..
+ if(stringPrm == null || stringPrm.isEmpty()){
+ localFileName = "analysis-results.tar";
+ }else{
+ localFileName = stringPrm.substring(stringPrm.lastIndexOf("/")+1);
+ }
+ String outputLocation = downloadLocation+File.separator+localFileName;
+ FileDownloader fileDownloader = new FileDownloader("output/"+stringPrm,outputLocation, Mode.overwrite);
+ try {
+ fileDownloader.perform(storageClient);
+ ((StringParameterType) outParam.getType()).setValue(outputLocation);
+ stringMap.put(paramKey, outParam);
+ } catch (Exception e) {
+ throw new GFacProviderException(e.getLocalizedMessage(),e);
+ }
+ }
+
+ else if ("StringArray".equals(paramDataType)) {
+ String[] valueArray = ((StringArrayType) outParam.getType())
+ .getValueArray();
+ for (String v : valueArray) {
+ String localFileName = v.substring(v.lastIndexOf("/")+1);;
+ String outputLocation = downloadLocation+File.separator+localFileName;
+ FileDownloader fileDownloader = new FileDownloader("output/"+v,outputLocation, Mode.overwrite);
+ try {
+ fileDownloader.perform(storageClient);
+ ((StringParameterType) outParam.getType()).setValue(outputLocation);
+ stringMap.put(paramKey, outParam);
+ } catch (Exception e) {
+ throw new GFacProviderException(e.getLocalizedMessage(),e);
+ }
+ }
+ }
+ }
+ if (stringMap == null || stringMap.isEmpty()) {
+ throw new GFacProviderException("Empty Output returned from the Application, Double check the application" +
+ "and ApplicationDescriptor output Parameter Names");
+ }
+
+ downloadStdOuts();
+ }
+
+
+ public void downloadStdOuts() throws GFacProviderException{
+ String downloadLocation = getDownloadLocation();
+ File file = new File(downloadLocation);
+ if(!file.exists()){
+ file.mkdirs();
+ }
+
+ HpcApplicationDeploymentType appDepType = (HpcApplicationDeploymentType) jobContext
+ .getApplicationContext().getApplicationDeploymentDescription()
+ .getType();
+
+ String stdout = appDepType.getStandardOutput();
+ String stderr = appDepType.getStandardError();
+ if(stdout != null) {
+ stdout = stdout.substring(stdout.lastIndexOf('/')+1);
+ }
+
+ if(stderr != null) {
+ stderr = stderr.substring(stderr.lastIndexOf('/')+1);
+ }
+
+ String stdoutFileName = (stdout == null || stdout.equals("")) ? "stdout"
+ : stdout;
+ String stderrFileName = (stdout == null || stderr.equals("")) ? "stderr"
+ : stderr;
+
+ ApplicationDescription application = jobContext.getApplicationContext().getApplicationDeploymentDescription();
+ ApplicationDeploymentDescriptionType appDesc = application.getType();
+
+ String stdoutLocation = downloadLocation+File.separator+stdoutFileName;
+ FileDownloader f1 = new FileDownloader("output/"+stdoutFileName,stdoutLocation, Mode.overwrite);
+ try {
+ f1.perform(storageClient);
+ String stdoutput = readFile(stdoutLocation);
+ appDesc.setStandardOutput(stdoutput);
+ } catch (Exception e) {
+ throw new GFacProviderException(e.getLocalizedMessage(),e);
+ }
+ String stderrLocation = downloadLocation+File.separator+stderrFileName;
+ FileDownloader f2 = new FileDownloader("output/"+stderrFileName,stderrLocation, Mode.overwrite);
+ try {
+ f2.perform(storageClient);
+ String stderror = readFile(stderrLocation);
+ appDesc.setStandardError(stderror);
+ } catch (Exception e) {
+ throw new GFacProviderException(e.getLocalizedMessage(),e);
+ }
+ }
+
+ private String readFile(String localFile) throws IOException {
+ BufferedReader instream = new BufferedReader(new FileReader(localFile));
+ StringBuffer buff = new StringBuffer();
+ String temp = null;
+ while ((temp = instream.readLine()) != null) {
+ buff.append(temp);
+ buff.append(Constants.NEWLINE);
+ }
+
+ log.info("finish read file:" + localFile);
+
+ return buff.toString();
+ }
+
+ private String getDownloadLocation() {
+ TaskDetails taskData = jobContext.getTaskData();
+ if (taskData != null && taskData.getAdvancedOutputDataHandling() != null) {
+ String outputDataDirectory = taskData.getAdvancedOutputDataHandling().getOutputDataDir();
+ return outputDataDirectory;
+ }
+ return null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/FileDownloader.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/FileDownloader.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/FileDownloader.java
new file mode 100644
index 0000000..680aa51
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/FileDownloader.java
@@ -0,0 +1,256 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.bes.utils;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Map;
+
+import org.unigrids.services.atomic.types.GridFileType;
+import org.unigrids.services.atomic.types.ProtocolType;
+
+import de.fzj.unicore.uas.client.FileTransferClient;
+import de.fzj.unicore.uas.client.StorageClient;
+import de.fzj.unicore.uas.client.UFTPConstants;
+import de.fzj.unicore.uas.client.UFTPFileTransferClient;
+import de.fzj.unicore.uas.fts.FiletransferOptions.IMonitorable;
+import de.fzj.unicore.uas.fts.FiletransferOptions.SupportsPartialRead;
+
+/**
+ * helper that exports remote files from a UNICORE Storage
+ * to the local client machine.<br/>
+ * Simple wildcards ("*" and "?") and download of
+ * directories are supported.
+ *
+ * TODO this should be refactored so the single-file download logic
+ * is separated from the wildcard/directory/provided outputStream logic
+ *
+ * @author schuller
+ */
+public class FileDownloader extends FileTransferBase{
+
+ private boolean showProgress=true;
+
+ private boolean forceFileOnly=false;
+
+ private OutputStream targetStream=null;
+
+ public FileDownloader(String from, String to, Mode mode){
+ this(from,to,mode,true);
+ }
+
+ public FileDownloader(String from, String to, Mode mode, boolean failOnError){
+ this.to=to;
+ this.from=from;
+ this.mode=mode;
+ this.failOnError=failOnError;
+ }
+
+ public void perform(StorageClient sms)throws Exception{
+ boolean isWildcard=hasWildCards(from);
+ boolean isDirectory=false;
+ GridFileType gridSource=null;
+ if(isWildcard){
+ performWildCardExport(sms);
+ }
+ else {
+ //check if source is a directory
+ gridSource=sms.listProperties(from);
+ isDirectory=gridSource.getIsDirectory();
+ if(isDirectory){
+ if(forceFileOnly){
+ throw new IOException("Source is a directory");
+ }
+ performDirectoryExport(gridSource, new File(to), sms);
+ }
+ else{
+ download(gridSource,new File(to),sms);
+ }
+ }
+ }
+
+ protected void performDirectoryExport(GridFileType directory, File targetDirectory, StorageClient sms)throws Exception{
+ if(!targetDirectory.exists()|| !targetDirectory.canWrite()){
+ throw new IOException("Target directory <"+to+"> does not exist or is not writable!");
+ }
+ if(!targetDirectory.isDirectory()){
+ throw new IOException("Target <"+to+"> is not a directory!");
+ }
+ GridFileType[]gridFiles=sms.listDirectory(directory.getPath());
+ for(GridFileType file: gridFiles){
+ if(file.getIsDirectory()){
+ if(!recurse) {
+ System.out.println("Skipping directory "+file.getPath());
+ continue;
+ }
+ else{
+ File newTargetDirectory=new File(targetDirectory,getName(file.getPath()));
+ boolean success=newTargetDirectory.mkdirs();
+ if(!success)throw new IOException("Can create directory: "+newTargetDirectory.getAbsolutePath());
+ performDirectoryExport(file, newTargetDirectory, sms);
+ continue;
+ }
+ }
+ download(file, new File(targetDirectory,getName(file.getPath())), sms);
+ }
+ }
+
+ protected void performWildCardExport(StorageClient sms)throws Exception{
+ String dir=getDir(from);
+ if(dir==null)dir="/";
+ GridFileType[] files=sms.find(dir, false, from, false, null, null);
+ File targetDir=targetStream==null?new File(to):null;
+ if(targetStream==null){
+ if(!targetDir.isDirectory())throw new IOException("Target is not a directory.");
+ }
+ for(GridFileType f: files){
+ download(f, targetDir, sms);
+ }
+ }
+
+ private String getDir(String path){
+ return new File(path).getParent();
+ }
+
+ private String getName(String path){
+ return new File(path).getName();
+ }
+
+ /**
+ * download a single regular file
+ *
+ * @param source - grid file descriptor
+ * @param localFile - local file or directory to write to
+ * @param sms
+ * @throws Exception
+ */
+ private void download(GridFileType source, File localFile, StorageClient sms)throws Exception{
+ if(source==null || source.getIsDirectory()){
+ throw new IllegalStateException("Source="+source);
+ }
+
+ OutputStream os=targetStream!=null?targetStream:null;
+ FileTransferClient ftc=null;
+ try{
+ String path=source.getPath();
+ if(targetStream==null){
+ if(localFile.isDirectory()){
+ localFile=new File(localFile,getName(source.getPath()));
+ }
+ if(mode.equals(Mode.nooverwrite) && localFile.exists()){
+ System.out.println("File exists and creation mode was set to 'nooverwrite'.");
+ return;
+ }
+ System.out.println("Downloading remote file '"+sms.getUrl()+"#/"+path+"' -> "+localFile.getAbsolutePath());
+ os=new FileOutputStream(localFile.getAbsolutePath(), mode.equals(Mode.append));
+ }
+
+ chosenProtocol=sms.findSupportedProtocol(preferredProtocols.toArray(new ProtocolType.Enum[preferredProtocols.size()]));
+ Map<String,String>extraParameters=makeExtraParameters(chosenProtocol);
+ ftc=sms.getExport(path,extraParameters,chosenProtocol);
+ configure(ftc, extraParameters);
+ System.out.println("DEB:File transfer URL : "+ftc.getUrl());
+// ProgressBar p=null;
+ if(ftc instanceof IMonitorable && showProgress){
+ long size=ftc.getSourceFileSize();
+ if(isRange()){
+ size=getRangeSize();
+ }
+// p=new ProgressBar(localFile.getName(),size,msg);
+// ((IMonitorable) ftc).setProgressListener(p);
+ }
+ long startTime=System.currentTimeMillis();
+ if(isRange()){
+ if(!(ftc instanceof SupportsPartialRead)){
+ throw new Exception("Byte range is defined but protocol does not allow " +
+ "partial read! Please choose a different protocol!");
+ }
+ System.out.println("Byte range: "+startByte+" - "+(getRangeSize()>0?endByte:""));
+ SupportsPartialRead pReader=(SupportsPartialRead)ftc;
+ pReader.readPartial(startByte, endByte-startByte+1, os);
+ }
+ else{
+ ftc.readAllData(os);
+ }
+// if(p!=null){
+// p.finish();
+// }
+ if(timing){
+ long duration=System.currentTimeMillis()-startTime;
+ double rate=(double)localFile.length()/(double)duration;
+ System.out.println("Rate: " +rate+ " kB/sec.");
+ }
+ if(targetStream==null)copyProperties(source, localFile);
+ }
+ finally{
+ try{
+ if(targetStream==null && os!=null){
+ os.close();
+ }
+ }catch(Exception ignored){}
+ if(ftc!=null){
+ try{
+ ftc.destroy();
+ }catch(Exception e1){
+// System.out.println("Could not destroy the filetransfer client",e1);
+ }
+ }
+ }
+ }
+
+ /**
+ * if possible, copy the remote executable flag to the local file
+ * @throws Exception
+ */
+ private void copyProperties(GridFileType source, File localFile)throws Exception{
+ try{
+ localFile.setExecutable(source.getPermissions().getExecutable());
+ }
+ catch(Exception ex){
+ //TODO: logging
+// ("Can't set 'executable' flag for "+localFile.getName(), ex);
+ }
+ }
+
+ private void configure(FileTransferClient ftc, Map<String,String>params){
+ if(ftc instanceof UFTPFileTransferClient){
+ UFTPFileTransferClient u=(UFTPFileTransferClient)ftc;
+ String secret=params.get(UFTPConstants.PARAM_SECRET);
+ u.setSecret(secret);
+ }
+ }
+
+ public void setShowProgress(boolean showProgress) {
+ this.showProgress = showProgress;
+ }
+
+ public void setForceFileOnly(boolean forceFileOnly) {
+ this.forceFileOnly = forceFileOnly;
+ }
+
+ public void setTargetStream(OutputStream targetStream) {
+ this.targetStream = targetStream;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/49b6987f/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/FileTransferBase.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/FileTransferBase.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/FileTransferBase.java
new file mode 100644
index 0000000..ef46feb
--- /dev/null
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/FileTransferBase.java
@@ -0,0 +1,227 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.airavata.gfac.bes.utils;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.regex.Pattern;
+
+import org.unigrids.services.atomic.types.GridFileType;
+import org.unigrids.services.atomic.types.ProtocolType;
+
+import de.fzj.unicore.uas.client.StorageClient;
+import de.fzj.unicore.uas.util.PropertyHelper;
+public class FileTransferBase {
+
+ protected Properties extraParameterSource;
+
+ protected boolean timing=false;
+
+ protected boolean recurse=false;
+
+ protected String from;
+
+ protected String to;
+
+ //index of first byte to download
+ protected Long startByte;
+
+ //index of last byte to download
+ protected Long endByte;
+
+ /**
+ * the creation mode
+ */
+ protected Mode mode;
+
+ /**
+ * whether the job processing should fail if an error occurs
+ */
+ protected boolean failOnError;
+
+ protected List<ProtocolType.Enum> preferredProtocols=new ArrayList<ProtocolType.Enum>();
+
+ public FileTransferBase(){
+ preferredProtocols.add(ProtocolType.BFT);
+ }
+
+ protected Map<String,String>makeExtraParameters(ProtocolType.Enum protocol){
+ Map<String, String> res;
+ if(extraParameterSource==null){
+ res=new HashMap<String, String>();
+ }
+ else{
+ String p=String.valueOf(protocol);
+ PropertyHelper ph=new PropertyHelper(extraParameterSource, new String[]{p,p.toLowerCase()});
+ res= ph.getFilteredMap();
+ }
+ if(res.size()>0){
+ // TODO: change it to logger
+ System.out.println("Have "+res.size()+" extra parameters for protocol "+protocol);
+ }
+ return res;
+ }
+
+
+ public String getTo() {
+ return to;
+ }
+
+ public String getFrom() {
+ return from;
+ }
+
+ public void setTo(String to) {
+ this.to = to;
+ }
+
+ public void setFrom(String from) {
+ this.from = from;
+ }
+
+ public Mode getMode() {
+ return mode;
+ }
+
+ public boolean isFailOnError() {
+ return failOnError;
+ }
+
+ public boolean isTiming() {
+ return timing;
+ }
+
+ public void setTiming(boolean timing) {
+ this.timing = timing;
+ }
+
+ public void setFailOnError(boolean failOnError) {
+ this.failOnError = failOnError;
+ }
+
+ public List<ProtocolType.Enum> getPreferredProtocols() {
+ return preferredProtocols;
+ }
+
+ public void setPreferredProtocols(List<ProtocolType.Enum> preferredProtocols) {
+ this.preferredProtocols = preferredProtocols;
+ }
+
+ public void setExtraParameterSource(Properties properties){
+ this.extraParameterSource=properties;
+ }
+
+ public void setRecurse(boolean recurse) {
+ this.recurse = recurse;
+ }
+ /**
+ * check if the given path denotes a valid remote directory
+ * @param remotePath - the path
+ * @param sms - the storage
+ * @return <code>true</code> if the remote directory exists and is a directory
+ */
+ protected boolean isValidDirectory(String remotePath, StorageClient sms){
+ boolean result=false;
+ if(! ("/".equals(remotePath) || ".".equals(remotePath)) ){
+ try{
+ GridFileType gft=sms.listProperties(remotePath);
+ result=gft.getIsDirectory();
+ }catch(Exception ex){
+ result=false;
+ }
+ }
+ else result=true;
+
+ return result;
+ }
+
+ public File[] resolveWildCards(File original){
+ final String name=original.getName();
+ if(!hasWildCards(original))return new File[]{original};
+ File parent=original.getParentFile();
+ if(parent==null)parent=new File(".");
+ FilenameFilter filter=new FilenameFilter(){
+ Pattern p=createPattern(name);
+ public boolean accept(File file, String name){
+ return p.matcher(name).matches();
+ }
+ };
+ return parent.listFiles(filter);
+ }
+
+ protected boolean hasWildCards(File file){
+ return hasWildCards(file.getName());
+ }
+
+ public boolean hasWildCards(String name){
+ return name.contains("*") || name.contains("?");
+ }
+
+ private Pattern createPattern(String nameWithWildcards){
+ String regex=nameWithWildcards.replace("?",".").replace("*", ".*");
+ return Pattern.compile(regex);
+ }
+
+ protected ProtocolType.Enum chosenProtocol=null;
+
+ public ProtocolType.Enum getChosenProtocol(){
+ return chosenProtocol;
+ }
+
+ public Long getStartByte() {
+ return startByte;
+ }
+
+ public void setStartByte(Long startByte) {
+ this.startByte = startByte;
+ }
+
+ public Long getEndByte() {
+ return endByte;
+ }
+
+ public void setEndByte(Long endByte) {
+ this.endByte = endByte;
+ }
+
+ /**
+ * checks if a byte range is defined
+ * @return <code>true</code> iff both startByte and endByte are defined
+ */
+ protected boolean isRange(){
+ return startByte!=null && endByte!=null;
+ }
+
+ /**
+ * get the number of bytes in the byte range, or "-1" if the range is open-ended
+ * @return
+ */
+ protected long getRangeSize(){
+ if(Long.MAX_VALUE==endByte)return -1;
+ return endByte-startByte;
+ }
+}