You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ms...@apache.org on 2015/03/30 12:17:10 UTC
airavata git commit: Create and launch sample now independent;
better file output handling; some bits of us3 and juropa testing
Repository: airavata
Updated Branches:
refs/heads/master f5a2eac62 -> e0df621b6
Create and launch sample now independent; better file output handling;
some bits of us3 and juropa testing
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/e0df621b
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/e0df621b
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/e0df621b
Branch: refs/heads/master
Commit: e0df621b6f4af5147824d4c5d61d155b134828ff
Parents: f5a2eac
Author: msmemon <sh...@gmail.com>
Authored: Mon Mar 30 12:16:20 2015 +0200
Committer: msmemon <sh...@gmail.com>
Committed: Mon Mar 30 12:16:20 2015 +0200
----------------------------------------------------------------------
.../client/samples/CreateLaunchBES.java | 487 +++++++++++++++++++
.../tools/RegisterSampleApplications.java | 38 +-
.../tools/RegisterSampleApplicationsUtils.java | 3 +-
.../gfac/bes/utils/ApplicationProcessor.java | 58 ++-
.../airavata/gfac/bes/utils/BESConstants.java | 4 +
.../gfac/bes/utils/DataTransferrer.java | 7 +-
.../gfac/bes/utils/ResourceProcessor.java | 8 +-
.../gfac/bes/utils/UASDataStagingProcessor.java | 5 +-
8 files changed, 588 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/e0df621b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchBES.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchBES.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchBES.java
new file mode 100644
index 0000000..44ebfc7
--- /dev/null
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchBES.java
@@ -0,0 +1,487 @@
+package org.apache.airavata.client.samples;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.airavata.api.Airavata;
+import org.apache.airavata.api.client.AiravataClientFactory;
+import org.apache.airavata.client.tools.RegisterSampleApplications;
+import org.apache.airavata.client.tools.RegisterSampleApplicationsUtils;
+import org.apache.airavata.model.appcatalog.appinterface.InputDataObjectType;
+import org.apache.airavata.model.appcatalog.appinterface.OutputDataObjectType;
+import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
+import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionInterface;
+import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
+import org.apache.airavata.model.appcatalog.computeresource.SecurityProtocol;
+import org.apache.airavata.model.appcatalog.computeresource.UnicoreJobSubmission;
+import org.apache.airavata.model.error.AiravataClientException;
+import org.apache.airavata.model.error.AiravataErrorType;
+import org.apache.airavata.model.error.AiravataSystemException;
+import org.apache.airavata.model.error.ExperimentNotFoundException;
+import org.apache.airavata.model.error.InvalidRequestException;
+import org.apache.airavata.model.util.ExperimentModelUtil;
+import org.apache.airavata.model.workspace.Gateway;
+import org.apache.airavata.model.workspace.Project;
+import org.apache.airavata.model.workspace.experiment.AdvancedOutputDataHandling;
+import org.apache.airavata.model.workspace.experiment.ComputationalResourceScheduling;
+import org.apache.airavata.model.workspace.experiment.ErrorDetails;
+import org.apache.airavata.model.workspace.experiment.Experiment;
+import org.apache.airavata.model.workspace.experiment.ExperimentSummary;
+import org.apache.airavata.model.workspace.experiment.UserConfigurationData;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CreateLaunchBES {
+
+ public static final String THRIFT_SERVER_HOST = "localhost";
+ public static final int THRIFT_SERVER_PORT = 8930;
+
+ private final static Logger logger = LoggerFactory.getLogger(CreateLaunchExperiment.class);
+ private static final String DEFAULT_USER = "default.registry.user";
+ private static final String DEFAULT_GATEWAY = "php_reference_gateway";
+ private static Airavata.Client airavataClient;
+
+ private static String echoAppId = "Echo_5dd52cd4-f9a0-459f-9baf-f8e715e44548";
+ private static String mpiAppId = "HelloMPI_f0bb3b56-914e-4752-bb7b-dd18ef9dce00";
+
+ private static String unicoreHostName = "fsd-cloud15.zam.kfa-juelich.de";
+
+ private static String gatewayId;
+
+ // unicore service endpoint url
+ private static final String unicoreEndPointURL = "https://deisa-unic.fz-juelich.de:9111/FZJ_JUROPA/services/BESFactory?res=default_bes_factory";
+// private static final String unicoreEndPointURL = "https://fsd-cloud15.zam.kfa-juelich.de:7000/INTEROP1/services/BESFactory?res=default_bes_factory";
+
+ public static void main(String[] args) throws Exception {
+ airavataClient = AiravataClientFactory.createAiravataClient(THRIFT_SERVER_HOST, THRIFT_SERVER_PORT);
+ System.out.println("API version is " + airavataClient.getAPIVersion());
+// createGateway();
+// getGateway("testGatewayId");
+// registerApplications(); // run this only the first time
+ createAndLaunchExp();
+ }
+
+ private static String fsdResourceId;
+
+
+ public static void getAvailableAppInterfaceComputeResources(String appInterfaceId) {
+ try {
+ Map<String, String> availableAppInterfaceComputeResources = airavataClient.getAvailableAppInterfaceComputeResources(appInterfaceId);
+ for (String key : availableAppInterfaceComputeResources.keySet()) {
+ System.out.println("id : " + key);
+ System.out.println("name : " + availableAppInterfaceComputeResources.get(key));
+ }
+ } catch (AiravataSystemException e) {
+ e.printStackTrace();
+ } catch (InvalidRequestException e) {
+ e.printStackTrace();
+ } catch (AiravataClientException e) {
+ e.printStackTrace();
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+
+ public static void createGateway() {
+ try {
+ Gateway gateway = new Gateway();
+ gateway.setGatewayId("testGatewayId2");
+ gateway.setGatewayName("testGateway2");
+ gatewayId = airavataClient.addGateway(gateway);
+ System.out.println(gatewayId);
+ } catch (AiravataSystemException e) {
+ e.printStackTrace();
+ } catch (InvalidRequestException e) {
+ e.printStackTrace();
+ } catch (AiravataClientException e) {
+ e.printStackTrace();
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+ public static void getGateway(String gatewayId) {
+ try {
+ Gateway gateway = airavataClient.getGateway(gatewayId);
+ gateway.setDomain("testDomain");
+ airavataClient.updateGateway(gatewayId, gateway);
+ List<Gateway> allGateways = airavataClient.getAllGateways();
+ System.out.println(allGateways.size());
+ if (airavataClient.isGatewayExist(gatewayId)) {
+ Gateway gateway1 = airavataClient.getGateway(gatewayId);
+ System.out.println(gateway1.getGatewayName());
+ }
+ boolean b = airavataClient.deleteGateway("testGatewayId2");
+ System.out.println(b);
+ } catch (AiravataSystemException e) {
+ e.printStackTrace();
+ } catch (InvalidRequestException e) {
+ e.printStackTrace();
+ } catch (AiravataClientException e) {
+ e.printStackTrace();
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+
+ public static void createAndLaunchExp() throws TException {
+ List<String> experimentIds = new ArrayList<String>();
+ try {
+ for (int i = 0; i < 1; i++) {
+// final String expId = createEchoExperimentForFSD(airavataClient);
+ final String expId = createMPIExperimentForFSD(airavataClient);
+ experimentIds.add(expId);
+ System.out.println("Experiment ID : " + expId);
+// updateExperiment(airavata, expId);
+ launchExperiment(airavataClient, expId);
+ }
+
+ Thread.sleep(10000);
+ for (String exId : experimentIds) {
+ Experiment experiment = airavataClient.getExperiment(exId);
+ System.out.println(experiment.getExperimentID() + " " + experiment.getExperimentStatus().getExperimentState().name());
+ }
+
+
+ } catch (Exception e) {
+ logger.error("Error while connecting with server", e.getMessage());
+ e.printStackTrace();
+
+ }
+ }
+
+ public static void launchExperiment(Airavata.Client client, String expId)
+ throws TException {
+ try {
+ String tokenId = "-0bbb-403b-a88a-42b6dbe198e9";
+ client.launchExperiment(expId, tokenId);
+ } catch (ExperimentNotFoundException e) {
+ logger.error("Error occured while launching the experiment...", e.getMessage());
+ throw new ExperimentNotFoundException(e);
+ } catch (AiravataSystemException e) {
+ logger.error("Error occured while launching the experiment...", e.getMessage());
+ throw new AiravataSystemException(e);
+ } catch (InvalidRequestException e) {
+ logger.error("Error occured while launching the experiment...", e.getMessage());
+ throw new InvalidRequestException(e);
+ } catch (AiravataClientException e) {
+ logger.error("Error occured while launching the experiment...", e.getMessage());
+ throw new AiravataClientException(e);
+ } catch (TException e) {
+ logger.error("Error occured while launching the experiment...", e.getMessage());
+ throw new TException(e);
+ }
+ }
+
+ public static void registerApplications() {
+ RegisterSampleApplications registerSampleApplications = new RegisterSampleApplications(airavataClient);
+
+ //Register all compute hosts
+ registerSampleApplications.registerXSEDEHosts();
+
+
+ //Register Gateway Resource Preferences
+ registerSampleApplications.registerGatewayResourceProfile();
+
+ //Register all application modules
+ registerSampleApplications.registerAppModules();
+
+ //Register all application deployments
+ registerSampleApplications.registerAppDeployments();
+
+ //Register all application interfaces
+ registerSampleApplications.registerAppInterfaces();
+ }
+
+ public static String registerUnicoreEndpoint(String hostName, String hostDesc, JobSubmissionProtocol protocol, SecurityProtocol securityProtocol) throws TException {
+
+ ComputeResourceDescription computeResourceDescription = RegisterSampleApplicationsUtils
+ .createComputeResourceDescription(hostName, hostDesc, null, null);
+
+ fsdResourceId = airavataClient.registerComputeResource(computeResourceDescription);
+
+ if (fsdResourceId.isEmpty())
+ throw new AiravataClientException();
+
+ System.out.println("FSD Compute ResourceID: " + fsdResourceId);
+
+ JobSubmissionInterface jobSubmission = RegisterSampleApplicationsUtils.createJobSubmissionInterface(fsdResourceId, protocol, 2);
+ UnicoreJobSubmission ucrJobSubmission = new UnicoreJobSubmission();
+ ucrJobSubmission.setSecurityProtocol(securityProtocol);
+ ucrJobSubmission.setUnicoreEndPointURL(unicoreEndPointURL);
+
+ return jobSubmission.getJobSubmissionInterfaceId();
+ }
+
+ public static String createEchoExperimentForFSD(Airavata.Client client) throws TException {
+ try {
+ List<InputDataObjectType> exInputs = client.getApplicationInputs(echoAppId);
+ for (InputDataObjectType inputDataObjectType : exInputs) {
+ if (inputDataObjectType.getName().equalsIgnoreCase("Input_to_Echo")) {
+ inputDataObjectType.setValue("Hello World");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("Input_to_Echo2")) {
+ inputDataObjectType.setValue("http://www.textfiles.com/100/ad.txt");
+ } else if (inputDataObjectType.getName().equalsIgnoreCase("Input_to_Echo3")) {
+ inputDataObjectType.setValue("file:///tmp/test.txt");
+ }
+ }
+ List<OutputDataObjectType> exOut = client.getApplicationOutputs(echoAppId);
+
+ Experiment simpleExperiment =
+ ExperimentModelUtil.createSimpleExperiment("default", "admin", "echoExperiment", "SimpleEcho2", echoAppId, exInputs);
+ simpleExperiment.setExperimentOutputs(exOut);
+
+
+ Map<String, String> computeResources = airavataClient.getAvailableAppInterfaceComputeResources(echoAppId);
+ if (computeResources != null && computeResources.size() != 0) {
+ for (String id : computeResources.keySet()) {
+ String resourceName = computeResources.get(id);
+ if (resourceName.equals(unicoreHostName)) {
+ ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling(id, 1, 1, 1, "normal", 30, 0, 1048576, "sds128");
+ UserConfigurationData userConfigurationData = new UserConfigurationData();
+ userConfigurationData.setAiravataAutoSchedule(false);
+ userConfigurationData.setOverrideManualScheduledParams(false);
+ userConfigurationData.setComputationalResourceScheduling(scheduling);
+
+ userConfigurationData.setGenerateCert(false);
+ userConfigurationData.setUserDN("");
+
+ // set output directory
+ AdvancedOutputDataHandling dataHandling = new AdvancedOutputDataHandling();
+ dataHandling.setOutputDataDir("/tmp/airavata/output/" + UUID.randomUUID().toString() + "/");
+ userConfigurationData.setAdvanceOutputDataHandling(dataHandling);
+ simpleExperiment.setUserConfigurationData(userConfigurationData);
+
+ return client.createExperiment(DEFAULT_GATEWAY, simpleExperiment);
+ }
+ }
+ }
+ } catch (AiravataSystemException e) {
+ logger.error("Error occured while creating the experiment...", e.getMessage());
+ throw new AiravataSystemException(e);
+ } catch (InvalidRequestException e) {
+ logger.error("Error occured while creating the experiment...", e.getMessage());
+ throw new InvalidRequestException(e);
+ } catch (AiravataClientException e) {
+ logger.error("Error occured while creating the experiment...", e.getMessage());
+ throw new AiravataClientException(e);
+ } catch (TException e) {
+ logger.error("Error occured while creating the experiment...", e.getMessage());
+ throw new TException(e);
+ }
+ return null;
+ }
+
+
+ public static String createMPIExperimentForFSD(Airavata.Client client) throws TException {
+ try {
+ List<InputDataObjectType> exInputs = client.getApplicationInputs(mpiAppId);
+ for (InputDataObjectType inputDataObjectType : exInputs) {
+// if (inputDataObjectType.getName().equalsIgnoreCase("Sample_Input")) {
+// inputDataObjectType.setValue("");
+// }
+ if (inputDataObjectType.getName().equalsIgnoreCase("NumberOfProcesses")) {
+ inputDataObjectType.setValue("32");
+ }
+ if (inputDataObjectType.getName().equalsIgnoreCase("US3INPUT")) {
+ inputDataObjectType.setValue("file://home/m.memon/us3input/smallerdata/hpcinput-uslims3.uthscsa.edu-uslims3_cauma3-01594.tar");
+ }
+ if (inputDataObjectType.getName().equalsIgnoreCase("US3INPUTARG")) {
+ inputDataObjectType.setValue("hpcinput-uslims3.uthscsa.edu-uslims3_cauma3-01594.tar");
+ }
+ }
+
+ List<OutputDataObjectType> exOut = client.getApplicationOutputs(mpiAppId);
+
+ for (OutputDataObjectType outputDataObjectType : exOut) {
+ if(outputDataObjectType.getName().equals("US3OUT")){
+ outputDataObjectType.setValue("output/analysis-results.tar");
+ }
+ }
+
+ Experiment simpleExperiment =
+ ExperimentModelUtil.createSimpleExperiment("default", "admin", "mpiExperiment", "HelloMPI", mpiAppId, exInputs);
+ simpleExperiment.setExperimentOutputs(exOut);
+
+
+ Map<String, String> computeResources = airavataClient.getAvailableAppInterfaceComputeResources(mpiAppId);
+ if (computeResources != null && computeResources.size() != 0) {
+ for (String id : computeResources.keySet()) {
+ String resourceName = computeResources.get(id);
+ if (resourceName.equals(unicoreHostName)) {
+ ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling(id, 0, 4, 0, null, 10, 0, 0, null);
+ UserConfigurationData userConfigurationData = new UserConfigurationData();
+ userConfigurationData.setAiravataAutoSchedule(false);
+ userConfigurationData.setOverrideManualScheduledParams(false);
+ userConfigurationData.setComputationalResourceScheduling(scheduling);
+
+ userConfigurationData.setGenerateCert(true);
+ userConfigurationData.setUserDN("CN=m.memon, O=Ultrascan Gateway, C=DE");
+
+ // set output directory
+ AdvancedOutputDataHandling dataHandling = new AdvancedOutputDataHandling();
+ dataHandling.setOutputDataDir("/tmp/airavata/output/" + UUID.randomUUID().toString() + "/");
+ userConfigurationData.setAdvanceOutputDataHandling(dataHandling);
+ simpleExperiment.setUserConfigurationData(userConfigurationData);
+
+ return client.createExperiment(DEFAULT_GATEWAY, simpleExperiment);
+ }
+ }
+ }
+ } catch (AiravataSystemException e) {
+ logger.error("Error occured while creating the experiment...", e.getMessage());
+ throw new AiravataSystemException(e);
+ } catch (InvalidRequestException e) {
+ logger.error("Error occured while creating the experiment...", e.getMessage());
+ throw new InvalidRequestException(e);
+ } catch (AiravataClientException e) {
+ logger.error("Error occured while creating the experiment...", e.getMessage());
+ throw new AiravataClientException(e);
+ } catch (TException e) {
+ logger.error("Error occured while creating the experiment...", e.getMessage());
+ throw new TException(e);
+ }
+ return null;
+ }
+
+
+ public static List<Experiment> getExperimentsForUser(Airavata.Client client, String user) {
+ try {
+ return client.getAllUserExperiments(DEFAULT_GATEWAY, user);
+ } catch (AiravataSystemException e) {
+ e.printStackTrace();
+ } catch (InvalidRequestException e) {
+ e.printStackTrace();
+ } catch (AiravataClientException e) {
+ e.printStackTrace();
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ public static List<Project> getAllUserProject(Airavata.Client client, String user) {
+ try {
+ return client.getAllUserProjects(DEFAULT_GATEWAY, user);
+ } catch (AiravataSystemException e) {
+ e.printStackTrace();
+ } catch (InvalidRequestException e) {
+ e.printStackTrace();
+ } catch (AiravataClientException e) {
+ e.printStackTrace();
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ public static List<Project> searchProjectsByProjectName(Airavata.Client client, String user, String projectName) {
+ try {
+ return client.searchProjectsByProjectName(DEFAULT_GATEWAY, user, projectName);
+ } catch (AiravataSystemException e) {
+ e.printStackTrace();
+ } catch (InvalidRequestException e) {
+ e.printStackTrace();
+ } catch (AiravataClientException e) {
+ e.printStackTrace();
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ public static List<Project> searchProjectsByProjectDesc(Airavata.Client client, String user, String desc) {
+ try {
+ return client.searchProjectsByProjectDesc(DEFAULT_GATEWAY, user, desc);
+ } catch (AiravataSystemException e) {
+ e.printStackTrace();
+ } catch (InvalidRequestException e) {
+ e.printStackTrace();
+ } catch (AiravataClientException e) {
+ e.printStackTrace();
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+
+ public static List<ExperimentSummary> searchExperimentsByName(Airavata.Client client, String user, String expName) {
+ try {
+ return client.searchExperimentsByName(DEFAULT_GATEWAY, user, expName);
+ } catch (AiravataSystemException e) {
+ e.printStackTrace();
+ } catch (InvalidRequestException e) {
+ e.printStackTrace();
+ } catch (AiravataClientException e) {
+ e.printStackTrace();
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ public static List<ExperimentSummary> searchExperimentsByDesc(Airavata.Client client, String user, String desc) {
+ try {
+ return client.searchExperimentsByDesc(DEFAULT_GATEWAY, user, desc);
+ } catch (AiravataSystemException e) {
+ e.printStackTrace();
+ } catch (InvalidRequestException e) {
+ e.printStackTrace();
+ } catch (AiravataClientException e) {
+ e.printStackTrace();
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ public static List<ExperimentSummary> searchExperimentsByApplication(Airavata.Client client, String user, String app) {
+ try {
+ return client.searchExperimentsByApplication(DEFAULT_GATEWAY, user, app);
+ } catch (AiravataSystemException e) {
+ e.printStackTrace();
+ } catch (InvalidRequestException e) {
+ e.printStackTrace();
+ } catch (AiravataClientException e) {
+ e.printStackTrace();
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ public static void getExperiment(Airavata.Client client, String expId) throws Exception {
+ try {
+ Experiment experiment = client.getExperiment(expId);
+ List<ErrorDetails> errors = experiment.getErrors();
+ if (errors != null && !errors.isEmpty()) {
+ for (ErrorDetails error : errors) {
+ System.out.println("ERROR MESSAGE : " + error.getActualErrorMessage());
+ }
+ }
+
+ } catch (ExperimentNotFoundException e) {
+ logger.error("Experiment does not exist", e);
+ throw new ExperimentNotFoundException("Experiment does not exist");
+ } catch (AiravataSystemException e) {
+ logger.error("Error while retrieving experiment", e);
+ throw new AiravataSystemException(AiravataErrorType.INTERNAL_ERROR);
+ } catch (InvalidRequestException e) {
+ logger.error("Error while retrieving experiment", e);
+ throw new InvalidRequestException("Error while retrieving experiment");
+ } catch (AiravataClientException e) {
+ logger.error("Error while retrieving experiment", e);
+ throw new AiravataClientException(AiravataErrorType.INTERNAL_ERROR);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/e0df621b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
index e10dde2..19999e0 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
@@ -29,6 +29,7 @@ import java.util.*;
import org.apache.airavata.api.Airavata;
import org.apache.airavata.api.client.AiravataClientFactory;
import org.apache.airavata.model.appcatalog.appdeployment.ApplicationParallelismType;
+import org.apache.airavata.model.appcatalog.appdeployment.SetEnvPaths;
import org.apache.airavata.model.appcatalog.appinterface.*;
import org.apache.airavata.model.appcatalog.computeresource.*;
import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference;
@@ -60,7 +61,8 @@ public class RegisterSampleApplications {
private static String fsdResourceId;
// unicore service endpoint url
- private static final String unicoreEndPointURL = "https://fsd-cloud15.zam.kfa-juelich.de:7000/INTEROP1/services/BESFactory?res=default_bes_factory";
+// private static final String unicoreEndPointURL = "https://fsd-cloud15.zam.kfa-juelich.de:7000/INTEROP1/services/BESFactory?res=default_bes_factory";
+ private static final String unicoreEndPointURL = "https://deisa-unic.fz-juelich.de:9111/FZJ_JUROPA/services/BESFactory?res=default_bes_factory";
//Appplication Names
private static final String echoName = "Echo";
@@ -565,17 +567,40 @@ public class RegisterSampleApplications {
InputDataObjectType input1 = RegisterSampleApplicationsUtils.createAppInput("Sample_Input", "",
DataType.STRING, null, 1,true, false, false, "An optional MPI source file", null);
+ InputDataObjectType input11 = RegisterSampleApplicationsUtils.createAppInput("US3INPUT", "",
+ DataType.URI, null, 1,true, false, false, "Input US3 file", null);
+
+ InputDataObjectType input12 = RegisterSampleApplicationsUtils.createAppInput("US3INPUTARG", "",
+ DataType.STRING, null, 1,true, false, false, "Input US3 Arg", null);
+
+
+ InputDataObjectType input2 = RegisterSampleApplicationsUtils.createAppInput("NumberOfProcesses", "",
+ DataType.INTEGER, null, 2,false, true, false, "Number Of Processes", null);
+
+
+ InputDataObjectType input3 = RegisterSampleApplicationsUtils.createAppInput("ProcessesPerHost", "",
+ DataType.INTEGER, null, 3,false, true, false, "Processes per host", null);
+
+
List<InputDataObjectType> applicationInputs = new ArrayList<InputDataObjectType>();
applicationInputs.add(input1);
+ applicationInputs.add(input2);
+ applicationInputs.add(input3);
+ applicationInputs.add(input11);
+ applicationInputs.add(input12);
OutputDataObjectType output1 = RegisterSampleApplicationsUtils.createAppOutput("STDOutput",
"", DataType.STDOUT, true, true);
OutputDataObjectType output2 = RegisterSampleApplicationsUtils.createAppOutput("STDErr",
"", DataType.STDERR, true, true);
+ OutputDataObjectType output3 = RegisterSampleApplicationsUtils.createAppOutput("US3OUT",
+ "", DataType.STRING, true, false);
+
List<OutputDataObjectType> applicationOutputs = new ArrayList<OutputDataObjectType>();
applicationOutputs.add(output1);
applicationOutputs.add(output2);
+ applicationOutputs.add(output3);
mpiInterfaceId = airavataClient.registerApplicationInterface(DEFAULT_GATEWAY,
@@ -1263,11 +1288,18 @@ public class RegisterSampleApplications {
RegisterSampleApplicationsUtils.createApplicationDeployment(echoModuleId, fsdResourceId,
"/bin/echo", ApplicationParallelismType.SERIAL, echoDescription, null, null, null));
System.out.println("Echo on FSD deployment Id: " + echoAppDeployId);
-
+
//Register MPI
+// String mpiAppDeployId = airavataClient.registerApplicationDeployment(DEFAULT_GATEWAY,
+// RegisterSampleApplicationsUtils.createApplicationDeployment(mpiModuleId, fsdResourceId,
+// "/home/bes/hellompi", ApplicationParallelismType.OPENMP_MPI, mpiDescription, null, null, null));
+
+
String mpiAppDeployId = airavataClient.registerApplicationDeployment(DEFAULT_GATEWAY,
RegisterSampleApplicationsUtils.createApplicationDeployment(mpiModuleId, fsdResourceId,
- "/home/bes/hellompi", ApplicationParallelismType.OPENMP_MPI, mpiDescription, null, null, null));
+ "us_mpi_analysis", ApplicationParallelismType.MPI, mpiDescription, null, null, null));
+
+
System.out.println("MPI on FSD deployment Id: " + mpiAppDeployId);
} catch (TException e) {
http://git-wip-us.apache.org/repos/asf/airavata/blob/e0df621b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplicationsUtils.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplicationsUtils.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplicationsUtils.java
index 86f7d1a..5ef1eae 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplicationsUtils.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplicationsUtils.java
@@ -24,6 +24,7 @@ package org.apache.airavata.client.tools;
import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
import org.apache.airavata.model.appcatalog.appdeployment.ApplicationModule;
import org.apache.airavata.model.appcatalog.appdeployment.ApplicationParallelismType;
+import org.apache.airavata.model.appcatalog.appdeployment.SetEnvPaths;
import org.apache.airavata.model.appcatalog.appinterface.*;
import org.apache.airavata.model.appcatalog.computeresource.*;
import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference;
@@ -73,7 +74,7 @@ public class RegisterSampleApplicationsUtils {
return deployment;
}
- public static ApplicationModule createApplicationModule(String appModuleName,
+ public static ApplicationModule createApplicationModule(String appModuleName,
String appModuleVersion, String appModuleDescription) {
ApplicationModule module = new ApplicationModule();
module.setAppModuleDescription(appModuleDescription);
http://git-wip-us.apache.org/repos/asf/airavata/blob/e0df621b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java
index 7c7a59e..bc91e49 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ApplicationProcessor.java
@@ -21,9 +21,13 @@
package org.apache.airavata.gfac.bes.utils;
+import java.util.Iterator;
+import java.util.List;
+
import org.apache.airavata.gfac.core.context.JobExecutionContext;
import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
import org.apache.airavata.model.appcatalog.appdeployment.ApplicationParallelismType;
+import org.apache.airavata.model.appcatalog.appinterface.InputDataObjectType;
import org.ggf.schemas.jsdl.x2005.x11.jsdl.ApplicationType;
import org.ggf.schemas.jsdl.x2005.x11.jsdl.JobDefinitionType;
import org.ggf.schemas.jsdl.x2005.x11.jsdlPosix.FileNameType;
@@ -77,24 +81,35 @@ public class ApplicationProcessor {
}else if (parallelism.equals(ApplicationParallelismType.MPI)){
JSDLUtils.getSPMDApplication(value).setSPMDVariation(SPMDVariations.MPI.value());
}
+
+ // setting number of processes
+ try {
+ String np = getInputAsString(context, BESConstants.NUMBER_OF_PROCESSES);
+ if((np != null) && (Integer.parseInt(np) > 0)){
+ NumberOfProcessesType num = NumberOfProcessesType.Factory.newInstance();
+ num.setStringValue(np);
+ JSDLUtils.getSPMDApplication(value).setNumberOfProcesses(num);
+ }
+
+ }catch(RuntimeException np) {
+ // do nothing
+ }
- int totalCPUCount = context.getTaskData().getTaskScheduling().getTotalCPUCount();
- if(totalCPUCount > 0){
- NumberOfProcessesType num = NumberOfProcessesType.Factory.newInstance();
- num.setStringValue(String.valueOf(totalCPUCount));
- JSDLUtils.getSPMDApplication(value).setNumberOfProcesses(num);
- }
- int totalNodeCount = context.getTaskData().getTaskScheduling().getNodeCount();
- if(totalNodeCount > 0){
- int ppn = totalCPUCount / totalNodeCount;
- ProcessesPerHostType pph = ProcessesPerHostType.Factory.newInstance();
- pph.setStringValue(String.valueOf(ppn));
- JSDLUtils.getSPMDApplication(value).setProcessesPerHost(pph);
+ try {
+ // setting processes per host
+ String pphost = getInputAsString(context, BESConstants.PROCESSES_PER_HOST);
+ if((pphost != null) && (Integer.parseInt(pphost) > 0)){
+ ProcessesPerHostType pph = ProcessesPerHostType.Factory.newInstance();
+ pph.setStringValue(String.valueOf(pphost));
+ JSDLUtils.getSPMDApplication(value).setProcessesPerHost(pph);
+ }
+ }catch(RuntimeException np) {
+ // do nothing
}
-
+
int totalThreadCount = context.getTaskData().getTaskScheduling().getNumberOfThreads();
-
+ // we take it as threads per processes
if(totalThreadCount > 0){
ThreadsPerProcessType tpp = ThreadsPerProcessType.Factory.newInstance();
tpp.setStringValue(String.valueOf(totalThreadCount));
@@ -187,5 +202,20 @@ public class ApplicationProcessor {
return isParallel;
}
+ private static String getInputAsString(JobExecutionContext context, String name) {
+ List<InputDataObjectType> inputList = context.getTaskData().getApplicationInputs();
+ String value = null;
+ for (Iterator<InputDataObjectType> iterator = inputList.iterator(); iterator.hasNext();) {
+ InputDataObjectType inputDataObjectType = iterator
+ .next();
+ if (inputDataObjectType.getName().equals(name)) {
+ value = inputDataObjectType.getValue();
+ break;
+ }
+ }
+ return value;
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/e0df621b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/BESConstants.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/BESConstants.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/BESConstants.java
index ffbf62e..6e87e28 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/BESConstants.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/BESConstants.java
@@ -36,6 +36,10 @@ public interface BESConstants {
public static final String PROP_CA_KEY_PATH = "bes.ca.key.path";
public static final String PROP_CA_KEY_PASS = "bes.ca.key.pass";
+
+ public static final String NUMBER_OF_PROCESSES = "NumberOfProcesses";
+
+ public static final String PROCESSES_PER_HOST = "ProcessesPerHost";
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/e0df621b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java
index cb036fa..d7f3244 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/DataTransferrer.java
@@ -37,6 +37,8 @@ import org.apache.airavata.model.appcatalog.appinterface.DataType;
import org.apache.airavata.model.appcatalog.appinterface.InputDataObjectType;
import org.apache.airavata.model.appcatalog.appinterface.OutputDataObjectType;
import org.apache.airavata.model.workspace.experiment.TaskDetails;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -91,9 +93,10 @@ public class DataTransferrer {
file.mkdirs();
}
List<String> outPrms = extractOutParams(jobContext);
+
for (String outPrm : outPrms) {
- String outputLocation = downloadLocation+File.separator+outPrm;
- FileDownloader fileDownloader = new FileDownloader(outPrm,outputLocation, Mode.overwrite);
+ if("".equals(outPrm)) continue;
+ FileDownloader fileDownloader = new FileDownloader(outPrm,downloadLocation, Mode.overwrite);
try {
fileDownloader.perform(storageClient);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/airavata/blob/e0df621b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java
index fce0c31..cc3ad8b 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/ResourceProcessor.java
@@ -34,10 +34,13 @@ public class ResourceProcessor {
TaskDetails taskData = context.getTaskData();
+
+
if(taskData != null && taskData.isSetTaskScheduling()){
try {
- ComputationalResourceScheduling crs = taskData.getTaskScheduling();
+ ComputationalResourceScheduling crs = taskData.getTaskScheduling();
+
if (crs.getTotalPhysicalMemory() > 0) {
RangeValueType rangeType = new RangeValueType();
rangeType.setLowerBound(Double.NaN);
@@ -52,6 +55,8 @@ public class ResourceProcessor {
rangeType.setUpperBound(Double.NaN);
rangeType.setExact(crs.getNodeCount());
JSDLUtils.setTotalResourceCountRequirements(value, rangeType);
+ // set totalcpu count to -1 as we dont need that
+ crs.setTotalCPUCount(0);
}
if(crs.getWallTimeLimit() > 0) {
@@ -70,6 +75,7 @@ public class ResourceProcessor {
rangeType.setExact(crs.getTotalCPUCount());
JSDLUtils.setTotalCPUCountRequirements(value, rangeType);
}
+
} catch (NullPointerException npe) {
new GFacProviderException("No value set for resource requirements.",npe);
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/e0df621b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java
index e82edef..09c6e11 100644
--- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java
+++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/utils/UASDataStagingProcessor.java
@@ -136,7 +136,10 @@ public class UASDataStagingProcessor {
ApplicationProcessor.addApplicationArgument(value, context, input.getValue());
}
else if (input.getType().equals(DataType.FLOAT) || input.getType().equals(DataType.INTEGER)){
- ApplicationProcessor.addApplicationArgument(value, context, String.valueOf(input.getValue()));
+ if(! (input.getName().equals(BESConstants.NUMBER_OF_PROCESSES) || input.getName().equals(BESConstants.PROCESSES_PER_HOST))) {
+ // temp avoid environ going to app args
+ ApplicationProcessor.addApplicationArgument(value, context, String.valueOf(input.getValue()));
+ }
}
}
}