You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ra...@apache.org on 2014/07/25 22:41:09 UTC
git commit: Adding samples for ultrascan application.
Repository: airavata
Updated Branches:
refs/heads/master 03ecceab1 -> 0f10016dd
Adding samples for ultrascan application.
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/0f10016d
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/0f10016d
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/0f10016d
Branch: refs/heads/master
Commit: 0f10016ddd1e61870f7b32de12f7c0278b2bbf15
Parents: 03eccea
Author: raminder <ra...@apache.org>
Authored: Fri Jul 25 16:40:51 2014 -0400
Committer: raminder <ra...@apache.org>
Committed: Fri Jul 25 16:40:51 2014 -0400
----------------------------------------------------------------------
.../samples/CreateLaunchExperimentUS3.java | 72 +++---
.../client/tools/RegisterUS3Application.java | 244 +++++++++++++++++++
2 files changed, 279 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/0f10016d/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperimentUS3.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperimentUS3.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperimentUS3.java
index ac1e7f5..25f27dc 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperimentUS3.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/samples/CreateLaunchExperimentUS3.java
@@ -48,7 +48,7 @@ import org.slf4j.LoggerFactory;
public class CreateLaunchExperimentUS3 {
//FIXME: Read from a config file
- public static final String THRIFT_SERVER_HOST = "gw111.iu.xsede.org";
+ public static final String THRIFT_SERVER_HOST = "gridfarm005.ucs.indiana.edu";
public static final int THRIFT_SERVER_PORT = 8930;
private final static Logger logger = LoggerFactory.getLogger(CreateLaunchExperiment.class);
private static final String DEFAULT_USER = "default.registry.user";
@@ -59,10 +59,8 @@ public class CreateLaunchExperimentUS3 {
final Airavata.Client airavata = AiravataClientFactory.createAiravataClient(THRIFT_SERVER_HOST, THRIFT_SERVER_PORT);
System.out.println("API version is " + airavata.getAPIVersion());
// addDescriptors();
-// final String expId = createExperimentForTrestles(airavata);
- final String expId = createUS3ExperimentForTrestles(airavata);
-// final String expId = createExperimentForStampede(airavata);
-// final String expId = createUS3ExperimentForStampede(airavata);
+// final String expId = createUS3ExperimentForTrestles(airavata);
+ final String expId = createUS3ExperimentForStampede(airavata);
System.out.println("Experiment ID : " + expId);
launchExperiment(airavata, expId);
System.out.println("Launched successfully");
@@ -202,30 +200,30 @@ public class CreateLaunchExperimentUS3 {
List<DataObjectType> exOut = new ArrayList<DataObjectType>();
DataObjectType output = new DataObjectType();
output.setKey("output");
- output.setType(DataType.URI);
- output.setValue("");
- DataObjectType output1 = new DataObjectType();
- output1.setKey("stdout");
- output1.setType(DataType.STDOUT);
- output1.setValue("");
- DataObjectType output2 = new DataObjectType();
- output2.setKey("stderr");
- output2.setType(DataType.STDERR);
- output2.setValue("");
+// output.setType(DataType.URI);
+ output.setValue("");
+// DataObjectType output1 = new DataObjectType();
+// output1.setKey("stdout");
+// output1.setType(DataType.STDOUT);
+// output1.setValue("");
+// DataObjectType output2 = new DataObjectType();
+// output2.setKey("stderr");
+// output2.setType(DataType.STDERR);
+// output2.setValue("");
exOut.add(output);
- exOut.add(output1);
- exOut.add(output2);
+// exOut.add(output1);
+// exOut.add(output2);
- Project project = ProjectModelUtil.createProject("project1", "admin", "test project");
- String projectId = client.createProject(project);
+ // Project project = ProjectModelUtil.createProject("ultrascan", "ultrascan", "test project");
+ //String projectId = client.createProject(project);
- Experiment simpleExperiment = ExperimentModelUtil.createSimpleExperiment(projectId, "admin", "US3ExperimentTrestles", "US3AppTrestles", "US3AppTrestles", exInputs);
+ Experiment simpleExperiment = ExperimentModelUtil.createSimpleExperiment("ultrascan_41574ef5-b054-4d03-ab20-2cfe768d5096", "ultrascan", "US3ExperimentTrestles", "US3AppTrestles", "ultrascan_68d397d9-ffc2-470e-bdf7-8d7b4f1cab2e", exInputs);
simpleExperiment.setExperimentOutputs(exOut);
- ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling("trestles.sdsc.edu", 2, 32, 0, "shared", 0, 0, 0, "uot111");
+ ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling("trestles.sdsc.xsede.org_fcf21cd7-d8ce-4359-bd7e-49062ce80265", 32, 2, 0, "shared", 30, 0, 0, "uot111");
UserConfigurationData userConfigurationData = new UserConfigurationData();
- scheduling.setResourceHostId("gsissh-trestles");
+ scheduling.setResourceHostId("trestles.sdsc.xsede.org_fcf21cd7-d8ce-4359-bd7e-49062ce80265");
userConfigurationData.setAiravataAutoSchedule(false);
userConfigurationData.setOverrideManualScheduledParams(false);
@@ -264,27 +262,27 @@ public class CreateLaunchExperimentUS3 {
output.setKey("output");
output.setType(DataType.URI);
output.setValue("");
- DataObjectType output1 = new DataObjectType();
- output1.setKey("stdout");
- output1.setType(DataType.STDOUT);
- output1.setValue("");
- DataObjectType output2 = new DataObjectType();
- output2.setKey("stderr");
- output2.setType(DataType.STDERR);
- output2.setValue("");
+// DataObjectType output1 = new DataObjectType();
+// output1.setKey("stdout");
+// output1.setType(DataType.STDOUT);
+// output1.setValue("");
+// DataObjectType output2 = new DataObjectType();
+// output2.setKey("stderr");
+// output2.setType(DataType.STDERR);
+// output2.setValue("");
exOut.add(output);
- exOut.add(output1);
- exOut.add(output2);
+// exOut.add(output1);
+// exOut.add(output2);
- Project project = ProjectModelUtil.createProject("project1", "admin", "test project");
- String projectId = client.createProject(project);
+// Project project = ProjectModelUtil.createProject("project1", "admin", "test project");
+// String projectId = client.createProject(project);
- Experiment simpleExperiment = ExperimentModelUtil.createSimpleExperiment(projectId, "admin", "US3ExperimentStampede", "US3AppStampede", "US3AppStampede", exInputs);
+ Experiment simpleExperiment = ExperimentModelUtil.createSimpleExperiment("ultrascan_41574ef5-b054-4d03-ab20-2cfe768d5096", "ultrascan", "US3ExperimentStampede", "US3AppStampede", "ultrascan_68d397d9-ffc2-470e-bdf7-8d7b4f1cab2e", exInputs);
simpleExperiment.setExperimentOutputs(exOut);
- ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling("stampede.tacc.xsede.org", 2, 32, 0, "normal", 0, 0, 0, "TG-MCB070039N");
+ ComputationalResourceScheduling scheduling = ExperimentModelUtil.createComputationResourceScheduling("stampede.tacc.xsede.org_7e291aa8-319a-4c70-a3b5-b2e6f91c8f5d", 16, 2, 0, "normal", 30, 0, 0, "TG-MCB070039N");
- scheduling.setResourceHostId("gsissh-stampede");
+ scheduling.setResourceHostId("stampede.tacc.xsede.org_7e291aa8-319a-4c70-a3b5-b2e6f91c8f5d");
UserConfigurationData userConfigurationData = new UserConfigurationData();
userConfigurationData.setAiravataAutoSchedule(false);
http://git-wip-us.apache.org/repos/asf/airavata/blob/0f10016d/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterUS3Application.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterUS3Application.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterUS3Application.java
new file mode 100644
index 0000000..1339e29
--- /dev/null
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterUS3Application.java
@@ -0,0 +1,244 @@
+package org.apache.airavata.client.tools;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.airavata.api.Airavata;
+import org.apache.airavata.api.client.AiravataClientFactory;
+import org.apache.airavata.model.appcatalog.appdeployment.ApplicationParallelismType;
+import org.apache.airavata.model.appcatalog.appinterface.DataType;
+import org.apache.airavata.model.appcatalog.appinterface.InputDataObjectType;
+import org.apache.airavata.model.appcatalog.appinterface.OutputDataObjectType;
+import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
+import org.apache.airavata.model.appcatalog.computeresource.JobManagerCommand;
+import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManager;
+import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManagerType;
+import org.apache.airavata.model.appcatalog.computeresource.SCPDataMovement;
+import org.apache.airavata.model.appcatalog.computeresource.SSHJobSubmission;
+import org.apache.airavata.model.appcatalog.computeresource.SecurityProtocol;
+import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference;
+import org.apache.airavata.model.appcatalog.gatewayprofile.GatewayResourceProfile;
+import org.apache.airavata.model.error.AiravataClientException;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class RegisterUS3Application {
+
+ private final static Logger logger = LoggerFactory.getLogger(RegisterSampleApplications.class);
+
+ public static final String THRIFT_SERVER_HOST = "gridfarm005.ucs.indiana.edu";
+ public static final int THRIFT_SERVER_PORT = 8930;
+ private static final String DEFAULT_GATEWAY = "default";
+ private static Airavata.Client airavataClient;
+
+ //Host Id's
+ private static String stampedeResourceId = "stampede.tacc.xsede.org_7e291aa8-319a-4c70-a3b5-b2e6f91c8f5d";
+ private static String trestlesResourceId = "trestles.sdsc.xsede.org_fcf21cd7-d8ce-4359-bd7e-49062ce80265";
+ private static String lonestarResourceId = "lonestar.tacc.teragrid.org_fe9b698a-4f68-459b-acc4-09c457a769b2";
+ private static String alamoResourceId = "alamo.uthscsa.edu_2ff1f99d-565a-48c6-9f7d-a2dc5019d238";
+
+ private static String ultrascanModuleId = "ultrascan_c393abae-f75d-442f-8ad3-f0b59376131a";
+ private static String ultrascanAppId = "ultrascan_68d397d9-ffc2-470e-bdf7-8d7b4f1cab2e";
+
+ public static void main(String[] args) {
+ try {
+ airavataClient = AiravataClientFactory.createAiravataClient(THRIFT_SERVER_HOST, THRIFT_SERVER_PORT);
+ System.out.println("API version is " + airavataClient.getAPIVersion());
+
+ //Register all compute hosts
+// registerXSEDEHosts();
+
+ //Register module
+// registerAppModules();
+
+ //Register Application
+// registerUltrascanInterface();
+
+ //Register Deployment
+// registerApplicationDeployment();
+ registerGatewayResourceProfile();
+
+ } catch (Exception e) {
+ logger.error("Error while connecting with server", e.getMessage());
+ e.printStackTrace();
+ }
+
+ }
+ public static void registerXSEDEHosts() {
+ try {
+ System.out.println("\n #### Registering XSEDE Computational Resources #### \n");
+
+ //Register Stampede
+ stampedeResourceId = registerComputeHost("stampede.tacc.xsede.org", "TACC Stampede Cluster",
+ ResourceJobManagerType.SLURM, "push", "/usr/bin", SecurityProtocol.GSI, 2222, null);
+ System.out.println("Stampede Resource Id is " + stampedeResourceId);
+
+ //Register Trestles
+ trestlesResourceId = registerComputeHost("trestles.sdsc.xsede.org", "SDSC Trestles Cluster",
+ ResourceJobManagerType.PBS, "push", "/opt/torque/bin/", SecurityProtocol.GSI, 22, null);
+ System.out.println("Trestles Resource Id is " + trestlesResourceId);
+
+ //Register Lonestar
+ lonestarResourceId = registerComputeHost("lonestar.tacc.teragrid.org", "TACC Lonestar Cluster",
+ ResourceJobManagerType.UGE, "push", "/opt/torque/bin/", SecurityProtocol.GSI, 22, null);
+ System.out.println("Lonestar Resource Id is " + lonestarResourceId);
+
+ //Register Alamo
+ alamoResourceId = registerComputeHost("alamo.uthscsa.edu", "Alamo Cluster",
+ ResourceJobManagerType.PBS, "push", "/opt/torque/bin/", SecurityProtocol.SSH_KEYS, 22, null);
+ System.out.println("Alamo Cluster " + alamoResourceId);
+
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+
+ }
+ public static void registerAppModules() {
+ try {
+ System.out.println("\n #### Registering Application Modules #### \n");
+
+ //Register Echo
+ ultrascanModuleId = airavataClient.registerApplicationModule(
+ RegisterSampleApplicationsUtils.createApplicationModule(
+ "ultrascan", "1.0", "ultrascan application"));
+ System.out.println("Ultrascan Module Id " + ultrascanModuleId);
+ }catch(Exception e){
+ e.getLocalizedMessage();
+ }
+ }
+ public static void registerUltrascanInterface() {
+ try {
+ System.out.println("#### Registering WRF Interface #### \n");
+
+ List<String> appModules = new ArrayList<String>();
+ appModules.add(ultrascanModuleId);
+
+ InputDataObjectType input1 = RegisterSampleApplicationsUtils.createAppInput("input", null,
+ DataType.URI, null, false, "input tar file", null);
+
+ List<InputDataObjectType> applicationInputs = new ArrayList<InputDataObjectType>();
+ applicationInputs.add(input1);
+
+ OutputDataObjectType output1 = RegisterSampleApplicationsUtils.createAppOutput("output",
+ "", DataType.URI);
+
+// OutputDataObjectType output2 = RegisterSampleApplicationsUtils.createAppOutput("stdout",
+// "", DataType.URI);
+// OutputDataObjectType output3 = RegisterSampleApplicationsUtils.createAppOutput("stderr",
+// "", DataType.URI);
+
+ List<OutputDataObjectType> applicationOutputs = new ArrayList<OutputDataObjectType>();
+ applicationOutputs.add(output1);
+
+ ultrascanAppId = airavataClient.registerApplicationInterface(
+ RegisterSampleApplicationsUtils.createApplicationInterfaceDescription("ultrascan", "ultrascan application",
+ appModules, applicationInputs, applicationOutputs));
+ System.out.println("Ultrascan Application Interface Id " + ultrascanAppId);
+
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+ }
+
+ public static void registerApplicationDeployment() {
+ try {
+ System.out.println("#### Registering Application Deployments on Stampede #### \n");
+
+ // Register Stampede
+ String ultascanStamplede = airavataClient.registerApplicationDeployment(RegisterSampleApplicationsUtils.createApplicationDeployment(ultrascanModuleId,
+ stampedeResourceId, "/home1/01623/us3/bin/us_mpi_analysis", ApplicationParallelismType.MPI, "ultrascan application"));
+ System.out.println("Ultrascan on stampede deployment Id " + ultascanStamplede);
+
+ String ultascanTrestles = airavataClient.registerApplicationDeployment(RegisterSampleApplicationsUtils.createApplicationDeployment(ultrascanModuleId,
+ trestlesResourceId, "/home/us3/trestles/bin/us_mpi_analysis", ApplicationParallelismType.MPI, "ultrascan application"));
+ System.out.println("Ultrascan on trestles deployment Id " + ultascanTrestles);
+ String ultascanLonestar = airavataClient.registerApplicationDeployment(RegisterSampleApplicationsUtils.createApplicationDeployment(ultrascanModuleId,
+ lonestarResourceId, "/home1/01623/us3/bin/us_mpi_analysis", ApplicationParallelismType.MPI, "ultrascan application"));
+ System.out.println("Ultrascan on lonestar deployment Id " + ultascanLonestar);
+ String ultascanAlamo = airavataClient.registerApplicationDeployment(RegisterSampleApplicationsUtils.createApplicationDeployment(ultrascanModuleId,
+ alamoResourceId, "/home/us3/bin/us_mpi_analysis.sh", ApplicationParallelismType.MPI, "ultrascan application"));
+ System.out.println("Ultrascan on alamo deployment Id " + ultascanAlamo);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ public static void registerGatewayResourceProfile() {
+
+ try {
+ System.out.println("#### Registering Gateway proflie #### \n");
+
+
+ ComputeResourcePreference stampedeResourcePreferences = RegisterSampleApplicationsUtils.
+ createComputeResourcePreference(stampedeResourceId, "TG-MCB070039N", false, null, null, null,
+ "/scratch/01623/us3/jobs/");
+
+ ComputeResourcePreference trestlesResourcePreferences = RegisterSampleApplicationsUtils.
+ createComputeResourcePreference(trestlesResourceId, "uot111", false, null, null, null,
+ "/oasis/projects/nsf/uot111/us3/airavata-workdirs/");
+
+ ComputeResourcePreference lonestarResourcePreferences = RegisterSampleApplicationsUtils.
+ createComputeResourcePreference(lonestarResourceId, "ULTRASCAN", false, null, null, null,
+ "/scratch/01623/us3/airavata/");
+
+ ComputeResourcePreference alamoResourcePreferences = RegisterSampleApplicationsUtils.
+ createComputeResourcePreference(lonestarResourceId, null, false, null, null, null,
+ "/mnt/glusterfs/work/");
+
+ GatewayResourceProfile gatewayResourceProfile = new GatewayResourceProfile();
+ gatewayResourceProfile.setGatewayID(DEFAULT_GATEWAY);
+ gatewayResourceProfile.setGatewayName(DEFAULT_GATEWAY);
+ gatewayResourceProfile.addToComputeResourcePreferences(stampedeResourcePreferences);
+ gatewayResourceProfile.addToComputeResourcePreferences(trestlesResourcePreferences);
+ gatewayResourceProfile.addToComputeResourcePreferences(lonestarResourcePreferences);
+ gatewayResourceProfile.addToComputeResourcePreferences(alamoResourcePreferences);
+
+ String gatewayProfile = airavataClient.registerGatewayResourceProfile(gatewayResourceProfile);
+ System.out.println("Gateway Profile is registered with Id " + gatewayProfile);
+
+ } catch (TException e) {
+ e.printStackTrace();
+ }
+ }
+ public static String registerComputeHost(String hostName, String hostDesc, ResourceJobManagerType resourceJobManagerType, String monitoringEndPoint,
+ String jobMangerBinPath, SecurityProtocol securityProtocol, int portNumber, String jobManagerCommand) throws TException {
+
+ ComputeResourceDescription computeResourceDescription = RegisterSampleApplicationsUtils
+ .createComputeResourceDescription(hostName, hostDesc, null, null);
+
+ String computeResourceId = airavataClient.registerComputeResource(computeResourceDescription);
+
+ if (computeResourceId.isEmpty())
+ throw new AiravataClientException();
+
+ ResourceJobManager resourceJobManager = RegisterSampleApplicationsUtils.createResourceJobManager(resourceJobManagerType, monitoringEndPoint,
+ jobMangerBinPath, null);
+
+ if (jobManagerCommand != null) {
+ Map<JobManagerCommand, String> jobManagerCommandStringMap = new HashMap<JobManagerCommand, String>();
+ jobManagerCommandStringMap.put(JobManagerCommand.SUBMISSION, jobManagerCommand);
+ resourceJobManager.setJobManagerCommands(jobManagerCommandStringMap);
+ }
+
+ SSHJobSubmission sshJobSubmission = new SSHJobSubmission();
+ sshJobSubmission.setResourceJobManager(resourceJobManager);
+ sshJobSubmission.setSecurityProtocol(securityProtocol);
+ sshJobSubmission.setSshPort(portNumber);
+ boolean sshAddStatus = airavataClient.addSSHJobSubmissionDetails(computeResourceId, 1, sshJobSubmission);
+
+ if (!sshAddStatus)
+ throw new AiravataClientException();
+
+ SCPDataMovement scpDataMovement = new SCPDataMovement();
+ scpDataMovement.setSecurityProtocol(securityProtocol);
+ scpDataMovement.setSshPort(portNumber);
+ boolean scpAddStatus = airavataClient.addSCPDataMovementDetails(computeResourceId, 1, scpDataMovement);
+
+ if (!scpAddStatus)
+ throw new AiravataClientException();
+
+ return computeResourceId;
+ }
+}